ipn/ipnserver, log/filelogger, logpolicy: when tailscaled is running as a windows service, ensure the service's log messages are written to file

This patch moves the Windows-only initialization of the filelogger into logpolicy.
Previously we only did it when babysitting the tailscaled subprocess, but this meant
that log messages from the service itself never made it to disk. This meant that
if logtail could not dial out, its log messages would be lost.

I modified filelogger.New to work a bit differently and added a `maybeWrapForPlatform`
to logpolicy to ensure that the filelogger is plugged in front of logtail ASAP.

Fixes https://github.com/tailscale/tailscale/issues/3570

Signed-off-by: Aaron Klotz <aaron@tailscale.com>

.gitattributes

@@ -1 +1,2 @@
 go.mod filter=go-mod
+*.go  diff=golang

.github/ISSUE_TEMPLATE/config.yml

@@ -1,5 +1,8 @@
 blank_issues_enabled: true
 contact_links:
-  - name: Support requests and Troubleshooting
+  - name: Support
+    url: https://tailscale.com/contact/support/
+    about: Contact us for support
+  - name: Troubleshooting
     url: https://tailscale.com/kb/1023/troubleshooting
-    about: Troubleshoot common issues. Contact us by email at support@tailscale.com.
+    about: Troubleshoot common issues

.github/ISSUE_TEMPLATE/feature_request.yml

@@ -7,14 +7,7 @@ body:
     attributes:
       value: |
         Please check if your feature request is [already filed](https://github.com/tailscale/tailscale/issues).
-  - type: input
-    id: request
-    attributes:
-      label: Tell us about your idea!
-      description: What is your feature request?
-      placeholder: e.g., A pet pangolin
-    validations:
-      required: true
+        Tell us about your idea!
   - type: textarea
     id: problem
     attributes:

.github/dependabot.yml

@@ -2,15 +2,20 @@
 # https://docs.github.com/en/code-security/supply-chain-security/keeping-your-dependencies-updated-automatically/configuration-options-for-dependency-updates
 version: 2
 updates:
-  - package-ecosystem: "gomod"
-    directory: "/"
-    schedule:
-      interval: "daily"
-    commit-message:
-      prefix: "go.mod:"
+  ## Disabled between releases. We reenable it briefly after every
+  ## stable release, pull in all changes, and close it again so that
+  ## the tree remains more stable during development and the upstream
+  ## changes have time to soak before the next release.
+  # - package-ecosystem: "gomod"
+  #   directory: "/"
+  #   schedule:
+  #     interval: "daily"
+  #   commit-message:
+  #     prefix: "go.mod:"
+  #   open-pull-requests-limit: 100
   - package-ecosystem: "github-actions"
     directory: "/"
     schedule:
-      interval: "daily"
+      interval: "weekly"
     commit-message:
       prefix: ".github:"

.github/workflows/cifuzz.yml

@@ -0,0 +1,26 @@
+name: CIFuzz
+on: [pull_request]
+jobs:
+  Fuzzing:
+    runs-on: ubuntu-latest
+    steps:
+    - name: Build Fuzzers
+      id: build
+      uses: google/oss-fuzz/infra/cifuzz/actions/build_fuzzers@master
+      with:
+        oss-fuzz-project-name: 'tailscale'
+        dry-run: false
+        language: go
+    - name: Run Fuzzers
+      uses: google/oss-fuzz/infra/cifuzz/actions/run_fuzzers@master
+      with:
+        oss-fuzz-project-name: 'tailscale'
+        fuzz-seconds: 300
+        dry-run: false
+        language: go
+    - name: Upload Crash
+      uses: actions/upload-artifact@v2.2.4
+      if: failure() && steps.build.outcome == 'success'
+      with:
+        name: artifacts
+        path: ./out/artifacts

.github/workflows/linux-race.yml

@@ -31,6 +31,21 @@ jobs:
     - name: Run tests and benchmarks with -race flag on linux
       run: go test -race -bench=. -benchtime=1x ./...
 
+    - name: Check that no tracked files in the repo have been modified
+      run: git diff --no-ext-diff --name-only --exit-code || (echo "Build/test modified the files above."; exit 1)
+
+    - name: Check that no files have been added to the repo
+      run: |
+        # Note: The "error: pathspec..." you see below is normal!
+        # In the success case in which there are no new untracked files,
+        # git ls-files complains about the pathspec not matching anything.
+        # That's OK. It's not worth the effort to suppress. Please ignore it.
+        if git ls-files --others --exclude-standard --directory --no-empty-directory --error-unmatch -- ':/*'
+        then
+          echo "Build/test created untracked files in the repo (file names above)."
+          exit 1
+        fi
+
     - uses: k0kubun/action-slack@v2.0.0
       with:
         payload: |

.github/workflows/linux.yml

@@ -28,9 +28,33 @@ jobs:
     - name: Basic build
       run: go build ./cmd/...
 
+    - name: Get QEMU
+      run: |
+        # The qemu in Ubuntu 20.04 (Focal) is too old; we need 5.x something
+        # to run Go binaries. 5.2.0 (Debian bullseye) empirically works, and
+        # use this PPA which brings in a modern qemu.
+        sudo add-apt-repository -y ppa:jacob/virtualisation
+        sudo apt-get -y update
+        sudo apt-get -y install qemu-user
+
     - name: Run tests on linux
       run: go test -bench=. -benchtime=1x ./...
 
+    - name: Check that no tracked files in the repo have been modified
+      run: git diff --no-ext-diff --name-only --exit-code || (echo "Build/test modified the files above."; exit 1)
+
+    - name: Check that no files have been added to the repo
+      run: |
+        # Note: The "error: pathspec..." you see below is normal!
+        # In the success case in which there are no new untracked files,
+        # git ls-files complains about the pathspec not matching anything.
+        # That's OK. It's not worth the effort to suppress. Please ignore it.
+        if git ls-files --others --exclude-standard --directory --no-empty-directory --error-unmatch -- ':/*'
+        then
+          echo "Build/test created untracked files in the repo (file names above)."
+          exit 1
+        fi
+
     - uses: k0kubun/action-slack@v2.0.0
       with:
         payload: |

.github/workflows/linux32.yml

@@ -31,6 +31,21 @@ jobs:
     - name: Run tests on linux
       run: GOARCH=386 go test -bench=. -benchtime=1x ./...
 
+    - name: Check that no tracked files in the repo have been modified
+      run: git diff --no-ext-diff --name-only --exit-code || (echo "Build/test modified the files above."; exit 1)
+
+    - name: Check that no files have been added to the repo
+      run: |
+        # Note: The "error: pathspec..." you see below is normal!
+        # In the success case in which there are no new untracked files,
+        # git ls-files complains about the pathspec not matching anything.
+        # That's OK. It's not worth the effort to suppress. Please ignore it.
+        if git ls-files --others --exclude-standard --directory --no-empty-directory --error-unmatch -- ':/*'
+        then
+          echo "Build/test created untracked files in the repo (file names above)."
+          exit 1
+        fi
+
     - uses: k0kubun/action-slack@v2.0.0
       with:
         payload: |

Dockerfile

@@ -48,13 +48,13 @@ ARG VERSION_SHORT=""
 ENV VERSION_SHORT=$VERSION_SHORT
 ARG VERSION_GIT_HASH=""
 ENV VERSION_GIT_HASH=$VERSION_GIT_HASH
+ARG TARGETARCH
 
-RUN go install -tags=xversion -ldflags="\
+RUN GOARCH=$TARGETARCH go install -tags=xversion -ldflags="\
       -X tailscale.com/version.Long=$VERSION_LONG \
       -X tailscale.com/version.Short=$VERSION_SHORT \
       -X tailscale.com/version.GitCommit=$VERSION_GIT_HASH" \
       -v ./cmd/tailscale ./cmd/tailscaled
 
-FROM alpine:3.14
-RUN apk add --no-cache ca-certificates iptables iproute2 ip6tables
+FROM ghcr.io/tailscale/alpine-base:3.14
 COPY --from=build-env /go/bin/* /usr/local/bin/

Dockerfile.base

@@ -0,0 +1,6 @@
+# Copyright (c) 2021 Tailscale Inc & AUTHORS All rights reserved.
+# Use of this source code is governed by a BSD-style
+# license that can be found in the LICENSE file.
+
+FROM alpine:3.14
+RUN apk add --no-cache ca-certificates iptables iproute2 ip6tables

Makefile

@@ -1,3 +1,5 @@
+IMAGE_REPO ?= tailscale/tailscale
+
 usage:
 	echo "See Makefile"
 
@@ -21,7 +23,18 @@ build386:
 buildlinuxarm:
 	GOOS=linux GOARCH=arm go install tailscale.com/cmd/tailscale tailscale.com/cmd/tailscaled
 
+buildmultiarchimage:
+	./build_docker.sh
+
 check: staticcheck vet depaware buildwindows build386 buildlinuxarm
 
 staticcheck:
 	go run honnef.co/go/tools/cmd/staticcheck -- $$(go list ./... | grep -v tempfork)
+
+spk:
+	go run github.com/tailscale/tailscale-synology@main --version=build -o tailscale.spk --source=.
+
+pushspk: spk
+	echo "Pushing SPKG to root@${SYNOHOST} (env var SYNOHOST) ..."
+	scp tailscale.spk root@${SYNOHOST}:
+	ssh root@${SYNOHOST} /usr/syno/bin/synopkg install tailscale.spk

README.md

@@ -8,11 +8,12 @@ Private WireGuard® networks made easy
 
 This repository contains all the open source Tailscale client code and
 the `tailscaled` daemon and `tailscale` CLI tool. The `tailscaled`
-daemon runs primarily on Linux; it also works to varying degrees on
-FreeBSD, OpenBSD, Darwin, and Windows.
+daemon runs on Linux, Windows and [macOS](https://tailscale.com/kb/1065/macos-variants/), and to varying degrees on FreeBSD, OpenBSD, and Darwin. (The Tailscale iOS and Android apps use this repo's code, but this repo doesn't contain the mobile GUI code.)
 
 The Android app is at https://github.com/tailscale/tailscale-android
 
+The Synology package is at https://github.com/tailscale/tailscale-synology
+
 ## Using
 
 We serve packages for a variety of distros at

VERSION.txt

@@ -1 +1 @@
-1.17.0
+1.19.0

build_dist.sh

@@ -30,12 +30,14 @@ else
 fi
 
 long_suffix="$change_suffix-t$short_hash"
-SHORT="$major.$minor.$patch"
+MINOR="$major.$minor"
+SHORT="$MINOR.$patch"
 LONG="${SHORT}$long_suffix"
 GIT_HASH="$git_hash"
 
 if [ "$1" = "shellvars" ]; then
 	cat <<EOF
+VERSION_MINOR="$MINOR"
 VERSION_SHORT="$SHORT"
 VERSION_LONG="$LONG"
 VERSION_GIT_HASH="$GIT_HASH"

build_docker.sh

@@ -21,8 +21,15 @@ set -eu
 
 eval $(./build_dist.sh shellvars)
 
-docker build \
-  --build-arg VERSION_LONG=$VERSION_LONG \
-  --build-arg VERSION_SHORT=$VERSION_SHORT \
-  --build-arg VERSION_GIT_HASH=$VERSION_GIT_HASH \
-  -t tailscale:$VERSION_SHORT -t tailscale:latest .
+go run github.com/tailscale/mkctr@latest \
+  --base="ghcr.io/tailscale/alpine-base:3.14" \
+  --gopaths="\
+    tailscale.com/cmd/tailscale:/usr/local/bin/tailscale, \
+    tailscale.com/cmd/tailscaled:/usr/local/bin/tailscaled" \
+  --ldflags="\
+    -X tailscale.com/version.Long=${VERSION_LONG} \
+    -X tailscale.com/version.Short=${VERSION_SHORT} \
+    -X tailscale.com/version.GitCommit=${VERSION_GIT_HASH}" \
+  --tags="v${VERSION_SHORT},v${VERSION_MINOR}" \
+  --repos="tailscale/tailscale,ghcr.io/tailscale/tailscale" \
+  --push

client/tailscale/tailscale.go

@@ -38,6 +38,9 @@ var (
 	// TailscaledSocket is the tailscaled Unix socket. It's used by the TailscaledDialer.
 	TailscaledSocket = paths.DefaultTailscaledSocket()
 
+	// TailscaledSocketSetExplicitly reports whether the user explicitly set TailscaledSocket.
+	TailscaledSocketSetExplicitly bool
+
 	// TailscaledDialer is the DialContext func that connects to the local machine's
 	// tailscaled or equivalent.
 	TailscaledDialer = defaultDialer
@@ -47,7 +50,8 @@ func defaultDialer(ctx context.Context, network, addr string) (net.Conn, error)
 	if addr != "local-tailscaled.sock:80" {
 		return nil, fmt.Errorf("unexpected URL address %q", addr)
 	}
-	if TailscaledSocket == paths.DefaultTailscaledSocket() {
+	// TODO: make this part of a safesocket.ConnectionStrategy
+	if !TailscaledSocketSetExplicitly {
 		// On macOS, when dialing from non-sandboxed program to sandboxed GUI running
 		// a TCP server on a random port, find the random port. For HTTP connections,
 		// we don't send the token. It gets added in an HTTP Basic-Auth header.
@@ -56,7 +60,11 @@ func defaultDialer(ctx context.Context, network, addr string) (net.Conn, error)
 			return d.DialContext(ctx, "tcp", "localhost:"+strconv.Itoa(port))
 		}
 	}
-	return safesocket.Connect(TailscaledSocket, 41112)
+	s := safesocket.DefaultConnectionStrategy(TailscaledSocket)
+	// The user provided a non-default tailscaled socket address.
+	// Connect only to exactly what they provided.
+	s.UseFallback(false)
+	return safesocket.Connect(s)
 }
 
 var (
@@ -90,6 +98,27 @@ func DoLocalRequest(req *http.Request) (*http.Response, error) {
 	return tsClient.Do(req)
 }
 
+func doLocalRequestNiceError(req *http.Request) (*http.Response, error) {
+	res, err := DoLocalRequest(req)
+	if err == nil {
+		if server := res.Header.Get("Tailscale-Version"); server != "" && server != version.Long && onVersionMismatch != nil {
+			onVersionMismatch(version.Long, server)
+		}
+		return res, nil
+	}
+	if ue, ok := err.(*url.Error); ok {
+		if oe, ok := ue.Err.(*net.OpError); ok && oe.Op == "dial" {
+			path := req.URL.Path
+			pathPrefix := path
+			if i := strings.Index(path, "?"); i != -1 {
+				pathPrefix = path[:i]
+			}
+			return nil, fmt.Errorf("Failed to connect to local Tailscale daemon for %s; %s Error: %w", pathPrefix, tailscaledConnectHint(), oe)
+		}
+	}
+	return nil, err
+}
+
 type errorJSON struct {
 	Error string
 }
@@ -140,23 +169,11 @@ func send(ctx context.Context, method, path string, wantStatus int, body io.Read
 	if err != nil {
 		return nil, err
 	}
-	res, err := DoLocalRequest(req)
+	res, err := doLocalRequestNiceError(req)
 	if err != nil {
-		if ue, ok := err.(*url.Error); ok {
-			if oe, ok := ue.Err.(*net.OpError); ok && oe.Op == "dial" {
-				pathPrefix := path
-				if i := strings.Index(path, "?"); i != -1 {
-					pathPrefix = path[:i]
-				}
-				return nil, fmt.Errorf("Failed to connect to local Tailscale daemon for %s; %s Error: %w", pathPrefix, tailscaledConnectHint(), oe)
-			}
-		}
 		return nil, err
 	}
 	defer res.Body.Close()
-	if server := res.Header.Get("Tailscale-Version"); server != "" && server != version.Long && onVersionMismatch != nil {
-		onVersionMismatch(version.Long, server)
-	}
 	slurp, err := ioutil.ReadAll(res.Body)
 	if err != nil {
 		return nil, err
@@ -196,6 +213,12 @@ func Goroutines(ctx context.Context) ([]byte, error) {
 	return get200(ctx, "/localapi/v0/goroutines")
 }
 
+// DaemonMetrics returns the Tailscale daemon's metrics in
+// the Prometheus text exposition format.
+func DaemonMetrics(ctx context.Context) ([]byte, error) {
+	return get200(ctx, "/localapi/v0/metrics")
+}
+
 // Profile returns a pprof profile of the Tailscale daemon.
 func Profile(ctx context.Context, pprofType string, sec int) ([]byte, error) {
 	var secArg string
@@ -222,7 +245,7 @@ func Status(ctx context.Context) (*ipnstate.Status, error) {
 	return status(ctx, "")
 }
 
-// StatusWithPeers returns the Tailscale daemon's status, without the peer info.
+// StatusWithoutPeers returns the Tailscale daemon's status, without the peer info.
 func StatusWithoutPeers(ctx context.Context) (*ipnstate.Status, error) {
 	return status(ctx, "?peers=false")
 }
@@ -289,6 +312,30 @@ func FileTargets(ctx context.Context) ([]apitype.FileTarget, error) {
 	return fts, nil
 }
 
+// PushFile sends Taildrop file r to target.
+//
+// A size of -1 means unknown.
+// The name parameter is the original filename, not escaped.
+func PushFile(ctx context.Context, target tailcfg.StableNodeID, size int64, name string, r io.Reader) error {
+	req, err := http.NewRequestWithContext(ctx, "PUT", "http://local-tailscaled.sock/localapi/v0/file-put/"+string(target)+"/"+url.PathEscape(name), r)
+	if err != nil {
+		return err
+	}
+	if size != -1 {
+		req.ContentLength = size
+	}
+	res, err := doLocalRequestNiceError(req)
+	if err != nil {
+		return err
+	}
+	if res.StatusCode == 200 {
+		io.Copy(io.Discard, res.Body)
+		return nil
+	}
+	all, _ := io.ReadAll(res.Body)
+	return fmt.Errorf("%s: %s", res.Status, all)
+}
+
 func CheckIPForwarding(ctx context.Context) error {
 	body, err := get200(ctx, "/localapi/v0/check-ip-forwarding")
 	if err != nil {

cmd/derper/derper.go

@@ -12,6 +12,7 @@ import (
 	"errors"
 	"expvar"
 	"flag"
+	"fmt"
 	"io"
 	"io/ioutil"
 	"log"
@@ -36,6 +37,7 @@ import (
 var (
 	dev           = flag.Bool("dev", false, "run in localhost development mode")
 	addr          = flag.String("a", ":443", "server address")
+	httpPort      = flag.Int("http-port", 80, "The port on which to serve HTTP. Set to -1 to disable")
 	configPath    = flag.String("c", "", "config file path")
 	certMode      = flag.String("certmode", "letsencrypt", "mode for getting a cert. possible options: manual, letsencrypt")
 	certDir       = flag.String("certdir", tsweb.DefaultCertDir("derper-certs"), "directory to store LetsEncrypt certs, if addr's port is :443")
@@ -141,7 +143,7 @@ func main() {
 
 	cfg := loadConfig()
 
-	serveTLS := tsweb.IsProd443(*addr)
+	serveTLS := tsweb.IsProd443(*addr) || *certMode == "manual"
 
 	s := derp.NewServer(cfg.PrivateKey, log.Printf)
 	s.SetVerifyClient(*verifyClients)
@@ -235,24 +237,41 @@ func main() {
 			cert.Certificate = append(cert.Certificate, s.MetaCert())
 			return cert, nil
 		}
-		go func() {
-			port80srv := &http.Server{
-				Addr:        net.JoinHostPort(listenHost, "80"),
-				Handler:     certManager.HTTPHandler(tsweb.Port80Handler{Main: mux}),
-				ReadTimeout: 30 * time.Second,
-				// Crank up WriteTimeout a bit more than usually
-				// necessary just so we can do long CPU profiles
-				// and not hit net/http/pprof's "profile
-				// duration exceeds server's WriteTimeout".
-				WriteTimeout: 5 * time.Minute,
-			}
-			err := port80srv.ListenAndServe()
-			if err != nil {
-				if err != http.ErrServerClosed {
-					log.Fatal(err)
+		httpsrv.Handler = http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+			// Set HTTP headers to appease automated security scanners.
+			//
+			// Security automation gets cranky when HTTPS sites don't
+			// set HSTS, and when they don't specify a content
+			// security policy for XSS mitigation.
+			//
+			// DERP's HTTP interface is only ever used for debug
+			// access (for which trivial safe policies work just
+			// fine), and by DERP clients which don't obey any of
+			// these browser-centric headers anyway.
+			w.Header().Set("Strict-Transport-Security", "max-age=63072000; includeSubDomains")
+			w.Header().Set("Content-Security-Policy", "default-src 'none'; frame-ancestors 'none'; form-action 'none'; base-uri 'self'; block-all-mixed-content; plugin-types 'none'")
+			mux.ServeHTTP(w, r)
+		})
+		if *httpPort > -1 {
+			go func() {
+				port80srv := &http.Server{
+					Addr:        net.JoinHostPort(listenHost, fmt.Sprintf("%d", *httpPort)),
+					Handler:     certManager.HTTPHandler(tsweb.Port80Handler{Main: mux}),
+					ReadTimeout: 30 * time.Second,
+					// Crank up WriteTimeout a bit more than usually
+					// necessary just so we can do long CPU profiles
+					// and not hit net/http/pprof's "profile
+					// duration exceeds server's WriteTimeout".
+					WriteTimeout: 5 * time.Minute,
 				}
-			}
-		}()
+				err := port80srv.ListenAndServe()
+				if err != nil {
+					if err != http.ErrServerClosed {
+						log.Fatal(err)
+					}
+				}
+			}()
+		}
 		err = httpsrv.ListenAndServeTLS("", "")
 	} else {
 		log.Printf("derper: serving on %s", *addr)

cmd/tailscale/cli/cli.go

@@ -164,6 +164,11 @@ change in the future.
 	}
 
 	tailscale.TailscaledSocket = rootArgs.socket
+	rootfs.Visit(func(f *flag.Flag) {
+		if f.Name == "socket" {
+			tailscale.TailscaledSocketSetExplicitly = true
+		}
+	})
 
 	err := rootCmd.Run(context.Background())
 	if errors.Is(err, flag.ErrHelp) {
@@ -191,7 +196,8 @@ var rootArgs struct {
 var gotSignal syncs.AtomicBool
 
 func connect(ctx context.Context) (net.Conn, *ipn.BackendClient, context.Context, context.CancelFunc) {
-	c, err := safesocket.Connect(rootArgs.socket, 41112)
+	s := safesocket.DefaultConnectionStrategy(rootArgs.socket)
+	c, err := safesocket.Connect(s)
 	if err != nil {
 		if runtime.GOOS != "windows" && rootArgs.socket == "" {
 			fatalf("--socket cannot be empty")

cmd/tailscale/cli/cli_test.go

@@ -18,8 +18,10 @@ import (
 	"tailscale.com/ipn"
 	"tailscale.com/ipn/ipnstate"
 	"tailscale.com/tstest"
+	"tailscale.com/types/key"
 	"tailscale.com/types/persist"
 	"tailscale.com/types/preftype"
+	"tailscale.com/version/distro"
 )
 
 // geese is a collection of gooses. It need not be complete.
@@ -57,6 +59,7 @@ func TestCheckForAccidentalSettingReverts(t *testing.T) {
 		curExitNodeIP netaddr.IP
 		curUser       string // os.Getenv("USER") on the client side
 		goos          string // empty means "linux"
+		distro        distro.Distro
 
 		want string
 	}{
@@ -313,6 +316,7 @@ func TestCheckForAccidentalSettingReverts(t *testing.T) {
 				ControlURL:       ipn.DefaultControlURL,
 				AllowSingleHosts: true,
 				CorpDNS:          true,
+				RouteAll:         true,
 
 				// And assume this no-op accidental pre-1.8 value:
 				NoSNAT: true,
@@ -329,7 +333,7 @@ func TestCheckForAccidentalSettingReverts(t *testing.T) {
 
 				NetfilterMode: preftype.NetfilterNoDivert, // we never had this bug, but pretend it got set non-zero on Windows somehow
 			},
-			goos: "windows",
+			goos: "openbsd",
 			want: "", // not an error
 		},
 		{
@@ -405,6 +409,21 @@ func TestCheckForAccidentalSettingReverts(t *testing.T) {
 			},
 			want: accidentalUpPrefix + " --hostname=foo --exit-node=100.64.5.7",
 		},
+		{
+			name:          "error_exit_node_and_allow_lan_omit_with_id_pref", // Isue 3480
+			flags:         []string{"--hostname=foo"},
+			curExitNodeIP: netaddr.MustParseIP("100.2.3.4"),
+			curPrefs: &ipn.Prefs{
+				ControlURL:       ipn.DefaultControlURL,
+				AllowSingleHosts: true,
+				CorpDNS:          true,
+				NetfilterMode:    preftype.NetfilterOn,
+
+				ExitNodeAllowLANAccess: true,
+				ExitNodeID:             "some_stable_id",
+			},
+			want: accidentalUpPrefix + " --hostname=foo --exit-node-allow-lan-access --exit-node=100.2.3.4",
+		},
 		{
 			name:  "ignore_login_server_synonym",
 			flags: []string{"--login-server=https://controlplane.tailscale.com"},
@@ -427,6 +446,38 @@ func TestCheckForAccidentalSettingReverts(t *testing.T) {
 			},
 			want: accidentalUpPrefix + " --netfilter-mode=off --accept-dns=false",
 		},
+		{
+			// Issue 3176: on Synology, don't require --accept-routes=false because user
+			// migth've had old an install, and we don't support --accept-routes anyway.
+			name:  "synology_permit_omit_accept_routes",
+			flags: []string{"--hostname=foo"},
+			curPrefs: &ipn.Prefs{
+				ControlURL:       "https://login.tailscale.com",
+				CorpDNS:          true,
+				AllowSingleHosts: true,
+				RouteAll:         true,
+				NetfilterMode:    preftype.NetfilterOn,
+			},
+			goos:   "linux",
+			distro: distro.Synology,
+			want:   "",
+		},
+		{
+			// Same test case as "synology_permit_omit_accept_routes" above, but
+			// on non-Synology distro.
+			name:  "not_synology_dont_permit_omit_accept_routes",
+			flags: []string{"--hostname=foo"},
+			curPrefs: &ipn.Prefs{
+				ControlURL:       "https://login.tailscale.com",
+				CorpDNS:          true,
+				AllowSingleHosts: true,
+				RouteAll:         true,
+				NetfilterMode:    preftype.NetfilterOn,
+			},
+			goos:   "linux",
+			distro: "", // not Synology
+			want:   accidentalUpPrefix + " --hostname=foo --accept-routes",
+		},
 	}
 	for _, tt := range tests {
 		t.Run(tt.name, func(t *testing.T) {
@@ -447,6 +498,7 @@ func TestCheckForAccidentalSettingReverts(t *testing.T) {
 				goos:          goos,
 				flagSet:       flagSet,
 				curExitNodeIP: tt.curExitNodeIP,
+				distro:        tt.distro,
 			}); err != nil {
 				got = err.Error()
 			}
@@ -495,6 +547,7 @@ func TestPrefsFromUpArgs(t *testing.T) {
 				WantRunning:      true,
 				CorpDNS:          true,
 				AllowSingleHosts: true,
+				RouteAll:         true,
 				NetfilterMode:    preftype.NetfilterOn,
 			},
 		},
@@ -532,7 +585,7 @@ func TestPrefsFromUpArgs(t *testing.T) {
 			args: upArgsT{
 				exitNodeIP: "foo",
 			},
-			wantErr: `invalid IP address "foo" for --exit-node: ParseIP("foo"): unable to parse IP`,
+			wantErr: `invalid value "foo" for --exit-node; must be IP or unique node name`,
 		},
 		{
 			name: "error_exit_node_allow_lan_without_exit_node",
@@ -806,3 +859,133 @@ func TestUpdatePrefs(t *testing.T) {
 		})
 	}
 }
+
+func TestExitNodeIPOfArg(t *testing.T) {
+	mustIP := netaddr.MustParseIP
+	tests := []struct {
+		name    string
+		arg     string
+		st      *ipnstate.Status
+		want    netaddr.IP
+		wantErr string
+	}{
+		{
+			name: "ip_while_stopped_okay",
+			arg:  "1.2.3.4",
+			st: &ipnstate.Status{
+				BackendState: "Stopped",
+			},
+			want: mustIP("1.2.3.4"),
+		},
+		{
+			name: "ip_not_found",
+			arg:  "1.2.3.4",
+			st: &ipnstate.Status{
+				BackendState: "Running",
+			},
+			wantErr: `no node found in netmap with IP 1.2.3.4`,
+		},
+		{
+			name: "ip_not_exit",
+			arg:  "1.2.3.4",
+			st: &ipnstate.Status{
+				BackendState: "Running",
+				Peer: map[key.NodePublic]*ipnstate.PeerStatus{
+					key.NewNode().Public(): {
+						TailscaleIPs: []netaddr.IP{mustIP("1.2.3.4")},
+					},
+				},
+			},
+			wantErr: `node 1.2.3.4 is not advertising an exit node`,
+		},
+		{
+			name: "ip",
+			arg:  "1.2.3.4",
+			st: &ipnstate.Status{
+				BackendState: "Running",
+				Peer: map[key.NodePublic]*ipnstate.PeerStatus{
+					key.NewNode().Public(): {
+						TailscaleIPs:   []netaddr.IP{mustIP("1.2.3.4")},
+						ExitNodeOption: true,
+					},
+				},
+			},
+			want: mustIP("1.2.3.4"),
+		},
+		{
+			name:    "no_match",
+			arg:     "unknown",
+			st:      &ipnstate.Status{MagicDNSSuffix: ".foo"},
+			wantErr: `invalid value "unknown" for --exit-node; must be IP or unique node name`,
+		},
+		{
+			name: "name",
+			arg:  "skippy",
+			st: &ipnstate.Status{
+				MagicDNSSuffix: ".foo",
+				Peer: map[key.NodePublic]*ipnstate.PeerStatus{
+					key.NewNode().Public(): {
+						DNSName:        "skippy.foo.",
+						TailscaleIPs:   []netaddr.IP{mustIP("1.0.0.2")},
+						ExitNodeOption: true,
+					},
+				},
+			},
+			want: mustIP("1.0.0.2"),
+		},
+		{
+			name: "name_not_exit",
+			arg:  "skippy",
+			st: &ipnstate.Status{
+				MagicDNSSuffix: ".foo",
+				Peer: map[key.NodePublic]*ipnstate.PeerStatus{
+					key.NewNode().Public(): {
+						DNSName:      "skippy.foo.",
+						TailscaleIPs: []netaddr.IP{mustIP("1.0.0.2")},
+					},
+				},
+			},
+			wantErr: `node "skippy" is not advertising an exit node`,
+		},
+		{
+			name: "ambiguous",
+			arg:  "skippy",
+			st: &ipnstate.Status{
+				MagicDNSSuffix: ".foo",
+				Peer: map[key.NodePublic]*ipnstate.PeerStatus{
+					key.NewNode().Public(): {
+						DNSName:        "skippy.foo.",
+						TailscaleIPs:   []netaddr.IP{mustIP("1.0.0.2")},
+						ExitNodeOption: true,
+					},
+					key.NewNode().Public(): {
+						DNSName:        "SKIPPY.foo.",
+						TailscaleIPs:   []netaddr.IP{mustIP("1.0.0.2")},
+						ExitNodeOption: true,
+					},
+				},
+			},
+			wantErr: `ambiguous exit node name "skippy"`,
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			got, err := exitNodeIPOfArg(tt.arg, tt.st)
+			if err != nil {
+				if err.Error() == tt.wantErr {
+					return
+				}
+				if tt.wantErr == "" {
+					t.Fatal(err)
+				}
+				t.Fatalf("error = %#q; want %#q", err, tt.wantErr)
+			}
+			if tt.wantErr != "" {
+				t.Fatalf("got %v; want error %#q", got, tt.wantErr)
+			}
+			if got != tt.want {
+				t.Fatalf("got %v; want %v", got, tt.want)
+			}
+		})
+	}
+}

cmd/tailscale/cli/debug.go

@@ -5,6 +5,8 @@
 package cli
 
 import (
+	"bufio"
+	"bytes"
 	"context"
 	"encoding/json"
 	"errors"
@@ -14,7 +16,9 @@ import (
 	"log"
 	"os"
 	"runtime"
+	"strconv"
 	"strings"
+	"time"
 
 	"github.com/peterbourgon/ff/v3/ffcli"
 	"tailscale.com/client/tailscale"
@@ -24,39 +28,76 @@ import (
 )
 
 var debugCmd = &ffcli.Command{
-	Name: "debug",
-	Exec: runDebug,
+	Name:     "debug",
+	Exec:     runDebug,
+	LongHelp: `"tailscale debug" contains misc debug facilities; it is not a stable interface.`,
 	FlagSet: (func() *flag.FlagSet {
 		fs := newFlagSet("debug")
-		fs.BoolVar(&debugArgs.goroutines, "daemon-goroutines", false, "If true, dump the tailscaled daemon's goroutines")
-		fs.BoolVar(&debugArgs.ipn, "ipn", false, "If true, subscribe to IPN notifications")
-		fs.BoolVar(&debugArgs.prefs, "prefs", false, "If true, dump active prefs")
-		fs.BoolVar(&debugArgs.derpMap, "derp", false, "If true, dump DERP map")
-		fs.BoolVar(&debugArgs.pretty, "pretty", false, "If true, pretty-print output (for --prefs)")
-		fs.BoolVar(&debugArgs.netMap, "netmap", true, "whether to include netmap in --ipn mode")
-		fs.BoolVar(&debugArgs.env, "env", false, "dump environment")
-		fs.BoolVar(&debugArgs.localCreds, "local-creds", false, "print how to connect to local tailscaled")
 		fs.StringVar(&debugArgs.file, "file", "", "get, delete:NAME, or NAME")
 		fs.StringVar(&debugArgs.cpuFile, "cpu-profile", "", "if non-empty, grab a CPU profile for --profile-sec seconds and write it to this file; - for stdout")
 		fs.StringVar(&debugArgs.memFile, "mem-profile", "", "if non-empty, grab a memory profile and write it to this file; - for stdout")
 		fs.IntVar(&debugArgs.cpuSec, "profile-seconds", 15, "number of seconds to run a CPU profile for, when --cpu-profile is non-empty")
 		return fs
 	})(),
+	Subcommands: []*ffcli.Command{
+		{
+			Name:      "derp-map",
+			Exec:      runDERPMap,
+			ShortHelp: "print DERP map",
+		},
+		{
+			Name:      "daemon-goroutines",
+			Exec:      runDaemonGoroutines,
+			ShortHelp: "print tailscaled's goroutines",
+		},
+		{
+			Name:      "metrics",
+			Exec:      runDaemonMetrics,
+			ShortHelp: "print tailscaled's metrics",
+			FlagSet: (func() *flag.FlagSet {
+				fs := newFlagSet("metrics")
+				fs.BoolVar(&metricsArgs.watch, "watch", false, "print JSON dump of delta values")
+				return fs
+			})(),
+		},
+		{
+			Name:      "env",
+			Exec:      runEnv,
+			ShortHelp: "print cmd/tailscale environment",
+		},
+		{
+			Name:      "local-creds",
+			Exec:      runLocalCreds,
+			ShortHelp: "print how to access Tailscale local API",
+		},
+		{
+			Name:      "prefs",
+			Exec:      runPrefs,
+			ShortHelp: "print prefs",
+			FlagSet: (func() *flag.FlagSet {
+				fs := newFlagSet("prefs")
+				fs.BoolVar(&prefsArgs.pretty, "pretty", false, "If true, pretty-print output")
+				return fs
+			})(),
+		},
+		{
+			Name:      "watch-ipn",
+			Exec:      runWatchIPN,
+			ShortHelp: "subscribe to IPN message bus",
+			FlagSet: (func() *flag.FlagSet {
+				fs := newFlagSet("watch-ipn")
+				fs.BoolVar(&watchIPNArgs.netmap, "netmap", true, "include netmap in messages")
+				return fs
+			})(),
+		},
+	},
 }
 
 var debugArgs struct {
-	env        bool
-	localCreds bool
-	goroutines bool
-	ipn        bool
-	netMap     bool
-	derpMap    bool
-	file       string
-	prefs      bool
-	pretty     bool
-	cpuSec     int
-	cpuFile    string
-	memFile    string
+	file    string
+	cpuSec  int
+	cpuFile string
+	memFile string
 }
 
 func writeProfile(dst string, v []byte) error {
@@ -81,26 +122,9 @@ func runDebug(ctx context.Context, args []string) error {
 	if len(args) > 0 {
 		return errors.New("unknown arguments")
 	}
-	if debugArgs.env {
-		for _, e := range os.Environ() {
-			outln(e)
-		}
-		return nil
-	}
-	if debugArgs.localCreds {
-		port, token, err := safesocket.LocalTCPPortAndToken()
-		if err == nil {
-			printf("curl -u:%s http://localhost:%d/localapi/v0/status\n", token, port)
-			return nil
-		}
-		if runtime.GOOS == "windows" {
-			printf("curl http://localhost:41112/localapi/v0/status\n")
-			return nil
-		}
-		printf("curl --unix-socket %s http://foo/localapi/v0/status\n", paths.DefaultTailscaledSocket())
-		return nil
-	}
+	var usedFlag bool
 	if out := debugArgs.cpuFile; out != "" {
+		usedFlag = true // TODO(bradfitz): add "profile" subcommand
 		log.Printf("Capturing CPU profile for %v seconds ...", debugArgs.cpuSec)
 		if v, err := tailscale.Profile(ctx, "profile", debugArgs.cpuSec); err != nil {
 			return err
@@ -112,6 +136,7 @@ func runDebug(ctx context.Context, args []string) error {
 		}
 	}
 	if out := debugArgs.memFile; out != "" {
+		usedFlag = true // TODO(bradfitz): add "profile" subcommand
 		log.Printf("Capturing memory profile ...")
 		if v, err := tailscale.Profile(ctx, "heap", 0); err != nil {
 			return err
@@ -122,55 +147,8 @@ func runDebug(ctx context.Context, args []string) error {
 			log.Printf("Memory profile written to %s", outName(out))
 		}
 	}
-	if debugArgs.prefs {
-		prefs, err := tailscale.GetPrefs(ctx)
-		if err != nil {
-			return err
-		}
-		if debugArgs.pretty {
-			outln(prefs.Pretty())
-		} else {
-			j, _ := json.MarshalIndent(prefs, "", "\t")
-			outln(string(j))
-		}
-		return nil
-	}
-	if debugArgs.goroutines {
-		goroutines, err := tailscale.Goroutines(ctx)
-		if err != nil {
-			return err
-		}
-		Stdout.Write(goroutines)
-		return nil
-	}
-	if debugArgs.derpMap {
-		dm, err := tailscale.CurrentDERPMap(ctx)
-		if err != nil {
-			return fmt.Errorf(
-				"failed to get local derp map, instead `curl %s/derpmap/default`: %w", ipn.DefaultControlURL, err,
-			)
-		}
-		enc := json.NewEncoder(Stdout)
-		enc.SetIndent("", "\t")
-		enc.Encode(dm)
-		return nil
-	}
-	if debugArgs.ipn {
-		c, bc, ctx, cancel := connect(ctx)
-		defer cancel()
-
-		bc.SetNotifyCallback(func(n ipn.Notify) {
-			if !debugArgs.netMap {
-				n.NetMap = nil
-			}
-			j, _ := json.MarshalIndent(n, "", "\t")
-			printf("%s\n", j)
-		})
-		bc.RequestEngineStatus()
-		pump(ctx, bc, c)
-		return errors.New("exit")
-	}
 	if debugArgs.file != "" {
+		usedFlag = true // TODO(bradfitz): add "file" subcommand
 		if debugArgs.file == "get" {
 			wfs, err := tailscale.WaitingFiles(ctx)
 			if err != nil {
@@ -193,5 +171,148 @@ func runDebug(ctx context.Context, args []string) error {
 		io.Copy(Stdout, rc)
 		return nil
 	}
+	if usedFlag {
+		// TODO(bradfitz): delete this path when all debug flags are migrated
+		// to subcommands.
+		return nil
+	}
+	return errors.New("see 'tailscale debug --help")
+}
+
+func runLocalCreds(ctx context.Context, args []string) error {
+	port, token, err := safesocket.LocalTCPPortAndToken()
+	if err == nil {
+		printf("curl -u:%s http://localhost:%d/localapi/v0/status\n", token, port)
+		return nil
+	}
+	if runtime.GOOS == "windows" {
+		printf("curl http://localhost:%v/localapi/v0/status\n", safesocket.WindowsLocalPort)
+		return nil
+	}
+	printf("curl --unix-socket %s http://foo/localapi/v0/status\n", paths.DefaultTailscaledSocket())
 	return nil
 }
+
+var prefsArgs struct {
+	pretty bool
+}
+
+func runPrefs(ctx context.Context, args []string) error {
+	prefs, err := tailscale.GetPrefs(ctx)
+	if err != nil {
+		return err
+	}
+	if prefsArgs.pretty {
+		outln(prefs.Pretty())
+	} else {
+		j, _ := json.MarshalIndent(prefs, "", "\t")
+		outln(string(j))
+	}
+	return nil
+}
+
+var watchIPNArgs struct {
+	netmap bool
+}
+
+func runWatchIPN(ctx context.Context, args []string) error {
+	c, bc, ctx, cancel := connect(ctx)
+	defer cancel()
+
+	bc.SetNotifyCallback(func(n ipn.Notify) {
+		if !watchIPNArgs.netmap {
+			n.NetMap = nil
+		}
+		j, _ := json.MarshalIndent(n, "", "\t")
+		printf("%s\n", j)
+	})
+	bc.RequestEngineStatus()
+	pump(ctx, bc, c)
+	return errors.New("exit")
+}
+
+func runDERPMap(ctx context.Context, args []string) error {
+	dm, err := tailscale.CurrentDERPMap(ctx)
+	if err != nil {
+		return fmt.Errorf(
+			"failed to get local derp map, instead `curl %s/derpmap/default`: %w", ipn.DefaultControlURL, err,
+		)
+	}
+	enc := json.NewEncoder(Stdout)
+	enc.SetIndent("", "\t")
+	enc.Encode(dm)
+	return nil
+}
+
+func runEnv(ctx context.Context, args []string) error {
+	for _, e := range os.Environ() {
+		outln(e)
+	}
+	return nil
+}
+
+func runDaemonGoroutines(ctx context.Context, args []string) error {
+	goroutines, err := tailscale.Goroutines(ctx)
+	if err != nil {
+		return err
+	}
+	Stdout.Write(goroutines)
+	return nil
+}
+
+var metricsArgs struct {
+	watch bool
+}
+
+func runDaemonMetrics(ctx context.Context, args []string) error {
+	last := map[string]int64{}
+	for {
+		out, err := tailscale.DaemonMetrics(ctx)
+		if err != nil {
+			return err
+		}
+		if !metricsArgs.watch {
+			Stdout.Write(out)
+			return nil
+		}
+		bs := bufio.NewScanner(bytes.NewReader(out))
+		type change struct {
+			name     string
+			from, to int64
+		}
+		var changes []change
+		var maxNameLen int
+		for bs.Scan() {
+			line := bytes.TrimSpace(bs.Bytes())
+			if len(line) == 0 || line[0] == '#' {
+				continue
+			}
+			f := strings.Fields(string(line))
+			if len(f) != 2 {
+				continue
+			}
+			name := f[0]
+			n, _ := strconv.ParseInt(f[1], 10, 64)
+			prev, ok := last[name]
+			if ok && prev == n {
+				continue
+			}
+			last[name] = n
+			if !ok {
+				continue
+			}
+			changes = append(changes, change{name, prev, n})
+			if len(name) > maxNameLen {
+				maxNameLen = len(name)
+			}
+		}
+		if len(changes) > 0 {
+			format := fmt.Sprintf("%%-%ds %%+5d => %%v\n", maxNameLen)
+			for _, c := range changes {
+				fmt.Fprintf(Stdout, format, c.name, c.to-c.from, c.to)
+			}
+			io.WriteString(Stdout, "\n")
+		}
+		time.Sleep(time.Second)
+	}
+}

cmd/tailscale/cli/diag.go

@@ -25,23 +25,23 @@ func fixTailscaledConnectError(origErr error) error {
 	if err != nil {
 		return fmt.Errorf("failed to connect to local Tailscaled process and failed to enumerate processes while looking for it")
 	}
-	found := false
+	var foundProc ps.Process
 	for _, proc := range procs {
 		base := filepath.Base(proc.Executable())
 		if base == "tailscaled" {
-			found = true
+			foundProc = proc
 			break
 		}
 		if runtime.GOOS == "darwin" && base == "IPNExtension" {
-			found = true
+			foundProc = proc
 			break
 		}
 		if runtime.GOOS == "windows" && strings.EqualFold(base, "tailscaled.exe") {
-			found = true
+			foundProc = proc
 			break
 		}
 	}
-	if !found {
+	if foundProc == nil {
 		switch runtime.GOOS {
 		case "windows":
 			return fmt.Errorf("failed to connect to local tailscaled process; is the Tailscale service running?")
@@ -52,5 +52,5 @@ func fixTailscaledConnectError(origErr error) error {
 		}
 		return fmt.Errorf("failed to connect to local tailscaled process; it doesn't appear to be running")
 	}
-	return fmt.Errorf("failed to connect to local tailscaled (which appears to be running). Got error: %w", origErr)
+	return fmt.Errorf("failed to connect to local tailscaled (which appears to be running as %v, pid %v). Got error: %w", foundProc.Executable(), foundProc.Pid(), origErr)
 }

cmd/tailscale/cli/file.go

@@ -11,11 +11,9 @@ import (
 	"flag"
 	"fmt"
 	"io"
-	"io/ioutil"
 	"log"
 	"mime"
 	"net/http"
-	"net/url"
 	"os"
 	"path/filepath"
 	"strconv"
@@ -30,6 +28,7 @@ import (
 	"tailscale.com/client/tailscale/apitype"
 	"tailscale.com/ipn"
 	"tailscale.com/net/tsaddr"
+	"tailscale.com/tailcfg"
 	"tailscale.com/version"
 )
 
@@ -96,7 +95,7 @@ func runCp(ctx context.Context, args []string) error {
 		return err
 	}
 
-	peerAPIBase, isOffline, err := discoverPeerAPIBase(ctx, ip)
+	stableID, isOffline, err := getTargetStableID(ctx, ip)
 	if err != nil {
 		return fmt.Errorf("can't send to %s: %v", target, err)
 	}
@@ -154,32 +153,21 @@ func runCp(ctx context.Context, args []string) error {
 			}
 		}
 
-		dstURL := peerAPIBase + "/v0/put/" + url.PathEscape(name)
-		req, err := http.NewRequestWithContext(ctx, "PUT", dstURL, fileContents)
-		if err != nil {
-			return err
-		}
-		req.ContentLength = contentLength
 		if cpArgs.verbose {
-			log.Printf("sending to %v ...", dstURL)
+			log.Printf("sending %q to %v/%v/%v ...", name, target, ip, stableID)
 		}
-		res, err := http.DefaultClient.Do(req)
+		err := tailscale.PushFile(ctx, stableID, contentLength, name, fileContents)
 		if err != nil {
 			return err
 		}
-		if res.StatusCode == 200 {
-			io.Copy(ioutil.Discard, res.Body)
-			res.Body.Close()
-			continue
+		if cpArgs.verbose {
+			log.Printf("sent %q", name)
 		}
-		io.Copy(Stdout, res.Body)
-		res.Body.Close()
-		return errors.New(res.Status)
 	}
 	return nil
 }
 
-func discoverPeerAPIBase(ctx context.Context, ipStr string) (base string, isOffline bool, err error) {
+func getTargetStableID(ctx context.Context, ipStr string) (id tailcfg.StableNodeID, isOffline bool, err error) {
 	ip, err := netaddr.ParseIP(ipStr)
 	if err != nil {
 		return "", false, err
@@ -195,7 +183,7 @@ func discoverPeerAPIBase(ctx context.Context, ipStr string) (base string, isOffl
 				continue
 			}
 			isOffline = n.Online != nil && !*n.Online
-			return ft.PeerAPIURL, isOffline, nil
+			return n.StableID, isOffline, nil
 		}
 	}
 	return "", false, fileTargetErrorDetail(ctx, ip)

cmd/tailscale/cli/ip.go

@@ -18,12 +18,13 @@ import (
 
 var ipCmd = &ffcli.Command{
 	Name:       "ip",
-	ShortUsage: "ip [-4] [-6] [peername]",
-	ShortHelp:  "Show current Tailscale IP address(es)",
-	LongHelp:   "Shows the Tailscale IP address of the current machine without an argument. With an argument, it shows the IP of a named peer.",
+	ShortUsage: "ip [-1] [-4] [-6] [peer hostname or ip address]",
+	ShortHelp:  "Show Tailscale IP addresses",
+	LongHelp:   "Show Tailscale IP addresses for peer. Peer defaults to the current machine.",
 	Exec:       runIP,
 	FlagSet: (func() *flag.FlagSet {
 		fs := newFlagSet("ip")
+		fs.BoolVar(&ipArgs.want1, "1", false, "only print one IP address")
 		fs.BoolVar(&ipArgs.want4, "4", false, "only print IPv4 address")
 		fs.BoolVar(&ipArgs.want6, "6", false, "only print IPv6 address")
 		return fs
@@ -31,13 +32,14 @@ var ipCmd = &ffcli.Command{
 }
 
 var ipArgs struct {
+	want1 bool
 	want4 bool
 	want6 bool
 }
 
 func runIP(ctx context.Context, args []string) error {
 	if len(args) > 1 {
-		return errors.New("unknown arguments")
+		return errors.New("too many arguments, expected at most one peer")
 	}
 	var of string
 	if len(args) == 1 {
@@ -45,8 +47,14 @@ func runIP(ctx context.Context, args []string) error {
 	}
 
 	v4, v6 := ipArgs.want4, ipArgs.want6
-	if v4 && v6 {
-		return errors.New("tailscale ip -4 and -6 are mutually exclusive")
+	nflags := 0
+	for _, b := range []bool{ipArgs.want1, v4, v6} {
+		if b {
+			nflags++
+		}
+	}
+	if nflags > 1 {
+		return errors.New("tailscale ip -1, -4, and -6 are mutually exclusive")
 	}
 	if !v4 && !v6 {
 		v4, v6 = true, true
@@ -71,6 +79,9 @@ func runIP(ctx context.Context, args []string) error {
 		return fmt.Errorf("no current Tailscale IPs; state: %v", st.BackendState)
 	}
 
+	if ipArgs.want1 {
+		ips = ips[:1]
+	}
 	match := false
 	for _, ip := range ips {
 		if ip.Is4() && v4 || ip.Is6() && v6 {

cmd/tailscale/cli/status.go

@@ -29,7 +29,22 @@ var statusCmd = &ffcli.Command{
 	Name:       "status",
 	ShortUsage: "status [--active] [--web] [--json]",
 	ShortHelp:  "Show state of tailscaled and its connections",
-	Exec:       runStatus,
+	LongHelp: strings.TrimSpace(`
+
+JSON FORMAT
+
+Warning: this format has changed between releases and might change more
+in the future.
+
+For a description of the fields, see the "type Status" declaration at:
+
+https://github.com/tailscale/tailscale/blob/main/ipn/ipnstate/ipnstate.go
+
+(and be sure to select branch/tag that corresponds to the version
+ of Tailscale you're running)
+
+`),
+	Exec: runStatus,
 	FlagSet: (func() *flag.FlagSet {
 		fs := newFlagSet("status")
 		fs.BoolVar(&statusArgs.json, "json", false, "output in JSON format (WARNING: format subject to change)")
@@ -145,11 +160,19 @@ func runStatus(ctx context.Context, args []string) error {
 		)
 		relay := ps.Relay
 		anyTraffic := ps.TxBytes != 0 || ps.RxBytes != 0
+		var offline string
+		if !ps.Online {
+			offline = "; offline"
+		}
 		if !ps.Active {
 			if ps.ExitNode {
-				f("idle; exit node")
+				f("idle; exit node" + offline)
+			} else if ps.ExitNodeOption {
+				f("idle; offers exit node" + offline)
 			} else if anyTraffic {
-				f("idle")
+				f("idle" + offline)
+			} else if !ps.Online {
+				f("offline")
 			} else {
 				f("-")
 			}
@@ -157,12 +180,17 @@ func runStatus(ctx context.Context, args []string) error {
 			f("active; ")
 			if ps.ExitNode {
 				f("exit node; ")
+			} else if ps.ExitNodeOption {
+				f("offers exit node; ")
 			}
 			if relay != "" && ps.CurAddr == "" {
 				f("relay %q", relay)
 			} else if ps.CurAddr != "" {
 				f("direct %s", ps.CurAddr)
 			}
+			if !ps.Online {
+				f("; offline")
+			}
 		}
 		if anyTraffic {
 			f(", tx %d rx %d", ps.TxBytes, ps.RxBytes)

cmd/tailscale/cli/up.go

@@ -6,6 +6,8 @@ package cli
 
 import (
 	"context"
+	"encoding/base64"
+	"encoding/json"
 	"errors"
 	"flag"
 	"fmt"
@@ -28,6 +30,8 @@ import (
 	"tailscale.com/tailcfg"
 	"tailscale.com/types/logger"
 	"tailscale.com/types/preftype"
+	"tailscale.com/util/dnsname"
+	"tailscale.com/version"
 	"tailscale.com/version/distro"
 )
 
@@ -46,8 +50,10 @@ down").
 
 If flags are specified, the flags must be the complete set of desired
 settings. An error is returned if any setting would be changed as a
-result of an unspecified flag's default value, unless the --reset
-flag is also used.
+result of an unspecified flag's default value, unless the --reset flag
+is also used. (The flags --authkey, --force-reauth, and --qr are not
+considered settings that need to be re-specified when modifying
+settings.)
 `),
 	FlagSet: upFlagSet,
 	Exec:    runUp,
@@ -60,20 +66,34 @@ func effectiveGOOS() string {
 	return runtime.GOOS
 }
 
+// acceptRouteDefault returns the CLI's default value of --accept-routes as
+// a function of the platform it's running on.
+func acceptRouteDefault(goos string) bool {
+	switch goos {
+	case "windows":
+		return true
+	case "darwin":
+		return version.IsSandboxedMacOS()
+	default:
+		return false
+	}
+}
+
 var upFlagSet = newUpFlagSet(effectiveGOOS(), &upArgs)
 
 func newUpFlagSet(goos string, upArgs *upArgsT) *flag.FlagSet {
 	upf := newFlagSet("up")
 
 	upf.BoolVar(&upArgs.qr, "qr", false, "show QR code for login URLs")
+	upf.BoolVar(&upArgs.json, "json", false, "output in JSON format (WARNING: format subject to change)")
 	upf.BoolVar(&upArgs.forceReauth, "force-reauth", false, "force reauthentication")
 	upf.BoolVar(&upArgs.reset, "reset", false, "reset unspecified settings to their default values")
 
 	upf.StringVar(&upArgs.server, "login-server", ipn.DefaultControlURL, "base URL of control server")
-	upf.BoolVar(&upArgs.acceptRoutes, "accept-routes", false, "accept routes advertised by other Tailscale nodes")
+	upf.BoolVar(&upArgs.acceptRoutes, "accept-routes", acceptRouteDefault(goos), "accept routes advertised by other Tailscale nodes")
 	upf.BoolVar(&upArgs.acceptDNS, "accept-dns", true, "accept DNS configuration from the admin panel")
 	upf.BoolVar(&upArgs.singleRoutes, "host-routes", true, "install host routes to other Tailscale nodes")
-	upf.StringVar(&upArgs.exitNodeIP, "exit-node", "", "Tailscale IP of the exit node for internet traffic, or empty string to not use an exit node")
+	upf.StringVar(&upArgs.exitNodeIP, "exit-node", "", "Tailscale exit node (IP or base name) for internet traffic, or empty string to not use an exit node")
 	upf.BoolVar(&upArgs.exitNodeAllowLANAccess, "exit-node-allow-lan-access", false, "Allow direct access to the local network when routing traffic via an exit node")
 	upf.BoolVar(&upArgs.shieldsUp, "shields-up", false, "don't allow incoming connections")
 	upf.StringVar(&upArgs.advertiseTags, "advertise-tags", "", "comma-separated ACL tags to request; each must start with \"tag:\" (e.g. \"tag:eng,tag:montreal,tag:ssh\")")
@@ -121,6 +141,7 @@ type upArgsT struct {
 	authKeyOrFile          string // "secret" or "file:/path/to/secret"
 	hostname               string
 	opUser                 string
+	json                   bool
 }
 
 func (a upArgsT) getAuthKey() (string, error) {
@@ -138,6 +159,33 @@ func (a upArgsT) getAuthKey() (string, error) {
 
 var upArgs upArgsT
 
+// Fields output when `tailscale up --json` is used. Two JSON blocks will be output.
+//
+// When "tailscale up" is run it first outputs a block with AuthURL and QR populated,
+// providing the link for where to authenticate this client. BackendState would be
+// valid but boring, as it will almost certainly be "NeedsLogin". Error would be
+// populated if something goes badly wrong.
+//
+// When the client is authenticated by having someone visit the AuthURL, a second
+// JSON block will be output. The AuthURL and QR fields will not be present, the
+// BackendState and Error fields will give the result of the authentication.
+// Ex:
+// {
+//    "AuthURL": "https://login.tailscale.com/a/0123456789abcdef",
+//    "QR": "data:image/png;base64,0123...cdef"
+//    "BackendState": "NeedsLogin"
+// }
+// {
+//    "BackendState": "Running"
+// }
+//
+type upOutputJSON struct {
+	AuthURL      string `json:",omitempty"` // Authentication URL of the form https://login.tailscale.com/a/0123456789
+	QR           string `json:",omitempty"` // a DataURL (base64) PNG of a QR code AuthURL
+	BackendState string `json:",omitempty"` // name of state like Running or NeedsMachineAuth
+	Error        string `json:",omitempty"` // description of an error
+}
+
 func warnf(format string, args ...interface{}) {
 	printf("Warning: "+format+"\n", args...)
 }
@@ -190,6 +238,65 @@ func calcAdvertiseRoutes(advertiseRoutes string, advertiseDefaultRoute bool) ([]
 	return routes, nil
 }
 
+// peerWithTailscaleIP returns the peer in st with the provided
+// Tailscale IP.
+func peerWithTailscaleIP(st *ipnstate.Status, ip netaddr.IP) (ps *ipnstate.PeerStatus, ok bool) {
+	for _, ps := range st.Peer {
+		for _, ip2 := range ps.TailscaleIPs {
+			if ip == ip2 {
+				return ps, true
+			}
+		}
+	}
+	return nil, false
+}
+
+// exitNodeIPOfArg maps from a user-provided CLI flag value to an IP
+// address they want to use as an exit node.
+func exitNodeIPOfArg(arg string, st *ipnstate.Status) (ip netaddr.IP, err error) {
+	if arg == "" {
+		return ip, errors.New("invalid use of exitNodeIPOfArg with empty string")
+	}
+	ip, err = netaddr.ParseIP(arg)
+	if err == nil {
+		// If we're online already and have a netmap, double check that the IP
+		// address specified is valid.
+		if st.BackendState == "Running" {
+			ps, ok := peerWithTailscaleIP(st, ip)
+			if !ok {
+				return ip, fmt.Errorf("no node found in netmap with IP %v", ip)
+			}
+			if !ps.ExitNodeOption {
+				return ip, fmt.Errorf("node %v is not advertising an exit node", ip)
+			}
+		}
+		return ip, err
+	}
+	match := 0
+	for _, ps := range st.Peer {
+		baseName := dnsname.TrimSuffix(ps.DNSName, st.MagicDNSSuffix)
+		if !strings.EqualFold(arg, baseName) {
+			continue
+		}
+		match++
+		if len(ps.TailscaleIPs) == 0 {
+			return ip, fmt.Errorf("node %q has no Tailscale IP?", arg)
+		}
+		if !ps.ExitNodeOption {
+			return ip, fmt.Errorf("node %q is not advertising an exit node", arg)
+		}
+		ip = ps.TailscaleIPs[0]
+	}
+	switch match {
+	case 0:
+		return ip, fmt.Errorf("invalid value %q for --exit-node; must be IP or unique node name", arg)
+	case 1:
+		return ip, nil
+	default:
+		return ip, fmt.Errorf("ambiguous exit node name %q", arg)
+	}
+}
+
 // prefsFromUpArgs returns the ipn.Prefs for the provided args.
 //
 // Note that the parameters upArgs and warnf are named intentionally
@@ -205,9 +312,9 @@ func prefsFromUpArgs(upArgs upArgsT, warnf logger.Logf, st *ipnstate.Status, goo
 	var exitNodeIP netaddr.IP
 	if upArgs.exitNodeIP != "" {
 		var err error
-		exitNodeIP, err = netaddr.ParseIP(upArgs.exitNodeIP)
+		exitNodeIP, err = exitNodeIPOfArg(upArgs.exitNodeIP, st)
 		if err != nil {
-			return nil, fmt.Errorf("invalid IP address %q for --exit-node: %v", upArgs.exitNodeIP, err)
+			return nil, err
 		}
 	} else if upArgs.exitNodeAllowLANAccess {
 		return nil, fmt.Errorf("--exit-node-allow-lan-access can only be used with --exit-node")
@@ -380,11 +487,12 @@ func runUp(ctx context.Context, args []string) error {
 
 	env := upCheckEnv{
 		goos:          effectiveGOOS(),
+		distro:        distro.Get(),
 		user:          os.Getenv("USER"),
 		flagSet:       upFlagSet,
 		upArgs:        upArgs,
 		backendState:  st.BackendState,
-		curExitNodeIP: exitNodeIP(prefs, st),
+		curExitNodeIP: exitNodeIP(curPrefs, st),
 	}
 	simpleUp, justEditMP, err := updatePrefs(prefs, curPrefs, env)
 	if err != nil {
@@ -435,10 +543,16 @@ func runUp(ctx context.Context, args []string) error {
 				startLoginInteractive()
 			case ipn.NeedsMachineAuth:
 				printed = true
-				fmt.Fprintf(Stderr, "\nTo authorize your machine, visit (as admin):\n\n\t%s\n\n", prefs.AdminPageURL())
+				if env.upArgs.json {
+					printUpDoneJSON(ipn.NeedsMachineAuth, "")
+				} else {
+					fmt.Fprintf(Stderr, "\nTo authorize your machine, visit (as admin):\n\n\t%s\n\n", prefs.AdminPageURL())
+				}
 			case ipn.Running:
 				// Done full authentication process
-				if printed {
+				if env.upArgs.json {
+					printUpDoneJSON(ipn.Running, "")
+				} else if printed {
 					// Only need to print an update if we printed the "please click" message earlier.
 					fmt.Fprintf(Stderr, "Success.\n")
 				}
@@ -451,15 +565,33 @@ func runUp(ctx context.Context, args []string) error {
 		}
 		if url := n.BrowseToURL; url != nil && printAuthURL(*url) {
 			printed = true
-			fmt.Fprintf(Stderr, "\nTo authenticate, visit:\n\n\t%s\n\n", *url)
-			if upArgs.qr {
+			if upArgs.json {
+				js := &upOutputJSON{AuthURL: *url, BackendState: st.BackendState}
+
 				q, err := qrcode.New(*url, qrcode.Medium)
-				if err != nil {
-					log.Printf("QR code error: %v", err)
-				} else {
-					fmt.Fprintf(Stderr, "%s\n", q.ToString(false))
+				if err == nil {
+					png, err := q.PNG(128)
+					if err == nil {
+						js.QR = "data:image/png;base64," + base64.StdEncoding.EncodeToString(png)
+					}
 				}
 
+				data, err := json.MarshalIndent(js, "", "\t")
+				if err != nil {
+					log.Printf("upOutputJSON marshalling error: %v", err)
+				} else {
+					fmt.Println(string(data))
+				}
+			} else {
+				fmt.Fprintf(Stderr, "\nTo authenticate, visit:\n\n\t%s\n\n", *url)
+				if upArgs.qr {
+					q, err := qrcode.New(*url, qrcode.Medium)
+					if err != nil {
+						log.Printf("QR code error: %v", err)
+					} else {
+						fmt.Fprintf(Stderr, "%s\n", q.ToString(false))
+					}
+				}
 			}
 		}
 	})
@@ -546,6 +678,16 @@ func runUp(ctx context.Context, args []string) error {
 	}
 }
 
+func printUpDoneJSON(state ipn.State, errorString string) {
+	js := &upOutputJSON{BackendState: state.String(), Error: errorString}
+	data, err := json.MarshalIndent(js, "", "  ")
+	if err != nil {
+		log.Printf("printUpDoneJSON marshalling error: %v", err)
+	} else {
+		fmt.Println(string(data))
+	}
+}
+
 var (
 	prefsOfFlag = map[string][]string{} // "exit-node" => ExitNodeIP, ExitNodeID
 )
@@ -588,7 +730,7 @@ func addPrefFlagMapping(flagName string, prefNames ...string) {
 // correspond to an ipn.Pref.
 func preflessFlag(flagName string) bool {
 	switch flagName {
-	case "authkey", "force-reauth", "reset", "qr":
+	case "authkey", "force-reauth", "reset", "qr", "json":
 		return true
 	}
 	return false
@@ -622,6 +764,7 @@ type upCheckEnv struct {
 	upArgs        upArgsT
 	backendState  string
 	curExitNodeIP netaddr.IP
+	distro        distro.Distro
 }
 
 // checkForAccidentalSettingReverts (the "up checker") checks for
@@ -672,6 +815,10 @@ func checkForAccidentalSettingReverts(newPrefs, curPrefs *ipn.Prefs, env upCheck
 		if flagName == "login-server" && ipn.IsLoginServerSynonym(valCur) && ipn.IsLoginServerSynonym(valNew) {
 			continue
 		}
+		if flagName == "accept-routes" && valNew == false && env.goos == "linux" && env.distro == distro.Synology {
+			// Issue 3176. Old prefs had 'RouteAll: true' on disk, so ignore that.
+			continue
+		}
 		missing = append(missing, fmtFlagValueArg(flagName, valCur))
 	}
 	if len(missing) == 0 {

cmd/tailscale/depaware.txt

@@ -3,6 +3,7 @@ tailscale.com/cmd/tailscale dependencies: (generated by github.com/tailscale/dep
    W 💣 github.com/alexbrainman/sspi                                 from github.com/alexbrainman/sspi/negotiate+
    W    github.com/alexbrainman/sspi/internal/common                 from github.com/alexbrainman/sspi/negotiate
    W 💣 github.com/alexbrainman/sspi/negotiate                       from tailscale.com/net/tshttpproxy
+        github.com/golang/groupcache/lru                             from tailscale.com/net/dnscache
         github.com/kballard/go-shellquote                            from tailscale.com/cmd/tailscale/cli
    L    github.com/klauspost/compress/flate                          from nhooyr.io/websocket
      💣 github.com/mitchellh/go-ps                                   from tailscale.com/cmd/tailscale/cli+
@@ -72,6 +73,7 @@ tailscale.com/cmd/tailscale dependencies: (generated by github.com/tailscale/dep
         tailscale.com/types/persist                                  from tailscale.com/ipn
         tailscale.com/types/preftype                                 from tailscale.com/cmd/tailscale/cli+
         tailscale.com/types/structs                                  from tailscale.com/ipn+
+        tailscale.com/util/clientmetric                              from tailscale.com/net/netcheck+
         tailscale.com/util/dnsname                                   from tailscale.com/cmd/tailscale/cli+
    W    tailscale.com/util/endian                                    from tailscale.com/net/netns
         tailscale.com/util/groupmember                               from tailscale.com/cmd/tailscale/cli
@@ -90,7 +92,7 @@ tailscale.com/cmd/tailscale dependencies: (generated by github.com/tailscale/dep
         golang.org/x/crypto/nacl/secretbox                           from golang.org/x/crypto/nacl/box
         golang.org/x/crypto/poly1305                                 from golang.org/x/crypto/chacha20poly1305
         golang.org/x/crypto/salsa20/salsa                            from golang.org/x/crypto/nacl/box+
-        golang.org/x/net/dns/dnsmessage                              from net
+        golang.org/x/net/dns/dnsmessage                              from net+
         golang.org/x/net/http/httpguts                               from net/http+
         golang.org/x/net/http/httpproxy                              from net/http
         golang.org/x/net/http2/hpack                                 from net/http
@@ -102,7 +104,7 @@ tailscale.com/cmd/tailscale dependencies: (generated by github.com/tailscale/dep
         golang.org/x/sys/cpu                                         from golang.org/x/crypto/blake2b+
   LD    golang.org/x/sys/unix                                        from tailscale.com/net/netns+
    W    golang.org/x/sys/windows                                     from golang.org/x/sys/windows/registry+
-   W    golang.org/x/sys/windows/registry                            from golang.zx2c4.com/wireguard/windows/tunnel/winipcfg
+   W    golang.org/x/sys/windows/registry                            from golang.zx2c4.com/wireguard/windows/tunnel/winipcfg+
         golang.org/x/text/secure/bidirule                            from golang.org/x/net/idna
         golang.org/x/text/transform                                  from golang.org/x/text/secure/bidirule+
         golang.org/x/text/unicode/bidi                               from golang.org/x/net/idna+

cmd/tailscaled/depaware.txt

@@ -25,7 +25,8 @@ tailscale.com/cmd/tailscaled dependencies: (generated by github.com/tailscale/de
    L    github.com/aws/aws-sdk-go-v2/credentials/stscreds            from github.com/aws/aws-sdk-go-v2/config
    L    github.com/aws/aws-sdk-go-v2/feature/ec2/imds                from github.com/aws/aws-sdk-go-v2/config+
    L    github.com/aws/aws-sdk-go-v2/feature/ec2/imds/internal/config from github.com/aws/aws-sdk-go-v2/feature/ec2/imds
-   L    github.com/aws/aws-sdk-go-v2/internal/endpoints              from github.com/aws/aws-sdk-go-v2/service/ssm/internal/endpoints+
+   L    github.com/aws/aws-sdk-go-v2/internal/configsources          from github.com/aws/aws-sdk-go-v2/service/ssm+
+   L    github.com/aws/aws-sdk-go-v2/internal/endpoints/v2           from github.com/aws/aws-sdk-go-v2/service/ssm/internal/endpoints+
    L    github.com/aws/aws-sdk-go-v2/internal/ini                    from github.com/aws/aws-sdk-go-v2/config
    L    github.com/aws/aws-sdk-go-v2/internal/rand                   from github.com/aws/aws-sdk-go-v2/aws+
    L    github.com/aws/aws-sdk-go-v2/internal/sdk                    from github.com/aws/aws-sdk-go-v2/aws+
@@ -59,10 +60,10 @@ tailscale.com/cmd/tailscaled dependencies: (generated by github.com/tailscale/de
    L    github.com/aws/smithy-go/transport/http/internal/io          from github.com/aws/smithy-go/transport/http
    L    github.com/aws/smithy-go/waiter                              from github.com/aws/aws-sdk-go-v2/service/ssm
    L    github.com/coreos/go-iptables/iptables                       from tailscale.com/wgengine/router
-        github.com/go-multierror/multierror                          from tailscale.com/cmd/tailscaled+
    W 💣 github.com/go-ole/go-ole                                     from github.com/go-ole/go-ole/oleutil+
    W 💣 github.com/go-ole/go-ole/oleutil                             from tailscale.com/wgengine/winnet
    L 💣 github.com/godbus/dbus/v5                                    from tailscale.com/net/dns
+        github.com/golang/groupcache/lru                             from tailscale.com/net/dnscache
         github.com/google/btree                                      from inet.af/netstack/tcpip/header+
    L    github.com/insomniacslk/dhcp/dhcpv4                          from tailscale.com/net/tstun
    L    github.com/insomniacslk/dhcp/iana                            from github.com/insomniacslk/dhcp/dhcpv4
@@ -92,35 +93,38 @@ tailscale.com/cmd/tailscaled dependencies: (generated by github.com/tailscale/de
         github.com/tailscale/goupnp/scpd                             from github.com/tailscale/goupnp
         github.com/tailscale/goupnp/soap                             from github.com/tailscale/goupnp+
         github.com/tailscale/goupnp/ssdp                             from github.com/tailscale/goupnp
+   L 💣 github.com/tailscale/netlink                                 from tailscale.com/wgengine/router
         github.com/tcnksm/go-httpstat                                from tailscale.com/net/netcheck
    L    github.com/u-root/uio/rand                                   from github.com/insomniacslk/dhcp/dhcpv4
    L    github.com/u-root/uio/ubinary                                from github.com/u-root/uio/uio
    L    github.com/u-root/uio/uio                                    from github.com/insomniacslk/dhcp/dhcpv4+
-   L 💣 github.com/vishvananda/netlink                               from tailscale.com/wgengine/router
-   L 💣 github.com/vishvananda/netlink/nl                            from github.com/vishvananda/netlink
-   L    github.com/vishvananda/netns                                 from github.com/vishvananda/netlink+
+   L 💣 github.com/vishvananda/netlink/nl                            from github.com/tailscale/netlink
+   L    github.com/vishvananda/netns                                 from github.com/tailscale/netlink+
      💣 go4.org/intern                                               from inet.af/netaddr
      💣 go4.org/mem                                                  from tailscale.com/client/tailscale+
         go4.org/unsafe/assume-no-moving-gc                           from go4.org/intern
+   W 💣 golang.zx2c4.com/wintun                                      from golang.zx2c4.com/wireguard/tun
      💣 golang.zx2c4.com/wireguard/conn                              from golang.zx2c4.com/wireguard/device+
    W 💣 golang.zx2c4.com/wireguard/conn/winrio                       from golang.zx2c4.com/wireguard/conn
      💣 golang.zx2c4.com/wireguard/device                            from tailscale.com/net/tstun+
      💣 golang.zx2c4.com/wireguard/ipc                               from golang.zx2c4.com/wireguard/device
-   W 💣 golang.zx2c4.com/wireguard/ipc/winpipe                       from golang.zx2c4.com/wireguard/ipc
+   W 💣 golang.zx2c4.com/wireguard/ipc/namedpipe                     from golang.zx2c4.com/wireguard/ipc
         golang.zx2c4.com/wireguard/ratelimiter                       from golang.zx2c4.com/wireguard/device
         golang.zx2c4.com/wireguard/replay                            from golang.zx2c4.com/wireguard/device
         golang.zx2c4.com/wireguard/rwcancel                          from golang.zx2c4.com/wireguard/device+
         golang.zx2c4.com/wireguard/tai64n                            from golang.zx2c4.com/wireguard/device
      💣 golang.zx2c4.com/wireguard/tun                               from golang.zx2c4.com/wireguard/device+
-   W 💣 golang.zx2c4.com/wireguard/tun/wintun                        from golang.zx2c4.com/wireguard/tun
    W 💣 golang.zx2c4.com/wireguard/windows/tunnel/winipcfg           from tailscale.com/cmd/tailscaled+
         inet.af/netaddr                                              from inet.af/wf+
         inet.af/netstack/atomicbitops                                from inet.af/netstack/tcpip+
      💣 inet.af/netstack/buffer                                      from inet.af/netstack/tcpip/stack
+        inet.af/netstack/context                                     from inet.af/netstack/refs+
      💣 inet.af/netstack/gohacks                                     from inet.af/netstack/state/wire+
         inet.af/netstack/linewriter                                  from inet.af/netstack/log
         inet.af/netstack/log                                         from inet.af/netstack/state+
         inet.af/netstack/rand                                        from inet.af/netstack/tcpip/network/hash+
+        inet.af/netstack/refs                                        from inet.af/netstack/refsvfs2
+        inet.af/netstack/refsvfs2                                    from inet.af/netstack/tcpip/stack
      💣 inet.af/netstack/sleep                                       from inet.af/netstack/tcpip/transport/tcp
      💣 inet.af/netstack/state                                       from inet.af/netstack/atomicbitops+
         inet.af/netstack/state/wire                                  from inet.af/netstack/state
@@ -131,6 +135,7 @@ tailscale.com/cmd/tailscaled dependencies: (generated by github.com/tailscale/de
         inet.af/netstack/tcpip/hash/jenkins                          from inet.af/netstack/tcpip/stack+
         inet.af/netstack/tcpip/header                                from inet.af/netstack/tcpip/header/parse+
         inet.af/netstack/tcpip/header/parse                          from inet.af/netstack/tcpip/network/ipv4+
+        inet.af/netstack/tcpip/internal/tcp                          from inet.af/netstack/tcpip/stack+
         inet.af/netstack/tcpip/link/channel                          from tailscale.com/wgengine/netstack
         inet.af/netstack/tcpip/network/hash                          from inet.af/netstack/tcpip/network/ipv4+
         inet.af/netstack/tcpip/network/internal/fragmentation        from inet.af/netstack/tcpip/network/ipv4+
@@ -140,7 +145,10 @@ tailscale.com/cmd/tailscaled dependencies: (generated by github.com/tailscale/de
         inet.af/netstack/tcpip/ports                                 from inet.af/netstack/tcpip/stack+
         inet.af/netstack/tcpip/seqnum                                from inet.af/netstack/tcpip/header+
      💣 inet.af/netstack/tcpip/stack                                 from inet.af/netstack/tcpip/adapters/gonet+
+        inet.af/netstack/tcpip/transport                             from inet.af/netstack/tcpip/transport/icmp+
         inet.af/netstack/tcpip/transport/icmp                        from tailscale.com/wgengine/netstack
+        inet.af/netstack/tcpip/transport/internal/network            from inet.af/netstack/tcpip/transport/icmp+
+        inet.af/netstack/tcpip/transport/internal/noop               from inet.af/netstack/tcpip/transport/raw
         inet.af/netstack/tcpip/transport/packet                      from inet.af/netstack/tcpip/transport/raw
         inet.af/netstack/tcpip/transport/raw                         from inet.af/netstack/tcpip/transport/icmp+
      💣 inet.af/netstack/tcpip/transport/tcp                         from inet.af/netstack/tcpip/adapters/gonet+
@@ -173,30 +181,31 @@ tailscale.com/cmd/tailscaled dependencies: (generated by github.com/tailscale/de
         tailscale.com/ipn/policy                                     from tailscale.com/ipn/ipnlocal
         tailscale.com/ipn/store/aws                                  from tailscale.com/ipn/ipnserver
         tailscale.com/kube                                           from tailscale.com/ipn
-        tailscale.com/log/filelogger                                 from tailscale.com/ipn/ipnserver
+   W    tailscale.com/log/filelogger                                 from tailscale.com/logpolicy
         tailscale.com/log/logheap                                    from tailscale.com/control/controlclient
         tailscale.com/logpolicy                                      from tailscale.com/cmd/tailscaled
-        tailscale.com/logtail                                        from tailscale.com/logpolicy
+        tailscale.com/logtail                                        from tailscale.com/logpolicy+
         tailscale.com/logtail/backoff                                from tailscale.com/cmd/tailscaled+
         tailscale.com/logtail/filch                                  from tailscale.com/logpolicy
      💣 tailscale.com/metrics                                        from tailscale.com/derp
         tailscale.com/net/dns                                        from tailscale.com/cmd/tailscaled+
         tailscale.com/net/dns/resolver                               from tailscale.com/net/dns+
         tailscale.com/net/dnscache                                   from tailscale.com/control/controlclient+
-        tailscale.com/net/dnsfallback                                from tailscale.com/control/controlclient
+        tailscale.com/net/dnsfallback                                from tailscale.com/control/controlclient+
         tailscale.com/net/flowtrack                                  from tailscale.com/net/packet+
      💣 tailscale.com/net/interfaces                                 from tailscale.com/cmd/tailscaled+
         tailscale.com/net/netcheck                                   from tailscale.com/wgengine/magicsock
-        tailscale.com/net/netknob                                    from tailscale.com/ipn/localapi+
+        tailscale.com/net/netknob                                    from tailscale.com/logpolicy+
         tailscale.com/net/netns                                      from tailscale.com/cmd/tailscaled+
      💣 tailscale.com/net/netstat                                    from tailscale.com/ipn/ipnserver
         tailscale.com/net/packet                                     from tailscale.com/net/tstun+
         tailscale.com/net/portmapper                                 from tailscale.com/cmd/tailscaled+
-        tailscale.com/net/socks5                                     from tailscale.com/net/socks5/tssocks
-        tailscale.com/net/socks5/tssocks                             from tailscale.com/cmd/tailscaled
+        tailscale.com/net/proxymux                                   from tailscale.com/cmd/tailscaled
+        tailscale.com/net/socks5                                     from tailscale.com/cmd/tailscaled
         tailscale.com/net/stun                                       from tailscale.com/net/netcheck+
         tailscale.com/net/tlsdial                                    from tailscale.com/control/controlclient+
         tailscale.com/net/tsaddr                                     from tailscale.com/ipn/ipnlocal+
+        tailscale.com/net/tsdial                                     from tailscale.com/cmd/tailscaled+
      💣 tailscale.com/net/tshttpproxy                                from tailscale.com/cmd/tailscaled+
         tailscale.com/net/tstun                                      from tailscale.com/cmd/tailscaled+
      💣 tailscale.com/paths                                          from tailscale.com/client/tailscale+
@@ -218,16 +227,18 @@ tailscale.com/cmd/tailscaled dependencies: (generated by github.com/tailscale/de
         tailscale.com/types/netmap                                   from tailscale.com/control/controlclient+
         tailscale.com/types/nettype                                  from tailscale.com/wgengine/magicsock
         tailscale.com/types/opt                                      from tailscale.com/control/controlclient+
-        tailscale.com/types/pad32                                    from tailscale.com/derp+
+        tailscale.com/types/pad32                                    from tailscale.com/derp
         tailscale.com/types/persist                                  from tailscale.com/control/controlclient+
         tailscale.com/types/preftype                                 from tailscale.com/ipn+
         tailscale.com/types/structs                                  from tailscale.com/control/controlclient+
+        tailscale.com/util/clientmetric                              from tailscale.com/ipn/localapi+
    L    tailscale.com/util/cmpver                                    from tailscale.com/net/dns
      💣 tailscale.com/util/deephash                                  from tailscale.com/ipn/ipnlocal+
         tailscale.com/util/dnsname                                   from tailscale.com/hostinfo+
   LW    tailscale.com/util/endian                                    from tailscale.com/net/dns+
         tailscale.com/util/groupmember                               from tailscale.com/ipn/ipnserver
         tailscale.com/util/lineread                                  from tailscale.com/hostinfo+
+        tailscale.com/util/multierr                                  from tailscale.com/cmd/tailscaled+
         tailscale.com/util/osshare                                   from tailscale.com/cmd/tailscaled+
         tailscale.com/util/pidowner                                  from tailscale.com/ipn/ipnserver
         tailscale.com/util/racebuild                                 from tailscale.com/logpolicy
@@ -242,7 +253,7 @@ tailscale.com/cmd/tailscaled dependencies: (generated by github.com/tailscale/de
         tailscale.com/wgengine/filter                                from tailscale.com/control/controlclient+
         tailscale.com/wgengine/magicsock                             from tailscale.com/wgengine+
         tailscale.com/wgengine/monitor                               from tailscale.com/cmd/tailscaled+
-        tailscale.com/wgengine/netstack                              from tailscale.com/cmd/tailscaled+
+        tailscale.com/wgengine/netstack                              from tailscale.com/cmd/tailscaled
         tailscale.com/wgengine/router                                from tailscale.com/cmd/tailscaled+
         tailscale.com/wgengine/wgcfg                                 from tailscale.com/ipn/ipnlocal+
         tailscale.com/wgengine/wgcfg/nmcfg                           from tailscale.com/ipn/ipnlocal
@@ -265,7 +276,9 @@ tailscale.com/cmd/tailscaled dependencies: (generated by github.com/tailscale/de
         golang.org/x/net/dns/dnsmessage                              from net+
         golang.org/x/net/http/httpguts                               from net/http+
         golang.org/x/net/http/httpproxy                              from net/http
-        golang.org/x/net/http2/hpack                                 from net/http
+        golang.org/x/net/http2                                       from golang.org/x/net/http2/h2c+
+        golang.org/x/net/http2/h2c                                   from tailscale.com/ipn/ipnlocal
+        golang.org/x/net/http2/hpack                                 from net/http+
         golang.org/x/net/idna                                        from golang.org/x/net/http/httpguts+
         golang.org/x/net/ipv4                                        from golang.zx2c4.com/wireguard/device
         golang.org/x/net/ipv6                                        from golang.zx2c4.com/wireguard/device+
@@ -356,7 +369,7 @@ tailscale.com/cmd/tailscaled dependencies: (generated by github.com/tailscale/de
         path                                                         from github.com/aws/aws-sdk-go-v2/credentials/ec2rolecreds+
         path/filepath                                                from crypto/x509+
         reflect                                                      from crypto/x509+
-        regexp                                                       from github.com/aws/aws-sdk-go-v2/internal/endpoints+
+        regexp                                                       from github.com/aws/aws-sdk-go-v2/internal/endpoints/v2+
         regexp/syntax                                                from regexp
         runtime/debug                                                from github.com/klauspost/compress/zstd+
         runtime/pprof                                                from net/http/pprof+

cmd/tailscaled/tailscaled.go

@@ -20,6 +20,7 @@ import (
 	"net/http/pprof"
 	"os"
 	"os/signal"
+	"path/filepath"
 	"runtime"
 	"runtime/debug"
 	"strconv"
@@ -27,17 +28,23 @@ import (
 	"syscall"
 	"time"
 
-	"github.com/go-multierror/multierror"
+	"inet.af/netaddr"
 	"tailscale.com/ipn"
 	"tailscale.com/ipn/ipnserver"
 	"tailscale.com/logpolicy"
+	"tailscale.com/logtail"
 	"tailscale.com/net/dns"
 	"tailscale.com/net/netns"
-	"tailscale.com/net/socks5/tssocks"
+	"tailscale.com/net/proxymux"
+	"tailscale.com/net/socks5"
+	"tailscale.com/net/tsdial"
 	"tailscale.com/net/tstun"
 	"tailscale.com/paths"
+	"tailscale.com/safesocket"
 	"tailscale.com/types/flagtype"
 	"tailscale.com/types/logger"
+	"tailscale.com/util/clientmetric"
+	"tailscale.com/util/multierr"
 	"tailscale.com/util/osshare"
 	"tailscale.com/version"
 	"tailscale.com/version/distro"
@@ -78,6 +85,7 @@ var args struct {
 	debug          string
 	port           uint16
 	statepath      string
+	statedir       string
 	socketpath     string
 	birdSocketPath string
 	verbose        int
@@ -114,7 +122,8 @@ func main() {
 	flag.StringVar(&args.httpProxyAddr, "outbound-http-proxy-listen", "", `optional [ip]:port to run an outbound HTTP proxy (e.g. "localhost:8080")`)
 	flag.StringVar(&args.tunname, "tun", defaultTunName(), `tunnel interface name; use "userspace-networking" (beta) to not use TUN`)
 	flag.Var(flagtype.PortValue(&args.port, 0), "port", "UDP port to listen on for WireGuard and peer-to-peer traffic; 0 means automatically select")
-	flag.StringVar(&args.statepath, "state", paths.DefaultTailscaledStateFile(), "path of state file; use 'kube:<secret-name>' to use Kubernetes secrets or 'arn:aws:ssm:...' to store in AWS SSM")
+	flag.StringVar(&args.statepath, "state", paths.DefaultTailscaledStateFile(), "absolute path of state file; use 'kube:<secret-name>' to use Kubernetes secrets or 'arn:aws:ssm:...' to store in AWS SSM. If empty and --statedir is provided, the default is <statedir>/tailscaled.state")
+	flag.StringVar(&args.statedir, "statedir", "", "path to directory for storage of config state, TLS certs, temporary incoming Taildrop files, etc. If empty, it's derived from --state when possible.")
 	flag.StringVar(&args.socketpath, "socket", paths.DefaultTailscaledSocket(), "path of the service unix socket")
 	flag.StringVar(&args.birdSocketPath, "bird-socket", "", "path of the bird unix socket")
 	flag.BoolVar(&printVersion, "version", false, "print version information and exit")
@@ -169,8 +178,7 @@ func main() {
 	osshare.SetFileSharingEnabled(false, logger.Discard)
 
 	if err != nil {
-		// No need to log; the func already did
-		os.Exit(1)
+		log.Fatal(err)
 	}
 }
 
@@ -202,6 +210,16 @@ func trySynologyMigration(p string) error {
 	return nil
 }
 
+func statePathOrDefault() string {
+	if args.statepath != "" {
+		return args.statepath
+	}
+	if args.statedir != "" {
+		return filepath.Join(args.statedir, "tailscaled.state")
+	}
+	return ""
+}
+
 func ipnServerOpts() (o ipnserver.Options) {
 	// Allow changing the OS-specific IPN behavior for tests
 	// so we can e.g. test Windows-specific behaviors on Linux.
@@ -210,9 +228,15 @@ func ipnServerOpts() (o ipnserver.Options) {
 		goos = runtime.GOOS
 	}
 
-	o.Port = 41112
-	o.StatePath = args.statepath
-	o.SocketPath = args.socketpath // even for goos=="windows", for tests
+	o.VarRoot = args.statedir
+
+	// If an absolute --state is provided but not --statedir, try to derive
+	// a state directory.
+	if o.VarRoot == "" && filepath.IsAbs(args.statepath) {
+		if dir := filepath.Dir(args.statepath); strings.EqualFold(filepath.Base(dir), "tailscale") {
+			o.VarRoot = dir
+		}
+	}
 
 	switch goos {
 	default:
@@ -227,7 +251,7 @@ func ipnServerOpts() (o ipnserver.Options) {
 func run() error {
 	var err error
 
-	pol := logpolicy.New("tailnode.log.tailscale.io")
+	pol := logpolicy.New(logtail.CollectionNode)
 	pol.SetVerbosityLevel(args.verbose)
 	defer func() {
 		// Finish uploading logs after closing everything else.
@@ -261,10 +285,10 @@ func run() error {
 		return nil
 	}
 
-	if args.statepath == "" {
-		log.Fatalf("--state is required")
+	if args.statepath == "" && args.statedir == "" {
+		log.Fatalf("--statedir (or at least --state) is required")
 	}
-	if err := trySynologyMigration(args.statepath); err != nil {
+	if err := trySynologyMigration(statePathOrDefault()); err != nil {
 		log.Printf("error in synology migration: %v", err)
 	}
 
@@ -276,36 +300,55 @@ func run() error {
 
 	linkMon, err := monitor.New(logf)
 	if err != nil {
-		log.Fatalf("creating link monitor: %v", err)
+		return fmt.Errorf("monitor.New: %w", err)
 	}
 	pol.Logtail.SetLinkMonitor(linkMon)
 
-	socksListener := mustStartTCPListener("SOCKS5", args.socksAddr)
-	httpProxyListener := mustStartTCPListener("HTTP proxy", args.httpProxyAddr)
+	socksListener, httpProxyListener := mustStartProxyListeners(args.socksAddr, args.httpProxyAddr)
 
-	e, useNetstack, err := createEngine(logf, linkMon)
+	dialer := new(tsdial.Dialer) // mutated below (before used)
+	e, useNetstack, err := createEngine(logf, linkMon, dialer)
 	if err != nil {
-		logf("wgengine.New: %v", err)
-		return err
+		return fmt.Errorf("createEngine: %w", err)
+	}
+	if _, ok := e.(wgengine.ResolvingEngine).GetResolver(); !ok {
+		panic("internal error: exit node resolver not wired up")
 	}
 
-	var ns *netstack.Impl
-	if useNetstack || wrapNetstack {
-		onlySubnets := wrapNetstack && !useNetstack
-		ns = mustStartNetstack(logf, e, onlySubnets)
+	ns, err := newNetstack(logf, dialer, e)
+	if err != nil {
+		return fmt.Errorf("newNetstack: %w", err)
+	}
+	ns.ProcessLocalIPs = useNetstack
+	ns.ProcessSubnets = useNetstack || wrapNetstack
+	if err := ns.Start(); err != nil {
+		return fmt.Errorf("failed to start netstack: %w", err)
+	}
+
+	if useNetstack {
+		dialer.UseNetstackForIP = func(ip netaddr.IP) bool {
+			_, ok := e.PeerForIP(ip)
+			return ok
+		}
+		dialer.NetstackDialTCP = func(ctx context.Context, dst netaddr.IPPort) (net.Conn, error) {
+			return ns.DialContextTCP(ctx, dst)
+		}
 	}
 
 	if socksListener != nil || httpProxyListener != nil {
-		srv := tssocks.NewServer(logger.WithPrefix(logf, "socks5: "), e, ns)
 		if httpProxyListener != nil {
-			hs := &http.Server{Handler: httpProxyHandler(srv.Dialer)}
+			hs := &http.Server{Handler: httpProxyHandler(dialer.UserDial)}
 			go func() {
 				log.Fatalf("HTTP proxy exited: %v", hs.Serve(httpProxyListener))
 			}()
 		}
 		if socksListener != nil {
+			ss := &socks5.Server{
+				Logf:   logger.WithPrefix(logf, "socks5: "),
+				Dialer: dialer.UserDial,
+			}
 			go func() {
-				log.Fatalf("SOCKS5 server exited: %v", srv.Serve(socksListener))
+				log.Fatalf("SOCKS5 server exited: %v", ss.Serve(socksListener))
 			}()
 		}
 	}
@@ -332,32 +375,49 @@ func run() error {
 	}()
 
 	opts := ipnServerOpts()
-	opts.DebugMux = debugMux
-	err = ipnserver.Run(ctx, logf, pol.PublicID.String(), ipnserver.FixedEngine(e), opts)
+
+	store, err := ipnserver.StateStore(statePathOrDefault(), logf)
+	if err != nil {
+		return fmt.Errorf("ipnserver.StateStore: %w", err)
+	}
+	srv, err := ipnserver.New(logf, pol.PublicID.String(), store, e, dialer, nil, opts)
+	if err != nil {
+		return fmt.Errorf("ipnserver.New: %w", err)
+	}
+
+	if debugMux != nil {
+		debugMux.HandleFunc("/debug/ipn", srv.ServeHTMLStatus)
+	}
+
+	ln, _, err := safesocket.Listen(args.socketpath, safesocket.WindowsLocalPort)
+	if err != nil {
+		return fmt.Errorf("safesocket.Listen: %v", err)
+	}
+
+	err = srv.Run(ctx, ln)
 	// Cancelation is not an error: it is the only way to stop ipnserver.
 	if err != nil && err != context.Canceled {
-		logf("ipnserver.Run: %v", err)
-		return err
+		return fmt.Errorf("ipnserver.Run: %w", err)
 	}
 
 	return nil
 }
 
-func createEngine(logf logger.Logf, linkMon *monitor.Mon) (e wgengine.Engine, useNetstack bool, err error) {
+func createEngine(logf logger.Logf, linkMon *monitor.Mon, dialer *tsdial.Dialer) (e wgengine.Engine, useNetstack bool, err error) {
 	if args.tunname == "" {
 		return nil, false, errors.New("no --tun value specified")
 	}
 	var errs []error
 	for _, name := range strings.Split(args.tunname, ",") {
 		logf("wgengine.NewUserspaceEngine(tun %q) ...", name)
-		e, useNetstack, err = tryEngine(logf, linkMon, name)
+		e, useNetstack, err = tryEngine(logf, linkMon, dialer, name)
 		if err == nil {
 			return e, useNetstack, nil
 		}
 		logf("wgengine.NewUserspaceEngine(tun %q) error: %v", name, err)
 		errs = append(errs, err)
 	}
-	return nil, false, multierror.New(errs)
+	return nil, false, multierr.New(errs...)
 }
 
 var wrapNetstack = shouldWrapNetstack()
@@ -382,10 +442,11 @@ func shouldWrapNetstack() bool {
 	return false
 }
 
-func tryEngine(logf logger.Logf, linkMon *monitor.Mon, name string) (e wgengine.Engine, useNetstack bool, err error) {
+func tryEngine(logf logger.Logf, linkMon *monitor.Mon, dialer *tsdial.Dialer, name string) (e wgengine.Engine, useNetstack bool, err error) {
 	conf := wgengine.Config{
 		ListenPort:  args.port,
 		LinkMonitor: linkMon,
+		Dialer:      dialer,
 	}
 
 	useNetstack = name == "userspace-networking"
@@ -395,14 +456,14 @@ func tryEngine(logf logger.Logf, linkMon *monitor.Mon, name string) (e wgengine.
 		log.Printf("Connecting to BIRD at %s ...", args.birdSocketPath)
 		conf.BIRDClient, err = createBIRDClient(args.birdSocketPath)
 		if err != nil {
-			return nil, false, err
+			return nil, false, fmt.Errorf("createBIRDClient: %w", err)
 		}
 	}
 	if !useNetstack {
 		dev, devName, err := tstun.New(logf, name)
 		if err != nil {
 			tstun.Diagnose(logf, name)
-			return nil, false, err
+			return nil, false, fmt.Errorf("tstun.New(%q): %w", name, err)
 		}
 		conf.Tun = dev
 		if strings.HasPrefix(name, "tap:") {
@@ -414,11 +475,11 @@ func tryEngine(logf logger.Logf, linkMon *monitor.Mon, name string) (e wgengine.
 		r, err := router.New(logf, dev, linkMon)
 		if err != nil {
 			dev.Close()
-			return nil, false, err
+			return nil, false, fmt.Errorf("creating router: %w", err)
 		}
 		d, err := dns.NewOSConfigurator(logf, devName)
 		if err != nil {
-			return nil, false, err
+			return nil, false, fmt.Errorf("dns.NewOSConfigurator: %w", err)
 		}
 		conf.DNS = d
 		conf.Router = r
@@ -435,6 +496,7 @@ func tryEngine(logf logger.Logf, linkMon *monitor.Mon, name string) (e wgengine.
 
 func newDebugMux() *http.ServeMux {
 	mux := http.NewServeMux()
+	mux.HandleFunc("/debug/metrics", servePrometheusMetrics)
 	mux.HandleFunc("/debug/pprof/", pprof.Index)
 	mux.HandleFunc("/debug/pprof/cmdline", pprof.Cmdline)
 	mux.HandleFunc("/debug/pprof/profile", pprof.Profile)
@@ -443,6 +505,11 @@ func newDebugMux() *http.ServeMux {
 	return mux
 }
 
+func servePrometheusMetrics(w http.ResponseWriter, r *http.Request) {
+	w.Header().Set("Content-Type", "text/plain")
+	clientmetric.WritePrometheusExpositionFormat(w)
+}
+
 func runDebugServer(mux *http.ServeMux, addr string) {
 	srv := &http.Server{
 		Addr:    addr,
@@ -453,33 +520,54 @@ func runDebugServer(mux *http.ServeMux, addr string) {
 	}
 }
 
-func mustStartNetstack(logf logger.Logf, e wgengine.Engine, onlySubnets bool) *netstack.Impl {
+func newNetstack(logf logger.Logf, dialer *tsdial.Dialer, e wgengine.Engine) (*netstack.Impl, error) {
 	tunDev, magicConn, ok := e.(wgengine.InternalsGetter).GetInternals()
 	if !ok {
-		log.Fatalf("%T is not a wgengine.InternalsGetter", e)
+		return nil, fmt.Errorf("%T is not a wgengine.InternalsGetter", e)
 	}
-	ns, err := netstack.Create(logf, tunDev, e, magicConn, onlySubnets)
-	if err != nil {
-		log.Fatalf("netstack.Create: %v", err)
-	}
-	if err := ns.Start(); err != nil {
-		log.Fatalf("failed to start netstack: %v", err)
-	}
-	return ns
+	return netstack.Create(logf, tunDev, e, magicConn, dialer)
 }
 
-func mustStartTCPListener(name, addr string) net.Listener {
-	if addr == "" {
-		return nil
+// mustStartProxyListeners creates listeners for local SOCKS and HTTP
+// proxies, if the respective addresses are not empty. socksAddr and
+// httpAddr can be the same, in which case socksListener will receive
+// connections that look like they're speaking SOCKS and httpListener
+// will receive everything else.
+//
+// socksListener and httpListener can be nil, if their respective
+// addrs are empty.
+func mustStartProxyListeners(socksAddr, httpAddr string) (socksListener, httpListener net.Listener) {
+	if socksAddr == httpAddr && socksAddr != "" && !strings.HasSuffix(socksAddr, ":0") {
+		ln, err := net.Listen("tcp", socksAddr)
+		if err != nil {
+			log.Fatalf("proxy listener: %v", err)
+		}
+		return proxymux.SplitSOCKSAndHTTP(ln)
 	}
-	ln, err := net.Listen("tcp", addr)
-	if err != nil {
-		log.Fatalf("%v listener: %v", name, err)
+
+	var err error
+	if socksAddr != "" {
+		socksListener, err = net.Listen("tcp", socksAddr)
+		if err != nil {
+			log.Fatalf("SOCKS5 listener: %v", err)
+		}
+		if strings.HasSuffix(socksAddr, ":0") {
+			// Log kernel-selected port number so integration tests
+			// can find it portably.
+			log.Printf("SOCKS5 listening on %v", socksListener.Addr())
+		}
 	}
-	if strings.HasSuffix(addr, ":0") {
-		// Log kernel-selected port number so integration tests
-		// can find it portably.
-		log.Printf("%v listening on %v", name, ln.Addr())
+	if httpAddr != "" {
+		httpListener, err = net.Listen("tcp", httpAddr)
+		if err != nil {
+			log.Fatalf("HTTP proxy listener: %v", err)
+		}
+		if strings.HasSuffix(httpAddr, ":0") {
+			// Log kernel-selected port number so integration tests
+			// can find it portably.
+			log.Printf("HTTP proxy listening on %v", httpListener.Addr())
+		}
 	}
-	return ln
+
+	return socksListener, httpListener
 }

cmd/tailscaled/tailscaled_windows.go

@@ -32,12 +32,15 @@ import (
 	"tailscale.com/ipn/ipnserver"
 	"tailscale.com/logpolicy"
 	"tailscale.com/net/dns"
+	"tailscale.com/net/tsdial"
 	"tailscale.com/net/tstun"
+	"tailscale.com/safesocket"
 	"tailscale.com/types/logger"
 	"tailscale.com/util/winutil"
 	"tailscale.com/version"
 	"tailscale.com/wf"
 	"tailscale.com/wgengine"
+	"tailscale.com/wgengine/monitor"
 	"tailscale.com/wgengine/netstack"
 	"tailscale.com/wgengine/router"
 )
@@ -74,7 +77,14 @@ func (service *ipnService) Execute(args []string, r <-chan svc.ChangeRequest, ch
 	go func() {
 		defer close(doneCh)
 		args := []string{"/subproc", service.Policy.PublicID.String()}
-		ipnserver.BabysitProc(ctx, args, log.Printf)
+		// Make a logger without a date prefix, as filelogger
+		// and logtail both already add their own. All we really want
+		// from the log package is the automatic newline.
+		// We start with log.Default().Writer(), which is the logtail
+		// writer that logpolicy already installed as the global
+		// output.
+		logger := log.New(log.Default().Writer(), "", 0)
+		ipnserver.BabysitProc(ctx, args, logger.Printf)
 	}()
 
 	changes <- svc.Status{State: svc.Running, Accepts: svcAccepts}
@@ -109,6 +119,9 @@ func beWindowsSubprocess() bool {
 	}
 	logid := os.Args[2]
 
+	// Remove the date/time prefix; the logtail + file logggers add it.
+	log.SetFlags(0)
+
 	log.Printf("Program starting: v%v: %#v", version.Long, os.Args)
 	log.Printf("subproc mode: logid=%v", logid)
 
@@ -172,6 +185,12 @@ func beFirewallKillswitch() bool {
 func startIPNServer(ctx context.Context, logid string) error {
 	var logf logger.Logf = log.Printf
 
+	linkMon, err := monitor.New(logf)
+	if err != nil {
+		return err
+	}
+	dialer := new(tsdial.Dialer)
+
 	getEngineRaw := func() (wgengine.Engine, error) {
 		dev, devName, err := tstun.New(logf, "Tailscale")
 		if err != nil {
@@ -192,19 +211,26 @@ func startIPNServer(ctx context.Context, logid string) error {
 			return nil, fmt.Errorf("DNS: %w", err)
 		}
 		eng, err := wgengine.NewUserspaceEngine(logf, wgengine.Config{
-			Tun:        dev,
-			Router:     r,
-			DNS:        d,
-			ListenPort: 41641,
+			Tun:         dev,
+			Router:      r,
+			DNS:         d,
+			ListenPort:  41641,
+			LinkMonitor: linkMon,
+			Dialer:      dialer,
 		})
 		if err != nil {
 			r.Close()
 			dev.Close()
 			return nil, fmt.Errorf("engine: %w", err)
 		}
-		onlySubnets := true
-		if wrapNetstack {
-			mustStartNetstack(logf, eng, onlySubnets)
+		ns, err := newNetstack(logf, dialer, eng)
+		if err != nil {
+			return nil, fmt.Errorf("newNetstack: %w", err)
+		}
+		ns.ProcessLocalIPs = false
+		ns.ProcessSubnets = wrapNetstack
+		if err := ns.Start(); err != nil {
+			return nil, fmt.Errorf("failed to start netstack: %w", err)
 		}
 		return wgengine.NewWatchdog(eng), nil
 	}
@@ -266,7 +292,18 @@ func startIPNServer(ctx context.Context, logid string) error {
 			return nil, fmt.Errorf("%w\n\nlogid: %v", res.Err, logid)
 		}
 	}
-	err := ipnserver.Run(ctx, logf, logid, getEngine, ipnServerOpts())
+
+	store, err := ipnserver.StateStore(statePathOrDefault(), logf)
+	if err != nil {
+		return err
+	}
+
+	ln, _, err := safesocket.Listen(args.socketpath, safesocket.WindowsLocalPort)
+	if err != nil {
+		return fmt.Errorf("safesocket.Listen: %v", err)
+	}
+
+	err = ipnserver.Run(ctx, logf, ln, store, linkMon, dialer, logid, getEngine, ipnServerOpts())
 	if err != nil {
 		logf("ipnserver.Run: %v", err)
 	}

cmd/tsshd/tsshd.go

@@ -157,8 +157,9 @@ func handleSSH(s ssh.Session) {
 		cmd.Process.Kill()
 		if err := cmd.Wait(); err != nil {
 			s.Exit(1)
+		} else {
+			s.Exit(0)
 		}
-		s.Exit(0)
 		return
 	}
 

control/controlclient/auto.go

@@ -339,11 +339,9 @@ func (c *Auto) authRoutine() {
 				continue
 			}
 			if url != "" {
-				if goal.url != "" {
-					err = fmt.Errorf("[unexpected] server required a new URL?")
-					report(err, "WaitLoginURL")
-				}
-
+				// goal.url ought to be empty here.
+				// However, not all control servers get this right,
+				// and logging about it here just generates noise.
 				c.mu.Lock()
 				c.loginGoal = &LoginGoal{
 					wantLoggedIn: true,

control/controlclient/client.go

@@ -79,3 +79,9 @@ type Client interface {
 	// requesting a DNS record be created or updated.
 	SetDNS(context.Context, *tailcfg.SetDNSRequest) error
 }
+
+// UserVisibleError is an error that should be shown to users.
+type UserVisibleError string
+
+func (e UserVisibleError) Error() string            { return string(e) }
+func (e UserVisibleError) UserVisibleError() string { return string(e) }

control/controlclient/direct.go

@@ -46,6 +46,7 @@ import (
 	"tailscale.com/types/netmap"
 	"tailscale.com/types/opt"
 	"tailscale.com/types/persist"
+	"tailscale.com/util/clientmetric"
 	"tailscale.com/util/systemd"
 	"tailscale.com/wgengine/monitor"
 )
@@ -60,7 +61,7 @@ type Direct struct {
 	keepAlive              bool
 	logf                   logger.Logf
 	linkMon                *monitor.Mon // or nil
-	discoPubKey            tailcfg.DiscoKey
+	discoPubKey            key.DiscoPublic
 	getMachinePrivKey      func() (key.MachinePrivate, error)
 	debugFlags             []string
 	keepSharerAndUserSplit bool
@@ -88,7 +89,7 @@ type Options struct {
 	AuthKey              string                             // optional node auth key for auto registration
 	TimeNow              func() time.Time                   // time.Now implementation used by Client
 	Hostinfo             *tailcfg.Hostinfo                  // non-nil passes ownership, nil means to use default using os.Hostname, etc
-	DiscoPublicKey       tailcfg.DiscoKey
+	DiscoPublicKey       key.DiscoPublic
 	NewDecompressor      func() (Decompressor, error)
 	KeepAlive            bool
 	Logf                 logger.Logf
@@ -146,13 +147,20 @@ func NewDirect(opts Options) (*Direct, error) {
 	}
 
 	httpc := opts.HTTPTestClient
+	if httpc == nil && runtime.GOOS == "js" {
+		// In js/wasm, net/http.Transport (as of Go 1.18) will
+		// only use the browser's Fetch API if you're using
+		// the DefaultClient (or a client without dial hooks
+		// etc set).
+		httpc = http.DefaultClient
+	}
 	if httpc == nil {
 		dnsCache := &dnscache.Resolver{
 			Forward:          dnscache.Get().Forward, // use default cache's forwarder
 			UseLastGood:      true,
 			LookupIPFallback: dnsfallback.Lookup,
 		}
-		dialer := netns.NewDialer()
+		dialer := netns.NewDialer(opts.Logf)
 		tr := http.DefaultTransport.(*http.Transport).Clone()
 		tr.Proxy = tshttpproxy.ProxyFromEnvironment
 		tshttpproxy.SetTransportGetProxyConnectHeader(tr)
@@ -284,8 +292,8 @@ func (c *Direct) doLogin(ctx context.Context, opt loginOpt) (mustRegen bool, new
 	tryingNewKey := c.tryingNewKey
 	serverKey := c.serverKey
 	authKey := c.authKey
-	hostinfo := c.hostinfo.Clone()
-	backendLogID := hostinfo.BackendLogID
+	hi := c.hostinfo.Clone()
+	backendLogID := hi.BackendLogID
 	expired := c.expiry != nil && !c.expiry.IsZero() && c.expiry.Before(c.timeNow())
 	c.mu.Unlock()
 
@@ -357,9 +365,9 @@ func (c *Direct) doLogin(ctx context.Context, opt loginOpt) (mustRegen bool, new
 	now := time.Now().Round(time.Second)
 	request := tailcfg.RegisterRequest{
 		Version:    1,
-		OldNodeKey: tailcfg.NodeKeyFromNodePublic(oldNodeKey),
-		NodeKey:    tailcfg.NodeKeyFromNodePublic(tryingNewKey.Public()),
-		Hostinfo:   hostinfo,
+		OldNodeKey: oldNodeKey,
+		NodeKey:    tryingNewKey.Public(),
+		Hostinfo:   hi,
 		Followup:   opt.URL,
 		Timestamp:  &now,
 		Ephemeral:  (opt.Flags & LoginEphemeral) != 0,
@@ -431,7 +439,7 @@ func (c *Direct) doLogin(ctx context.Context, opt loginOpt) (mustRegen bool, new
 		resp.NodeKeyExpired, resp.MachineAuthorized, resp.AuthURL != "")
 
 	if resp.Error != "" {
-		return false, "", errors.New(resp.Error)
+		return false, "", UserVisibleError(resp.Error)
 	}
 	if resp.NodeKeyExpired {
 		if regen {
@@ -551,12 +559,21 @@ const pollTimeout = 120 * time.Second
 
 // cb nil means to omit peers.
 func (c *Direct) sendMapRequest(ctx context.Context, maxPolls int, cb func(*netmap.NetworkMap)) error {
+	metricMapRequests.Add(1)
+	metricMapRequestsActive.Add(1)
+	defer metricMapRequestsActive.Add(-1)
+	if maxPolls == -1 {
+		metricMapRequestsPoll.Add(1)
+	} else {
+		metricMapRequestsLite.Add(1)
+	}
+
 	c.mu.Lock()
 	persist := c.persist
 	serverURL := c.serverURL
 	serverKey := c.serverKey
-	hostinfo := c.hostinfo.Clone()
-	backendLogID := hostinfo.BackendLogID
+	hi := c.hostinfo.Clone()
+	backendLogID := hi.BackendLogID
 	localPort := c.localPort
 	var epStrs []string
 	var epTypes []tailcfg.EndpointType
@@ -595,18 +612,18 @@ func (c *Direct) sendMapRequest(ctx context.Context, maxPolls int, cb func(*netm
 	request := &tailcfg.MapRequest{
 		Version:       tailcfg.CurrentMapRequestVersion,
 		KeepAlive:     c.keepAlive,
-		NodeKey:       tailcfg.NodeKeyFromNodePublic(persist.PrivateNodeKey.Public()),
+		NodeKey:       persist.PrivateNodeKey.Public(),
 		DiscoKey:      c.discoPubKey,
 		Endpoints:     epStrs,
 		EndpointTypes: epTypes,
 		Stream:        allowStream,
-		Hostinfo:      hostinfo,
+		Hostinfo:      hi,
 		DebugFlags:    c.debugFlags,
 		OmitPeers:     cb == nil,
 	}
 	var extraDebugFlags []string
-	if hostinfo != nil && c.linkMon != nil && !c.skipIPForwardingCheck &&
-		ipForwardingBroken(hostinfo.RoutableIPs, c.linkMon.InterfaceState()) {
+	if hi != nil && c.linkMon != nil && !c.skipIPForwardingCheck &&
+		ipForwardingBroken(hi.RoutableIPs, c.linkMon.InterfaceState()) {
 		extraDebugFlags = append(extraDebugFlags, "warn-ip-forwarding-off")
 	}
 	if health.RouterHealth() != nil {
@@ -615,6 +632,9 @@ func (c *Direct) sendMapRequest(ctx context.Context, maxPolls int, cb func(*netm
 	if health.NetworkCategoryHealth() != nil {
 		extraDebugFlags = append(extraDebugFlags, "warn-network-category-unhealthy")
 	}
+	if hostinfo.DisabledEtcAptSource() {
+		extraDebugFlags = append(extraDebugFlags, "warn-etc-apt-source-disabled")
+	}
 	if len(extraDebugFlags) > 0 {
 		old := request.DebugFlags
 		request.DebugFlags = append(old[:len(old):len(old)], extraDebugFlags...)
@@ -737,11 +757,14 @@ func (c *Direct) sendMapRequest(ctx context.Context, maxPolls int, cb func(*netm
 			return err
 		}
 
+		metricMapResponseMessages.Add(1)
+
 		if allowStream {
 			health.GotStreamedMapResponse()
 		}
 
 		if pr := resp.PingRequest; pr != nil && c.isUniquePingRequest(pr) {
+			metricMapResponsePings.Add(1)
 			go answerPing(c.logf, c.httpc, pr)
 		}
 
@@ -758,13 +781,23 @@ func (c *Direct) sendMapRequest(ctx context.Context, maxPolls int, cb func(*netm
 			return ctx.Err()
 		}
 		if resp.KeepAlive {
+			metricMapResponseKeepAlives.Add(1)
 			continue
 		}
 
+		metricMapResponseMap.Add(1)
+		if i > 0 {
+			metricMapResponseMapDelta.Add(1)
+		}
+
 		hasDebug := resp.Debug != nil
 		// being conservative here, if Debug not present set to False
 		controlknobs.SetDisableUPnP(hasDebug && resp.Debug.DisableUPnP.EqualBool(true))
 		if hasDebug {
+			if code := resp.Debug.Exit; code != nil {
+				c.logf("exiting process with status %v per controlplane", *code)
+				os.Exit(*code)
+			}
 			if resp.Debug.LogHeapPprof {
 				go logheap.LogHeap(resp.Debug.LogHeapURL)
 			}
@@ -1167,7 +1200,13 @@ func sleepAsRequested(ctx context.Context, logf logger.Logf, timeoutReset chan<-
 
 // SetDNS sends the SetDNSRequest request to the control plane server,
 // requesting a DNS record be created or updated.
-func (c *Direct) SetDNS(ctx context.Context, req *tailcfg.SetDNSRequest) error {
+func (c *Direct) SetDNS(ctx context.Context, req *tailcfg.SetDNSRequest) (err error) {
+	metricSetDNS.Add(1)
+	defer func() {
+		if err != nil {
+			metricSetDNSError.Add(1)
+		}
+	}()
 	c.mu.Lock()
 	serverKey := c.serverKey
 	c.mu.Unlock()
@@ -1267,3 +1306,20 @@ func postPingResult(now time.Time, logf logger.Logf, c *http.Client, pr *tailcfg
 	}
 	return nil
 }
+
+var (
+	metricMapRequestsActive = clientmetric.NewGauge("controlclient_map_requests_active")
+
+	metricMapRequests     = clientmetric.NewCounter("controlclient_map_requests")
+	metricMapRequestsLite = clientmetric.NewCounter("controlclient_map_requests_lite")
+	metricMapRequestsPoll = clientmetric.NewCounter("controlclient_map_requests_poll")
+
+	metricMapResponseMessages   = clientmetric.NewCounter("controlclient_map_response_message") // any message type
+	metricMapResponsePings      = clientmetric.NewCounter("controlclient_map_response_ping")
+	metricMapResponseKeepAlives = clientmetric.NewCounter("controlclient_map_response_keepalive")
+	metricMapResponseMap        = clientmetric.NewCounter("controlclient_map_response_map")       // any non-keepalive map response
+	metricMapResponseMapDelta   = clientmetric.NewCounter("controlclient_map_response_map_delta") // 2nd+ non-keepalive map response
+
+	metricSetDNS      = clientmetric.NewCounter("controlclient_setdns")
+	metricSetDNSError = clientmetric.NewCounter("controlclient_setdns_error")
+)

control/controlclient/map.go

@@ -110,7 +110,7 @@ func (ms *mapSession) netmapForResponse(resp *tailcfg.MapResponse) *netmap.Netwo
 	}
 
 	nm := &netmap.NetworkMap{
-		NodeKey:         tailcfg.NodeKeyFromNodePublic(ms.privateNodeKey.Public()),
+		NodeKey:         ms.privateNodeKey.Public(),
 		PrivateKey:      ms.privateNodeKey,
 		MachineKey:      ms.machinePubKey,
 		Peers:           resp.Peers,

control/noise/conn.go

@@ -0,0 +1,359 @@
+// Copyright (c) 2021 Tailscale Inc & AUTHORS All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+// Package noise implements the base transport of the Tailscale 2021
+// control protocol.
+//
+// The base transport implements Noise IK, instantiated with
+// Curve25519, ChaCha20Poly1305 and BLAKE2s.
+package noise
+
+import (
+	"crypto/cipher"
+	"encoding/binary"
+	"fmt"
+	"net"
+	"sync"
+	"time"
+
+	"golang.org/x/crypto/blake2s"
+	chp "golang.org/x/crypto/chacha20poly1305"
+	"tailscale.com/types/key"
+)
+
+const (
+	// maxMessageSize is the maximum size of a protocol frame on the
+	// wire, including header and payload.
+	maxMessageSize = 4096
+	// maxCiphertextSize is the maximum amount of ciphertext bytes
+	// that one protocol frame can carry, after framing.
+	maxCiphertextSize = maxMessageSize - 3
+	// maxPlaintextSize is the maximum amount of plaintext bytes that
+	// one protocol frame can carry, after encryption and framing.
+	maxPlaintextSize = maxCiphertextSize - chp.Overhead
+)
+
+// A Conn is a secured Noise connection. It implements the net.Conn
+// interface, with the unusual trait that any write error (including a
+// SetWriteDeadline induced i/o timeout) causes all future writes to
+// fail.
+type Conn struct {
+	conn          net.Conn
+	version       uint16
+	peer          key.MachinePublic
+	handshakeHash [blake2s.Size]byte
+	rx            rxState
+	tx            txState
+}
+
+// rxState is all the Conn state that Read uses.
+type rxState struct {
+	sync.Mutex
+	cipher    cipher.AEAD
+	nonce     nonce
+	buf       [maxMessageSize]byte
+	n         int    // number of valid bytes in buf
+	next      int    // offset of next undecrypted packet
+	plaintext []byte // slice into buf of decrypted bytes
+}
+
+// txState is all the Conn state that Write uses.
+type txState struct {
+	sync.Mutex
+	cipher cipher.AEAD
+	nonce  nonce
+	buf    [maxMessageSize]byte
+	err    error // records the first partial write error for all future calls
+}
+
+// ProtocolVersion returns the protocol version that was used to
+// establish this Conn.
+func (c *Conn) ProtocolVersion() int {
+	return int(c.version)
+}
+
+// HandshakeHash returns the Noise handshake hash for the connection,
+// which can be used to bind other messages to this connection
+// (i.e. to ensure that the message wasn't replayed from a different
+// connection).
+func (c *Conn) HandshakeHash() [blake2s.Size]byte {
+	return c.handshakeHash
+}
+
+// Peer returns the peer's long-term public key.
+func (c *Conn) Peer() key.MachinePublic {
+	return c.peer
+}
+
+// readNLocked reads into c.rx.buf until buf contains at least total
+// bytes. Returns a slice of the total bytes in rxBuf, or an
+// error if fewer than total bytes are available.
+func (c *Conn) readNLocked(total int) ([]byte, error) {
+	if total > maxMessageSize {
+		return nil, errReadTooBig{total}
+	}
+	for {
+		if total <= c.rx.n {
+			return c.rx.buf[:total], nil
+		}
+
+		n, err := c.conn.Read(c.rx.buf[c.rx.n:])
+		c.rx.n += n
+		if err != nil {
+			return nil, err
+		}
+	}
+}
+
+// decryptLocked decrypts msg (which is header+ciphertext) in-place
+// and sets c.rx.plaintext to the decrypted bytes.
+func (c *Conn) decryptLocked(msg []byte) (err error) {
+	if msgType := msg[0]; msgType != msgTypeRecord {
+		return fmt.Errorf("received message with unexpected type %d, want %d", msgType, msgTypeRecord)
+	}
+	// We don't check the length field here, because the caller
+	// already did in order to figure out how big the msg slice should
+	// be.
+	ciphertext := msg[headerLen:]
+
+	if !c.rx.nonce.Valid() {
+		return errCipherExhausted{}
+	}
+
+	c.rx.plaintext, err = c.rx.cipher.Open(ciphertext[:0], c.rx.nonce[:], ciphertext, nil)
+	c.rx.nonce.Increment()
+
+	if err != nil {
+		// Once a decryption has failed, our Conn is no longer
+		// synchronized with our peer. Nuke the cipher state to be
+		// safe, so that no further decryptions are attempted. Future
+		// read attempts will return net.ErrClosed.
+		c.rx.cipher = nil
+	}
+	return err
+}
+
+// encryptLocked encrypts plaintext into c.tx.buf (including the
+// packet header) and returns a slice of the ciphertext, or an error
+// if the cipher is exhausted (i.e. can no longer be used safely).
+func (c *Conn) encryptLocked(plaintext []byte) ([]byte, error) {
+	if !c.tx.nonce.Valid() {
+		// Received 2^64-1 messages on this cipher state. Connection
+		// is no longer usable.
+		return nil, errCipherExhausted{}
+	}
+
+	c.tx.buf[0] = msgTypeRecord
+	binary.BigEndian.PutUint16(c.tx.buf[1:headerLen], uint16(len(plaintext)+chp.Overhead))
+	ret := c.tx.cipher.Seal(c.tx.buf[:headerLen], c.tx.nonce[:], plaintext, nil)
+	c.tx.nonce.Increment()
+
+	return ret, nil
+}
+
+// wholeMessageLocked returns a slice of one whole Noise transport
+// message from c.rx.buf, if one whole message is available, and
+// advances the read state to the next Noise message in the
+// buffer. Returns nil without advancing read state if there isn't one
+// whole message in c.rx.buf.
+func (c *Conn) wholeMessageLocked() []byte {
+	available := c.rx.n - c.rx.next
+	if available < headerLen {
+		return nil
+	}
+	bs := c.rx.buf[c.rx.next:c.rx.n]
+	totalSize := headerLen + int(binary.BigEndian.Uint16(bs[1:3]))
+	if len(bs) < totalSize {
+		return nil
+	}
+	c.rx.next += totalSize
+	return bs[:totalSize]
+}
+
+// decryptOneLocked decrypts one Noise transport message, reading from
+// c.conn as needed, and sets c.rx.plaintext to point to the decrypted
+// bytes. c.rx.plaintext is only valid if err == nil.
+func (c *Conn) decryptOneLocked() error {
+	c.rx.plaintext = nil
+
+	// Fast path: do we have one whole ciphertext frame buffered
+	// already?
+	if bs := c.wholeMessageLocked(); bs != nil {
+		return c.decryptLocked(bs)
+	}
+
+	if c.rx.next != 0 {
+		// To simplify the read logic, move the remainder of the
+		// buffered bytes back to the head of the buffer, so we can
+		// grow it without worrying about wraparound.
+		c.rx.n = copy(c.rx.buf[:], c.rx.buf[c.rx.next:c.rx.n])
+		c.rx.next = 0
+	}
+
+	bs, err := c.readNLocked(headerLen)
+	if err != nil {
+		return err
+	}
+	// The rest of the header (besides the length field) gets verified
+	// in decryptLocked, not here.
+	messageLen := headerLen + int(binary.BigEndian.Uint16(bs[1:3]))
+	bs, err = c.readNLocked(messageLen)
+	if err != nil {
+		return err
+	}
+
+	c.rx.next = len(bs)
+
+	return c.decryptLocked(bs)
+}
+
+// Read implements io.Reader.
+func (c *Conn) Read(bs []byte) (int, error) {
+	c.rx.Lock()
+	defer c.rx.Unlock()
+
+	if c.rx.cipher == nil {
+		return 0, net.ErrClosed
+	}
+	// If no plaintext is buffered, decrypt incoming frames until we
+	// have some plaintext. Zero-byte Noise frames are allowed in this
+	// protocol, which is why we have to loop here rather than decrypt
+	// a single additional frame.
+	for len(c.rx.plaintext) == 0 {
+		if err := c.decryptOneLocked(); err != nil {
+			return 0, err
+		}
+	}
+	n := copy(bs, c.rx.plaintext)
+	c.rx.plaintext = c.rx.plaintext[n:]
+	return n, nil
+}
+
+// Write implements io.Writer.
+func (c *Conn) Write(bs []byte) (n int, err error) {
+	c.tx.Lock()
+	defer c.tx.Unlock()
+
+	if c.tx.err != nil {
+		return 0, c.tx.err
+	}
+	defer func() {
+		if err != nil {
+			// All write errors are fatal for this conn, so clear the
+			// cipher state whenever an error happens.
+			c.tx.cipher = nil
+		}
+		if c.tx.err == nil {
+			// Only set c.tx.err if not nil so that we can return one
+			// error on the first failure, and a different one for
+			// subsequent calls. See the error handling around Write
+			// below for why.
+			c.tx.err = err
+		}
+	}()
+
+	if c.tx.cipher == nil {
+		return 0, net.ErrClosed
+	}
+
+	var sent int
+	for len(bs) > 0 {
+		toSend := bs
+		if len(toSend) > maxPlaintextSize {
+			toSend = bs[:maxPlaintextSize]
+		}
+		bs = bs[len(toSend):]
+
+		ciphertext, err := c.encryptLocked(toSend)
+		if err != nil {
+			return 0, err
+		}
+
+		n, err := c.conn.Write(ciphertext)
+		sent += n
+		if err != nil {
+			// Return the raw error on the Write that actually
+			// failed. For future writes, return that error wrapped in
+			// a desync error.
+			c.tx.err = errPartialWrite{err}
+			return sent, err
+		}
+	}
+	return sent, nil
+}
+
+// Close implements io.Closer.
+func (c *Conn) Close() error {
+	closeErr := c.conn.Close() // unblocks any waiting reads or writes
+
+	// Remove references to live cipher state. Strictly speaking this
+	// is unnecessary, but we want to try and hand the active cipher
+	// state to the garbage collector promptly, to preserve perfect
+	// forward secrecy as much as we can.
+	c.rx.Lock()
+	c.rx.cipher = nil
+	c.rx.Unlock()
+	c.tx.Lock()
+	c.tx.cipher = nil
+	c.tx.Unlock()
+	return closeErr
+}
+
+func (c *Conn) LocalAddr() net.Addr                { return c.conn.LocalAddr() }
+func (c *Conn) RemoteAddr() net.Addr               { return c.conn.RemoteAddr() }
+func (c *Conn) SetDeadline(t time.Time) error      { return c.conn.SetDeadline(t) }
+func (c *Conn) SetReadDeadline(t time.Time) error  { return c.conn.SetReadDeadline(t) }
+func (c *Conn) SetWriteDeadline(t time.Time) error { return c.conn.SetWriteDeadline(t) }
+
+// errCipherExhausted is the error returned when we run out of nonces
+// on a cipher.
+type errCipherExhausted struct{}
+
+func (errCipherExhausted) Error() string {
+	return "cipher exhausted, no more nonces available for current key"
+}
+func (errCipherExhausted) Timeout() bool   { return false }
+func (errCipherExhausted) Temporary() bool { return false }
+
+// errPartialWrite is the error returned when the cipher state has
+// become unusable due to a past partial write.
+type errPartialWrite struct {
+	err error
+}
+
+func (e errPartialWrite) Error() string {
+	return fmt.Sprintf("cipher state desynchronized due to partial write (%v)", e.err)
+}
+func (e errPartialWrite) Unwrap() error   { return e.err }
+func (e errPartialWrite) Temporary() bool { return false }
+func (e errPartialWrite) Timeout() bool   { return false }
+
+// errReadTooBig is the error returned when the peer sent an
+// unacceptably large Noise frame.
+type errReadTooBig struct {
+	requested int
+}
+
+func (e errReadTooBig) Error() string {
+	return fmt.Sprintf("requested read of %d bytes exceeds max allowed Noise frame size", e.requested)
+}
+func (e errReadTooBig) Temporary() bool {
+	// permanent error because this error only occurs when our peer
+	// sends us a frame so large we're unwilling to ever decode it.
+	return false
+}
+func (e errReadTooBig) Timeout() bool { return false }
+
+type nonce [chp.NonceSize]byte
+
+func (n *nonce) Valid() bool {
+	return binary.BigEndian.Uint32(n[:4]) == 0 && binary.BigEndian.Uint64(n[4:]) != invalidNonce
+}
+
+func (n *nonce) Increment() {
+	if !n.Valid() {
+		panic("increment of invalid nonce")
+	}
+	binary.BigEndian.PutUint64(n[4:], 1+binary.BigEndian.Uint64(n[4:]))
+}

control/noise/conn_test.go

@@ -0,0 +1,339 @@
+// Copyright (c) 2021 Tailscale Inc & AUTHORS All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package noise
+
+import (
+	"bufio"
+	"bytes"
+	"context"
+	"crypto/rand"
+	"encoding/binary"
+	"fmt"
+	"io"
+	"net"
+	"strings"
+	"sync"
+	"testing"
+	"testing/iotest"
+
+	chp "golang.org/x/crypto/chacha20poly1305"
+	"golang.org/x/net/nettest"
+	tsnettest "tailscale.com/net/nettest"
+	"tailscale.com/types/key"
+)
+
+func TestMessageSize(t *testing.T) {
+	// This test is a regression guard against someone looking at
+	// maxCiphertextSize, going "huh, we could be more efficient if it
+	// were larger, and accidentally violating the Noise spec. Do not
+	// change this max value, it's a deliberate limitation of the
+	// cryptographic protocol we use (see Section 3 "Message Format"
+	// of the Noise spec).
+	const max = 65535
+	if maxCiphertextSize > max {
+		t.Fatalf("max ciphertext size is %d, which is larger than the maximum noise message size %d", maxCiphertextSize, max)
+	}
+}
+
+func TestConnBasic(t *testing.T) {
+	client, server := pair(t)
+
+	sb := sinkReads(server)
+
+	want := "test"
+	if _, err := io.WriteString(client, want); err != nil {
+		t.Fatalf("client write failed: %v", err)
+	}
+	client.Close()
+
+	if got := sb.String(4); got != want {
+		t.Fatalf("wrong content received: got %q, want %q", got, want)
+	}
+	if err := sb.Error(); err != io.EOF {
+		t.Fatal("client close wasn't seen by server")
+	}
+	if sb.Total() != 4 {
+		t.Fatalf("wrong amount of bytes received: got %d, want 4", sb.Total())
+	}
+}
+
+// bufferedWriteConn wraps a net.Conn and gives control over how
+// Writes get batched out.
+type bufferedWriteConn struct {
+	net.Conn
+	w           *bufio.Writer
+	manualFlush bool
+}
+
+func (c *bufferedWriteConn) Write(bs []byte) (int, error) {
+	n, err := c.w.Write(bs)
+	if err == nil && !c.manualFlush {
+		err = c.w.Flush()
+	}
+	return n, err
+}
+
+// TestFastPath exercises the Read codepath that can receive multiple
+// Noise frames at once and decode each in turn without making another
+// syscall.
+func TestFastPath(t *testing.T) {
+	s1, s2 := tsnettest.NewConn("noise", 128000)
+	b := &bufferedWriteConn{s1, bufio.NewWriterSize(s1, 10000), false}
+	client, server := pairWithConns(t, b, s2)
+
+	b.manualFlush = true
+
+	sb := sinkReads(server)
+
+	const packets = 10
+	s := "test"
+	for i := 0; i < packets; i++ {
+		// Many separate writes, to force separate Noise frames that
+		// all get buffered up and then all sent as a single slice to
+		// the server.
+		if _, err := io.WriteString(client, s); err != nil {
+			t.Fatalf("client write1 failed: %v", err)
+		}
+	}
+	if err := b.w.Flush(); err != nil {
+		t.Fatalf("client flush failed: %v", err)
+	}
+	client.Close()
+
+	want := strings.Repeat(s, packets)
+	if got := sb.String(len(want)); got != want {
+		t.Fatalf("wrong content received: got %q, want %q", got, want)
+	}
+	if err := sb.Error(); err != io.EOF {
+		t.Fatalf("client close wasn't seen by server")
+	}
+}
+
+// Writes things larger than a single Noise frame, to check the
+// chunking on the encoder and decoder.
+func TestBigData(t *testing.T) {
+	client, server := pair(t)
+
+	serverReads := sinkReads(server)
+	clientReads := sinkReads(client)
+
+	const sz = 15 * 1024 // 15KiB
+	clientStr := strings.Repeat("abcde", sz/5)
+	serverStr := strings.Repeat("fghij", sz/5*2)
+
+	if _, err := io.WriteString(client, clientStr); err != nil {
+		t.Fatalf("writing client>server: %v", err)
+	}
+	if _, err := io.WriteString(server, serverStr); err != nil {
+		t.Fatalf("writing server>client: %v", err)
+	}
+
+	if serverGot := serverReads.String(sz); serverGot != clientStr {
+		t.Error("server didn't receive what client sent")
+	}
+	if clientGot := clientReads.String(2 * sz); clientGot != serverStr {
+		t.Error("client didn't receive what server sent")
+	}
+
+	getNonce := func(n [chp.NonceSize]byte) uint64 {
+		if binary.BigEndian.Uint32(n[:4]) != 0 {
+			panic("unexpected nonce")
+		}
+		return binary.BigEndian.Uint64(n[4:])
+	}
+
+	// Reach into the Conns and verify the cipher nonces advanced as
+	// expected.
+	if getNonce(client.tx.nonce) != getNonce(server.rx.nonce) {
+		t.Error("desynchronized client tx nonce")
+	}
+	if getNonce(server.tx.nonce) != getNonce(client.rx.nonce) {
+		t.Error("desynchronized server tx nonce")
+	}
+	if n := getNonce(client.tx.nonce); n != 4 {
+		t.Errorf("wrong client tx nonce, got %d want 4", n)
+	}
+	if n := getNonce(server.tx.nonce); n != 8 {
+		t.Errorf("wrong client tx nonce, got %d want 8", n)
+	}
+}
+
+// readerConn wraps a net.Conn and routes its Reads through a separate
+// io.Reader.
+type readerConn struct {
+	net.Conn
+	r io.Reader
+}
+
+func (c readerConn) Read(bs []byte) (int, error) { return c.r.Read(bs) }
+
+// Check that the receiver can handle not being able to read an entire
+// frame in a single syscall.
+func TestDataTrickle(t *testing.T) {
+	s1, s2 := tsnettest.NewConn("noise", 128000)
+	client, server := pairWithConns(t, s1, readerConn{s2, iotest.OneByteReader(s2)})
+	serverReads := sinkReads(server)
+
+	const sz = 10000
+	clientStr := strings.Repeat("abcde", sz/5)
+	if _, err := io.WriteString(client, clientStr); err != nil {
+		t.Fatalf("writing client>server: %v", err)
+	}
+
+	serverGot := serverReads.String(sz)
+	if serverGot != clientStr {
+		t.Error("server didn't receive what client sent")
+	}
+}
+
+func TestConnStd(t *testing.T) {
+	// You can run this test manually, and noise.Conn should pass all
+	// of them except for TestConn/PastTimeout,
+	// TestConn/FutureTimeout, TestConn/ConcurrentMethods, because
+	// those tests assume that write errors are recoverable, and
+	// they're not on our Conn due to cipher security.
+	t.Skip("not all tests can pass on this Conn, see https://github.com/golang/go/issues/46977")
+	nettest.TestConn(t, func() (c1 net.Conn, c2 net.Conn, stop func(), err error) {
+		s1, s2 := tsnettest.NewConn("noise", 4096)
+		controlKey := key.NewMachine()
+		machineKey := key.NewMachine()
+		serverErr := make(chan error, 1)
+		go func() {
+			var err error
+			c2, err = Server(context.Background(), s2, controlKey)
+			serverErr <- err
+		}()
+		c1, err = Client(context.Background(), s1, machineKey, controlKey.Public())
+		if err != nil {
+			s1.Close()
+			s2.Close()
+			return nil, nil, nil, fmt.Errorf("connecting client: %w", err)
+		}
+		if err := <-serverErr; err != nil {
+			c1.Close()
+			s1.Close()
+			s2.Close()
+			return nil, nil, nil, fmt.Errorf("connecting server: %w", err)
+		}
+		return c1, c2, func() {
+			c1.Close()
+			c2.Close()
+		}, nil
+	})
+}
+
+// mkConns creates synthetic Noise Conns wrapping the given net.Conns.
+// This function is for testing just the Conn transport logic without
+// having to muck about with Noise handshakes.
+func mkConns(s1, s2 net.Conn) (*Conn, *Conn) {
+	var k1, k2 [chp.KeySize]byte
+	if _, err := rand.Read(k1[:]); err != nil {
+		panic(err)
+	}
+	if _, err := rand.Read(k2[:]); err != nil {
+		panic(err)
+	}
+
+	ret1 := &Conn{
+		conn: s1,
+		tx:   txState{cipher: newCHP(k1)},
+		rx:   rxState{cipher: newCHP(k2)},
+	}
+	ret2 := &Conn{
+		conn: s2,
+		tx:   txState{cipher: newCHP(k2)},
+		rx:   rxState{cipher: newCHP(k1)},
+	}
+
+	return ret1, ret2
+}
+
+type readSink struct {
+	r io.Reader
+
+	cond *sync.Cond
+	sync.Mutex
+	bs  bytes.Buffer
+	err error
+}
+
+func sinkReads(r io.Reader) *readSink {
+	ret := &readSink{
+		r: r,
+	}
+	ret.cond = sync.NewCond(&ret.Mutex)
+	go func() {
+		var buf [4096]byte
+		for {
+			n, err := r.Read(buf[:])
+			ret.Lock()
+			ret.bs.Write(buf[:n])
+			if err != nil {
+				ret.err = err
+			}
+			ret.cond.Broadcast()
+			ret.Unlock()
+			if err != nil {
+				return
+			}
+		}
+	}()
+	return ret
+}
+
+func (s *readSink) String(total int) string {
+	s.Lock()
+	defer s.Unlock()
+	for s.bs.Len() < total && s.err == nil {
+		s.cond.Wait()
+	}
+	if s.err != nil {
+		total = s.bs.Len()
+	}
+	return string(s.bs.Bytes()[:total])
+}
+
+func (s *readSink) Error() error {
+	s.Lock()
+	defer s.Unlock()
+	for s.err == nil {
+		s.cond.Wait()
+	}
+	return s.err
+}
+
+func (s *readSink) Total() int {
+	s.Lock()
+	defer s.Unlock()
+	return s.bs.Len()
+}
+
+func pairWithConns(t *testing.T, clientConn, serverConn net.Conn) (*Conn, *Conn) {
+	var (
+		controlKey = key.NewMachine()
+		machineKey = key.NewMachine()
+		server     *Conn
+		serverErr  = make(chan error, 1)
+	)
+	go func() {
+		var err error
+		server, err = Server(context.Background(), serverConn, controlKey)
+		serverErr <- err
+	}()
+
+	client, err := Client(context.Background(), clientConn, machineKey, controlKey.Public())
+	if err != nil {
+		t.Fatalf("client connection failed: %v", err)
+	}
+	if err := <-serverErr; err != nil {
+		t.Fatalf("server connection failed: %v", err)
+	}
+	return client, server
+}
+
+func pair(t *testing.T) (*Conn, *Conn) {
+	s1, s2 := tsnettest.NewConn("noise", 128000)
+	return pairWithConns(t, s1, s2)
+}

control/noise/handshake.go

@@ -0,0 +1,443 @@
+// Copyright (c) 2021 Tailscale Inc & AUTHORS All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package noise
+
+import (
+	"context"
+	"crypto/cipher"
+	"encoding/binary"
+	"errors"
+	"fmt"
+	"hash"
+	"io"
+	"net"
+	"strconv"
+	"time"
+
+	"go4.org/mem"
+	"golang.org/x/crypto/blake2s"
+	chp "golang.org/x/crypto/chacha20poly1305"
+	"golang.org/x/crypto/curve25519"
+	"golang.org/x/crypto/hkdf"
+	"tailscale.com/types/key"
+)
+
+const (
+	// protocolName is the name of the specific instantiation of Noise
+	// that the control protocol uses. This string's value is fixed by
+	// the Noise spec, and shouldn't be changed unless we're updating
+	// the control protocol to use a different Noise instance.
+	protocolName = "Noise_IK_25519_ChaChaPoly_BLAKE2s"
+	// protocolVersion is the version of the control protocol that
+	// Client will use when initiating a handshake.
+	protocolVersion uint16 = 1
+	// protocolVersionPrefix is the name portion of the protocol
+	// name+version string that gets mixed into the handshake as a
+	// prologue.
+	//
+	// This mixing verifies that both clients agree that they're
+	// executing the control protocol at a specific version that
+	// matches the advertised version in the cleartext packet header.
+	protocolVersionPrefix = "Tailscale Control Protocol v"
+	invalidNonce          = ^uint64(0)
+)
+
+func protocolVersionPrologue(version uint16) []byte {
+	ret := make([]byte, 0, len(protocolVersionPrefix)+5) // 5 bytes is enough to encode all possible version numbers.
+	ret = append(ret, protocolVersionPrefix...)
+	return strconv.AppendUint(ret, uint64(version), 10)
+}
+
+// Client initiates a control client handshake, returning the resulting
+// control connection.
+//
+// The context deadline, if any, covers the entire handshaking
+// process. Any preexisting Conn deadline is removed.
+func Client(ctx context.Context, conn net.Conn, machineKey key.MachinePrivate, controlKey key.MachinePublic) (*Conn, error) {
+	if deadline, ok := ctx.Deadline(); ok {
+		if err := conn.SetDeadline(deadline); err != nil {
+			return nil, fmt.Errorf("setting conn deadline: %w", err)
+		}
+		defer func() {
+			conn.SetDeadline(time.Time{})
+		}()
+	}
+
+	var s symmetricState
+	s.Initialize()
+
+	// prologue
+	s.MixHash(protocolVersionPrologue(protocolVersion))
+
+	// <- s
+	// ...
+	s.MixHash(controlKey.UntypedBytes())
+
+	// -> e, es, s, ss
+	init := mkInitiationMessage()
+	machineEphemeral := key.NewMachine()
+	machineEphemeralPub := machineEphemeral.Public()
+	copy(init.EphemeralPub(), machineEphemeralPub.UntypedBytes())
+	s.MixHash(machineEphemeralPub.UntypedBytes())
+	cipher, err := s.MixDH(machineEphemeral, controlKey)
+	if err != nil {
+		return nil, fmt.Errorf("computing es: %w", err)
+	}
+	machineKeyPub := machineKey.Public()
+	s.EncryptAndHash(cipher, init.MachinePub(), machineKeyPub.UntypedBytes())
+	cipher, err = s.MixDH(machineKey, controlKey)
+	if err != nil {
+		return nil, fmt.Errorf("computing ss: %w", err)
+	}
+	s.EncryptAndHash(cipher, init.Tag(), nil) // empty message payload
+
+	if _, err := conn.Write(init[:]); err != nil {
+		return nil, fmt.Errorf("writing initiation: %w", err)
+	}
+
+	// Read in the payload and look for errors/protocol violations from the server.
+	var resp responseMessage
+	if _, err := io.ReadFull(conn, resp.Header()); err != nil {
+		return nil, fmt.Errorf("reading response header: %w", err)
+	}
+	if resp.Type() != msgTypeResponse {
+		if resp.Type() != msgTypeError {
+			return nil, fmt.Errorf("unexpected response message type %d", resp.Type())
+		}
+		msg := make([]byte, resp.Length())
+		if _, err := io.ReadFull(conn, msg); err != nil {
+			return nil, err
+		}
+		return nil, fmt.Errorf("server error: %q", msg)
+	}
+	if resp.Length() != len(resp.Payload()) {
+		return nil, fmt.Errorf("wrong length %d received for handshake response", resp.Length())
+	}
+	if _, err := io.ReadFull(conn, resp.Payload()); err != nil {
+		return nil, err
+	}
+
+	// <- e, ee, se
+	controlEphemeralPub := key.MachinePublicFromRaw32(mem.B(resp.EphemeralPub()))
+	s.MixHash(controlEphemeralPub.UntypedBytes())
+	if _, err = s.MixDH(machineEphemeral, controlEphemeralPub); err != nil {
+		return nil, fmt.Errorf("computing ee: %w", err)
+	}
+	cipher, err = s.MixDH(machineKey, controlEphemeralPub)
+	if err != nil {
+		return nil, fmt.Errorf("computing se: %w", err)
+	}
+	if err := s.DecryptAndHash(cipher, nil, resp.Tag()); err != nil {
+		return nil, fmt.Errorf("decrypting payload: %w", err)
+	}
+
+	c1, c2, err := s.Split()
+	if err != nil {
+		return nil, fmt.Errorf("finalizing handshake: %w", err)
+	}
+
+	c := &Conn{
+		conn:          conn,
+		version:       protocolVersion,
+		peer:          controlKey,
+		handshakeHash: s.h,
+		tx: txState{
+			cipher: c1,
+		},
+		rx: rxState{
+			cipher: c2,
+		},
+	}
+	return c, nil
+}
+
+// Server initiates a control server handshake, returning the resulting
+// control connection.
+//
+// The context deadline, if any, covers the entire handshaking
+// process.
+func Server(ctx context.Context, conn net.Conn, controlKey key.MachinePrivate) (*Conn, error) {
+	if deadline, ok := ctx.Deadline(); ok {
+		if err := conn.SetDeadline(deadline); err != nil {
+			return nil, fmt.Errorf("setting conn deadline: %w", err)
+		}
+		defer func() {
+			conn.SetDeadline(time.Time{})
+		}()
+	}
+
+	// Deliberately does not support formatting, so that we don't echo
+	// attacker-controlled input back to them.
+	sendErr := func(msg string) error {
+		if len(msg) >= 1<<16 {
+			msg = msg[:1<<16]
+		}
+		var hdr [headerLen]byte
+		hdr[0] = msgTypeError
+		binary.BigEndian.PutUint16(hdr[1:3], uint16(len(msg)))
+		if _, err := conn.Write(hdr[:]); err != nil {
+			return fmt.Errorf("sending %q error to client: %w", msg, err)
+		}
+		if _, err := io.WriteString(conn, msg); err != nil {
+			return fmt.Errorf("sending %q error to client: %w", msg, err)
+		}
+		return fmt.Errorf("refused client handshake: %q", msg)
+	}
+
+	var s symmetricState
+	s.Initialize()
+
+	var init initiationMessage
+	if _, err := io.ReadFull(conn, init.Header()); err != nil {
+		return nil, err
+	}
+	if init.Version() != protocolVersion {
+		return nil, sendErr("unsupported protocol version")
+	}
+	if init.Type() != msgTypeInitiation {
+		return nil, sendErr("unexpected handshake message type")
+	}
+	if init.Length() != len(init.Payload()) {
+		return nil, sendErr("wrong handshake initiation length")
+	}
+	if _, err := io.ReadFull(conn, init.Payload()); err != nil {
+		return nil, err
+	}
+
+	// prologue. Can only do this once we at least think the client is
+	// handshaking using a supported version.
+	s.MixHash(protocolVersionPrologue(protocolVersion))
+
+	// <- s
+	// ...
+	controlKeyPub := controlKey.Public()
+	s.MixHash(controlKeyPub.UntypedBytes())
+
+	// -> e, es, s, ss
+	machineEphemeralPub := key.MachinePublicFromRaw32(mem.B(init.EphemeralPub()))
+	s.MixHash(machineEphemeralPub.UntypedBytes())
+	cipher, err := s.MixDH(controlKey, machineEphemeralPub)
+	if err != nil {
+		return nil, fmt.Errorf("computing es: %w", err)
+	}
+	var machineKeyBytes [32]byte
+	if err := s.DecryptAndHash(cipher, machineKeyBytes[:], init.MachinePub()); err != nil {
+		return nil, fmt.Errorf("decrypting machine key: %w", err)
+	}
+	machineKey := key.MachinePublicFromRaw32(mem.B(machineKeyBytes[:]))
+	cipher, err = s.MixDH(controlKey, machineKey)
+	if err != nil {
+		return nil, fmt.Errorf("computing ss: %w", err)
+	}
+	if err := s.DecryptAndHash(cipher, nil, init.Tag()); err != nil {
+		return nil, fmt.Errorf("decrypting initiation tag: %w", err)
+	}
+
+	// <- e, ee, se
+	resp := mkResponseMessage()
+	controlEphemeral := key.NewMachine()
+	controlEphemeralPub := controlEphemeral.Public()
+	copy(resp.EphemeralPub(), controlEphemeralPub.UntypedBytes())
+	s.MixHash(controlEphemeralPub.UntypedBytes())
+	if _, err := s.MixDH(controlEphemeral, machineEphemeralPub); err != nil {
+		return nil, fmt.Errorf("computing ee: %w", err)
+	}
+	cipher, err = s.MixDH(controlEphemeral, machineKey)
+	if err != nil {
+		return nil, fmt.Errorf("computing se: %w", err)
+	}
+	s.EncryptAndHash(cipher, resp.Tag(), nil) // empty message payload
+
+	c1, c2, err := s.Split()
+	if err != nil {
+		return nil, fmt.Errorf("finalizing handshake: %w", err)
+	}
+
+	if _, err := conn.Write(resp[:]); err != nil {
+		return nil, err
+	}
+
+	c := &Conn{
+		conn:          conn,
+		version:       protocolVersion,
+		peer:          machineKey,
+		handshakeHash: s.h,
+		tx: txState{
+			cipher: c2,
+		},
+		rx: rxState{
+			cipher: c1,
+		},
+	}
+	return c, nil
+}
+
+// symmetricState contains the state of an in-flight handshake.
+type symmetricState struct {
+	finished bool
+
+	h  [blake2s.Size]byte // hash of currently-processed handshake state
+	ck [blake2s.Size]byte // chaining key used to construct session keys at the end of the handshake
+}
+
+func (s *symmetricState) checkFinished() {
+	if s.finished {
+		panic("attempted to use symmetricState after Split was called")
+	}
+}
+
+// Initialize sets s to the initial handshake state, prior to
+// processing any handshake messages.
+func (s *symmetricState) Initialize() {
+	s.checkFinished()
+	s.h = blake2s.Sum256([]byte(protocolName))
+	s.ck = s.h
+}
+
+// MixHash updates s.h to be BLAKE2s(s.h || data), where || is
+// concatenation.
+func (s *symmetricState) MixHash(data []byte) {
+	s.checkFinished()
+	h := newBLAKE2s()
+	h.Write(s.h[:])
+	h.Write(data)
+	h.Sum(s.h[:0])
+}
+
+// MixDH updates s.ck with the result of X25519(priv, pub) and returns
+// a singleUseCHP that can be used to encrypt or decrypt handshake
+// data.
+//
+// MixDH corresponds to MixKey(X25519(...))) in the spec. Implementing
+// it as a single function allows for strongly-typed arguments that
+// reduce the risk of error in the caller (e.g. invoking X25519 with
+// two private keys, or two public keys), and thus producing the wrong
+// calculation.
+func (s *symmetricState) MixDH(priv key.MachinePrivate, pub key.MachinePublic) (*singleUseCHP, error) {
+	s.checkFinished()
+	keyData, err := curve25519.X25519(priv.UntypedBytes(), pub.UntypedBytes())
+	if err != nil {
+		return nil, fmt.Errorf("computing X25519: %w", err)
+	}
+
+	r := hkdf.New(newBLAKE2s, keyData, s.ck[:], nil)
+	if _, err := io.ReadFull(r, s.ck[:]); err != nil {
+		return nil, fmt.Errorf("extracting ck: %w", err)
+	}
+	var k [chp.KeySize]byte
+	if _, err := io.ReadFull(r, k[:]); err != nil {
+		return nil, fmt.Errorf("extracting k: %w", err)
+	}
+	return newSingleUseCHP(k), nil
+}
+
+// EncryptAndHash encrypts plaintext into ciphertext (which must be
+// the correct size to hold the encrypted plaintext) using cipher,
+// mixes the ciphertext into s.h, and returns the ciphertext.
+func (s *symmetricState) EncryptAndHash(cipher *singleUseCHP, ciphertext, plaintext []byte) {
+	s.checkFinished()
+	if len(ciphertext) != len(plaintext)+chp.Overhead {
+		panic("ciphertext is wrong size for given plaintext")
+	}
+	ret := cipher.Seal(ciphertext[:0], plaintext, s.h[:])
+	s.MixHash(ret)
+}
+
+// DecryptAndHash decrypts the given ciphertext into plaintext (which
+// must be the correct size to hold the decrypted ciphertext) using
+// cipher. If decryption is successful, it mixes the ciphertext into
+// s.h.
+func (s *symmetricState) DecryptAndHash(cipher *singleUseCHP, plaintext, ciphertext []byte) error {
+	s.checkFinished()
+	if len(ciphertext) != len(plaintext)+chp.Overhead {
+		return errors.New("plaintext is wrong size for given ciphertext")
+	}
+	if _, err := cipher.Open(plaintext[:0], ciphertext, s.h[:]); err != nil {
+		return err
+	}
+	s.MixHash(ciphertext)
+	return nil
+}
+
+// Split returns two ChaCha20Poly1305 ciphers with keys derived from
+// the current handshake state. Methods on s cannot be used again
+// after calling Split.
+func (s *symmetricState) Split() (c1, c2 cipher.AEAD, err error) {
+	s.finished = true
+
+	var k1, k2 [chp.KeySize]byte
+	r := hkdf.New(newBLAKE2s, nil, s.ck[:], nil)
+	if _, err := io.ReadFull(r, k1[:]); err != nil {
+		return nil, nil, fmt.Errorf("extracting k1: %w", err)
+	}
+	if _, err := io.ReadFull(r, k2[:]); err != nil {
+		return nil, nil, fmt.Errorf("extracting k2: %w", err)
+	}
+	c1, err = chp.New(k1[:])
+	if err != nil {
+		return nil, nil, fmt.Errorf("constructing AEAD c1: %w", err)
+	}
+	c2, err = chp.New(k2[:])
+	if err != nil {
+		return nil, nil, fmt.Errorf("constructing AEAD c2: %w", err)
+	}
+	return c1, c2, nil
+}
+
+// newBLAKE2s returns a hash.Hash implementing BLAKE2s, or panics on
+// error.
+func newBLAKE2s() hash.Hash {
+	h, err := blake2s.New256(nil)
+	if err != nil {
+		// Should never happen, errors only happen when using BLAKE2s
+		// in MAC mode with a key.
+		panic(err)
+	}
+	return h
+}
+
+// newCHP returns a cipher.AEAD implementing ChaCha20Poly1305, or
+// panics on error.
+func newCHP(key [chp.KeySize]byte) cipher.AEAD {
+	aead, err := chp.New(key[:])
+	if err != nil {
+		// Can only happen if we passed a key of the wrong length. The
+		// function signature prevents that.
+		panic(err)
+	}
+	return aead
+}
+
+// singleUseCHP is an instance of ChaCha20Poly1305 that can be used
+// only once, either for encrypting or decrypting, but not both. The
+// chosen operation is always executed with an all-zeros
+// nonce. Subsequent calls to either Seal or Open panic.
+type singleUseCHP struct {
+	c cipher.AEAD
+}
+
+func newSingleUseCHP(key [chp.KeySize]byte) *singleUseCHP {
+	return &singleUseCHP{newCHP(key)}
+}
+
+func (c *singleUseCHP) Seal(dst, plaintext, additionalData []byte) []byte {
+	if c.c == nil {
+		panic("Attempted reuse of singleUseAEAD")
+	}
+	cipher := c.c
+	c.c = nil
+	var nonce [chp.NonceSize]byte
+	return cipher.Seal(dst, nonce[:], plaintext, additionalData)
+}
+
+func (c *singleUseCHP) Open(dst, ciphertext, additionalData []byte) ([]byte, error) {
+	if c.c == nil {
+		panic("Attempted reuse of singleUseAEAD")
+	}
+	cipher := c.c
+	c.c = nil
+	var nonce [chp.NonceSize]byte
+	return cipher.Open(dst, nonce[:], ciphertext, additionalData)
+}

control/noise/handshake_test.go

@@ -0,0 +1,299 @@
+// Copyright (c) 2021 Tailscale Inc & AUTHORS All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package noise
+
+import (
+	"bytes"
+	"context"
+	"io"
+	"strings"
+	"testing"
+	"time"
+
+	tsnettest "tailscale.com/net/nettest"
+	"tailscale.com/types/key"
+)
+
+func TestHandshake(t *testing.T) {
+	var (
+		clientConn, serverConn = tsnettest.NewConn("noise", 128000)
+		serverKey              = key.NewMachine()
+		clientKey              = key.NewMachine()
+		server                 *Conn
+		serverErr              = make(chan error, 1)
+	)
+	go func() {
+		var err error
+		server, err = Server(context.Background(), serverConn, serverKey)
+		serverErr <- err
+	}()
+
+	client, err := Client(context.Background(), clientConn, clientKey, serverKey.Public())
+	if err != nil {
+		t.Fatalf("client connection failed: %v", err)
+	}
+	if err := <-serverErr; err != nil {
+		t.Fatalf("server connection failed: %v", err)
+	}
+
+	if client.HandshakeHash() != server.HandshakeHash() {
+		t.Fatal("client and server disagree on handshake hash")
+	}
+
+	if client.ProtocolVersion() != int(protocolVersion) {
+		t.Fatalf("client reporting wrong protocol version %d, want %d", client.ProtocolVersion(), protocolVersion)
+	}
+	if client.ProtocolVersion() != server.ProtocolVersion() {
+		t.Fatalf("peers disagree on protocol version, client=%d server=%d", client.ProtocolVersion(), server.ProtocolVersion())
+	}
+	if client.Peer() != serverKey.Public() {
+		t.Fatal("client peer key isn't serverKey")
+	}
+	if server.Peer() != clientKey.Public() {
+		t.Fatal("client peer key isn't serverKey")
+	}
+}
+
+// Check that handshaking repeatedly with the same long-term keys
+// result in different handshake hashes and wire traffic.
+func TestNoReuse(t *testing.T) {
+	var (
+		hashes           = map[[32]byte]bool{}
+		clientHandshakes = map[[96]byte]bool{}
+		serverHandshakes = map[[48]byte]bool{}
+		packets          = map[[32]byte]bool{}
+	)
+	for i := 0; i < 10; i++ {
+		var (
+			clientRaw, serverRaw = tsnettest.NewConn("noise", 128000)
+			clientBuf, serverBuf bytes.Buffer
+			clientConn           = &readerConn{clientRaw, io.TeeReader(clientRaw, &clientBuf)}
+			serverConn           = &readerConn{serverRaw, io.TeeReader(serverRaw, &serverBuf)}
+			serverKey            = key.NewMachine()
+			clientKey            = key.NewMachine()
+			server               *Conn
+			serverErr            = make(chan error, 1)
+		)
+		go func() {
+			var err error
+			server, err = Server(context.Background(), serverConn, serverKey)
+			serverErr <- err
+		}()
+
+		client, err := Client(context.Background(), clientConn, clientKey, serverKey.Public())
+		if err != nil {
+			t.Fatalf("client connection failed: %v", err)
+		}
+		if err := <-serverErr; err != nil {
+			t.Fatalf("server connection failed: %v", err)
+		}
+
+		var clientHS [96]byte
+		copy(clientHS[:], serverBuf.Bytes())
+		if clientHandshakes[clientHS] {
+			t.Fatal("client handshake seen twice")
+		}
+		clientHandshakes[clientHS] = true
+
+		var serverHS [48]byte
+		copy(serverHS[:], clientBuf.Bytes())
+		if serverHandshakes[serverHS] {
+			t.Fatal("server handshake seen twice")
+		}
+		serverHandshakes[serverHS] = true
+
+		clientBuf.Reset()
+		serverBuf.Reset()
+		cb := sinkReads(client)
+		sb := sinkReads(server)
+
+		if hashes[client.HandshakeHash()] {
+			t.Fatalf("handshake hash %v seen twice", client.HandshakeHash())
+		}
+		hashes[client.HandshakeHash()] = true
+
+		// Sending 14 bytes turns into 32 bytes on the wire (+16 for
+		// the chacha20poly1305 overhead, +2 length header)
+		if _, err := io.WriteString(client, strings.Repeat("a", 14)); err != nil {
+			t.Fatalf("client>server write failed: %v", err)
+		}
+		if _, err := io.WriteString(server, strings.Repeat("b", 14)); err != nil {
+			t.Fatalf("server>client write failed: %v", err)
+		}
+
+		// Wait for the bytes to be read, so we know they've traveled end to end
+		cb.String(14)
+		sb.String(14)
+
+		var clientWire, serverWire [32]byte
+		copy(clientWire[:], clientBuf.Bytes())
+		copy(serverWire[:], serverBuf.Bytes())
+
+		if packets[clientWire] {
+			t.Fatalf("client wire traffic seen twice")
+		}
+		packets[clientWire] = true
+		if packets[serverWire] {
+			t.Fatalf("server wire traffic seen twice")
+		}
+		packets[serverWire] = true
+
+		server.Close()
+		client.Close()
+	}
+}
+
+// tamperReader wraps a reader and mutates the Nth byte.
+type tamperReader struct {
+	r     io.Reader
+	n     int
+	total int
+}
+
+func (r *tamperReader) Read(bs []byte) (int, error) {
+	n, err := r.r.Read(bs)
+	if off := r.n - r.total; off >= 0 && off < n {
+		bs[off] += 1
+	}
+	r.total += n
+	return n, err
+}
+
+func TestTampering(t *testing.T) {
+	// Tamper with every byte of the client initiation message.
+	for i := 0; i < 101; i++ {
+		var (
+			clientConn, serverRaw = tsnettest.NewConn("noise", 128000)
+			serverConn            = &readerConn{serverRaw, &tamperReader{serverRaw, i, 0}}
+			serverKey             = key.NewMachine()
+			clientKey             = key.NewMachine()
+			serverErr             = make(chan error, 1)
+		)
+		go func() {
+			_, err := Server(context.Background(), serverConn, serverKey)
+			// If the server failed, we have to close the Conn to
+			// unblock the client.
+			if err != nil {
+				serverConn.Close()
+			}
+			serverErr <- err
+		}()
+
+		_, err := Client(context.Background(), clientConn, clientKey, serverKey.Public())
+		if err == nil {
+			t.Fatal("client connection succeeded despite tampering")
+		}
+		if err := <-serverErr; err == nil {
+			t.Fatalf("server connection succeeded despite tampering")
+		}
+	}
+
+	// Tamper with every byte of the server response message.
+	for i := 0; i < 51; i++ {
+		var (
+			clientRaw, serverConn = tsnettest.NewConn("noise", 128000)
+			clientConn            = &readerConn{clientRaw, &tamperReader{clientRaw, i, 0}}
+			serverKey             = key.NewMachine()
+			clientKey             = key.NewMachine()
+			serverErr             = make(chan error, 1)
+		)
+		go func() {
+			_, err := Server(context.Background(), serverConn, serverKey)
+			serverErr <- err
+		}()
+
+		_, err := Client(context.Background(), clientConn, clientKey, serverKey.Public())
+		if err == nil {
+			t.Fatal("client connection succeeded despite tampering")
+		}
+		// The server shouldn't fail, because the tampering took place
+		// in its response.
+		if err := <-serverErr; err != nil {
+			t.Fatalf("server connection failed despite no tampering: %v", err)
+		}
+	}
+
+	// Tamper with every byte of the first server>client transport message.
+	for i := 0; i < 30; i++ {
+		var (
+			clientRaw, serverConn = tsnettest.NewConn("noise", 128000)
+			clientConn            = &readerConn{clientRaw, &tamperReader{clientRaw, 51 + i, 0}}
+			serverKey             = key.NewMachine()
+			clientKey             = key.NewMachine()
+			serverErr             = make(chan error, 1)
+		)
+		go func() {
+			server, err := Server(context.Background(), serverConn, serverKey)
+			serverErr <- err
+			_, err = io.WriteString(server, strings.Repeat("a", 14))
+			serverErr <- err
+		}()
+
+		client, err := Client(context.Background(), clientConn, clientKey, serverKey.Public())
+		if err != nil {
+			t.Fatalf("client handshake failed: %v", err)
+		}
+		// The server shouldn't fail, because the tampering took place
+		// in its response.
+		if err := <-serverErr; err != nil {
+			t.Fatalf("server handshake failed: %v", err)
+		}
+
+		// The client needs a timeout if the tampering is hitting the length header.
+		if i == 1 || i == 2 {
+			client.SetReadDeadline(time.Now().Add(10 * time.Millisecond))
+		}
+
+		var bs [100]byte
+		n, err := client.Read(bs[:])
+		if err == nil {
+			t.Fatal("read succeeded despite tampering")
+		}
+		if n != 0 {
+			t.Fatal("conn yielded some bytes despite tampering")
+		}
+	}
+
+	// Tamper with every byte of the first client>server transport message.
+	for i := 0; i < 30; i++ {
+		var (
+			clientConn, serverRaw = tsnettest.NewConn("noise", 128000)
+			serverConn            = &readerConn{serverRaw, &tamperReader{serverRaw, 101 + i, 0}}
+			serverKey             = key.NewMachine()
+			clientKey             = key.NewMachine()
+			serverErr             = make(chan error, 1)
+		)
+		go func() {
+			server, err := Server(context.Background(), serverConn, serverKey)
+			serverErr <- err
+			var bs [100]byte
+			// The server needs a timeout if the tampering is hitting the length header.
+			if i == 1 || i == 2 {
+				server.SetReadDeadline(time.Now().Add(10 * time.Millisecond))
+			}
+			n, err := server.Read(bs[:])
+			if n != 0 {
+				panic("server got bytes despite tampering")
+			} else {
+				serverErr <- err
+			}
+		}()
+
+		client, err := Client(context.Background(), clientConn, clientKey, serverKey.Public())
+		if err != nil {
+			t.Fatalf("client handshake failed: %v", err)
+		}
+		if err := <-serverErr; err != nil {
+			t.Fatalf("server handshake failed: %v", err)
+		}
+
+		if _, err := io.WriteString(client, strings.Repeat("a", 14)); err != nil {
+			t.Fatalf("client>server write failed: %v", err)
+		}
+		if err := <-serverErr; err == nil {
+			t.Fatal("server successfully received bytes despite tampering")
+		}
+	}
+}

control/noise/interop_test.go

@@ -0,0 +1,257 @@
+// Copyright (c) 2021 Tailscale Inc & AUTHORS All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package noise
+
+import (
+	"context"
+	"encoding/binary"
+	"errors"
+	"io"
+	"net"
+	"testing"
+
+	tsnettest "tailscale.com/net/nettest"
+	"tailscale.com/types/key"
+)
+
+// Can a reference Noise IK client talk to our server?
+func TestInteropClient(t *testing.T) {
+	var (
+		s1, s2      = tsnettest.NewConn("noise", 128000)
+		controlKey  = key.NewMachine()
+		machineKey  = key.NewMachine()
+		serverErr   = make(chan error, 2)
+		serverBytes = make(chan []byte, 1)
+		c2s         = "client>server"
+		s2c         = "server>client"
+	)
+
+	go func() {
+		server, err := Server(context.Background(), s2, controlKey)
+		serverErr <- err
+		if err != nil {
+			return
+		}
+		var buf [1024]byte
+		_, err = io.ReadFull(server, buf[:len(c2s)])
+		serverBytes <- buf[:len(c2s)]
+		if err != nil {
+			serverErr <- err
+			return
+		}
+		_, err = server.Write([]byte(s2c))
+		serverErr <- err
+	}()
+
+	gotS2C, err := noiseExplorerClient(s1, controlKey.Public(), machineKey, []byte(c2s))
+	if err != nil {
+		t.Fatalf("failed client interop: %v", err)
+	}
+	if string(gotS2C) != s2c {
+		t.Fatalf("server sent unexpected data %q, want %q", string(gotS2C), s2c)
+	}
+
+	if err := <-serverErr; err != nil {
+		t.Fatalf("server handshake failed: %v", err)
+	}
+	if err := <-serverErr; err != nil {
+		t.Fatalf("server read/write failed: %v", err)
+	}
+	if got := string(<-serverBytes); got != c2s {
+		t.Fatalf("server received %q, want %q", got, c2s)
+	}
+}
+
+// Can our client talk to a reference Noise IK server?
+func TestInteropServer(t *testing.T) {
+	var (
+		s1, s2      = tsnettest.NewConn("noise", 128000)
+		controlKey  = key.NewMachine()
+		machineKey  = key.NewMachine()
+		clientErr   = make(chan error, 2)
+		clientBytes = make(chan []byte, 1)
+		c2s         = "client>server"
+		s2c         = "server>client"
+	)
+
+	go func() {
+		client, err := Client(context.Background(), s1, machineKey, controlKey.Public())
+		clientErr <- err
+		if err != nil {
+			return
+		}
+		_, err = client.Write([]byte(c2s))
+		if err != nil {
+			clientErr <- err
+			return
+		}
+		var buf [1024]byte
+		_, err = io.ReadFull(client, buf[:len(s2c)])
+		clientBytes <- buf[:len(s2c)]
+		clientErr <- err
+	}()
+
+	gotC2S, err := noiseExplorerServer(s2, controlKey, machineKey.Public(), []byte(s2c))
+	if err != nil {
+		t.Fatalf("failed server interop: %v", err)
+	}
+	if string(gotC2S) != c2s {
+		t.Fatalf("server sent unexpected data %q, want %q", string(gotC2S), c2s)
+	}
+
+	if err := <-clientErr; err != nil {
+		t.Fatalf("client handshake failed: %v", err)
+	}
+	if err := <-clientErr; err != nil {
+		t.Fatalf("client read/write failed: %v", err)
+	}
+	if got := string(<-clientBytes); got != s2c {
+		t.Fatalf("client received %q, want %q", got, s2c)
+	}
+}
+
+// noiseExplorerClient uses the Noise Explorer implementation of Noise
+// IK to handshake as a Noise client on conn, transmit payload, and
+// read+return a payload from the peer.
+func noiseExplorerClient(conn net.Conn, controlKey key.MachinePublic, machineKey key.MachinePrivate, payload []byte) ([]byte, error) {
+	var mk keypair
+	copy(mk.private_key[:], machineKey.UntypedBytes())
+	copy(mk.public_key[:], machineKey.Public().UntypedBytes())
+	var peerKey [32]byte
+	copy(peerKey[:], controlKey.UntypedBytes())
+	session := InitSession(true, protocolVersionPrologue(protocolVersion), mk, peerKey)
+
+	_, msg1 := SendMessage(&session, nil)
+	var hdr [initiationHeaderLen]byte
+	binary.BigEndian.PutUint16(hdr[:2], protocolVersion)
+	hdr[2] = msgTypeInitiation
+	binary.BigEndian.PutUint16(hdr[3:5], 96)
+	if _, err := conn.Write(hdr[:]); err != nil {
+		return nil, err
+	}
+	if _, err := conn.Write(msg1.ne[:]); err != nil {
+		return nil, err
+	}
+	if _, err := conn.Write(msg1.ns); err != nil {
+		return nil, err
+	}
+	if _, err := conn.Write(msg1.ciphertext); err != nil {
+		return nil, err
+	}
+
+	var buf [1024]byte
+	if _, err := io.ReadFull(conn, buf[:51]); err != nil {
+		return nil, err
+	}
+	// ignore the header for this test, we're only checking the noise
+	// implementation.
+	msg2 := messagebuffer{
+		ciphertext: buf[35:51],
+	}
+	copy(msg2.ne[:], buf[3:35])
+	_, p, valid := RecvMessage(&session, &msg2)
+	if !valid {
+		return nil, errors.New("handshake failed")
+	}
+	if len(p) != 0 {
+		return nil, errors.New("non-empty payload")
+	}
+
+	_, msg3 := SendMessage(&session, payload)
+	hdr[0] = msgTypeRecord
+	binary.BigEndian.PutUint16(hdr[1:3], uint16(len(msg3.ciphertext)))
+	if _, err := conn.Write(hdr[:3]); err != nil {
+		return nil, err
+	}
+	if _, err := conn.Write(msg3.ciphertext); err != nil {
+		return nil, err
+	}
+
+	if _, err := io.ReadFull(conn, buf[:3]); err != nil {
+		return nil, err
+	}
+	// Ignore all of the header except the payload length
+	plen := int(binary.BigEndian.Uint16(buf[1:3]))
+	if _, err := io.ReadFull(conn, buf[:plen]); err != nil {
+		return nil, err
+	}
+
+	msg4 := messagebuffer{
+		ciphertext: buf[:plen],
+	}
+	_, p, valid = RecvMessage(&session, &msg4)
+	if !valid {
+		return nil, errors.New("transport message decryption failed")
+	}
+
+	return p, nil
+}
+
+func noiseExplorerServer(conn net.Conn, controlKey key.MachinePrivate, wantMachineKey key.MachinePublic, payload []byte) ([]byte, error) {
+	var mk keypair
+	copy(mk.private_key[:], controlKey.UntypedBytes())
+	copy(mk.public_key[:], controlKey.Public().UntypedBytes())
+	session := InitSession(false, protocolVersionPrologue(protocolVersion), mk, [32]byte{})
+
+	var buf [1024]byte
+	if _, err := io.ReadFull(conn, buf[:101]); err != nil {
+		return nil, err
+	}
+	// Ignore the header, we're just checking the noise implementation.
+	msg1 := messagebuffer{
+		ns:         buf[37:85],
+		ciphertext: buf[85:101],
+	}
+	copy(msg1.ne[:], buf[5:37])
+	_, p, valid := RecvMessage(&session, &msg1)
+	if !valid {
+		return nil, errors.New("handshake failed")
+	}
+	if len(p) != 0 {
+		return nil, errors.New("non-empty payload")
+	}
+
+	_, msg2 := SendMessage(&session, nil)
+	var hdr [headerLen]byte
+	hdr[0] = msgTypeResponse
+	binary.BigEndian.PutUint16(hdr[1:3], 48)
+	if _, err := conn.Write(hdr[:]); err != nil {
+		return nil, err
+	}
+	if _, err := conn.Write(msg2.ne[:]); err != nil {
+		return nil, err
+	}
+	if _, err := conn.Write(msg2.ciphertext[:]); err != nil {
+		return nil, err
+	}
+
+	if _, err := io.ReadFull(conn, buf[:3]); err != nil {
+		return nil, err
+	}
+	plen := int(binary.BigEndian.Uint16(buf[1:3]))
+	if _, err := io.ReadFull(conn, buf[:plen]); err != nil {
+		return nil, err
+	}
+
+	msg3 := messagebuffer{
+		ciphertext: buf[:plen],
+	}
+	_, p, valid = RecvMessage(&session, &msg3)
+	if !valid {
+		return nil, errors.New("transport message decryption failed")
+	}
+
+	_, msg4 := SendMessage(&session, payload)
+	hdr[0] = msgTypeRecord
+	binary.BigEndian.PutUint16(hdr[1:3], uint16(len(msg4.ciphertext)))
+	if _, err := conn.Write(hdr[:]); err != nil {
+		return nil, err
+	}
+	if _, err := conn.Write(msg4.ciphertext); err != nil {
+		return nil, err
+	}
+
+	return p, nil
+}

control/noise/messages.go

@@ -0,0 +1,88 @@
+// Copyright (c) 2021 Tailscale Inc & AUTHORS All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package noise
+
+import "encoding/binary"
+
+const (
+	// msgTypeInitiation frames carry a Noise IK handshake initiation message.
+	msgTypeInitiation = 1
+	// msgTypeResponse frames carry a Noise IK handshake response message.
+	msgTypeResponse = 2
+	// msgTypeError frames carry an unauthenticated human-readable
+	// error message.
+	//
+	// Errors reported in this message type must be treated as public
+	// hints only. They are not encrypted or authenticated, and so can
+	// be seen and tampered with on the wire.
+	msgTypeError = 3
+	// msgTypeRecord frames carry session data bytes.
+	msgTypeRecord = 4
+
+	// headerLen is the size of the header on all messages except msgTypeInitiation.
+	headerLen = 3
+	// initiationHeaderLen is the size of the header on all msgTypeInitiation messages.
+	initiationHeaderLen = 5
+)
+
+// initiationMessage is the protocol message sent from a client
+// machine to a control server.
+//
+// 2b: protocol version
+// 1b: message type (0x01)
+// 2b: payload length (96)
+// 5b: header (see headerLen for fields)
+// 32b: client ephemeral public key (cleartext)
+// 48b: client machine public key (encrypted)
+// 16b: message tag (authenticates the whole message)
+type initiationMessage [101]byte
+
+func mkInitiationMessage() initiationMessage {
+	var ret initiationMessage
+	binary.BigEndian.PutUint16(ret[:2], uint16(protocolVersion))
+	ret[2] = msgTypeInitiation
+	binary.BigEndian.PutUint16(ret[3:5], uint16(len(ret.Payload())))
+	return ret
+}
+
+func (m *initiationMessage) Header() []byte  { return m[:initiationHeaderLen] }
+func (m *initiationMessage) Payload() []byte { return m[initiationHeaderLen:] }
+
+func (m *initiationMessage) Version() uint16 { return binary.BigEndian.Uint16(m[:2]) }
+func (m *initiationMessage) Type() byte      { return m[2] }
+func (m *initiationMessage) Length() int     { return int(binary.BigEndian.Uint16(m[3:5])) }
+
+func (m *initiationMessage) EphemeralPub() []byte {
+	return m[initiationHeaderLen : initiationHeaderLen+32]
+}
+func (m *initiationMessage) MachinePub() []byte {
+	return m[initiationHeaderLen+32 : initiationHeaderLen+32+48]
+}
+func (m *initiationMessage) Tag() []byte { return m[initiationHeaderLen+32+48:] }
+
+// responseMessage is the protocol message sent from a control server
+// to a client machine.
+//
+// 1b: message type (0x02)
+// 2b: payload length (48)
+// 32b: control ephemeral public key (cleartext)
+// 16b: message tag (authenticates the whole message)
+type responseMessage [51]byte
+
+func mkResponseMessage() responseMessage {
+	var ret responseMessage
+	ret[0] = msgTypeResponse
+	binary.BigEndian.PutUint16(ret[1:], uint16(len(ret.Payload())))
+	return ret
+}
+
+func (m *responseMessage) Header() []byte  { return m[:headerLen] }
+func (m *responseMessage) Payload() []byte { return m[headerLen:] }
+
+func (m *responseMessage) Type() byte  { return m[0] }
+func (m *responseMessage) Length() int { return int(binary.BigEndian.Uint16(m[1:3])) }
+
+func (m *responseMessage) EphemeralPub() []byte { return m[headerLen : headerLen+32] }
+func (m *responseMessage) Tag() []byte          { return m[headerLen+32:] }

control/noise/noiseexplorer_test.go

@@ -0,0 +1,475 @@
+// This file contains the implementation of Noise IK from
+// https://noiseexplorer.com/ . Unlike the rest of this repository,
+// this file is licensed under the terms of the GNU GPL v3. See
+// https://source.symbolic.software/noiseexplorer/noiseexplorer for
+// more information.
+//
+// This file is used here to verify that Tailscale's implementation of
+// Noise IK is interoperable with another implementation.
+//lint:file-ignore SA4006 not our code.
+
+/*
+IK:
+  <- s
+  ...
+  -> e, es, s, ss
+  <- e, ee, se
+  ->
+  <-
+*/
+
+// Implementation Version: 1.0.2
+
+/* ---------------------------------------------------------------- *
+ * PARAMETERS                                                       *
+ * ---------------------------------------------------------------- */
+
+package noise
+
+import (
+	"crypto/rand"
+	"crypto/subtle"
+	"encoding/binary"
+	"hash"
+	"io"
+	"math"
+
+	"golang.org/x/crypto/blake2s"
+	"golang.org/x/crypto/chacha20poly1305"
+	"golang.org/x/crypto/curve25519"
+	"golang.org/x/crypto/hkdf"
+)
+
+/* ---------------------------------------------------------------- *
+ * TYPES                                                            *
+ * ---------------------------------------------------------------- */
+
+type keypair struct {
+	public_key  [32]byte
+	private_key [32]byte
+}
+
+type messagebuffer struct {
+	ne         [32]byte
+	ns         []byte
+	ciphertext []byte
+}
+
+type cipherstate struct {
+	k [32]byte
+	n uint32
+}
+
+type symmetricstate struct {
+	cs cipherstate
+	ck [32]byte
+	h  [32]byte
+}
+
+type handshakestate struct {
+	ss  symmetricstate
+	s   keypair
+	e   keypair
+	rs  [32]byte
+	re  [32]byte
+	psk [32]byte
+}
+
+type noisesession struct {
+	hs  handshakestate
+	h   [32]byte
+	cs1 cipherstate
+	cs2 cipherstate
+	mc  uint64
+	i   bool
+}
+
+/* ---------------------------------------------------------------- *
+ * CONSTANTS                                                        *
+ * ---------------------------------------------------------------- */
+
+var emptyKey = [32]byte{
+	0x00, 0x00, 0x00, 0x00,
+	0x00, 0x00, 0x00, 0x00,
+	0x00, 0x00, 0x00, 0x00,
+	0x00, 0x00, 0x00, 0x00,
+	0x00, 0x00, 0x00, 0x00,
+	0x00, 0x00, 0x00, 0x00,
+	0x00, 0x00, 0x00, 0x00,
+	0x00, 0x00, 0x00, 0x00,
+}
+
+var minNonce = uint32(0)
+
+/* ---------------------------------------------------------------- *
+ * UTILITY FUNCTIONS                                                *
+ * ---------------------------------------------------------------- */
+
+func getPublicKey(kp *keypair) [32]byte {
+	return kp.public_key
+}
+
+func isEmptyKey(k [32]byte) bool {
+	return subtle.ConstantTimeCompare(k[:], emptyKey[:]) == 1
+}
+
+func validatePublicKey(k []byte) bool {
+	forbiddenCurveValues := [12][]byte{
+		{0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0},
+		{1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0},
+		{224, 235, 122, 124, 59, 65, 184, 174, 22, 86, 227, 250, 241, 159, 196, 106, 218, 9, 141, 235, 156, 50, 177, 253, 134, 98, 5, 22, 95, 73, 184, 0},
+		{95, 156, 149, 188, 163, 80, 140, 36, 177, 208, 177, 85, 156, 131, 239, 91, 4, 68, 92, 196, 88, 28, 142, 134, 216, 34, 78, 221, 208, 159, 17, 87},
+		{236, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 127},
+		{237, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 127},
+		{238, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 127},
+		{205, 235, 122, 124, 59, 65, 184, 174, 22, 86, 227, 250, 241, 159, 196, 106, 218, 9, 141, 235, 156, 50, 177, 253, 134, 98, 5, 22, 95, 73, 184, 128},
+		{76, 156, 149, 188, 163, 80, 140, 36, 177, 208, 177, 85, 156, 131, 239, 91, 4, 68, 92, 196, 88, 28, 142, 134, 216, 34, 78, 221, 208, 159, 17, 215},
+		{217, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255},
+		{218, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255},
+		{219, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 25},
+	}
+
+	for _, testValue := range forbiddenCurveValues {
+		if subtle.ConstantTimeCompare(k[:], testValue[:]) == 1 {
+			panic("Invalid public key")
+		}
+	}
+	return true
+}
+
+/* ---------------------------------------------------------------- *
+ * PRIMITIVES                                                       *
+ * ---------------------------------------------------------------- */
+
+func incrementNonce(n uint32) uint32 {
+	return n + 1
+}
+
+func dh(private_key [32]byte, public_key [32]byte) [32]byte {
+	var ss [32]byte
+	curve25519.ScalarMult(&ss, &private_key, &public_key)
+	return ss
+}
+
+func generateKeypair() keypair {
+	var public_key [32]byte
+	var private_key [32]byte
+	_, _ = rand.Read(private_key[:])
+	curve25519.ScalarBaseMult(&public_key, &private_key)
+	if validatePublicKey(public_key[:]) {
+		return keypair{public_key, private_key}
+	}
+	return generateKeypair()
+}
+
+func generatePublicKey(private_key [32]byte) [32]byte {
+	var public_key [32]byte
+	curve25519.ScalarBaseMult(&public_key, &private_key)
+	return public_key
+}
+
+func encrypt(k [32]byte, n uint32, ad []byte, plaintext []byte) []byte {
+	var nonce [12]byte
+	var ciphertext []byte
+	enc, _ := chacha20poly1305.New(k[:])
+	binary.LittleEndian.PutUint32(nonce[4:], n)
+	ciphertext = enc.Seal(nil, nonce[:], plaintext, ad)
+	return ciphertext
+}
+
+func decrypt(k [32]byte, n uint32, ad []byte, ciphertext []byte) (bool, []byte, []byte) {
+	var nonce [12]byte
+	var plaintext []byte
+	enc, err := chacha20poly1305.New(k[:])
+	binary.LittleEndian.PutUint32(nonce[4:], n)
+	plaintext, err = enc.Open(nil, nonce[:], ciphertext, ad)
+	return (err == nil), ad, plaintext
+}
+
+func getHash(a []byte, b []byte) [32]byte {
+	return blake2s.Sum256(append(a, b...))
+}
+
+func hashProtocolName(protocolName []byte) [32]byte {
+	var h [32]byte
+	if len(protocolName) <= 32 {
+		copy(h[:], protocolName)
+	} else {
+		h = getHash(protocolName, []byte{})
+	}
+	return h
+}
+
+func blake2HkdfInterface() hash.Hash {
+	h, _ := blake2s.New256([]byte{})
+	return h
+}
+
+func getHkdf(ck [32]byte, ikm []byte) ([32]byte, [32]byte, [32]byte) {
+	var k1 [32]byte
+	var k2 [32]byte
+	var k3 [32]byte
+	output := hkdf.New(blake2HkdfInterface, ikm[:], ck[:], []byte{})
+	io.ReadFull(output, k1[:])
+	io.ReadFull(output, k2[:])
+	io.ReadFull(output, k3[:])
+	return k1, k2, k3
+}
+
+/* ---------------------------------------------------------------- *
+ * STATE MANAGEMENT                                                 *
+ * ---------------------------------------------------------------- */
+
+/* CipherState */
+func initializeKey(k [32]byte) cipherstate {
+	return cipherstate{k, minNonce}
+}
+
+func hasKey(cs *cipherstate) bool {
+	return !isEmptyKey(cs.k)
+}
+
+func setNonce(cs *cipherstate, newNonce uint32) *cipherstate {
+	cs.n = newNonce
+	return cs
+}
+
+func encryptWithAd(cs *cipherstate, ad []byte, plaintext []byte) (*cipherstate, []byte) {
+	e := encrypt(cs.k, cs.n, ad, plaintext)
+	cs = setNonce(cs, incrementNonce(cs.n))
+	return cs, e
+}
+
+func decryptWithAd(cs *cipherstate, ad []byte, ciphertext []byte) (*cipherstate, []byte, bool) {
+	valid, ad, plaintext := decrypt(cs.k, cs.n, ad, ciphertext)
+	cs = setNonce(cs, incrementNonce(cs.n))
+	return cs, plaintext, valid
+}
+
+func reKey(cs *cipherstate) *cipherstate {
+	e := encrypt(cs.k, math.MaxUint32, []byte{}, emptyKey[:])
+	copy(cs.k[:], e)
+	return cs
+}
+
+/* SymmetricState */
+
+func initializeSymmetric(protocolName []byte) symmetricstate {
+	h := hashProtocolName(protocolName)
+	ck := h
+	cs := initializeKey(emptyKey)
+	return symmetricstate{cs, ck, h}
+}
+
+func mixKey(ss *symmetricstate, ikm [32]byte) *symmetricstate {
+	ck, tempK, _ := getHkdf(ss.ck, ikm[:])
+	ss.cs = initializeKey(tempK)
+	ss.ck = ck
+	return ss
+}
+
+func mixHash(ss *symmetricstate, data []byte) *symmetricstate {
+	ss.h = getHash(ss.h[:], data)
+	return ss
+}
+
+func mixKeyAndHash(ss *symmetricstate, ikm [32]byte) *symmetricstate {
+	var tempH [32]byte
+	var tempK [32]byte
+	ss.ck, tempH, tempK = getHkdf(ss.ck, ikm[:])
+	ss = mixHash(ss, tempH[:])
+	ss.cs = initializeKey(tempK)
+	return ss
+}
+
+func getHandshakeHash(ss *symmetricstate) [32]byte {
+	return ss.h
+}
+
+func encryptAndHash(ss *symmetricstate, plaintext []byte) (*symmetricstate, []byte) {
+	var ciphertext []byte
+	if hasKey(&ss.cs) {
+		_, ciphertext = encryptWithAd(&ss.cs, ss.h[:], plaintext)
+	} else {
+		ciphertext = plaintext
+	}
+	ss = mixHash(ss, ciphertext)
+	return ss, ciphertext
+}
+
+func decryptAndHash(ss *symmetricstate, ciphertext []byte) (*symmetricstate, []byte, bool) {
+	var plaintext []byte
+	var valid bool
+	if hasKey(&ss.cs) {
+		_, plaintext, valid = decryptWithAd(&ss.cs, ss.h[:], ciphertext)
+	} else {
+		plaintext, valid = ciphertext, true
+	}
+	ss = mixHash(ss, ciphertext)
+	return ss, plaintext, valid
+}
+
+func split(ss *symmetricstate) (cipherstate, cipherstate) {
+	tempK1, tempK2, _ := getHkdf(ss.ck, []byte{})
+	cs1 := initializeKey(tempK1)
+	cs2 := initializeKey(tempK2)
+	return cs1, cs2
+}
+
+/* HandshakeState */
+
+func initializeInitiator(prologue []byte, s keypair, rs [32]byte, psk [32]byte) handshakestate {
+	var ss symmetricstate
+	var e keypair
+	var re [32]byte
+	name := []byte("Noise_IK_25519_ChaChaPoly_BLAKE2s")
+	ss = initializeSymmetric(name)
+	mixHash(&ss, prologue)
+	mixHash(&ss, rs[:])
+	return handshakestate{ss, s, e, rs, re, psk}
+}
+
+func initializeResponder(prologue []byte, s keypair, rs [32]byte, psk [32]byte) handshakestate {
+	var ss symmetricstate
+	var e keypair
+	var re [32]byte
+	name := []byte("Noise_IK_25519_ChaChaPoly_BLAKE2s")
+	ss = initializeSymmetric(name)
+	mixHash(&ss, prologue)
+	mixHash(&ss, s.public_key[:])
+	return handshakestate{ss, s, e, rs, re, psk}
+}
+
+func writeMessageA(hs *handshakestate, payload []byte) (*handshakestate, messagebuffer) {
+	ne, ns, ciphertext := emptyKey, []byte{}, []byte{}
+	hs.e = generateKeypair()
+	ne = hs.e.public_key
+	mixHash(&hs.ss, ne[:])
+	/* No PSK, so skipping mixKey */
+	mixKey(&hs.ss, dh(hs.e.private_key, hs.rs))
+	spk := make([]byte, len(hs.s.public_key))
+	copy(spk[:], hs.s.public_key[:])
+	_, ns = encryptAndHash(&hs.ss, spk)
+	mixKey(&hs.ss, dh(hs.s.private_key, hs.rs))
+	_, ciphertext = encryptAndHash(&hs.ss, payload)
+	messageBuffer := messagebuffer{ne, ns, ciphertext}
+	return hs, messageBuffer
+}
+
+func writeMessageB(hs *handshakestate, payload []byte) ([32]byte, messagebuffer, cipherstate, cipherstate) {
+	ne, ns, ciphertext := emptyKey, []byte{}, []byte{}
+	hs.e = generateKeypair()
+	ne = hs.e.public_key
+	mixHash(&hs.ss, ne[:])
+	/* No PSK, so skipping mixKey */
+	mixKey(&hs.ss, dh(hs.e.private_key, hs.re))
+	mixKey(&hs.ss, dh(hs.e.private_key, hs.rs))
+	_, ciphertext = encryptAndHash(&hs.ss, payload)
+	messageBuffer := messagebuffer{ne, ns, ciphertext}
+	cs1, cs2 := split(&hs.ss)
+	return hs.ss.h, messageBuffer, cs1, cs2
+}
+
+func writeMessageRegular(cs *cipherstate, payload []byte) (*cipherstate, messagebuffer) {
+	ne, ns, ciphertext := emptyKey, []byte{}, []byte{}
+	cs, ciphertext = encryptWithAd(cs, []byte{}, payload)
+	messageBuffer := messagebuffer{ne, ns, ciphertext}
+	return cs, messageBuffer
+}
+
+func readMessageA(hs *handshakestate, message *messagebuffer) (*handshakestate, []byte, bool) {
+	valid1 := true
+	if validatePublicKey(message.ne[:]) {
+		hs.re = message.ne
+	}
+	mixHash(&hs.ss, hs.re[:])
+	/* No PSK, so skipping mixKey */
+	mixKey(&hs.ss, dh(hs.s.private_key, hs.re))
+	_, ns, valid1 := decryptAndHash(&hs.ss, message.ns)
+	if valid1 && len(ns) == 32 && validatePublicKey(message.ns[:]) {
+		copy(hs.rs[:], ns)
+	}
+	mixKey(&hs.ss, dh(hs.s.private_key, hs.rs))
+	_, plaintext, valid2 := decryptAndHash(&hs.ss, message.ciphertext)
+	return hs, plaintext, (valid1 && valid2)
+}
+
+func readMessageB(hs *handshakestate, message *messagebuffer) ([32]byte, []byte, bool, cipherstate, cipherstate) {
+	valid1 := true
+	if validatePublicKey(message.ne[:]) {
+		hs.re = message.ne
+	}
+	mixHash(&hs.ss, hs.re[:])
+	/* No PSK, so skipping mixKey */
+	mixKey(&hs.ss, dh(hs.e.private_key, hs.re))
+	mixKey(&hs.ss, dh(hs.s.private_key, hs.re))
+	_, plaintext, valid2 := decryptAndHash(&hs.ss, message.ciphertext)
+	cs1, cs2 := split(&hs.ss)
+	return hs.ss.h, plaintext, (valid1 && valid2), cs1, cs2
+}
+
+func readMessageRegular(cs *cipherstate, message *messagebuffer) (*cipherstate, []byte, bool) {
+	/* No encrypted keys */
+	_, plaintext, valid2 := decryptWithAd(cs, []byte{}, message.ciphertext)
+	return cs, plaintext, valid2
+}
+
+/* ---------------------------------------------------------------- *
+ * PROCESSES                                                        *
+ * ---------------------------------------------------------------- */
+
+func InitSession(initiator bool, prologue []byte, s keypair, rs [32]byte) noisesession {
+	var session noisesession
+	psk := emptyKey
+	if initiator {
+		session.hs = initializeInitiator(prologue, s, rs, psk)
+	} else {
+		session.hs = initializeResponder(prologue, s, rs, psk)
+	}
+	session.i = initiator
+	session.mc = 0
+	return session
+}
+
+func SendMessage(session *noisesession, message []byte) (*noisesession, messagebuffer) {
+	var messageBuffer messagebuffer
+	if session.mc == 0 {
+		_, messageBuffer = writeMessageA(&session.hs, message)
+	}
+	if session.mc == 1 {
+		session.h, messageBuffer, session.cs1, session.cs2 = writeMessageB(&session.hs, message)
+		session.hs = handshakestate{}
+	}
+	if session.mc > 1 {
+		if session.i {
+			_, messageBuffer = writeMessageRegular(&session.cs1, message)
+		} else {
+			_, messageBuffer = writeMessageRegular(&session.cs2, message)
+		}
+	}
+	session.mc = session.mc + 1
+	return session, messageBuffer
+}
+
+func RecvMessage(session *noisesession, message *messagebuffer) (*noisesession, []byte, bool) {
+	var plaintext []byte
+	var valid bool
+	if session.mc == 0 {
+		_, plaintext, valid = readMessageA(&session.hs, message)
+	}
+	if session.mc == 1 {
+		session.h, plaintext, valid, session.cs1, session.cs2 = readMessageB(&session.hs, message)
+		session.hs = handshakestate{}
+	}
+	if session.mc > 1 {
+		if session.i {
+			_, plaintext, valid = readMessageRegular(&session.cs2, message)
+		} else {
+			_, plaintext, valid = readMessageRegular(&session.cs1, message)
+		}
+	}
+	session.mc = session.mc + 1
+	return session, plaintext, valid
+}
+
+func main() {}

derp/derp_client.go

@@ -210,12 +210,12 @@ func (c *Client) send(dstKey key.NodePublic, pkt []byte) (ret error) {
 	c.wmu.Lock()
 	defer c.wmu.Unlock()
 	if c.rate != nil {
-		pktLen := frameHeaderLen + dstKey.RawLen() + len(pkt)
+		pktLen := frameHeaderLen + key.NodePublicRawLen + len(pkt)
 		if !c.rate.AllowN(time.Now(), pktLen) {
 			return nil // drop
 		}
 	}
-	if err := writeFrameHeader(c.bw, frameSendPacket, uint32(dstKey.RawLen()+len(pkt))); err != nil {
+	if err := writeFrameHeader(c.bw, frameSendPacket, uint32(key.NodePublicRawLen+len(pkt))); err != nil {
 		return err
 	}
 	if _, err := c.bw.Write(dstKey.AppendTo(nil)); err != nil {

derp/derp_server.go

@@ -1017,7 +1017,7 @@ func (s *Server) verifyClient(clientKey key.NodePublic, info *clientInfo) error
 }
 
 func (s *Server) sendServerKey(lw *lazyBufioWriter) error {
-	buf := make([]byte, 0, len(magic)+s.publicKey.RawLen())
+	buf := make([]byte, 0, len(magic)+key.NodePublicRawLen)
 	buf = append(buf, magic...)
 	buf = s.publicKey.AppendTo(buf)
 	err := writeFrame(lw.bw(), frameServerKey, buf)
@@ -1469,7 +1469,7 @@ func (c *sclient) sendPacket(srcKey key.NodePublic, contents []byte) (err error)
 	withKey := !srcKey.IsZero()
 	pktLen := len(contents)
 	if withKey {
-		pktLen += srcKey.RawLen()
+		pktLen += key.NodePublicRawLen
 	}
 	if err = writeFrameHeader(c.bw.bw(), frameRecvPacket, uint32(pktLen)); err != nil {
 		return err

derp/derphttp/derphttp_client.go

@@ -429,7 +429,7 @@ func (c *Client) dialURL(ctx context.Context) (net.Conn, error) {
 		return c.dialer(ctx, "tcp", net.JoinHostPort(host, urlPort(c.url)))
 	}
 	hostOrIP := host
-	dialer := netns.NewDialer()
+	dialer := netns.NewDialer(c.logf)
 
 	if c.DNSCache != nil {
 		ip, _, _, err := c.DNSCache.LookupIP(ctx, host)
@@ -519,7 +519,7 @@ func (c *Client) DialRegionTLS(ctx context.Context, reg *tailcfg.DERPRegion) (tl
 }
 
 func (c *Client) dialContext(ctx context.Context, proto, addr string) (net.Conn, error) {
-	return netns.NewDialer().DialContext(ctx, proto, addr)
+	return netns.NewDialer(c.logf).DialContext(ctx, proto, addr)
 }
 
 // shouldDialProto reports whether an explicitly provided IPv4 or IPv6

disco/disco.go

@@ -25,8 +25,9 @@ import (
 	"fmt"
 	"net"
 
+	"go4.org/mem"
 	"inet.af/netaddr"
-	"tailscale.com/tailcfg"
+	"tailscale.com/types/key"
 )
 
 // Magic is the 6 byte header of all discovery messages.
@@ -115,19 +116,19 @@ type Ping struct {
 	// It shouldn't be trusted by itself, but can be combined with
 	// netmap data to reduce the discokey:nodekey relation from 1:N to
 	// 1:1.
-	NodeKey tailcfg.NodeKey
+	NodeKey key.NodePublic
 }
 
 func (m *Ping) AppendMarshal(b []byte) []byte {
 	dataLen := 12
 	hasKey := !m.NodeKey.IsZero()
 	if hasKey {
-		dataLen += len(m.NodeKey)
+		dataLen += key.NodePublicRawLen
 	}
 	ret, d := appendMsgHeader(b, TypePing, v0, dataLen)
 	n := copy(d, m.TxID[:])
 	if hasKey {
-		copy(d[n:], m.NodeKey[:])
+		m.NodeKey.AppendTo(d[:n])
 	}
 	return ret
 }
@@ -138,8 +139,10 @@ func parsePing(ver uint8, p []byte) (m *Ping, err error) {
 	}
 	m = new(Ping)
 	p = p[copy(m.TxID[:], p):]
-	if len(p) >= len(m.NodeKey) {
-		copy(m.NodeKey[:], p)
+	// Deliberately lax on longer-than-expected messages, for future
+	// compatibility.
+	if len(p) >= key.NodePublicRawLen {
+		m.NodeKey = key.NodePublicFromRaw32(mem.B(p[:key.NodePublicRawLen]))
 	}
 	return m, nil
 }

disco/disco_test.go

@@ -10,8 +10,9 @@ import (
 	"strings"
 	"testing"
 
+	"go4.org/mem"
 	"inet.af/netaddr"
-	"tailscale.com/tailcfg"
+	"tailscale.com/types/key"
 )
 
 func TestMarshalAndParse(t *testing.T) {
@@ -30,13 +31,8 @@ func TestMarshalAndParse(t *testing.T) {
 		{
 			name: "ping_with_nodekey_src",
 			m: &Ping{
-				TxID: [12]byte{1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12},
-				NodeKey: tailcfg.NodeKey{
-					1:  1,
-					2:  2,
-					30: 30,
-					31: 31,
-				},
+				TxID:    [12]byte{1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12},
+				NodeKey: key.NodePublicFromRaw32(mem.B([]byte{1: 1, 2: 2, 30: 30, 31: 31})),
 			},
 			want: "01 00 01 02 03 04 05 06 07 08 09 0a 0b 0c 00 01 02 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 1e 1f",
 		},

docs/k8s/README.md

@@ -32,7 +32,7 @@ There are quite a few ways of running Tailscale inside a Kubernetes Cluster, som
    ```
 
 ### Sample Sidecar
-Running as a sidecar allows you to directly expose a Kubernetes pod over Tailscale. This is particularly useful if you do not wish to expose a service on the public internet. This method allows bi-directional connectivty between the pod and other devices on the Tailnet. You can use [ACLs](https://tailscale.com/kb/1018/acls/) to control traffic flow.
+Running as a sidecar allows you to directly expose a Kubernetes pod over Tailscale. This is particularly useful if you do not wish to expose a service on the public internet. This method allows bi-directional connectivity between the pod and other devices on the Tailnet. You can use [ACLs](https://tailscale.com/kb/1018/acls/) to control traffic flow.
 
 1. Create and login to the sample nginx pod with a Tailscale sidecar
 
@@ -144,4 +144,4 @@ routes for the subnet-router are enabled.
    # INTERNAL_IP="$(kubectl get po <POD_NAME> -o=jsonpath='{.status.podIP}')" 
    INTERNAL_PORT=8080 
    curl http://$INTERNAL_IP:$INTERNAL_PORT
-   ```
+   ```

go.mod

@@ -4,209 +4,260 @@ go 1.17
 
 require (
 	filippo.io/mkcert v1.4.3
+	github.com/akutz/memconn v0.1.0
 	github.com/alexbrainman/sspi v0.0.0-20210105120005-909beea2cc74
 	github.com/anmitsu/go-shlex v0.0.0-20200514113438-38f4b401e2be // indirect
-	github.com/aws/aws-sdk-go v1.38.52
-	github.com/aws/aws-sdk-go-v2 v1.9.2
-	github.com/aws/aws-sdk-go-v2/config v1.8.3
-	github.com/aws/aws-sdk-go-v2/service/ssm v1.12.0
+	github.com/aws/aws-sdk-go-v2 v1.11.2
+	github.com/aws/aws-sdk-go-v2/config v1.11.0
+	github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.7.4
+	github.com/aws/aws-sdk-go-v2/service/s3 v1.21.0
+	github.com/aws/aws-sdk-go-v2/service/ssm v1.17.1
 	github.com/coreos/go-iptables v0.6.0
 	github.com/creack/pty v1.1.17
 	github.com/dave/jennifer v1.4.1
-	github.com/frankban/quicktest v1.13.1
+	github.com/frankban/quicktest v1.14.0
 	github.com/gliderlabs/ssh v0.3.3
-	github.com/go-multierror/multierror v1.0.2
-	github.com/go-ole/go-ole v1.2.6-0.20210915003542-8b1f7f90f6b1
-	github.com/godbus/dbus/v5 v5.0.5
+	github.com/go-ole/go-ole v1.2.6
+	github.com/godbus/dbus/v5 v5.0.6
+	github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da
 	github.com/google/go-cmp v0.5.6
 	github.com/google/uuid v1.3.0
 	github.com/goreleaser/nfpm v1.10.3
 	github.com/iancoleman/strcase v0.2.0
-	github.com/insomniacslk/dhcp v0.0.0-20210621130208-1cac67f12b1e
-	github.com/jsimonetti/rtnetlink v0.0.0-20210525051524-4cc836578190
+	github.com/insomniacslk/dhcp v0.0.0-20211026125128-ad197bcd36fd
+	github.com/jsimonetti/rtnetlink v0.0.0-20211203074127-fd9a11f42291
 	github.com/kballard/go-shellquote v0.0.0-20180428030007-95032a82bc51
 	github.com/klauspost/compress v1.13.6
-	github.com/mdlayher/netlink v1.4.1
+	github.com/mdlayher/netlink v1.4.2
 	github.com/mdlayher/sdnotify v0.0.0-20210228150836-ea3ec207d697
 	github.com/miekg/dns v1.1.43
 	github.com/mitchellh/go-ps v1.0.0
 	github.com/pborman/getopt v1.1.0
-	github.com/peterbourgon/ff/v3 v3.1.0
+	github.com/peterbourgon/ff/v3 v3.1.2
 	github.com/pkg/sftp v1.13.4
 	github.com/skip2/go-qrcode v0.0.0-20200617195104-da1b6568686e
 	github.com/tailscale/certstore v0.0.0-20210528134328-066c94b793d3
-	github.com/tailscale/depaware v0.0.0-20201214215404-77d1e9757027
+	github.com/tailscale/depaware v0.0.0-20210622194025-720c4b409502
 	github.com/tailscale/goexpect v0.0.0-20210902213824-6e8c725cea41
 	github.com/tailscale/goupnp v1.0.1-0.20210804011211-c64d0f06ea05
-	github.com/tailscale/hujson v0.0.0-20200924210142-dde312d0d6a2
+	github.com/tailscale/hujson v0.0.0-20211105212140-3a0adc019d83
+	github.com/tailscale/netlink v1.1.1-0.20211101221916-cabfb018fe85
 	github.com/tcnksm/go-httpstat v0.2.0
 	github.com/toqueteos/webbrowser v1.2.0
 	github.com/ulikunitz/xz v0.5.10 // indirect
-	github.com/vishvananda/netlink v1.1.0
-	go4.org/mem v0.0.0-20201119185036-c04c5a6ff174
-	golang.org/x/crypto v0.0.0-20210921155107-089bfa567519
-	golang.org/x/net v0.0.0-20211020060615-d418f374d309
+	github.com/vishvananda/netlink v1.1.1-0.20211101163509-b10eb8fe5cf6
+	go4.org/mem v0.0.0-20210711025021-927187094b94
+	golang.org/x/crypto v0.0.0-20211202192323-5770296d904e
+	golang.org/x/net v0.0.0-20211205041911-012df41ee64c
 	golang.org/x/sync v0.0.0-20210220032951-036812b2e83c
-	golang.org/x/sys v0.0.0-20211020174200-9d6173849985
-	golang.org/x/term v0.0.0-20210503060354-a79de5458b56
-	golang.org/x/time v0.0.0-20210611083556-38a9dc6acbc6
-	golang.org/x/tools v0.1.7
-	golang.zx2c4.com/wireguard v0.0.0-20211020205005-82e0b734e5d2
+	golang.org/x/sys v0.0.0-20211205182925-97ca703d548d
+	golang.org/x/term v0.0.0-20210927222741-03fcf44c2211
+	golang.org/x/time v0.0.0-20211116232009-f0f3c7e86c11
+	golang.org/x/tools v0.1.8
+	golang.zx2c4.com/wireguard v0.0.0-20211116201604-de7c702ace45
 	golang.zx2c4.com/wireguard/windows v0.4.10
-	honnef.co/go/tools v0.2.1
+	honnef.co/go/tools v0.2.2
 	inet.af/netaddr v0.0.0-20211027220019-c74959edd3b6
-	inet.af/netstack v0.0.0-20211027215559-ec21145de76b
-	inet.af/peercred v0.0.0-20210318190834-4259e17bb763
-	inet.af/wf v0.0.0-20210516214145-a5343001b756
+	inet.af/netstack v0.0.0-20211120045802-8aa80cf23d3c
+	inet.af/peercred v0.0.0-20210906144145-0893ea02156a
+	inet.af/wf v0.0.0-20211204062712-86aaea0a7310
 	nhooyr.io/websocket v1.8.7
 )
 
 require (
-	4d63.com/gochecknoglobals v0.0.0-20201008074935-acfc0b28355a // indirect
-	github.com/BurntSushi/toml v0.3.1 // indirect
+	4d63.com/gochecknoglobals v0.1.0 // indirect
+	github.com/Antonboom/errname v0.1.5 // indirect
+	github.com/Antonboom/nilnil v0.1.0 // indirect
+	github.com/BurntSushi/toml v0.4.1 // indirect
 	github.com/Djarvur/go-err113 v0.1.0 // indirect
-	github.com/Masterminds/goutils v1.1.0 // indirect
+	github.com/Masterminds/goutils v1.1.1 // indirect
 	github.com/Masterminds/semver v1.5.0 // indirect
 	github.com/Masterminds/semver/v3 v3.1.1 // indirect
 	github.com/Masterminds/sprig v2.22.0+incompatible // indirect
-	github.com/Microsoft/go-winio v0.4.16 // indirect
+	github.com/Microsoft/go-winio v0.5.1 // indirect
 	github.com/OpenPeeDeeP/depguard v1.0.1 // indirect
-	github.com/aws/aws-sdk-go-v2/credentials v1.4.3 // indirect
-	github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.6.0 // indirect
-	github.com/aws/aws-sdk-go-v2/internal/ini v1.2.4 // indirect
-	github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.3.2 // indirect
-	github.com/aws/aws-sdk-go-v2/service/sso v1.4.2 // indirect
-	github.com/aws/aws-sdk-go-v2/service/sts v1.7.2 // indirect
-	github.com/aws/smithy-go v1.8.0 // indirect
+	github.com/ProtonMail/go-crypto v0.0.0-20211112122917-428f8eabeeb3 // indirect
+	github.com/acomagu/bufpipe v1.0.3 // indirect
+	github.com/alexkohler/prealloc v1.0.0 // indirect
+	github.com/ashanbrown/forbidigo v1.2.0 // indirect
+	github.com/ashanbrown/makezero v0.0.0-20210520155254-b6261585ddde // indirect
+	github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.0.0 // indirect
+	github.com/aws/aws-sdk-go-v2/credentials v1.6.4 // indirect
+	github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.8.2 // indirect
+	github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.2 // indirect
+	github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.0.2 // indirect
+	github.com/aws/aws-sdk-go-v2/internal/ini v1.3.2 // indirect
+	github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.5.0 // indirect
+	github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.5.2 // indirect
+	github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.9.2 // indirect
+	github.com/aws/aws-sdk-go-v2/service/sso v1.6.2 // indirect
+	github.com/aws/aws-sdk-go-v2/service/sts v1.11.1 // indirect
+	github.com/aws/smithy-go v1.9.0 // indirect
+	github.com/beorn7/perks v1.0.1 // indirect
+	github.com/bkielbasa/cyclop v1.2.0 // indirect
 	github.com/blakesmith/ar v0.0.0-20190502131153-809d4375e1fb // indirect
-	github.com/bombsimon/wsl/v3 v3.1.0 // indirect
+	github.com/blizzy78/varnamelen v0.5.0 // indirect
+	github.com/bombsimon/wsl/v3 v3.3.0 // indirect
+	github.com/breml/bidichk v0.2.1 // indirect
+	github.com/butuzov/ireturn v0.1.1 // indirect
 	github.com/cavaliercoder/go-cpio v0.0.0-20180626203310-925f9528c45e // indirect
-	github.com/daixiang0/gci v0.2.7 // indirect
+	github.com/cespare/xxhash/v2 v2.1.2 // indirect
+	github.com/charithe/durationcheck v0.0.9 // indirect
+	github.com/chavacava/garif v0.0.0-20210405164556-e8a0a408d6af // indirect
+	github.com/daixiang0/gci v0.2.9 // indirect
 	github.com/davecgh/go-spew v1.1.1 // indirect
-	github.com/denis-tingajkin/go-header v0.3.1 // indirect
+	github.com/denis-tingajkin/go-header v0.4.2 // indirect
 	github.com/emirpasic/gods v1.12.0 // indirect
-	github.com/fatih/color v1.10.0 // indirect
-	github.com/fsnotify/fsnotify v1.4.9 // indirect
-	github.com/go-critic/go-critic v0.5.2 // indirect
+	github.com/esimonov/ifshort v1.0.3 // indirect
+	github.com/ettle/strcase v0.1.1 // indirect
+	github.com/fatih/color v1.13.0 // indirect
+	github.com/fatih/structtag v1.2.0 // indirect
+	github.com/fsnotify/fsnotify v1.5.1 // indirect
+	github.com/fzipp/gocyclo v0.3.1 // indirect
+	github.com/go-critic/go-critic v0.6.1 // indirect
 	github.com/go-git/gcfg v1.5.0 // indirect
-	github.com/go-git/go-billy/v5 v5.0.0 // indirect
-	github.com/go-git/go-git/v5 v5.2.0 // indirect
+	github.com/go-git/go-billy/v5 v5.3.1 // indirect
+	github.com/go-git/go-git/v5 v5.4.2 // indirect
 	github.com/go-toolsmith/astcast v1.0.0 // indirect
 	github.com/go-toolsmith/astcopy v1.0.0 // indirect
-	github.com/go-toolsmith/astequal v1.0.0 // indirect
+	github.com/go-toolsmith/astequal v1.0.1 // indirect
 	github.com/go-toolsmith/astfmt v1.0.0 // indirect
 	github.com/go-toolsmith/astp v1.0.0 // indirect
 	github.com/go-toolsmith/strparse v1.0.0 // indirect
 	github.com/go-toolsmith/typep v1.0.2 // indirect
-	github.com/go-xmlfmt/xmlfmt v0.0.0-20191208150333-d5b6f63a941b // indirect
+	github.com/go-xmlfmt/xmlfmt v0.0.0-20211206191508-7fd73a941850 // indirect
 	github.com/gobwas/glob v0.2.3 // indirect
-	github.com/gofrs/flock v0.8.0 // indirect
+	github.com/gofrs/flock v0.8.1 // indirect
+	github.com/golang/protobuf v1.5.2 // indirect
 	github.com/golangci/check v0.0.0-20180506172741-cfe4005ccda2 // indirect
 	github.com/golangci/dupl v0.0.0-20180902072040-3e9179ac440a // indirect
-	github.com/golangci/errcheck v0.0.0-20181223084120-ef45e06d44b6 // indirect
 	github.com/golangci/go-misc v0.0.0-20180628070357-927a3d87b613 // indirect
-	github.com/golangci/gocyclo v0.0.0-20180528144436-0a533e8fa43d // indirect
 	github.com/golangci/gofmt v0.0.0-20190930125516-244bba706f1a // indirect
-	github.com/golangci/golangci-lint v1.33.0 // indirect
-	github.com/golangci/ineffassign v0.0.0-20190609212857-42439a7714cc // indirect
+	github.com/golangci/golangci-lint v1.43.0 // indirect
 	github.com/golangci/lint-1 v0.0.0-20191013205115-297bf364a8e0 // indirect
 	github.com/golangci/maligned v0.0.0-20180506175553-b1d89398deca // indirect
 	github.com/golangci/misspell v0.3.5 // indirect
-	github.com/golangci/prealloc v0.0.0-20180630174525-215b22d4de21 // indirect
-	github.com/golangci/revgrep v0.0.0-20180812185044-276a5c0a1039 // indirect
+	github.com/golangci/revgrep v0.0.0-20210930125155-c22e5001d4f2 // indirect
 	github.com/golangci/unconvert v0.0.0-20180507085042-28b1c447d1f4 // indirect
 	github.com/google/btree v1.0.1 // indirect
-	github.com/google/goterm v0.0.0-20190703233501-fc88cf888a3f // indirect
+	github.com/google/goterm v0.0.0-20200907032337-555d40f16ae2 // indirect
 	github.com/google/rpmpack v0.0.0-20201206194719-59e495f2b7e1 // indirect
+	github.com/gordonklaus/ineffassign v0.0.0-20210914165742-4cc7213b9bc8 // indirect
 	github.com/goreleaser/chglog v0.1.2 // indirect
 	github.com/goreleaser/fileglob v0.3.1 // indirect
-	github.com/gostaticanalysis/analysisutil v0.6.1 // indirect
-	github.com/gostaticanalysis/comment v1.4.1 // indirect
+	github.com/gostaticanalysis/analysisutil v0.7.1 // indirect
+	github.com/gostaticanalysis/comment v1.4.2 // indirect
+	github.com/gostaticanalysis/forcetypeassert v0.1.0 // indirect
+	github.com/gostaticanalysis/nilerr v0.1.1 // indirect
+	github.com/hashicorp/errwrap v1.1.0 // indirect
+	github.com/hashicorp/go-multierror v1.1.1 // indirect
 	github.com/hashicorp/hcl v1.0.0 // indirect
 	github.com/huandu/xstrings v1.3.2 // indirect
-	github.com/imdario/mergo v0.3.11 // indirect
+	github.com/imdario/mergo v0.3.12 // indirect
 	github.com/inconshreveable/mousetrap v1.0.0 // indirect
 	github.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99 // indirect
-	github.com/jgautheron/goconst v0.0.0-20201117150253-ccae5bf973f3 // indirect
-	github.com/jingyugao/rowserrcheck v0.0.0-20191204022205-72ab7603b68a // indirect
+	github.com/jgautheron/goconst v1.5.1 // indirect
+	github.com/jingyugao/rowserrcheck v1.1.1 // indirect
 	github.com/jirfag/go-printf-func-name v0.0.0-20200119135958-7558a9eaa5af // indirect
 	github.com/jmespath/go-jmespath v0.4.0 // indirect
 	github.com/josharian/native v0.0.0-20200817173448-b6b71def0850 // indirect
-	github.com/kevinburke/ssh_config v0.0.0-20201106050909-4977a11b4351 // indirect
+	github.com/julz/importas v0.0.0-20210922140945-27e0a5d4dee2 // indirect
+	github.com/kevinburke/ssh_config v1.1.0 // indirect
+	github.com/kisielk/errcheck v1.6.0 // indirect
 	github.com/kisielk/gotool v1.0.0 // indirect
 	github.com/kr/fs v0.1.0 // indirect
 	github.com/kr/pretty v0.3.0 // indirect
 	github.com/kr/text v0.2.0 // indirect
-	github.com/kunwardeep/paralleltest v1.0.2 // indirect
+	github.com/kulti/thelper v0.4.0 // indirect
+	github.com/kunwardeep/paralleltest v1.0.3 // indirect
 	github.com/kyoh86/exportloopref v0.1.8 // indirect
-	github.com/magiconair/properties v1.8.4 // indirect
+	github.com/ldez/gomoddirectives v0.2.2 // indirect
+	github.com/ldez/tagliatelle v0.2.0 // indirect
+	github.com/magiconair/properties v1.8.5 // indirect
 	github.com/maratori/testpackage v1.0.1 // indirect
-	github.com/matoous/godox v0.0.0-20200801072554-4fb83dc2941e // indirect
-	github.com/mattn/go-colorable v0.1.8 // indirect
-	github.com/mattn/go-isatty v0.0.12 // indirect
-	github.com/mbilski/exhaustivestruct v1.1.0 // indirect
-	github.com/mdlayher/socket v0.0.0-20210307095302-262dc9984e00 // indirect
-	github.com/mitchellh/copystructure v1.0.0 // indirect
+	github.com/matoous/godox v0.0.0-20210227103229-6504466cf951 // indirect
+	github.com/mattn/go-colorable v0.1.12 // indirect
+	github.com/mattn/go-isatty v0.0.14 // indirect
+	github.com/mattn/go-runewidth v0.0.13 // indirect
+	github.com/matttproud/golang_protobuf_extensions v1.0.1 // indirect
+	github.com/mbilski/exhaustivestruct v1.2.0 // indirect
+	github.com/mdlayher/socket v0.0.0-20211102153432-57e3fa563ecb // indirect
+	github.com/mgechev/dots v0.0.0-20210922191527-e955255bf517 // indirect
+	github.com/mgechev/revive v1.1.2 // indirect
+	github.com/mitchellh/copystructure v1.2.0 // indirect
 	github.com/mitchellh/go-homedir v1.1.0 // indirect
-	github.com/mitchellh/mapstructure v1.4.0 // indirect
-	github.com/mitchellh/reflectwalk v1.0.1 // indirect
+	github.com/mitchellh/mapstructure v1.4.3 // indirect
+	github.com/mitchellh/reflectwalk v1.0.2 // indirect
 	github.com/moricho/tparallel v0.2.1 // indirect
-	github.com/nakabonne/nestif v0.3.0 // indirect
-	github.com/nbutton23/zxcvbn-go v0.0.0-20180912185939-ae427f1e4c1d // indirect
-	github.com/nishanths/exhaustive v0.1.0 // indirect
+	github.com/nakabonne/nestif v0.3.1 // indirect
+	github.com/nbutton23/zxcvbn-go v0.0.0-20210217022336-fa2cb2858354 // indirect
+	github.com/nishanths/exhaustive v0.7.11 // indirect
+	github.com/nishanths/predeclared v0.2.1 // indirect
+	github.com/olekukonko/tablewriter v0.0.5 // indirect
 	github.com/op/go-logging v0.0.0-20160315200505-970db520ece7 // indirect
-	github.com/pelletier/go-toml v1.8.1 // indirect
+	github.com/pelletier/go-toml v1.9.4 // indirect
 	github.com/phayes/checkstyle v0.0.0-20170904204023-bfd46e6a821d // indirect
-	github.com/pkg/diff v0.0.0-20200914180035-5b29258ca4f7 // indirect
+	github.com/pkg/diff v0.0.0-20210226163009-20ebb0f2a09e // indirect
 	github.com/pkg/errors v0.9.1 // indirect
 	github.com/pmezard/go-difflib v1.0.0 // indirect
-	github.com/polyfloyd/go-errorlint v0.0.0-20201127212506-19bd8db6546f // indirect
-	github.com/quasilyte/go-ruleguard v0.2.1 // indirect
-	github.com/quasilyte/regex/syntax v0.0.0-20200805063351-8f842688393c // indirect
-	github.com/rogpeppe/go-internal v1.6.2 // indirect
-	github.com/ryancurrah/gomodguard v1.1.0 // indirect
+	github.com/polyfloyd/go-errorlint v0.0.0-20211125173453-6d6d39c5bb8b // indirect
+	github.com/prometheus/client_golang v1.11.0 // indirect
+	github.com/prometheus/client_model v0.2.0 // indirect
+	github.com/prometheus/common v0.32.1 // indirect
+	github.com/prometheus/procfs v0.7.3 // indirect
+	github.com/quasilyte/go-ruleguard v0.3.13 // indirect
+	github.com/quasilyte/regex/syntax v0.0.0-20210819130434-b3f0c404a727 // indirect
+	github.com/rivo/uniseg v0.2.0 // indirect
+	github.com/rogpeppe/go-internal v1.8.1-0.20211023094830-115ce09fd6b4 // indirect
+	github.com/ryancurrah/gomodguard v1.2.3 // indirect
 	github.com/ryanrolds/sqlclosecheck v0.3.0 // indirect
+	github.com/sanposhiho/wastedassign/v2 v2.0.7 // indirect
 	github.com/sassoftware/go-rpmutils v0.0.0-20190420191620-a8f1baeba37b // indirect
-	github.com/securego/gosec/v2 v2.5.0 // indirect
-	github.com/sergi/go-diff v1.1.0 // indirect
+	github.com/securego/gosec/v2 v2.9.3 // indirect
+	github.com/sergi/go-diff v1.2.0 // indirect
 	github.com/shazow/go-diff v0.0.0-20160112020656-b6b7b6733b8c // indirect
-	github.com/sirupsen/logrus v1.7.0 // indirect
+	github.com/sirupsen/logrus v1.8.1 // indirect
+	github.com/sivchari/tenv v1.4.7 // indirect
 	github.com/sonatard/noctx v0.0.1 // indirect
 	github.com/sourcegraph/go-diff v0.6.1 // indirect
-	github.com/spf13/afero v1.5.1 // indirect
-	github.com/spf13/cast v1.3.1 // indirect
-	github.com/spf13/cobra v1.1.1 // indirect
+	github.com/spf13/afero v1.6.0 // indirect
+	github.com/spf13/cast v1.4.1 // indirect
+	github.com/spf13/cobra v1.2.1 // indirect
 	github.com/spf13/jwalterweatherman v1.1.0 // indirect
 	github.com/spf13/pflag v1.0.5 // indirect
-	github.com/spf13/viper v1.7.1 // indirect
-	github.com/ssgreg/nlreturn/v2 v2.1.0 // indirect
+	github.com/spf13/viper v1.9.0 // indirect
+	github.com/ssgreg/nlreturn/v2 v2.2.1 // indirect
 	github.com/stretchr/objx v0.3.0 // indirect
 	github.com/stretchr/testify v1.7.0 // indirect
 	github.com/subosito/gotenv v1.2.0 // indirect
-	github.com/tdakkota/asciicheck v0.0.0-20200416200610-e657995f937b // indirect
-	github.com/tetafro/godot v1.3.2 // indirect
-	github.com/timakin/bodyclose v0.0.0-20200424151742-cb6215831a94 // indirect
-	github.com/tomarrell/wrapcheck v0.0.0-20201130113247-1683564d9756 // indirect
-	github.com/tommy-muehle/go-mnd v1.3.1-0.20200224220436-e6f9a994e8fa // indirect
+	github.com/sylvia7788/contextcheck v1.0.4 // indirect
+	github.com/tdakkota/asciicheck v0.1.1 // indirect
+	github.com/tetafro/godot v1.4.11 // indirect
+	github.com/timakin/bodyclose v0.0.0-20210704033933-f49887972144 // indirect
+	github.com/tomarrell/wrapcheck/v2 v2.4.0 // indirect
+	github.com/tommy-muehle/go-mnd/v2 v2.4.0 // indirect
 	github.com/u-root/uio v0.0.0-20210528114334-82958018845c // indirect
 	github.com/ultraware/funlen v0.0.3 // indirect
 	github.com/ultraware/whitespace v0.0.4 // indirect
-	github.com/uudashr/gocognit v1.0.1 // indirect
-	github.com/vishvananda/netns v0.0.0-20191106174202-0a2b9b5464df // indirect
-	github.com/xanzy/ssh-agent v0.3.0 // indirect
+	github.com/uudashr/gocognit v1.0.5 // indirect
+	github.com/vishvananda/netns v0.0.0-20211101163701-50045581ed74 // indirect
+	github.com/xanzy/ssh-agent v0.3.1 // indirect
+	github.com/yeya24/promlinter v0.1.0 // indirect
 	go4.org/intern v0.0.0-20211027215823-ae77deb06f29 // indirect
 	go4.org/unsafe/assume-no-moving-gc v0.0.0-20211027215541-db492cf91b37 // indirect
-	golang.org/x/mod v0.4.2 // indirect
+	golang.org/x/mod v0.5.1 // indirect
 	golang.org/x/text v0.3.7 // indirect
 	golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 // indirect
-	gopkg.in/ini.v1 v1.62.0 // indirect
+	golang.zx2c4.com/wintun v0.0.0-20211104114900-415007cec224 // indirect
+	google.golang.org/protobuf v1.27.1 // indirect
+	gopkg.in/ini.v1 v1.66.2 // indirect
 	gopkg.in/warnings.v0 v0.1.2 // indirect
 	gopkg.in/yaml.v2 v2.4.0 // indirect
-	gopkg.in/yaml.v3 v3.0.0-20200615113413-eeeca48fe776 // indirect
-	howett.net/plist v0.0.0-20181124034731-591f970eefbb // indirect
-	mvdan.cc/gofumpt v0.0.0-20201129102820-5c11c50e9475 // indirect
+	gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b // indirect
+	howett.net/plist v1.0.0 // indirect
+	mvdan.cc/gofumpt v0.2.0 // indirect
 	mvdan.cc/interfacer v0.0.0-20180901003855-c20040233aed // indirect
 	mvdan.cc/lint v0.0.0-20170908181259-adc824a0674b // indirect
-	mvdan.cc/unparam v0.0.0-20200501210554-b37ab49443f7 // indirect
-	software.sslmate.com/src/go-pkcs12 v0.0.0-20180114231543-2291e8f0f237 // indirect
+	mvdan.cc/unparam v0.0.0-20211002134041-24922b6997ca // indirect
+	software.sslmate.com/src/go-pkcs12 v0.0.0-20210415151418-c5206de65a78 // indirect
 )

health/health.go

@@ -16,8 +16,8 @@ import (
 	"sync/atomic"
 	"time"
 
-	"github.com/go-multierror/multierror"
 	"tailscale.com/tailcfg"
+	"tailscale.com/util/multierr"
 )
 
 var (
@@ -58,6 +58,9 @@ const (
 	// SysDNS is the name of the net/dns subsystem.
 	SysDNS = Subsystem("dns")
 
+	// SysDNSOS is the name of the net/dns OSConfigurator subsystem.
+	SysDNSOS = Subsystem("dns-os")
+
 	// SysNetworkCategory is the name of the subsystem that sets
 	// the Windows network adapter's "category" (public, private, domain).
 	// If it's unhealthy, the Windows firewall rules won't match.
@@ -101,6 +104,12 @@ func SetDNSHealth(err error) { set(SysDNS, err) }
 // DNSHealth returns the net/dns.Manager error state.
 func DNSHealth() error { return get(SysDNS) }
 
+// SetDNSOSHealth sets the state of the net/dns.OSConfigurator
+func SetDNSOSHealth(err error) { set(SysDNSOS, err) }
+
+// DNSOSHealth returns the net/dns.OSConfigurator error state.
+func DNSOSHealth() error { return get(SysDNSOS) }
+
 // SetNetworkCategoryHealth sets the state of setting the network adaptor's category.
 // This only applies on Windows.
 func SetNetworkCategoryHealth(err error) { set(SysNetworkCategory, err) }
@@ -268,7 +277,7 @@ func selfCheckLocked() {
 // OverallError returns a summary of the health state.
 //
 // If there are multiple problems, the error will be of type
-// multierror.MultipleErrors.
+// multierr.Error.
 func OverallError() error {
 	mu.Lock()
 	defer mu.Unlock()
@@ -337,7 +346,7 @@ func overallErrorLocked() error {
 		// Not super efficient (stringifying these in a sort), but probably max 2 or 3 items.
 		return errs[i].Error() < errs[j].Error()
 	})
-	return multierror.New(errs)
+	return multierr.New(errs...)
 }
 
 var (

hostinfo/hostinfo.go

@@ -7,11 +7,14 @@
 package hostinfo
 
 import (
+	"bufio"
 	"io"
 	"os"
 	"path/filepath"
 	"runtime"
+	"strings"
 	"sync/atomic"
+	"time"
 
 	"go4.org/mem"
 	"tailscale.com/tailcfg"
@@ -84,6 +87,7 @@ const (
 	AWSFargate      = EnvType("fg")
 	FlyDotIo        = EnvType("fly")
 	Kubernetes      = EnvType("k8s")
+	DockerDesktop   = EnvType("dde")
 )
 
 var envType atomic.Value // of EnvType
@@ -141,6 +145,9 @@ func getEnvType() EnvType {
 	if inKubernetes() {
 		return Kubernetes
 	}
+	if inDockerDesktop() {
+		return DockerDesktop
+	}
 	return ""
 }
 
@@ -224,3 +231,62 @@ func inKubernetes() bool {
 	}
 	return false
 }
+
+func inDockerDesktop() bool {
+	if os.Getenv("TS_HOST_ENV") == "dde" {
+		return true
+	}
+	return false
+}
+
+type etcAptSrcResult struct {
+	mod      time.Time
+	disabled bool
+}
+
+var etcAptSrcCache atomic.Value // of etcAptSrcResult
+
+// DisabledEtcAptSource reports whether Ubuntu (or similar) has disabled
+// the /etc/apt/sources.list.d/tailscale.list file contents upon upgrade
+// to a new release of the distro.
+//
+// See https://github.com/tailscale/tailscale/issues/3177
+func DisabledEtcAptSource() bool {
+	if runtime.GOOS != "linux" {
+		return false
+	}
+	const path = "/etc/apt/sources.list.d/tailscale.list"
+	fi, err := os.Stat(path)
+	if err != nil || !fi.Mode().IsRegular() {
+		return false
+	}
+	mod := fi.ModTime()
+	if c, ok := etcAptSrcCache.Load().(etcAptSrcResult); ok && c.mod == mod {
+		return c.disabled
+	}
+	f, err := os.Open(path)
+	if err != nil {
+		return false
+	}
+	defer f.Close()
+	v := etcAptSourceFileIsDisabled(f)
+	etcAptSrcCache.Store(etcAptSrcResult{mod: mod, disabled: v})
+	return v
+}
+
+func etcAptSourceFileIsDisabled(r io.Reader) bool {
+	bs := bufio.NewScanner(r)
+	disabled := false // did we find the "disabled on upgrade" comment?
+	for bs.Scan() {
+		line := strings.TrimSpace(bs.Text())
+		if strings.Contains(line, "# disabled on upgrade") {
+			disabled = true
+		}
+		if line == "" || line[0] == '#' {
+			continue
+		}
+		// Well, it has some contents in it at least.
+		return false
+	}
+	return disabled
+}

hostinfo/hostinfo_test.go

@@ -6,6 +6,7 @@ package hostinfo
 
 import (
 	"encoding/json"
+	"strings"
 	"testing"
 )
 
@@ -27,3 +28,25 @@ func TestOSVersion(t *testing.T) {
 	}
 	t.Logf("Got: %#q", osVersion())
 }
+
+func TestEtcAptSourceFileIsDisabled(t *testing.T) {
+	tests := []struct {
+		name string
+		in   string
+		want bool
+	}{
+		{"empty", "", false},
+		{"normal", "deb foo\n", false},
+		{"normal-commented", "# deb foo\n", false},
+		{"normal-disabled-by-ubuntu", "# deb foo # disabled on upgrade to dingus\n", true},
+		{"normal-disabled-then-uncommented", "deb foo # disabled on upgrade to dingus\n", false},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			got := etcAptSourceFileIsDisabled(strings.NewReader(tt.in))
+			if got != tt.want {
+				t.Errorf("got %v; want %v", got, tt.want)
+			}
+		})
+	}
+}

hostinfo/hostinfo_windows.go

@@ -5,10 +5,11 @@
 package hostinfo
 
 import (
-	"os/exec"
-	"strings"
+	"fmt"
 	"sync/atomic"
-	"syscall"
+
+	"golang.org/x/sys/windows"
+	"golang.org/x/sys/windows/registry"
 )
 
 func init() {
@@ -21,19 +22,37 @@ func osVersionWindows() string {
 	if s, ok := winVerCache.Load().(string); ok {
 		return s
 	}
-	cmd := exec.Command("cmd", "/c", "ver")
-	cmd.SysProcAttr = &syscall.SysProcAttr{HideWindow: true}
-	out, _ := cmd.Output() // "\nMicrosoft Windows [Version 10.0.19041.388]\n\n"
-	s := strings.TrimSpace(string(out))
-	s = strings.TrimPrefix(s, "Microsoft Windows [")
-	s = strings.TrimSuffix(s, "]")
-
-	// "Version 10.x.y.z", with "Version" localized. Keep only stuff after the space.
-	if sp := strings.Index(s, " "); sp != -1 {
-		s = s[sp+1:]
+	major, minor, build := windows.RtlGetNtVersionNumbers()
+	s := fmt.Sprintf("%d.%d.%d", major, minor, build)
+	// Windows 11 still uses 10 as its major number internally
+	if major == 10 {
+		if ubr, err := getUBR(); err == nil {
+			s += fmt.Sprintf(".%d", ubr)
+		}
 	}
 	if s != "" {
 		winVerCache.Store(s)
 	}
 	return s // "10.0.19041.388", ideally
 }
+
+// getUBR obtains a fourth version field, the "Update Build Revision",
+// from the registry. This field is only available beginning with Windows 10.
+func getUBR() (uint32, error) {
+	key, err := registry.OpenKey(registry.LOCAL_MACHINE,
+		`SOFTWARE\Microsoft\Windows NT\CurrentVersion`, registry.QUERY_VALUE|registry.WOW64_64KEY)
+	if err != nil {
+		return 0, err
+	}
+	defer key.Close()
+
+	val, valType, err := key.GetIntegerValue("UBR")
+	if err != nil {
+		return 0, err
+	}
+	if valType != registry.DWORD {
+		return 0, registry.ErrUnexpectedType
+	}
+
+	return uint32(val), nil
+}

ipn/backend.go

@@ -12,6 +12,7 @@ import (
 	"tailscale.com/ipn/ipnstate"
 	"tailscale.com/tailcfg"
 	"tailscale.com/types/empty"
+	"tailscale.com/types/key"
 	"tailscale.com/types/netmap"
 	"tailscale.com/types/structs"
 )
@@ -48,7 +49,7 @@ type EngineStatus struct {
 	RBytes, WBytes int64
 	NumLive        int
 	LiveDERPs      int // number of active DERP connections
-	LivePeers      map[tailcfg.NodeKey]ipnstate.PeerStatusLite
+	LivePeers      map[key.NodePublic]ipnstate.PeerStatusLite
 }
 
 // Notify is a communication from a backend (e.g. tailscaled) to a frontend

ipn/ipnlocal/dnsconfig_test.go

@@ -232,32 +232,11 @@ func TestDNSConfigForNetmap(t *testing.T) {
 			},
 		},
 		{
-			name: "android_does_need_fallbacks",
-			os:   "android",
-			nm: &netmap.NetworkMap{
-				DNS: tailcfg.DNSConfig{
-					FallbackResolvers: []dnstype.Resolver{
-						{Addr: "8.8.4.4"},
-					},
-					Routes: map[string][]dnstype.Resolver{
-						"foo.com.": {{Addr: "1.2.3.4"}},
-					},
-				},
-			},
-			prefs: &ipn.Prefs{
-				CorpDNS: true,
-			},
-			want: &dns.Config{
-				Hosts: map[dnsname.FQDN][]netaddr.IP{},
-				DefaultResolvers: []dnstype.Resolver{
-					{Addr: "8.8.4.4:53"},
-				},
-				Routes: map[dnsname.FQDN][]dnstype.Resolver{
-					"foo.com.": {{Addr: "1.2.3.4:53"}},
-				},
-			},
-		},
-		{
+			// Prior to fixing https://github.com/tailscale/tailscale/issues/2116,
+			// Android had cases where it needed FallbackResolvers. This was the
+			// negative test for the case where Override-local-DNS was set, so the
+			// fallback resolvers did not need to be used. This test is still valid
+			// so we keep it, but the fallback test has been removed.
 			name: "android_does_NOT_need_fallbacks",
 			os:   "android",
 			nm: &netmap.NetworkMap{
@@ -344,3 +323,48 @@ func TestDNSConfigForNetmap(t *testing.T) {
 		})
 	}
 }
+
+func TestAllowExitNodeDNSProxyToServeName(t *testing.T) {
+	b := &LocalBackend{}
+	if b.allowExitNodeDNSProxyToServeName("google.com") {
+		t.Fatal("unexpected true on backend with nil NetMap")
+	}
+
+	b.netMap = &netmap.NetworkMap{
+		DNS: tailcfg.DNSConfig{
+			ExitNodeFilteredSet: []string{
+				".ts.net",
+				"some.exact.bad",
+			},
+		},
+	}
+	tests := []struct {
+		name string
+		want bool
+	}{
+		// Allow by default:
+		{"google.com", true},
+		{"GOOGLE.com", true},
+
+		// Rejected by suffix:
+		{"foo.TS.NET", false},
+		{"foo.ts.net", false},
+
+		// Suffix doesn't match
+		{"ts.net", true},
+
+		// Rejected by exact match:
+		{"some.exact.bad", false},
+		{"SOME.EXACT.BAD", false},
+
+		// But a prefix is okay.
+		{"prefix-okay.some.exact.bad", true},
+	}
+	for _, tt := range tests {
+		got := b.allowExitNodeDNSProxyToServeName(tt.name)
+		if got != tt.want {
+			t.Errorf("for %q = %v; want %v", tt.name, got, tt.want)
+		}
+	}
+
+}

ipn/ipnlocal/local.go

@@ -22,11 +22,8 @@ import (
 	"strings"
 	"sync"
 	"sync/atomic"
-	"syscall"
 	"time"
 
-	"github.com/go-multierror/multierror"
-	"go4.org/mem"
 	"inet.af/netaddr"
 	"tailscale.com/client/tailscale/apitype"
 	"tailscale.com/control/controlclient"
@@ -38,6 +35,7 @@ import (
 	"tailscale.com/net/dns"
 	"tailscale.com/net/interfaces"
 	"tailscale.com/net/tsaddr"
+	"tailscale.com/net/tsdial"
 	"tailscale.com/paths"
 	"tailscale.com/portlist"
 	"tailscale.com/tailcfg"
@@ -50,6 +48,7 @@ import (
 	"tailscale.com/types/preftype"
 	"tailscale.com/util/deephash"
 	"tailscale.com/util/dnsname"
+	"tailscale.com/util/multierr"
 	"tailscale.com/util/osshare"
 	"tailscale.com/util/systemd"
 	"tailscale.com/version"
@@ -89,6 +88,7 @@ type LocalBackend struct {
 	statsLogf             logger.Logf        // for printing peers stats on change
 	e                     wgengine.Engine
 	store                 ipn.StateStore
+	dialer                *tsdial.Dialer // non-nil
 	backendLogID          string
 	unregisterLinkMon     func()
 	unregisterHealthWatch func()
@@ -97,9 +97,12 @@ type LocalBackend struct {
 	gotPortPollRes        chan struct{}    // closed upon first readPoller result
 	serverURL             string           // tailcontrol URL
 	newDecompressor       func() (controlclient.Decompressor, error)
+	varRoot               string // or empty if SetVarRoot never called
 
 	filterHash deephash.Sum
 
+	filterAtomic atomic.Value // of *filter.Filter
+
 	// The mutex protects the following elements.
 	mu             sync.Mutex
 	httpTestClient *http.Client // for controlclient. nil by default, used by tests.
@@ -122,6 +125,7 @@ type LocalBackend struct {
 	engineStatus     ipn.EngineStatus
 	endpoints        []tailcfg.Endpoint
 	blocked          bool
+	keyExpired       bool
 	authURL          string // cleared on Notify
 	authURLSticky    string // not cleared on Notify
 	interact         bool
@@ -138,7 +142,11 @@ type LocalBackend struct {
 	// same as the Network Extension lifetime and we can thus avoid
 	// double-copying files by writing them to the right location
 	// immediately.
-	directFileRoot string
+	// It's also used on Synology & TrueNAS, but in that case DoFinalRename
+	// is also set true, which moves the *.partial file to its final
+	// name on completion.
+	directFileRoot          string
+	directFileDoFinalRename bool // false on macOS, true on Synology & TrueNAS
 
 	// statusLock must be held before calling statusChanged.Wait() or
 	// statusChanged.Broadcast().
@@ -152,16 +160,18 @@ type clientGen func(controlclient.Options) (controlclient.Client, error)
 
 // NewLocalBackend returns a new LocalBackend that is ready to run,
 // but is not actually running.
-func NewLocalBackend(logf logger.Logf, logid string, store ipn.StateStore, e wgengine.Engine) (*LocalBackend, error) {
+//
+// If dialer is nil, a new one is made.
+func NewLocalBackend(logf logger.Logf, logid string, store ipn.StateStore, dialer *tsdial.Dialer, e wgengine.Engine) (*LocalBackend, error) {
 	if e == nil {
-		panic("ipn.NewLocalBackend: wgengine must not be nil")
+		panic("ipn.NewLocalBackend: engine must not be nil")
+	}
+	if dialer == nil {
+		dialer = new(tsdial.Dialer)
 	}
 
 	osshare.SetFileSharingEnabled(false, logf)
 
-	// Default filter blocks everything and logs nothing, until Start() is called.
-	e.SetFilter(filter.NewAllowNone(logf, &netaddr.IPSet{}))
-
 	ctx, cancel := context.WithCancel(context.Background())
 	portpoll, err := portlist.NewPoller()
 	if err != nil {
@@ -176,11 +186,16 @@ func NewLocalBackend(logf logger.Logf, logid string, store ipn.StateStore, e wge
 		statsLogf:      logger.LogOnChange(logf, 5*time.Minute, time.Now),
 		e:              e,
 		store:          store,
+		dialer:         dialer,
 		backendLogID:   logid,
 		state:          ipn.NoState,
 		portpoll:       portpoll,
 		gotPortPollRes: make(chan struct{}),
 	}
+
+	// Default filter blocks everything and logs nothing, until Start() is called.
+	b.setFilter(filter.NewAllowNone(logf, &netaddr.IPSet{}))
+
 	b.statusChanged = sync.NewCond(&b.statusLock)
 	b.e.SetStatusCallback(b.setWgengineStatus)
 
@@ -207,6 +222,11 @@ func NewLocalBackend(logf logger.Logf, logid string, store ipn.StateStore, e wge
 	return b, nil
 }
 
+// Dialer returns the backend's dialer.
+func (b *LocalBackend) Dialer() *tsdial.Dialer {
+	return b.dialer
+}
+
 // SetDirectFileRoot sets the directory to download files to directly,
 // without buffering them through an intermediate daemon-owned
 // tailcfg.UserID-specific directory.
@@ -218,6 +238,17 @@ func (b *LocalBackend) SetDirectFileRoot(dir string) {
 	b.directFileRoot = dir
 }
 
+// SetDirectFileDoFinalRename sets whether the peerapi file server should rename
+// a received "name.partial" file to "name" when the download is complete.
+//
+// This only applies when SetDirectFileRoot is non-empty.
+// The default is false.
+func (b *LocalBackend) SetDirectFileDoFinalRename(v bool) {
+	b.mu.Lock()
+	defer b.mu.Unlock()
+	b.directFileDoFinalRename = v
+}
+
 // b.mu must be held.
 func (b *LocalBackend) maybePauseControlClientLocked() {
 	if b.cc == nil {
@@ -335,8 +366,8 @@ func (b *LocalBackend) updateStatus(sb *ipnstate.StatusBuilder, extraLocked func
 
 		if err := health.OverallError(); err != nil {
 			switch e := err.(type) {
-			case multierror.MultipleErrors:
-				for _, err := range e {
+			case multierr.Error:
+				for _, err := range e.Errors() {
 					s.Health = append(s.Health, err.Error())
 				}
 			default:
@@ -349,8 +380,18 @@ func (b *LocalBackend) updateStatus(sb *ipnstate.StatusBuilder, extraLocked func
 		}
 	})
 	sb.MutateSelfStatus(func(ss *ipnstate.PeerStatus) {
-		if b.netMap != nil && b.netMap.SelfNode != nil {
-			ss.ID = b.netMap.SelfNode.StableID
+		if b.netMap != nil {
+			ss.HostName = b.netMap.Hostinfo.Hostname
+			ss.DNSName = b.netMap.Name
+			ss.UserID = b.netMap.User
+			if sn := b.netMap.SelfNode; sn != nil {
+				ss.ID = sn.StableID
+				if c := sn.Capabilities; len(c) > 0 {
+					ss.Capabilities = append([]string(nil), c...)
+				}
+			}
+		} else {
+			ss.HostName, _ = os.Hostname()
 		}
 		for _, pln := range b.peerAPIListeners {
 			ss.PeerAPIURL = append(ss.PeerAPIURL, pln.urlStr)
@@ -376,33 +417,30 @@ func (b *LocalBackend) populatePeerStatusLocked(sb *ipnstate.StatusBuilder) {
 		if p.LastSeen != nil {
 			lastSeen = *p.LastSeen
 		}
-		var tailAddr4 string
 		var tailscaleIPs = make([]netaddr.IP, 0, len(p.Addresses))
 		for _, addr := range p.Addresses {
 			if addr.IsSingleIP() && tsaddr.IsTailscaleIP(addr.IP()) {
-				if addr.IP().Is4() && tailAddr4 == "" {
-					// The peer struct previously only allowed a single
-					// Tailscale IP address. For compatibility for a few releases starting
-					// with 1.8, keep it pulled out as IPv4-only for a bit.
-					tailAddr4 = addr.IP().String()
-				}
 				tailscaleIPs = append(tailscaleIPs, addr.IP())
 			}
 		}
-		sb.AddPeer(key.NodePublicFromRaw32(mem.B(p.Key[:])), &ipnstate.PeerStatus{
-			InNetworkMap:       true,
-			ID:                 p.StableID,
-			UserID:             p.User,
-			TailAddrDeprecated: tailAddr4,
-			TailscaleIPs:       tailscaleIPs,
-			HostName:           p.Hostinfo.Hostname,
-			DNSName:            p.Name,
-			OS:                 p.Hostinfo.OS,
-			KeepAlive:          p.KeepAlive,
-			Created:            p.Created,
-			LastSeen:           lastSeen,
-			ShareeNode:         p.Hostinfo.ShareeNode,
-			ExitNode:           p.StableID != "" && p.StableID == b.prefs.ExitNodeID,
+		exitNodeOption := tsaddr.PrefixesContainsFunc(p.AllowedIPs, func(r netaddr.IPPrefix) bool {
+			return r.Bits() == 0
+		})
+		sb.AddPeer(p.Key, &ipnstate.PeerStatus{
+			InNetworkMap:   true,
+			ID:             p.StableID,
+			UserID:         p.User,
+			TailscaleIPs:   tailscaleIPs,
+			HostName:       p.Hostinfo.Hostname,
+			DNSName:        p.Name,
+			OS:             p.Hostinfo.OS,
+			KeepAlive:      p.KeepAlive,
+			Created:        p.Created,
+			LastSeen:       lastSeen,
+			Online:         p.Online != nil && *p.Online,
+			ShareeNode:     p.Hostinfo.ShareeNode,
+			ExitNode:       p.StableID != "" && p.StableID == b.prefs.ExitNodeID,
+			ExitNodeOption: exitNodeOption,
 		})
 	}
 }
@@ -452,18 +490,35 @@ func (b *LocalBackend) setClientStatus(st controlclient.Status) {
 		// TODO(crawshaw): display in the UI.
 		if errors.Is(st.Err, io.EOF) {
 			b.logf("[v1] Received error: EOF")
-		} else {
-			b.logf("Received error: %v", st.Err)
-			e := st.Err.Error()
-			b.send(ipn.Notify{ErrMessage: &e})
+			return
+		}
+		b.logf("Received error: %v", st.Err)
+		var uerr controlclient.UserVisibleError
+		if errors.As(st.Err, &uerr) {
+			s := uerr.UserVisibleError()
+			b.send(ipn.Notify{ErrMessage: &s})
 		}
 		return
 	}
 
 	b.mu.Lock()
 	wasBlocked := b.blocked
+	keyExpiryExtended := false
+	if st.NetMap != nil {
+		wasExpired := b.keyExpired
+		isExpired := !st.NetMap.Expiry.IsZero() && st.NetMap.Expiry.Before(time.Now())
+		if wasExpired && !isExpired {
+			keyExpiryExtended = true
+		}
+		b.keyExpired = isExpired
+	}
 	b.mu.Unlock()
 
+	if keyExpiryExtended && wasBlocked {
+		// Key extended, unblock the engine
+		b.blockEngineUpdates(false)
+	}
+
 	if st.LoginFinished != nil && wasBlocked {
 		// Auth completed, unblock the engine
 		b.blockEngineUpdates(false)
@@ -570,6 +625,11 @@ func (b *LocalBackend) setClientStatus(st controlclient.Status) {
 // findExitNodeIDLocked updates b.prefs to reference an exit node by ID,
 // rather than by IP. It returns whether prefs was mutated.
 func (b *LocalBackend) findExitNodeIDLocked(nm *netmap.NetworkMap) (prefsChanged bool) {
+	if nm == nil {
+		// No netmap, can't resolve anything.
+		return false
+	}
+
 	// If we have a desired IP on file, try to find the corresponding
 	// node.
 	if b.prefs.ExitNodeIP.IsZero() {
@@ -847,7 +907,7 @@ func (b *LocalBackend) Start(opts ipn.Options) error {
 		})
 	}
 
-	var discoPublic tailcfg.DiscoKey
+	var discoPublic key.DiscoPublic
 	if controlclient.Debug.Disco {
 		discoPublic = b.e.DiscoPublicKey()
 	}
@@ -983,20 +1043,25 @@ func (b *LocalBackend) updateFilter(netMap *netmap.NetworkMap, prefs *ipn.Prefs)
 
 	if !haveNetmap {
 		b.logf("netmap packet filter: (not ready yet)")
-		b.e.SetFilter(filter.NewAllowNone(b.logf, logNets))
+		b.setFilter(filter.NewAllowNone(b.logf, logNets))
 		return
 	}
 
 	oldFilter := b.e.GetFilter()
 	if shieldsUp {
 		b.logf("netmap packet filter: (shields up)")
-		b.e.SetFilter(filter.NewShieldsUpFilter(localNets, logNets, oldFilter, b.logf))
+		b.setFilter(filter.NewShieldsUpFilter(localNets, logNets, oldFilter, b.logf))
 	} else {
 		b.logf("netmap packet filter: %v filters", len(packetFilter))
-		b.e.SetFilter(filter.New(packetFilter, localNets, logNets, oldFilter, b.logf))
+		b.setFilter(filter.New(packetFilter, localNets, logNets, oldFilter, b.logf))
 	}
 }
 
+func (b *LocalBackend) setFilter(f *filter.Filter) {
+	b.filterAtomic.Store(f)
+	b.e.SetFilter(f)
+}
+
 var removeFromDefaultRoute = []netaddr.IPPrefix{
 	// RFC1918 LAN ranges
 	netaddr.MustParseIPPrefix("192.168.0.0/16"),
@@ -1224,7 +1289,7 @@ func (b *LocalBackend) send(n ipn.Notify) {
 		return
 	}
 
-	if apiSrv != nil && apiSrv.hasFilesWaiting() {
+	if apiSrv.hasFilesWaiting() {
 		n.FilesWaiting = &empty.Message{}
 	}
 
@@ -1562,7 +1627,7 @@ func (b *LocalBackend) parseWgStatusLocked(s *wgengine.Status) (ret ipn.EngineSt
 	var peerStats, peerKeys strings.Builder
 
 	ret.LiveDERPs = s.DERPs
-	ret.LivePeers = map[tailcfg.NodeKey]ipnstate.PeerStatusLite{}
+	ret.LivePeers = map[key.NodePublic]ipnstate.PeerStatusLite{}
 	for _, p := range s.Peers {
 		if !p.LastHandshake.IsZero() {
 			fmt.Fprintf(&peerStats, "%d/%d ", p.RxBytes, p.TxBytes)
@@ -1633,7 +1698,7 @@ func (b *LocalBackend) SetPrefs(newp *ipn.Prefs) {
 }
 
 // setPrefsLockedOnEntry requires b.mu be held to call it, but it
-// unlocks b.mu when done.
+// unlocks b.mu when done. newp ownership passes to this function.
 func (b *LocalBackend) setPrefsLockedOnEntry(caller string, newp *ipn.Prefs) {
 	netMap := b.netMap
 	stateKey := b.stateKey
@@ -1641,6 +1706,10 @@ func (b *LocalBackend) setPrefsLockedOnEntry(caller string, newp *ipn.Prefs) {
 	oldp := b.prefs
 	newp.Persist = oldp.Persist // caller isn't allowed to override this
 	b.prefs = newp
+	// findExitNodeIDLocked returns whether it updated b.prefs, but
+	// everything in this function treats b.prefs as completely new
+	// anyway. No-op if no exit node resolution is needed.
+	b.findExitNodeIDLocked(netMap)
 	b.inServerMode = newp.ForceDaemon
 	// We do this to avoid holding the lock while doing everything else.
 	newp = b.prefs.Clone()
@@ -1676,7 +1745,7 @@ func (b *LocalBackend) setPrefsLockedOnEntry(caller string, newp *ipn.Prefs) {
 				// notified (to update its prefs/persist) on
 				// account switch.  Log this while we figure it
 				// out.
-				b.logf("active login: %s ([unexpected] corp#461, not %s)", newp.Persist.LoginName)
+				b.logf("active login: %q ([unexpected] corp#461, not %q)", newp.Persist.LoginName, login)
 			}
 		}
 	}
@@ -1718,15 +1787,24 @@ func (b *LocalBackend) getPeerAPIPortForTSMPPing(ip netaddr.IP) (port uint16, ok
 
 func (b *LocalBackend) peerAPIServicesLocked() (ret []tailcfg.Service) {
 	for _, pln := range b.peerAPIListeners {
-		proto := tailcfg.ServiceProto("peerapi4")
+		proto := tailcfg.PeerAPI4
 		if pln.ip.Is6() {
-			proto = "peerapi6"
+			proto = tailcfg.PeerAPI6
 		}
 		ret = append(ret, tailcfg.Service{
 			Proto: proto,
 			Port:  uint16(pln.port),
 		})
 	}
+	switch runtime.GOOS {
+	case "linux", "freebsd", "openbsd", "illumos", "darwin", "windows":
+		// These are the platforms currently supported by
+		// net/dns/resolver/tsdns.go:Resolver.HandleExitNodeDNSQuery.
+		ret = append(ret, tailcfg.Service{
+			Proto: tailcfg.PeerAPIDNS,
+			Port:  1, // version
+		})
+	}
 	return ret
 }
 
@@ -1771,6 +1849,12 @@ func (b *LocalBackend) NetMap() *netmap.NetworkMap {
 	return b.netMap
 }
 
+func (b *LocalBackend) isEngineBlocked() bool {
+	b.mu.Lock()
+	defer b.mu.Unlock()
+	return b.blocked
+}
+
 // blockEngineUpdate sets b.blocked to block, while holding b.mu. Its
 // indirect effect is to turn b.authReconfig() into a no-op if block
 // is true.
@@ -1821,6 +1905,15 @@ func (b *LocalBackend) authReconfig() {
 		}
 	}
 
+	// Keep the dialer updated about whether we're supposed to use
+	// an exit node's DNS server (so SOCKS5/HTTP outgoing dials
+	// can use it for name resolution)
+	if dohURL, ok := exitNodeCanProxyDNS(nm, prefs.ExitNodeID); ok {
+		b.dialer.SetExitDNSDoH(dohURL)
+	} else {
+		b.dialer.SetExitDNSDoH("")
+	}
+
 	cfg, err := nmcfg.WGCfg(nm, b.logf, flags, prefs.ExitNodeID)
 	if err != nil {
 		b.logf("wgcfg: %v", err)
@@ -1919,12 +2012,32 @@ func dnsConfigForNetmap(nm *netmap.NetworkMap, prefs *ipn.Prefs, logf logger.Log
 		return dcfg
 	}
 
+	for _, dom := range nm.DNS.Domains {
+		fqdn, err := dnsname.ToFQDN(dom)
+		if err != nil {
+			logf("[unexpected] non-FQDN search domain %q", dom)
+		}
+		dcfg.SearchDomains = append(dcfg.SearchDomains, fqdn)
+	}
+	if nm.DNS.Proxied { // actually means "enable MagicDNS"
+		for _, dom := range magicDNSRootDomains(nm) {
+			dcfg.Routes[dom] = nil // resolve internally with dcfg.Hosts
+		}
+	}
+
 	addDefault := func(resolvers []dnstype.Resolver) {
 		for _, r := range resolvers {
 			dcfg.DefaultResolvers = append(dcfg.DefaultResolvers, normalizeResolver(r))
 		}
 	}
 
+	// If we're using an exit node and that exit node is new enough (1.19.x+)
+	// to run a DoH DNS proxy, then send all our DNS traffic through it.
+	if dohURL, ok := exitNodeCanProxyDNS(nm, prefs.ExitNodeID); ok {
+		addDefault([]dnstype.Resolver{{Addr: dohURL}})
+		return dcfg
+	}
+
 	addDefault(nm.DNS.Resolvers)
 	for suffix, resolvers := range nm.DNS.Routes {
 		fqdn, err := dnsname.ToFQDN(suffix)
@@ -1946,18 +2059,6 @@ func dnsConfigForNetmap(nm *netmap.NetworkMap, prefs *ipn.Prefs, logf logger.Log
 			dcfg.Routes[fqdn] = append(dcfg.Routes[fqdn], normalizeResolver(r))
 		}
 	}
-	for _, dom := range nm.DNS.Domains {
-		fqdn, err := dnsname.ToFQDN(dom)
-		if err != nil {
-			logf("[unexpected] non-FQDN search domain %q", dom)
-		}
-		dcfg.SearchDomains = append(dcfg.SearchDomains, fqdn)
-	}
-	if nm.DNS.Proxied { // actually means "enable MagicDNS"
-		for _, dom := range magicDNSRootDomains(nm) {
-			dcfg.Routes[dom] = nil // resolve internally with dcfg.Hosts
-		}
-	}
 
 	// Set FallbackResolvers as the default resolvers in the
 	// scenarios that can't handle a purely split-DNS config. See
@@ -1981,9 +2082,6 @@ func dnsConfigForNetmap(nm *netmap.NetworkMap, prefs *ipn.Prefs, logf logger.Log
 		addDefault(nm.DNS.FallbackResolvers)
 	case len(dcfg.Routes) == 0:
 		// No settings requiring split DNS, no problem.
-	case versionOS == "android":
-		// We don't support split DNS at all on Android yet.
-		addDefault(nm.DNS.FallbackResolvers)
 	}
 
 	return dcfg
@@ -1999,34 +2097,29 @@ func normalizeResolver(cfg dnstype.Resolver) dnstype.Resolver {
 	return cfg
 }
 
+// SetVarRoot sets the root directory of Tailscale's writable
+// storage area . (e.g. "/var/lib/tailscale")
+//
+// It should only be called before the LocalBackend is used.
+func (b *LocalBackend) SetVarRoot(dir string) {
+	b.varRoot = dir
+}
+
 // TailscaleVarRoot returns the root directory of Tailscale's writable
 // storage area. (e.g. "/var/lib/tailscale")
 //
 // It returns an empty string if there's no configured or discovered
 // location.
 func (b *LocalBackend) TailscaleVarRoot() string {
+	if b.varRoot != "" {
+		return b.varRoot
+	}
 	switch runtime.GOOS {
 	case "ios", "android":
 		dir, _ := paths.AppSharedDir.Load().(string)
 		return dir
 	}
-	// Temporary (2021-09-27) transitional fix for #2927 (Synology
-	// cert dir) on the way towards a more complete fix
-	// (#2932). It fixes any case where the state file is provided
-	// to tailscaled explicitly when it's not in the default
-	// location.
-	if fs, ok := b.store.(*ipn.FileStore); ok {
-		if fp := fs.Path(); fp != "" {
-			if dir := filepath.Dir(fp); strings.EqualFold(filepath.Base(dir), "tailscale") {
-				return dir
-			}
-		}
-	}
-	stateFile := paths.DefaultTailscaledStateFile()
-	if stateFile == "" {
-		return ""
-	}
-	return filepath.Dir(stateFile)
+	return ""
 }
 
 func (b *LocalBackend) fileRootLocked(uid tailcfg.UserID) string {
@@ -2035,7 +2128,7 @@ func (b *LocalBackend) fileRootLocked(uid tailcfg.UserID) string {
 	}
 	varRoot := b.TailscaleVarRoot()
 	if varRoot == "" {
-		b.logf("peerapi disabled; no state directory")
+		b.logf("Taildrop disabled; no state directory")
 		return ""
 	}
 	baseDir := fmt.Sprintf("%s-uid-%d",
@@ -2043,7 +2136,7 @@ func (b *LocalBackend) fileRootLocked(uid tailcfg.UserID) string {
 		uid)
 	dir := filepath.Join(varRoot, "files", baseDir)
 	if err := os.MkdirAll(dir, 0700); err != nil {
-		b.logf("peerapi disabled; error making directory: %v", err)
+		b.logf("Taildrop disabled; error making directory: %v", err)
 		return ""
 	}
 	return dir
@@ -2106,22 +2199,20 @@ func (b *LocalBackend) initPeerAPIListener() {
 
 	fileRoot := b.fileRootLocked(selfNode.User)
 	if fileRoot == "" {
-		return
-	}
-
-	var tunName string
-	if ge, ok := b.e.(wgengine.InternalsGetter); ok {
-		if tunWrap, _, ok := ge.GetInternals(); ok {
-			tunName, _ = tunWrap.Name()
-		}
+		b.logf("peerapi starting without Taildrop directory configured")
 	}
 
 	ps := &peerAPIServer{
-		b:              b,
-		rootDir:        fileRoot,
-		tunName:        tunName,
-		selfNode:       selfNode,
-		directFileMode: b.directFileRoot != "",
+		b:                       b,
+		rootDir:                 fileRoot,
+		selfNode:                selfNode,
+		directFileMode:          b.directFileRoot != "",
+		directFileDoFinalRename: b.directFileDoFinalRename,
+	}
+	if re, ok := b.e.(wgengine.ResolvingEngine); ok {
+		if r, ok := re.GetResolver(); ok {
+			ps.resolver = r
+		}
 	}
 	b.peerAPIServer = ps
 
@@ -2398,6 +2489,7 @@ func (b *LocalBackend) nextState() ipn.State {
 		wantRunning = b.prefs.WantRunning
 		loggedOut   = b.prefs.LoggedOut
 		st          = b.engineStatus
+		keyExpired  = b.keyExpired
 	)
 	b.mu.Unlock()
 
@@ -2430,7 +2522,9 @@ func (b *LocalBackend) nextState() ipn.State {
 		}
 	case !wantRunning:
 		return ipn.Stopped
-	case !netMap.Expiry.IsZero() && time.Until(netMap.Expiry) <= 0:
+	case keyExpired:
+		// NetMap must be non-nil for us to get here.
+		// The node key expired, need to relogin.
 		return ipn.NeedsLogin
 	case netMap.MachineStatus != tailcfg.MachineAuthorized:
 		// TODO(crawshaw): handle tailcfg.MachineInvalid
@@ -2511,6 +2605,7 @@ func (b *LocalBackend) ResetForClientDisconnect() {
 	b.userID = ""
 	b.setNetMapLocked(nil)
 	b.prefs = new(ipn.Prefs)
+	b.keyExpired = false
 	b.authURL = ""
 	b.authURLSticky = ""
 	b.activeLogin = ""
@@ -2595,6 +2690,7 @@ func hasCapability(nm *netmap.NetworkMap, cap string) bool {
 }
 
 func (b *LocalBackend) setNetMapLocked(nm *netmap.NetworkMap) {
+	b.dialer.SetNetMap(nm)
 	var login string
 	if nm != nil {
 		login = nm.UserProfiles[nm.User].LoginName
@@ -2680,7 +2776,7 @@ func (b *LocalBackend) OperatorUserID() string {
 // TestOnlyPublicKeys returns the current machine and node public
 // keys. Used in tests only to facilitate automated node authorization
 // in the test harness.
-func (b *LocalBackend) TestOnlyPublicKeys() (machineKey key.MachinePublic, nodeKey tailcfg.NodeKey) {
+func (b *LocalBackend) TestOnlyPublicKeys() (machineKey key.MachinePublic, nodeKey key.NodePublic) {
 	b.mu.Lock()
 	prefs := b.prefs
 	machinePrivKey := b.machinePrivKey
@@ -2692,16 +2788,13 @@ func (b *LocalBackend) TestOnlyPublicKeys() (machineKey key.MachinePublic, nodeK
 
 	mk := machinePrivKey.Public()
 	nk := prefs.Persist.PrivateNodeKey.Public()
-	return mk, tailcfg.NodeKeyFromNodePublic(nk)
+	return mk, nk
 }
 
 func (b *LocalBackend) WaitingFiles() ([]apitype.WaitingFile, error) {
 	b.mu.Lock()
 	apiSrv := b.peerAPIServer
 	b.mu.Unlock()
-	if apiSrv == nil {
-		return nil, errors.New("peerapi disabled")
-	}
 	return apiSrv.WaitingFiles()
 }
 
@@ -2709,9 +2802,6 @@ func (b *LocalBackend) DeleteFile(name string) error {
 	b.mu.Lock()
 	apiSrv := b.peerAPIServer
 	b.mu.Unlock()
-	if apiSrv == nil {
-		return errors.New("peerapi disabled")
-	}
 	return apiSrv.DeleteFile(name)
 }
 
@@ -2719,9 +2809,6 @@ func (b *LocalBackend) OpenFile(name string) (rc io.ReadCloser, size int64, err
 	b.mu.Lock()
 	apiSrv := b.peerAPIServer
 	b.mu.Unlock()
-	if apiSrv == nil {
-		return nil, 0, errors.New("peerapi disabled")
-	}
 	return apiSrv.OpenFile(name)
 }
 
@@ -2782,7 +2869,7 @@ func (b *LocalBackend) SetDNS(ctx context.Context, name, value string) error {
 	b.mu.Lock()
 	cc := b.cc
 	if prefs := b.prefs; prefs != nil {
-		req.NodeKey = tailcfg.NodeKeyFromNodePublic(prefs.Persist.PrivateNodeKey.Public())
+		req.NodeKey = prefs.Persist.PrivateNodeKey.Public()
 	}
 	b.mu.Unlock()
 	if cc == nil {
@@ -2835,9 +2922,9 @@ func peerAPIBase(nm *netmap.NetworkMap, peer *tailcfg.Node) string {
 	var p4, p6 uint16
 	for _, s := range peer.Hostinfo.Services {
 		switch s.Proto {
-		case "peerapi4":
+		case tailcfg.PeerAPI4:
 			p4 = s.Port
-		case "peerapi6":
+		case tailcfg.PeerAPI6:
 			p6 = s.Port
 		}
 	}
@@ -2871,48 +2958,97 @@ func (b *LocalBackend) CheckIPForwarding() error {
 	if wgengine.IsNetstackRouter(b.e) {
 		return nil
 	}
-	if isBSD(runtime.GOOS) {
+
+	switch {
+	case isBSD(runtime.GOOS):
 		return fmt.Errorf("Subnet routing and exit nodes only work with additional manual configuration on %v, and is not currently officially supported.", runtime.GOOS)
+	case runtime.GOOS == "linux":
+		return checkIPForwardingLinux()
+	default:
+		// TODO: subnet routing and exit nodes probably don't work
+		// correctly on non-linux, non-netstack OSes either. Warn
+		// instead of being silent?
+		return nil
+	}
+}
+
+// checkIPForwardingLinux checks if IP forwarding is enabled correctly
+// for subnet routing and exit node functionality. Returns an error
+// describing configuration issues if the configuration is not
+// definitely good.
+func checkIPForwardingLinux() error {
+	const kbLink = "\nSee https://tailscale.com/kb/1104/enable-ip-forwarding/"
+
+	disabled, err := disabledSysctls("net.ipv4.ip_forward", "net.ipv6.conf.all.forwarding")
+	if err != nil {
+		return fmt.Errorf("Couldn't check system's IP forwarding configuration, subnet routing/exit nodes may not work: %w%s", err, kbLink)
 	}
 
-	var keys []string
-
-	if runtime.GOOS == "linux" {
-		keys = append(keys, "net.ipv4.ip_forward", "net.ipv6.conf.all.forwarding")
-	} else if isBSD(runtime.GOOS) {
-		keys = append(keys, "net.inet.ip.forwarding")
-	} else {
+	if len(disabled) == 0 {
+		// IP forwarding is enabled systemwide, all is well.
 		return nil
 	}
 
-	const suffix = "\nSubnet routes won't work without IP forwarding.\nSee https://tailscale.com/kb/1104/enable-ip-forwarding/"
-	for _, key := range keys {
-		bs, err := exec.Command("sysctl", "-n", key).Output()
-		if err != nil {
-			return fmt.Errorf("couldn't check %s (%v)%s", key, err, suffix)
+	// IP forwarding isn't enabled globally, but it might be enabled
+	// on a per-interface basis. Check if it's on for all interfaces,
+	// and warn appropriately if it's not.
+	ifaces, err := interfaces.GetList()
+	if err != nil {
+		return fmt.Errorf("Couldn't enumerate network interfaces, subnet routing/exit nodes may not work: %w%s", err, kbLink)
+	}
+
+	var (
+		warnings   []string
+		anyEnabled bool
+	)
+	for _, iface := range ifaces {
+		if iface.Name == "lo" {
+			continue
 		}
-		on, err := strconv.ParseBool(string(bytes.TrimSpace(bs)))
+		disabled, err = disabledSysctls(fmt.Sprintf("net.ipv4.conf.%s.forwarding", iface.Name), fmt.Sprintf("net.ipv6.conf.%s.forwarding", iface.Name))
 		if err != nil {
-			return fmt.Errorf("couldn't parse %s (%v)%s.", key, err, suffix)
+			return fmt.Errorf("Couldn't check system's IP forwarding configuration, subnet routing/exit nodes may not work: %w%s", err, kbLink)
 		}
-		if !on {
-			return fmt.Errorf("%s is disabled.%s", key, suffix)
+		if len(disabled) > 0 {
+			warnings = append(warnings, fmt.Sprintf("Traffic received on %s won't be forwarded (%s disabled)", iface.Name, strings.Join(disabled, ", ")))
+		} else {
+			anyEnabled = true
 		}
 	}
+	if !anyEnabled {
+		// IP forwarding is compeltely disabled, just say that rather
+		// than enumerate all the interfaces on the system.
+		return fmt.Errorf("IP forwarding is disabled, subnet routing/exit nodes will not work.%s", kbLink)
+	}
+	if len(warnings) > 0 {
+		// If partially enabled, enumerate the bits that won't work.
+		return fmt.Errorf("%s\nSubnet routes and exit nodes may not work correctly.%s", strings.Join(warnings, "\n"), kbLink)
+	}
+
 	return nil
 }
 
-// peerDialControlFunc is non-nil on platforms that require a way to
-// bind to dial out to other peers.
-var peerDialControlFunc func(*LocalBackend) func(network, address string, c syscall.RawConn) error
-
-// PeerDialControlFunc returns a net.Dialer.Control func (possibly nil) to use to
-// dial other Tailscale peers from the current environment.
-func (b *LocalBackend) PeerDialControlFunc() func(network, address string, c syscall.RawConn) error {
-	if peerDialControlFunc != nil {
-		return peerDialControlFunc(b)
+// disabledSysctls checks if the given sysctl keys are off, according
+// to strconv.ParseBool. Returns a list of keys that are disabled, or
+// err if something went wrong which prevented the lookups from
+// completing.
+func disabledSysctls(sysctls ...string) (disabled []string, err error) {
+	for _, k := range sysctls {
+		// TODO: on linux, we can get at these values via /proc/sys,
+		// rather than fork subcommands that may not be installed.
+		bs, err := exec.Command("sysctl", "-n", k).Output()
+		if err != nil {
+			return nil, fmt.Errorf("couldn't check %s (%v)", k, err)
+		}
+		on, err := strconv.ParseBool(string(bytes.TrimSpace(bs)))
+		if err != nil {
+			return nil, fmt.Errorf("couldn't parse %s (%v)", k, err)
+		}
+		if !on {
+			disabled = append(disabled, k)
+		}
 	}
-	return nil
+	return disabled, nil
 }
 
 // DERPMap returns the current DERPMap in use, or nil if not connected.
@@ -2924,3 +3060,77 @@ func (b *LocalBackend) DERPMap() *tailcfg.DERPMap {
 	}
 	return b.netMap.DERPMap
 }
+
+// OfferingExitNode reports whether b is currently offering exit node
+// access.
+func (b *LocalBackend) OfferingExitNode() bool {
+	b.mu.Lock()
+	defer b.mu.Unlock()
+	if b.prefs == nil {
+		return false
+	}
+	var def4, def6 bool
+	for _, r := range b.prefs.AdvertiseRoutes {
+		if r.Bits() != 0 {
+			continue
+		}
+		if r.IP().Is4() {
+			def4 = true
+		} else if r.IP().Is6() {
+			def6 = true
+		}
+	}
+	return def4 && def6
+}
+
+// allowExitNodeDNSProxyToServeName reports whether the Exit Node DNS
+// proxy is allowed to serve responses for the provided DNS name.
+func (b *LocalBackend) allowExitNodeDNSProxyToServeName(name string) bool {
+	b.mu.Lock()
+	defer b.mu.Unlock()
+	nm := b.netMap
+	if nm == nil {
+		return false
+	}
+	name = strings.ToLower(name)
+	for _, bad := range nm.DNS.ExitNodeFilteredSet {
+		if bad == "" {
+			// Invalid, ignore.
+			continue
+		}
+		if bad[0] == '.' {
+			// Entries beginning with a dot are suffix matches.
+			if dnsname.HasSuffix(name, bad) {
+				return false
+			}
+			continue
+		}
+		// Otherwise entries are exact matches. They're
+		// guaranteed to be lowercase already.
+		if name == bad {
+			return false
+		}
+	}
+	return true
+}
+
+// exitNodeCanProxyDNS reports the DoH base URL ("http://foo/dns-query") without query parameters
+// to exitNodeID's DoH service, if available.
+//
+// If exitNodeID is the zero valid, it returns "", false.
+func exitNodeCanProxyDNS(nm *netmap.NetworkMap, exitNodeID tailcfg.StableNodeID) (dohURL string, ok bool) {
+	if exitNodeID.IsZero() {
+		return "", false
+	}
+	for _, p := range nm.Peers {
+		if p.StableID != exitNodeID {
+			continue
+		}
+		for _, s := range p.Hostinfo.Services {
+			if s.Proto == tailcfg.PeerAPIDNS && s.Port >= 1 {
+				return peerAPIBase(nm, p) + "/dns-query", true
+			}
+		}
+	}
+	return "", false
+}

ipn/ipnlocal/local_test.go

@@ -92,14 +92,14 @@ func TestNetworkMapCompare(t *testing.T) {
 		},
 		{
 			"Node names identical",
-			&netmap.NetworkMap{Peers: []*tailcfg.Node{&tailcfg.Node{Name: "A"}}},
-			&netmap.NetworkMap{Peers: []*tailcfg.Node{&tailcfg.Node{Name: "A"}}},
+			&netmap.NetworkMap{Peers: []*tailcfg.Node{{Name: "A"}}},
+			&netmap.NetworkMap{Peers: []*tailcfg.Node{{Name: "A"}}},
 			true,
 		},
 		{
 			"Node names differ",
-			&netmap.NetworkMap{Peers: []*tailcfg.Node{&tailcfg.Node{Name: "A"}}},
-			&netmap.NetworkMap{Peers: []*tailcfg.Node{&tailcfg.Node{Name: "B"}}},
+			&netmap.NetworkMap{Peers: []*tailcfg.Node{{Name: "A"}}},
+			&netmap.NetworkMap{Peers: []*tailcfg.Node{{Name: "B"}}},
 			false,
 		},
 		{
@@ -117,8 +117,8 @@ func TestNetworkMapCompare(t *testing.T) {
 		{
 			"Node Users differ",
 			// User field is not checked.
-			&netmap.NetworkMap{Peers: []*tailcfg.Node{&tailcfg.Node{User: 0}}},
-			&netmap.NetworkMap{Peers: []*tailcfg.Node{&tailcfg.Node{User: 1}}},
+			&netmap.NetworkMap{Peers: []*tailcfg.Node{{User: 0}}},
+			&netmap.NetworkMap{Peers: []*tailcfg.Node{{User: 1}}},
 			true,
 		},
 	}
@@ -445,7 +445,7 @@ func TestLazyMachineKeyGeneration(t *testing.T) {
 		t.Fatalf("NewFakeUserspaceEngine: %v", err)
 	}
 	t.Cleanup(eng.Close)
-	lb, err := NewLocalBackend(logf, "logid", store, eng)
+	lb, err := NewLocalBackend(logf, "logid", store, nil, eng)
 	if err != nil {
 		t.Fatalf("NewLocalBackend: %v", err)
 	}

ipn/ipnlocal/loglines_test.go

@@ -54,7 +54,7 @@ func TestLocalLogLines(t *testing.T) {
 	}
 	t.Cleanup(e.Close)
 
-	lb, err := NewLocalBackend(logf, idA.String(), store, e)
+	lb, err := NewLocalBackend(logf, idA.String(), store, nil, e)
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -90,7 +90,7 @@ func TestLocalLogLines(t *testing.T) {
 			TxBytes:       10,
 			RxBytes:       10,
 			LastHandshake: time.Now(),
-			NodeKey:       tailcfg.NodeKeyFromNodePublic(key.NewNode().Public()),
+			NodeKey:       key.NewNode().Public(),
 		}},
 	})
 	lb.mu.Unlock()
@@ -105,7 +105,7 @@ func TestLocalLogLines(t *testing.T) {
 			TxBytes:       11,
 			RxBytes:       12,
 			LastHandshake: time.Now(),
-			NodeKey:       tailcfg.NodeKeyFromNodePublic(key.NewNode().Public()),
+			NodeKey:       key.NewNode().Public(),
 		}},
 	})
 	lb.mu.Unlock()

ipn/ipnlocal/peerapi.go

@@ -6,6 +6,8 @@ package ipnlocal
 
 import (
 	"context"
+	"encoding/base64"
+	"encoding/json"
 	"errors"
 	"fmt"
 	"hash/crc32"
@@ -27,32 +29,48 @@ import (
 	"unicode"
 	"unicode/utf8"
 
+	"golang.org/x/net/dns/dnsmessage"
 	"inet.af/netaddr"
 	"tailscale.com/client/tailscale/apitype"
+	"tailscale.com/hostinfo"
 	"tailscale.com/ipn"
 	"tailscale.com/logtail/backoff"
+	"tailscale.com/net/dns/resolver"
 	"tailscale.com/net/interfaces"
 	"tailscale.com/syncs"
 	"tailscale.com/tailcfg"
+	"tailscale.com/util/clientmetric"
 	"tailscale.com/wgengine"
+	"tailscale.com/wgengine/filter"
 )
 
 var initListenConfig func(*net.ListenConfig, netaddr.IP, *interfaces.State, string) error
 
+// addH2C is non-nil on platforms where we want to add H2C
+// ("cleartext" HTTP/2) support to the peerAPI.
+var addH2C func(*http.Server)
+
 type peerAPIServer struct {
 	b          *LocalBackend
-	rootDir    string
-	tunName    string
+	rootDir    string // empty means file receiving unavailable
 	selfNode   *tailcfg.Node
 	knownEmpty syncs.AtomicBool
+	resolver   *resolver.Resolver
 
 	// directFileMode is whether we're writing files directly to a
 	// download directory (as *.partial files), rather than making
 	// the frontend retrieve it over localapi HTTP and write it
-	// somewhere itself. This is used on GUI macOS version.
+	// somewhere itself. This is used on the GUI macOS versions
+	// and on Synology.
 	// In directFileMode, the peerapi doesn't do the final rename
-	// from "foo.jpg.partial" to "foo.jpg".
+	// from "foo.jpg.partial" to "foo.jpg" unless
+	// directFileDoFinalRename is set.
 	directFileMode bool
+
+	// directFileDoFinalRename is whether in directFileMode we
+	// additionally move the *.direct file to its final name after
+	// it's received.
+	directFileDoFinalRename bool
 }
 
 const (
@@ -69,6 +87,10 @@ const (
 	deletedSuffix = ".deleted"
 )
 
+func (s *peerAPIServer) canReceiveFiles() bool {
+	return s != nil && s.rootDir != ""
+}
+
 func validFilenameRune(r rune) bool {
 	switch r {
 	case '/':
@@ -115,7 +137,7 @@ func (s *peerAPIServer) diskPath(baseName string) (fullPath string, ok bool) {
 // hasFilesWaiting reports whether any files are buffered in the
 // tailscaled daemon storage.
 func (s *peerAPIServer) hasFilesWaiting() bool {
-	if s.rootDir == "" || s.directFileMode {
+	if s == nil || s.rootDir == "" || s.directFileMode {
 		return false
 	}
 	if s.knownEmpty.Get() {
@@ -175,8 +197,11 @@ func (s *peerAPIServer) hasFilesWaiting() bool {
 // As a side effect, it also does any lazy deletion of files as
 // required by Windows.
 func (s *peerAPIServer) WaitingFiles() (ret []apitype.WaitingFile, err error) {
+	if s == nil {
+		return nil, errNilPeerAPIServer
+	}
 	if s.rootDir == "" {
-		return nil, errors.New("peerapi disabled; no storage configured")
+		return nil, errNoTaildrop
 	}
 	if s.directFileMode {
 		return nil, nil
@@ -240,6 +265,11 @@ func (s *peerAPIServer) WaitingFiles() (ret []apitype.WaitingFile, err error) {
 	return ret, nil
 }
 
+var (
+	errNilPeerAPIServer = errors.New("peerapi unavailable; not listening")
+	errNoTaildrop       = errors.New("Taildrop disabled; no storage directory")
+)
+
 // tryDeleteAgain tries to delete path (and path+deletedSuffix) after
 // it failed earlier.  This happens on Windows when various anti-virus
 // tools hook into filesystem operations and have the file open still
@@ -255,8 +285,11 @@ func tryDeleteAgain(fullPath string) {
 }
 
 func (s *peerAPIServer) DeleteFile(baseName string) error {
+	if s == nil {
+		return errNilPeerAPIServer
+	}
 	if s.rootDir == "" {
-		return errors.New("peerapi disabled; no storage configured")
+		return errNoTaildrop
 	}
 	if s.directFileMode {
 		return errors.New("deletes not allowed in direct mode")
@@ -321,8 +354,11 @@ func touchFile(path string) error {
 }
 
 func (s *peerAPIServer) OpenFile(baseName string) (rc io.ReadCloser, size int64, err error) {
+	if s == nil {
+		return nil, 0, errNilPeerAPIServer
+	}
 	if s.rootDir == "" {
-		return nil, 0, errors.New("peerapi disabled; no storage configured")
+		return nil, 0, errNoTaildrop
 	}
 	if s.directFileMode {
 		return nil, 0, errors.New("opens not allowed in direct mode")
@@ -355,7 +391,7 @@ func (s *peerAPIServer) listen(ip netaddr.IP, ifState *interfaces.State) (ln net
 		// On iOS/macOS, this sets the lc.Control hook to
 		// setsockopt the interface index to bind to, to get
 		// out of the network sandbox.
-		if err := initListenConfig(&lc, ip, ifState, s.tunName); err != nil {
+		if err := initListenConfig(&lc, ip, ifState, s.b.dialer.TUNName()); err != nil {
 			return nil, err
 		}
 		if runtime.GOOS == "darwin" || runtime.GOOS == "ios" {
@@ -460,6 +496,9 @@ func (pln *peerAPIListener) serve() {
 		httpServer := &http.Server{
 			Handler: h,
 		}
+		if addH2C != nil {
+			addH2C(httpServer)
+		}
 		go httpServer.Serve(&oneConnListener{Listener: pln.ln, conn: c})
 	}
 }
@@ -500,9 +539,20 @@ func (h *peerAPIHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
 		h.handlePeerPut(w, r)
 		return
 	}
-	if r.URL.Path == "/v0/goroutines" {
+	if strings.HasPrefix(r.URL.Path, "/dns-query") {
+		h.handleDNSQuery(w, r)
+		return
+	}
+	switch r.URL.Path {
+	case "/v0/goroutines":
 		h.handleServeGoroutines(w, r)
 		return
+	case "/v0/env":
+		h.handleServeEnv(w, r)
+		return
+	case "/v0/metrics":
+		h.handleServeMetrics(w, r)
+		return
 	}
 	who := h.peerUser.DisplayName
 	fmt.Fprintf(w, `<html>
@@ -589,7 +639,7 @@ func (h *peerAPIHandler) handlePeerPut(w http.ResponseWriter, r *http.Request) {
 		return
 	}
 	if h.ps.rootDir == "" {
-		http.Error(w, "no rootdir", http.StatusInternalServerError)
+		http.Error(w, errNoTaildrop.Error(), http.StatusInternalServerError)
 		return
 	}
 	rawPath := r.URL.EscapedPath()
@@ -661,7 +711,7 @@ func (h *peerAPIHandler) handlePeerPut(w http.ResponseWriter, r *http.Request) {
 		http.Error(w, err.Error(), http.StatusInternalServerError)
 		return
 	}
-	if h.ps.directFileMode {
+	if h.ps.directFileMode && !h.ps.directFileDoFinalRename {
 		if inFile != nil { // non-zero length; TODO: notify even for zero length
 			inFile.markAndNotifyDone()
 		}
@@ -710,3 +760,247 @@ func (h *peerAPIHandler) handleServeGoroutines(w http.ResponseWriter, r *http.Re
 	}
 	w.Write(buf)
 }
+
+func (h *peerAPIHandler) handleServeEnv(w http.ResponseWriter, r *http.Request) {
+	if !h.isSelf {
+		http.Error(w, "not owner", http.StatusForbidden)
+		return
+	}
+	var data struct {
+		Hostinfo *tailcfg.Hostinfo
+		Uid      int
+		Args     []string
+		Env      []string
+	}
+	data.Hostinfo = hostinfo.New()
+	data.Uid = os.Getuid()
+	data.Args = os.Args
+	data.Env = os.Environ()
+
+	w.Header().Set("Content-Type", "application/json")
+	json.NewEncoder(w).Encode(data)
+}
+
+func (h *peerAPIHandler) handleServeMetrics(w http.ResponseWriter, r *http.Request) {
+	if !h.isSelf {
+		http.Error(w, "not owner", http.StatusForbidden)
+		return
+	}
+	w.Header().Set("Content-Type", "text/plain")
+	clientmetric.WritePrometheusExpositionFormat(w)
+}
+
+func (h *peerAPIHandler) replyToDNSQueries() bool {
+	if h.isSelf {
+		// If the peer is owned by the same user, just allow it
+		// without further checks.
+		return true
+	}
+	b := h.ps.b
+	if !b.OfferingExitNode() {
+		// If we're not an exit node, there's no point to
+		// being a DNS server for somebody.
+		return false
+	}
+	if !h.remoteAddr.IsValid() {
+		// This should never be the case if the peerAPIHandler
+		// was wired up correctly, but just in case.
+		return false
+	}
+	// Otherwise, we're an exit node but the peer is not us, so
+	// we need to check if they're allowed access to the internet.
+	// As peerapi bypasses wgengine/filter checks, we need to check
+	// ourselves. As a proxy for autogroup:internet access, we see
+	// if we would've accepted a packet to 0.0.0.0:53. We treat
+	// the IP 0.0.0.0 as being "the internet".
+	f, ok := b.filterAtomic.Load().(*filter.Filter)
+	if !ok {
+		return false
+	}
+	// Note: we check TCP here because the Filter type already had
+	// a CheckTCP method (for unit tests), but it's pretty
+	// arbitrary. DNS runs over TCP and UDP, so sure... we check
+	// TCP.
+	dstIP := netaddr.IPv4(0, 0, 0, 0)
+	remoteIP := h.remoteAddr.IP()
+	if remoteIP.Is6() {
+		// autogroup:internet for IPv6 is defined to start with 2000::/3,
+		// so use 2000::0 as the probe "the internet" address.
+		dstIP = netaddr.MustParseIP("2000::")
+	}
+	verdict := f.CheckTCP(remoteIP, dstIP, 53)
+	return verdict == filter.Accept
+}
+
+// handleDNSQuery implements a DoH server (RFC 8484) over the peerapi.
+// It's not over HTTPS as the spec dictates, but rather HTTP-over-WireGuard.
+func (h *peerAPIHandler) handleDNSQuery(w http.ResponseWriter, r *http.Request) {
+	if h.ps.resolver == nil {
+		http.Error(w, "DNS not wired up", http.StatusNotImplemented)
+		return
+	}
+	if !h.replyToDNSQueries() {
+		http.Error(w, "DNS access denied", http.StatusForbidden)
+		return
+	}
+	pretty := false // non-DoH debug mode for humans
+	q, publicError := dohQuery(r)
+	if publicError != "" && r.Method == "GET" {
+		if name := r.FormValue("q"); name != "" {
+			pretty = true
+			publicError = ""
+			q = dnsQueryForName(name, r.FormValue("t"))
+		}
+	}
+	if publicError != "" {
+		http.Error(w, publicError, http.StatusBadRequest)
+		return
+	}
+
+	// Some timeout that's short enough to be noticed by humans
+	// but long enough that it's longer than real DNS timeouts.
+	const arbitraryTimeout = 5 * time.Second
+
+	ctx, cancel := context.WithTimeout(r.Context(), arbitraryTimeout)
+	defer cancel()
+	res, err := h.ps.resolver.HandleExitNodeDNSQuery(ctx, q, h.remoteAddr, h.ps.b.allowExitNodeDNSProxyToServeName)
+	if err != nil {
+		h.logf("handleDNS fwd error: %v", err)
+		if err := ctx.Err(); err != nil {
+			http.Error(w, err.Error(), 500)
+		} else {
+			http.Error(w, "DNS forwarding error", 500)
+		}
+		return
+	}
+	if pretty {
+		// Non-standard response for interactive debugging.
+		w.Header().Set("Content-Type", "application/json")
+		writePrettyDNSReply(w, res)
+		return
+	}
+	w.Header().Set("Content-Type", "application/dns-message")
+	w.Header().Set("Content-Length", strconv.Itoa(len(res)))
+	w.Write(res)
+}
+
+func dohQuery(r *http.Request) (dnsQuery []byte, publicErr string) {
+	const maxQueryLen = 256 << 10
+	switch r.Method {
+	default:
+		return nil, "bad HTTP method"
+	case "GET":
+		q64 := r.FormValue("dns")
+		if q64 == "" {
+			return nil, "missing 'dns' parameter"
+		}
+		if base64.RawURLEncoding.DecodedLen(len(q64)) > maxQueryLen {
+			return nil, "query too large"
+		}
+		q, err := base64.RawURLEncoding.DecodeString(q64)
+		if err != nil {
+			return nil, "invalid 'dns' base64 encoding"
+		}
+		return q, ""
+	case "POST":
+		if r.Header.Get("Content-Type") != "application/dns-message" {
+			return nil, "unexpected Content-Type"
+		}
+		q, err := io.ReadAll(io.LimitReader(r.Body, maxQueryLen+1))
+		if err != nil {
+			return nil, "error reading post body with DNS query"
+		}
+		if len(q) > maxQueryLen {
+			return nil, "query too large"
+		}
+		return q, ""
+	}
+}
+
+func dnsQueryForName(name, typStr string) []byte {
+	typ := dnsmessage.TypeA
+	switch strings.ToLower(typStr) {
+	case "aaaa":
+		typ = dnsmessage.TypeAAAA
+	case "txt":
+		typ = dnsmessage.TypeTXT
+	}
+	b := dnsmessage.NewBuilder(nil, dnsmessage.Header{
+		OpCode:           0, // query
+		RecursionDesired: true,
+		ID:               0,
+	})
+	if !strings.HasSuffix(name, ".") {
+		name += "."
+	}
+	b.StartQuestions()
+	b.Question(dnsmessage.Question{
+		Name:  dnsmessage.MustNewName(name),
+		Type:  typ,
+		Class: dnsmessage.ClassINET,
+	})
+	msg, _ := b.Finish()
+	return msg
+}
+
+func writePrettyDNSReply(w io.Writer, res []byte) (err error) {
+	defer func() {
+		if err != nil {
+			j, _ := json.Marshal(struct {
+				Error string
+			}{err.Error()})
+			j = append(j, '\n')
+			w.Write(j)
+			return
+		}
+	}()
+	var p dnsmessage.Parser
+	hdr, err := p.Start(res)
+	if err != nil {
+		return err
+	}
+	if hdr.RCode != dnsmessage.RCodeSuccess {
+		return fmt.Errorf("DNS RCode = %v", hdr.RCode)
+	}
+	if err := p.SkipAllQuestions(); err != nil {
+		return err
+	}
+
+	var gotIPs []string
+	for {
+		h, err := p.AnswerHeader()
+		if err == dnsmessage.ErrSectionDone {
+			break
+		}
+		if err != nil {
+			return err
+		}
+		if h.Class != dnsmessage.ClassINET {
+			continue
+		}
+		switch h.Type {
+		case dnsmessage.TypeA:
+			r, err := p.AResource()
+			if err != nil {
+				return err
+			}
+			gotIPs = append(gotIPs, net.IP(r.A[:]).String())
+		case dnsmessage.TypeAAAA:
+			r, err := p.AAAAResource()
+			if err != nil {
+				return err
+			}
+			gotIPs = append(gotIPs, net.IP(r.AAAA[:]).String())
+		case dnsmessage.TypeTXT:
+			r, err := p.TXTResource()
+			if err != nil {
+				return err
+			}
+			gotIPs = append(gotIPs, r.TXT...)
+		}
+	}
+	j, _ := json.Marshal(gotIPs)
+	j = append(j, '\n')
+	w.Write(j)
+	return nil
+}

ipn/ipnlocal/peerapi_h2c.go

@@ -0,0 +1,22 @@
+// Copyright (c) 2021 Tailscale Inc & AUTHORS All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+//go:build !ios && !android
+// +build !ios,!android
+
+package ipnlocal
+
+import (
+	"net/http"
+
+	"golang.org/x/net/http2"
+	"golang.org/x/net/http2/h2c"
+)
+
+func init() {
+	addH2C = func(s *http.Server) {
+		h2s := &http2.Server{}
+		s.Handler = h2c.NewHandler(s.Handler, h2s)
+	}
+}

ipn/ipnlocal/peerapi_macios_ext.go

@@ -9,10 +9,8 @@
 package ipnlocal
 
 import (
-	"errors"
 	"fmt"
 	"net"
-	"syscall"
 
 	"inet.af/netaddr"
 	"tailscale.com/net/interfaces"
@@ -21,7 +19,6 @@ import (
 
 func init() {
 	initListenConfig = initListenConfigNetworkExtension
-	peerDialControlFunc = peerDialControlFuncNetworkExtension
 }
 
 // initListenConfigNetworkExtension configures nc for listening on IP
@@ -34,24 +31,3 @@ func initListenConfigNetworkExtension(nc *net.ListenConfig, ip netaddr.IP, st *i
 	}
 	return netns.SetListenConfigInterfaceIndex(nc, tunIf.Index)
 }
-
-func peerDialControlFuncNetworkExtension(b *LocalBackend) func(network, address string, c syscall.RawConn) error {
-	b.mu.Lock()
-	defer b.mu.Unlock()
-	st := b.prevIfState
-	pas := b.peerAPIServer
-	index := -1
-	if st != nil && pas != nil && pas.tunName != "" {
-		if tunIf, ok := st.Interface[pas.tunName]; ok {
-			index = tunIf.Index
-		}
-	}
-	var lc net.ListenConfig
-	netns.SetListenConfigInterfaceIndex(&lc, index)
-	return func(network, address string, c syscall.RawConn) error {
-		if index == -1 {
-			return errors.New("failed to find TUN interface to bind to")
-		}
-		return lc.Control(network, address, c)
-	}
-}

ipn/ipnlocal/peerapi_test.go

@@ -19,8 +19,13 @@ import (
 	"strings"
 	"testing"
 
+	"inet.af/netaddr"
+	"tailscale.com/ipn"
 	"tailscale.com/tailcfg"
 	"tailscale.com/tstest"
+	"tailscale.com/types/logger"
+	"tailscale.com/wgengine"
+	"tailscale.com/wgengine/filter"
 )
 
 type peerAPITestEnv struct {
@@ -174,7 +179,7 @@ func TestHandlePeerAPI(t *testing.T) {
 			req:        httptest.NewRequest("PUT", "/v0/put/foo", nil),
 			checks: checks(
 				httpStatus(http.StatusInternalServerError),
-				bodyContains("no rootdir"),
+				bodyContains("Taildrop disabled; no storage directory"),
 			),
 		},
 		{
@@ -568,3 +573,55 @@ func TestDeletedMarkers(t *testing.T) {
 	}
 
 }
+
+func TestPeerAPIReplyToDNSQueries(t *testing.T) {
+	var h peerAPIHandler
+
+	h.isSelf = true
+	if !h.replyToDNSQueries() {
+		t.Errorf("for isSelf = false; want true")
+	}
+	h.isSelf = false
+	h.remoteAddr = netaddr.MustParseIPPort("100.150.151.152:12345")
+
+	eng, _ := wgengine.NewFakeUserspaceEngine(logger.Discard, 0)
+	h.ps = &peerAPIServer{
+		b: &LocalBackend{
+			e: eng,
+		},
+	}
+	if h.ps.b.OfferingExitNode() {
+		t.Fatal("unexpectedly offering exit node")
+	}
+	h.ps.b.prefs = &ipn.Prefs{
+		AdvertiseRoutes: []netaddr.IPPrefix{
+			netaddr.MustParseIPPrefix("0.0.0.0/0"),
+			netaddr.MustParseIPPrefix("::/0"),
+		},
+	}
+	if !h.ps.b.OfferingExitNode() {
+		t.Fatal("unexpectedly not offering exit node")
+	}
+
+	if h.replyToDNSQueries() {
+		t.Errorf("unexpectedly doing DNS without filter")
+	}
+
+	h.ps.b.setFilter(filter.NewAllowNone(logger.Discard, new(netaddr.IPSet)))
+	if h.replyToDNSQueries() {
+		t.Errorf("unexpectedly doing DNS without filter")
+	}
+
+	f := filter.NewAllowAllForTest(logger.Discard)
+
+	h.ps.b.setFilter(f)
+	if !h.replyToDNSQueries() {
+		t.Errorf("unexpectedly deny; wanted to be a DNS server")
+	}
+
+	// Also test IPv6.
+	h.remoteAddr = netaddr.MustParseIPPort("[fe70::1]:12345")
+	if !h.replyToDNSQueries() {
+		t.Errorf("unexpectedly IPv6 deny; wanted to be a DNS server")
+	}
+}

ipn/ipnlocal/state_test.go

@@ -284,7 +284,7 @@ func TestStateMachine(t *testing.T) {
 	t.Cleanup(e.Close)
 
 	cc := newMockControl(t)
-	b, err := NewLocalBackend(logf, "logid", store, e)
+	b, err := NewLocalBackend(logf, "logid", store, nil, e)
 	if err != nil {
 		t.Fatalf("NewLocalBackend: %v", err)
 	}
@@ -867,6 +867,45 @@ func TestStateMachine(t *testing.T) {
 		// change either.
 		c.Assert(ipn.Starting, qt.Equals, b.State())
 	}
+	t.Logf("\n\nExpireKey")
+	notifies.expect(1)
+	cc.send(nil, "", false, &netmap.NetworkMap{
+		Expiry:        time.Now().Add(-time.Minute),
+		MachineStatus: tailcfg.MachineAuthorized,
+	})
+	{
+		nn := notifies.drain(1)
+		cc.assertCalls("unpause", "unpause")
+		c.Assert(nn[0].State, qt.IsNotNil)
+		c.Assert(ipn.NeedsLogin, qt.Equals, *nn[0].State)
+		c.Assert(ipn.NeedsLogin, qt.Equals, b.State())
+		c.Assert(b.isEngineBlocked(), qt.IsTrue)
+	}
+
+	t.Logf("\n\nExtendKey")
+	notifies.expect(1)
+	cc.send(nil, "", false, &netmap.NetworkMap{
+		Expiry:        time.Now().Add(time.Minute),
+		MachineStatus: tailcfg.MachineAuthorized,
+	})
+	{
+		nn := notifies.drain(1)
+		cc.assertCalls("unpause", "unpause", "unpause")
+		c.Assert(nn[0].State, qt.IsNotNil)
+		c.Assert(ipn.Starting, qt.Equals, *nn[0].State)
+		c.Assert(ipn.Starting, qt.Equals, b.State())
+		c.Assert(b.isEngineBlocked(), qt.IsFalse)
+	}
+	notifies.expect(1)
+	// Fake a DERP connection.
+	b.setWgengineStatus(&wgengine.Status{DERPs: 1}, nil)
+	{
+		nn := notifies.drain(1)
+		cc.assertCalls("unpause")
+		c.Assert(nn[0].State, qt.IsNotNil)
+		c.Assert(ipn.Running, qt.Equals, *nn[0].State)
+		c.Assert(ipn.Running, qt.Equals, b.State())
+	}
 }
 
 type testStateStorage struct {
@@ -902,7 +941,7 @@ func TestWGEngineStatusRace(t *testing.T) {
 	eng, err := wgengine.NewFakeUserspaceEngine(logf, 0)
 	c.Assert(err, qt.IsNil)
 	t.Cleanup(eng.Close)
-	b, err := NewLocalBackend(logf, "logid", new(ipn.MemoryStore), eng)
+	b, err := NewLocalBackend(logf, "logid", new(ipn.MemoryStore), nil, eng)
 	c.Assert(err, qt.IsNil)
 
 	cc := newMockControl(t)

ipn/ipnserver/server.go

@@ -35,9 +35,9 @@ import (
 	"tailscale.com/ipn/ipnlocal"
 	"tailscale.com/ipn/localapi"
 	"tailscale.com/ipn/store/aws"
-	"tailscale.com/log/filelogger"
 	"tailscale.com/logtail/backoff"
 	"tailscale.com/net/netstat"
+	"tailscale.com/net/tsdial"
 	"tailscale.com/paths"
 	"tailscale.com/safesocket"
 	"tailscale.com/smallzstd"
@@ -48,20 +48,18 @@ import (
 	"tailscale.com/version"
 	"tailscale.com/version/distro"
 	"tailscale.com/wgengine"
+	"tailscale.com/wgengine/monitor"
 )
 
 // Options is the configuration of the Tailscale node agent.
 type Options struct {
-	// SocketPath, on unix systems, is the unix socket path to listen
-	// on for frontend connections.
-	SocketPath string
-
-	// Port, on windows, is the localhost TCP port to listen on for
-	// frontend connections.
-	Port int
-
-	// StatePath is the path to the stored agent state.
-	StatePath string
+	// VarRoot is the the Tailscale daemon's private writable
+	// directory (usually "/var/lib/tailscale" on Linux) that
+	// contains the "tailscaled.state" file, the "certs" directory
+	// for TLS certs, and the "files" directory for incoming
+	// Taildrop files before they're moved to a user directory.
+	// If empty, Taildrop and TLS certs don't function.
+	VarRoot string
 
 	// AutostartStateKey, if non-empty, immediately starts the agent
 	// using the given StateKey. If empty, the agent stays idle and
@@ -83,10 +81,6 @@ type Options struct {
 	// the actual definition of "disconnect" is when the
 	// connection count transitions from 1 to 0.
 	SurviveDisconnects bool
-
-	// DebugMux, if non-nil, specifies an HTTP ServeMux in which
-	// to register a debug handler.
-	DebugMux *http.ServeMux
 }
 
 // Server is an IPN backend and its set of 0 or more active localhost
@@ -100,8 +94,8 @@ type Server struct {
 	// being run in "client mode" that requires an active GUI
 	// connection (such as on Windows by default).  Even if this
 	// is true, the ForceDaemon pref can override this.
-	resetOnZero bool
-	opts        Options
+	resetOnZero       bool
+	autostartStateKey ipn.StateKey
 
 	bsMu sync.Mutex // lock order: bsMu, then mu
 	bs   *ipn.BackendServer
@@ -114,6 +108,9 @@ type Server struct {
 	disconnectSub  map[chan<- struct{}]struct{} // keys are subscribers of disconnects
 }
 
+// LocalBackend returns the server's LocalBackend.
+func (s *Server) LocalBackend() *ipnlocal.LocalBackend { return s.b }
+
 // connIdentity represents the owner of a localhost TCP or unix socket connection.
 type connIdentity struct {
 	Conn       net.Conn
@@ -414,13 +411,16 @@ func (s *Server) checkConnIdentityLocked(ci connIdentity) error {
 //
 // s.mu must not be held.
 func (s *Server) localAPIPermissions(ci connIdentity) (read, write bool) {
-	if runtime.GOOS == "windows" {
+	switch runtime.GOOS {
+	case "windows":
 		s.mu.Lock()
 		defer s.mu.Unlock()
 		if s.checkConnIdentityLocked(ci) == nil {
 			return true, true
 		}
 		return false, false
+	case "js":
+		return true, true
 	}
 	if ci.IsUnixSock {
 		return true, !isReadonlyConn(ci, s.b.OperatorUserID(), logger.Discard)
@@ -606,18 +606,57 @@ func tryWindowsAppDataMigration(logf logger.Logf, path string) string {
 	return paths.TryConfigFileMigration(logf, oldFile, path)
 }
 
+// StateStore returns a StateStore from path.
+//
+// The path should be an absolute path to a file.
+//
+// Special cases:
+//
+//   * empty string means to use an in-memory store
+//   * if the string begins with "kube:", the suffix
+//     is a Kubernetes secret name
+//   * if the string begins with "arn:", the value is
+//     an AWS ARN for an SSM.
+func StateStore(path string, logf logger.Logf) (ipn.StateStore, error) {
+	if path == "" {
+		return &ipn.MemoryStore{}, nil
+	}
+	const kubePrefix = "kube:"
+	const arnPrefix = "arn:"
+	switch {
+	case strings.HasPrefix(path, kubePrefix):
+		secretName := strings.TrimPrefix(path, kubePrefix)
+		store, err := ipn.NewKubeStore(secretName)
+		if err != nil {
+			return nil, fmt.Errorf("ipn.NewKubeStore(%q): %v", secretName, err)
+		}
+		return store, nil
+	case strings.HasPrefix(path, arnPrefix):
+		store, err := aws.NewStore(path)
+		if err != nil {
+			return nil, fmt.Errorf("aws.NewStore(%q): %v", path, err)
+		}
+		return store, nil
+	}
+	if runtime.GOOS == "windows" {
+		path = tryWindowsAppDataMigration(logf, path)
+	}
+	store, err := ipn.NewFileStore(path)
+	if err != nil {
+		return nil, fmt.Errorf("ipn.NewFileStore(%q): %v", path, err)
+	}
+	return store, nil
+}
+
 // Run runs a Tailscale backend service.
 // The getEngine func is called repeatedly, once per connection, until it returns an engine successfully.
-func Run(ctx context.Context, logf logger.Logf, logid string, getEngine func() (wgengine.Engine, error), opts Options) error {
+//
+// Deprecated: use New and Server.Run instead.
+func Run(ctx context.Context, logf logger.Logf, ln net.Listener, store ipn.StateStore, linkMon *monitor.Mon, dialer *tsdial.Dialer, logid string, getEngine func() (wgengine.Engine, error), opts Options) error {
 	getEngine = getEngineUntilItWorksWrapper(getEngine)
 	runDone := make(chan struct{})
 	defer close(runDone)
 
-	listen, _, err := safesocket.Listen(opts.SocketPath, uint16(opts.Port))
-	if err != nil {
-		return fmt.Errorf("safesocket.Listen: %v", err)
-	}
-
 	var serverMu sync.Mutex
 	var serverOrNil *Server
 
@@ -633,57 +672,28 @@ func Run(ctx context.Context, logf logger.Logf, logid string, getEngine func() (
 			s.stopAll()
 		}
 		serverMu.Unlock()
-		listen.Close()
+		ln.Close()
 	}()
-	logf("Listening on %v", listen.Addr())
+	logf("Listening on %v", ln.Addr())
 
 	var serverModeUser *user.User
-	var store ipn.StateStore
-	if opts.StatePath != "" {
-		const kubePrefix = "kube:"
-		const arnPrefix = "arn:"
-		path := opts.StatePath
-		switch {
-		case strings.HasPrefix(path, kubePrefix):
-			secretName := strings.TrimPrefix(path, kubePrefix)
-			store, err = ipn.NewKubeStore(secretName)
-			if err != nil {
-				return fmt.Errorf("ipn.NewKubeStore(%q): %v", secretName, err)
-			}
-		case strings.HasPrefix(path, arnPrefix):
-			store, err = aws.NewStore(path)
-			if err != nil {
-				return fmt.Errorf("aws.NewStore(%q): %v", path, err)
-			}
-		default:
-			if runtime.GOOS == "windows" {
-				path = tryWindowsAppDataMigration(logf, path)
-			}
-			store, err = ipn.NewFileStore(path)
-			if err != nil {
-				return fmt.Errorf("ipn.NewFileStore(%q): %v", path, err)
-			}
+	if opts.AutostartStateKey == "" {
+		autoStartKey, err := store.ReadState(ipn.ServerModeStartKey)
+		if err != nil && err != ipn.ErrStateNotExist {
+			return fmt.Errorf("calling ReadState on state store: %w", err)
 		}
-		if opts.AutostartStateKey == "" {
-			autoStartKey, err := store.ReadState(ipn.ServerModeStartKey)
-			if err != nil && err != ipn.ErrStateNotExist {
-				return fmt.Errorf("calling ReadState on %s: %w", path, err)
-			}
-			key := string(autoStartKey)
-			if strings.HasPrefix(key, "user-") {
-				uid := strings.TrimPrefix(key, "user-")
-				u, err := lookupUserFromID(logf, uid)
-				if err != nil {
-					logf("ipnserver: found server mode auto-start key %q; failed to load user: %v", key, err)
-				} else {
-					logf("ipnserver: found server mode auto-start key %q (user %s)", key, u.Username)
-					serverModeUser = u
-				}
-				opts.AutostartStateKey = ipn.StateKey(key)
+		key := string(autoStartKey)
+		if strings.HasPrefix(key, "user-") {
+			uid := strings.TrimPrefix(key, "user-")
+			u, err := lookupUserFromID(logf, uid)
+			if err != nil {
+				logf("ipnserver: found server mode auto-start key %q; failed to load user: %v", key, err)
+			} else {
+				logf("ipnserver: found server mode auto-start key %q (user %s)", key, u.Username)
+				serverModeUser = u
 			}
+			opts.AutostartStateKey = ipn.StateKey(key)
 		}
-	} else {
-		store = &ipn.MemoryStore{}
 	}
 
 	bo := backoff.NewBackoff("ipnserver", logf, 30*time.Second)
@@ -693,7 +703,7 @@ func Run(ctx context.Context, logf logger.Logf, logid string, getEngine func() (
 	if err != nil {
 		logf("ipnserver: initial getEngine call: %v", err)
 		for i := 1; ctx.Err() == nil; i++ {
-			c, err := listen.Accept()
+			c, err := ln.Accept()
 			if err != nil {
 				logf("%d: Accept: %v", i, err)
 				bo.BackOff(ctx, err)
@@ -720,62 +730,107 @@ func Run(ctx context.Context, logf logger.Logf, logid string, getEngine func() (
 		}
 	}
 	if unservedConn != nil {
-		listen = &listenerWithReadyConn{
-			Listener: listen,
+		ln = &listenerWithReadyConn{
+			Listener: ln,
 			c:        unservedConn,
 		}
 	}
 
-	server, err := New(logf, logid, store, eng, serverModeUser, opts)
+	server, err := New(logf, logid, store, eng, dialer, serverModeUser, opts)
 	if err != nil {
 		return err
 	}
 	serverMu.Lock()
 	serverOrNil = server
 	serverMu.Unlock()
-	return server.Serve(ctx, listen)
+	return server.Run(ctx, ln)
 }
 
 // New returns a new Server.
 //
-// The opts.StatePath option is ignored; it's only used by Run.
-func New(logf logger.Logf, logid string, store ipn.StateStore, eng wgengine.Engine, serverModeUser *user.User, opts Options) (*Server, error) {
-	b, err := ipnlocal.NewLocalBackend(logf, logid, store, eng)
+// To start it, use the Server.Run method.
+func New(logf logger.Logf, logid string, store ipn.StateStore, eng wgengine.Engine, dialer *tsdial.Dialer, serverModeUser *user.User, opts Options) (*Server, error) {
+	b, err := ipnlocal.NewLocalBackend(logf, logid, store, dialer, eng)
 	if err != nil {
 		return nil, fmt.Errorf("NewLocalBackend: %v", err)
 	}
+	b.SetVarRoot(opts.VarRoot)
 	b.SetDecompressor(func() (controlclient.Decompressor, error) {
 		return smallzstd.NewDecoder(nil)
 	})
 
-	if opts.DebugMux != nil {
-		opts.DebugMux.HandleFunc("/debug/ipn", func(w http.ResponseWriter, r *http.Request) {
-			serveHTMLStatus(w, b)
-		})
+	dg := distro.Get()
+	switch dg {
+	case distro.Synology, distro.TrueNAS:
+		// See if they have a "Taildrop" share.
+		// See https://github.com/tailscale/tailscale/issues/2179#issuecomment-982821319
+		path, err := findTaildropDir(dg)
+		if err != nil {
+			logf("%s Taildrop support: %v", dg, err)
+		} else {
+			logf("%s Taildrop: using %v", dg, path)
+			b.SetDirectFileRoot(path)
+			b.SetDirectFileDoFinalRename(true)
+		}
+
+	}
+
+	if opts.AutostartStateKey == "" {
+		autoStartKey, err := store.ReadState(ipn.ServerModeStartKey)
+		if err != nil && err != ipn.ErrStateNotExist {
+			return nil, fmt.Errorf("calling ReadState on store: %w", err)
+		}
+		key := string(autoStartKey)
+		if strings.HasPrefix(key, "user-") {
+			uid := strings.TrimPrefix(key, "user-")
+			u, err := lookupUserFromID(logf, uid)
+			if err != nil {
+				logf("ipnserver: found server mode auto-start key %q; failed to load user: %v", key, err)
+			} else {
+				logf("ipnserver: found server mode auto-start key %q (user %s)", key, u.Username)
+				serverModeUser = u
+			}
+			opts.AutostartStateKey = ipn.StateKey(key)
+		}
 	}
 
 	server := &Server{
-		b:              b,
-		backendLogID:   logid,
-		logf:           logf,
-		resetOnZero:    !opts.SurviveDisconnects,
-		serverModeUser: serverModeUser,
-		opts:           opts,
+		b:                 b,
+		backendLogID:      logid,
+		logf:              logf,
+		resetOnZero:       !opts.SurviveDisconnects,
+		serverModeUser:    serverModeUser,
+		autostartStateKey: opts.AutostartStateKey,
 	}
 	server.bs = ipn.NewBackendServer(logf, b, server.writeToClients)
 	return server, nil
 }
 
-// Serve accepts connections from ln forever.
+// Run runs the server, accepting connections from ln forever.
 //
-// The context is only used to suppress errors
-func (s *Server) Serve(ctx context.Context, ln net.Listener) error {
+// If the context is done, the listener is closed.
+func (s *Server) Run(ctx context.Context, ln net.Listener) error {
 	defer s.b.Shutdown()
-	if s.opts.AutostartStateKey != "" {
+
+	runDone := make(chan struct{})
+	defer close(runDone)
+
+	// When the context is closed or when we return, whichever is first, close our listener
+	// and all open connections.
+	go func() {
+		select {
+		case <-ctx.Done():
+		case <-runDone:
+		}
+		s.stopAll()
+		ln.Close()
+	}()
+
+	if s.autostartStateKey != "" {
 		s.bs.GotCommand(ctx, &ipn.Command{
 			Version: version.Long,
 			Start: &ipn.StartArgs{
-				Opts: ipn.Options{StateKey: s.opts.AutostartStateKey},
+				Opts: ipn.Options{StateKey: s.autostartStateKey},
 			},
 		})
 	}
@@ -813,14 +868,6 @@ func BabysitProc(ctx context.Context, args []string, logf logger.Logf) {
 		panic("cannot determine executable: " + err.Error())
 	}
 
-	if runtime.GOOS == "windows" {
-		if len(args) != 2 && args[0] != "/subproc" {
-			panic(fmt.Sprintf("unexpected arguments %q", args))
-		}
-		logID := args[1]
-		logf = filelogger.New("tailscale-service", logID, logf)
-	}
-
 	var proc struct {
 		mu sync.Mutex
 		p  *os.Process
@@ -1013,13 +1060,13 @@ func (s *Server) localhostHandler(ci connIdentity) http.Handler {
 			io.WriteString(w, "<html><title>Tailscale</title><body><h1>Tailscale</h1>This is the local Tailscale daemon.")
 			return
 		}
-		serveHTMLStatus(w, s.b)
+		s.ServeHTMLStatus(w, r)
 	})
 }
 
-func serveHTMLStatus(w http.ResponseWriter, b *ipnlocal.LocalBackend) {
+func (s *Server) ServeHTMLStatus(w http.ResponseWriter, r *http.Request) {
 	w.Header().Set("Content-Type", "text/html; charset=utf-8")
-	st := b.Status()
+	st := s.b.Status()
 	// TODO(bradfitz): add LogID and opts to st?
 	st.WriteHTML(w)
 }
@@ -1074,3 +1121,50 @@ func (ln *listenerWithReadyConn) Accept() (net.Conn, error) {
 	}
 	return ln.Listener.Accept()
 }
+
+func findTaildropDir(dg distro.Distro) (string, error) {
+	const name = "Taildrop"
+	switch dg {
+	case distro.Synology:
+		return findSynologyTaildropDir(name)
+	case distro.TrueNAS:
+		return findTrueNASTaildropDir(name)
+	}
+	return "", fmt.Errorf("%s is an unsupported distro for Taildrop dir", dg)
+}
+
+// findSynologyTaildropDir looks for the first volume containing a
+// "Taildrop" directory.  We'd run "synoshare --get Taildrop" command
+// but on DSM7 at least, we lack permissions to run that.
+func findSynologyTaildropDir(name string) (dir string, err error) {
+	for i := 1; i <= 16; i++ {
+		dir = fmt.Sprintf("/volume%v/%s", i, name)
+		if fi, err := os.Stat(dir); err == nil && fi.IsDir() {
+			return dir, nil
+		}
+	}
+	return "", fmt.Errorf("shared folder %q not found", name)
+}
+
+// findTrueNASTaildropDir returns the first matching directory of
+// /mnt/{name} or /mnt/*/{name}
+func findTrueNASTaildropDir(name string) (dir string, err error) {
+	// If we're running in a jail, a mount point could just be added at /mnt/Taildrop
+	dir = fmt.Sprintf("/mnt/%s", name)
+	if fi, err := os.Stat(dir); err == nil && fi.IsDir() {
+		return dir, nil
+	}
+
+	// but if running on the host, it may be something like /mnt/Primary/Taildrop
+	fis, err := ioutil.ReadDir("/mnt")
+	if err != nil {
+		return "", fmt.Errorf("error reading /mnt: %w", err)
+	}
+	for _, fi := range fis {
+		dir = fmt.Sprintf("/mnt/%s/%s", fi.Name(), name)
+		if fi, err := os.Stat(dir); err == nil && fi.IsDir() {
+			return dir, nil
+		}
+	}
+	return "", fmt.Errorf("shared folder %q not found", name)
+}

ipn/ipnserver/server_test.go

@@ -13,6 +13,7 @@ import (
 
 	"tailscale.com/ipn"
 	"tailscale.com/ipn/ipnserver"
+	"tailscale.com/net/tsdial"
 	"tailscale.com/safesocket"
 	"tailscale.com/wgengine"
 )
@@ -32,10 +33,11 @@ func TestRunMultipleAccepts(t *testing.T) {
 		t.Logf(format, args...)
 	}
 
+	s := safesocket.DefaultConnectionStrategy(socketPath)
 	connect := func() {
 		for i := 1; i <= 2; i++ {
 			logf("connect %d ...", i)
-			c, err := safesocket.Connect(socketPath, 0)
+			c, err := safesocket.Connect(s)
 			if err != nil {
 				t.Fatalf("safesocket.Connect: %v\n", err)
 			}
@@ -62,10 +64,16 @@ func TestRunMultipleAccepts(t *testing.T) {
 	}
 	t.Cleanup(eng.Close)
 
-	opts := ipnserver.Options{
-		SocketPath: socketPath,
-	}
+	opts := ipnserver.Options{}
 	t.Logf("pre-Run")
-	err = ipnserver.Run(ctx, logTriggerTestf, "dummy_logid", ipnserver.FixedEngine(eng), opts)
+	store := new(ipn.MemoryStore)
+
+	ln, _, err := safesocket.Listen(socketPath, 0)
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer ln.Close()
+
+	err = ipnserver.Run(ctx, logTriggerTestf, ln, store, nil /* mon */, new(tsdial.Dialer), "dummy_logid", ipnserver.FixedEngine(eng), opts)
 	t.Logf("ipnserver.Run = %v", err)
 }

ipn/ipnstate/ipnstate.go

@@ -70,9 +70,14 @@ func (s *Status) Peers() []key.NodePublic {
 }
 
 type PeerStatusLite struct {
+	// TxBytes/RxBytes is the total number of bytes transmitted to/received from this peer.
 	TxBytes, RxBytes int64
-	LastHandshake    time.Time
-	NodeKey          tailcfg.NodeKey
+	// LastHandshake is the last time a handshake succeeded with this peer.
+	// (Or we got key confirmation via the first data message,
+	// which is approximately the same thing.)
+	LastHandshake time.Time
+	// NodeKey is this peer's public node key.
+	NodeKey key.NodePublic
 }
 
 type PeerStatus struct {
@@ -83,22 +88,23 @@ type PeerStatus struct {
 	OS        string // HostInfo.OS
 	UserID    tailcfg.UserID
 
-	TailAddrDeprecated string       `json:"TailAddr"` // Tailscale IP
-	TailscaleIPs       []netaddr.IP // Tailscale IP(s) assigned to this node
+	TailscaleIPs []netaddr.IP // Tailscale IP(s) assigned to this node
 
 	// Endpoints:
 	Addrs   []string
 	CurAddr string // one of Addrs, or unique if roaming
 	Relay   string // DERP region
 
-	RxBytes       int64
-	TxBytes       int64
-	Created       time.Time // time registered with tailcontrol
-	LastWrite     time.Time // time last packet sent
-	LastSeen      time.Time // last seen to tailcontrol
-	LastHandshake time.Time // with local wireguard
-	KeepAlive     bool
-	ExitNode      bool // true if this is the currently selected exit node.
+	RxBytes        int64
+	TxBytes        int64
+	Created        time.Time // time registered with tailcontrol
+	LastWrite      time.Time // time last packet sent
+	LastSeen       time.Time // last seen to tailcontrol; only present if offline
+	LastHandshake  time.Time // with local wireguard
+	Online         bool      // whether node is connected to the control plane
+	KeepAlive      bool
+	ExitNode       bool // true if this is the currently selected exit node.
+	ExitNodeOption bool // true if this node can be an exit node (offered && approved)
 
 	// Active is whether the node was recently active. The
 	// definition is somewhat undefined but has historically and
@@ -237,9 +243,6 @@ func (sb *StatusBuilder) AddPeer(peer key.NodePublic, st *PeerStatus) {
 	if v := st.UserID; v != 0 {
 		e.UserID = v
 	}
-	if v := st.TailAddrDeprecated; v != "" {
-		e.TailAddrDeprecated = v
-	}
 	if v := st.TailscaleIPs; v != nil {
 		e.TailscaleIPs = v
 	}
@@ -270,6 +273,9 @@ func (sb *StatusBuilder) AddPeer(peer key.NodePublic, st *PeerStatus) {
 	if v := st.LastWrite; !v.IsZero() {
 		e.LastWrite = v
 	}
+	if st.Online {
+		e.Online = true
+	}
 	if st.InNetworkMap {
 		e.InNetworkMap = true
 	}
@@ -285,6 +291,9 @@ func (sb *StatusBuilder) AddPeer(peer key.NodePublic, st *PeerStatus) {
 	if st.ExitNode {
 		e.ExitNode = true
 	}
+	if st.ExitNodeOption {
+		e.ExitNodeOption = true
+	}
 	if st.ShareeNode {
 		e.ShareeNode = true
 	}

ipn/localapi/localapi.go

@@ -12,7 +12,6 @@ import (
 	"errors"
 	"fmt"
 	"io"
-	"net"
 	"net/http"
 	"net/http/httputil"
 	"net/url"
@@ -20,7 +19,6 @@ import (
 	"runtime"
 	"strconv"
 	"strings"
-	"sync"
 	"time"
 
 	"inet.af/netaddr"
@@ -28,9 +26,9 @@ import (
 	"tailscale.com/ipn"
 	"tailscale.com/ipn/ipnlocal"
 	"tailscale.com/ipn/ipnstate"
-	"tailscale.com/net/netknob"
 	"tailscale.com/tailcfg"
 	"tailscale.com/types/logger"
+	"tailscale.com/util/clientmetric"
 	"tailscale.com/version"
 )
 
@@ -113,6 +111,8 @@ func (h *Handler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
 		h.serveSetDNS(w, r)
 	case "/localapi/v0/derpmap":
 		h.serveDERPMap(w, r)
+	case "/localapi/v0/metrics":
+		h.serveMetrics(w, r)
 	case "/":
 		io.WriteString(w, "tailscaled\n")
 	default:
@@ -184,6 +184,17 @@ func (h *Handler) serveGoroutines(w http.ResponseWriter, r *http.Request) {
 	w.Write(buf)
 }
 
+func (h *Handler) serveMetrics(w http.ResponseWriter, r *http.Request) {
+	// Require write access out of paranoia that the metrics
+	// might contain something sensitive.
+	if !h.PermitWrite {
+		http.Error(w, "metric access denied", http.StatusForbidden)
+		return
+	}
+	w.Header().Set("Content-Type", "text/plain")
+	clientmetric.WritePrometheusExpositionFormat(w)
+}
+
 // serveProfileFunc is the implementation of Handler.serveProfile, after auth,
 // for platforms where we want to link it in.
 var serveProfileFunc func(http.ResponseWriter, *http.Request)
@@ -362,6 +373,25 @@ func (h *Handler) serveFileTargets(w http.ResponseWriter, r *http.Request) {
 	json.NewEncoder(w).Encode(fts)
 }
 
+// serveFilePut sends a file to another node.
+//
+// It's sometimes possible for clients to do this themselves, without
+// tailscaled, except in the case of tailscaled running in
+// userspace-networking ("netstack") mode, in which case tailscaled
+// needs to a do a netstack dial out.
+//
+// Instead, the CLI also goes through tailscaled so it doesn't need to be
+// aware of the network mode in use.
+//
+// macOS/iOS have always used this localapi method to simplify the GUI
+// clients.
+//
+// The Windows client currently (2021-11-30) uses the peerapi (/v0/put/)
+// directly, as the Windows GUI always runs in tun mode anyway.
+//
+// URL format:
+//
+//    * PUT /localapi/v0/file-put/:stableID/:escaped-filename
 func (h *Handler) serveFilePut(w http.ResponseWriter, r *http.Request) {
 	if !h.PermitWrite {
 		http.Error(w, "file access denied", http.StatusForbidden)
@@ -409,7 +439,7 @@ func (h *Handler) serveFilePut(w http.ResponseWriter, r *http.Request) {
 	outReq.ContentLength = r.ContentLength
 
 	rp := httputil.NewSingleHostReverseProxy(dstURL)
-	rp.Transport = getDialPeerTransport(h.b)
+	rp.Transport = h.b.Dialer().PeerAPITransport()
 	rp.ServeHTTP(w, outReq)
 }
 
@@ -443,26 +473,6 @@ func (h *Handler) serveDERPMap(w http.ResponseWriter, r *http.Request) {
 	e.Encode(h.b.DERPMap())
 }
 
-var dialPeerTransportOnce struct {
-	sync.Once
-	v *http.Transport
-}
-
-func getDialPeerTransport(b *ipnlocal.LocalBackend) *http.Transport {
-	dialPeerTransportOnce.Do(func() {
-		t := http.DefaultTransport.(*http.Transport).Clone()
-		t.Dial = nil
-		dialer := net.Dialer{
-			Timeout:   30 * time.Second,
-			KeepAlive: netknob.PlatformTCPKeepAlive(),
-			Control:   b.PeerDialControlFunc(),
-		}
-		t.DialContext = dialer.DialContext
-		dialPeerTransportOnce.v = t
-	})
-	return dialPeerTransportOnce.v
-}
-
 func defBool(a string, def bool) bool {
 	if a == "" {
 		return def

ipn/policy/policy.go

@@ -14,7 +14,8 @@ import (
 // system (a version.OS value) is an interesting enough port to report
 // to our peer nodes for discovery purposes.
 func IsInterestingService(s tailcfg.Service, os string) bool {
-	if s.Proto == "peerapi4" || s.Proto == "peerapi6" {
+	switch s.Proto {
+	case tailcfg.PeerAPI4, tailcfg.PeerAPI6, tailcfg.PeerAPIDNS:
 		return true
 	}
 	if s.Proto != tailcfg.TCP {

ipn/store/aws/store_aws.go

@@ -23,7 +23,7 @@ import (
 )
 
 const (
-	parameterNameRxStr = `^parameter/(.*)`
+	parameterNameRxStr = `^parameter(/.*)`
 )
 
 var parameterNameRx = regexp.MustCompile(parameterNameRxStr)

log/filelogger/log.go

@@ -9,6 +9,7 @@ package filelogger
 import (
 	"bytes"
 	"fmt"
+	"io"
 	"io/ioutil"
 	"log"
 	"os"
@@ -26,30 +27,30 @@ const (
 	maxFiles = 50
 )
 
-// New returns a logf wrapper that appends to local disk log
+// New returns a Writer that appends to local disk log
 // files on Windows, rotating old log files as needed to stay under
 // file count & byte limits.
-func New(fileBasePrefix, logID string, logf logger.Logf) logger.Logf {
+func New(fileBasePrefix, logID string, inner *log.Logger) io.Writer {
 	if runtime.GOOS != "windows" {
 		panic("not yet supported on any platform except Windows")
 	}
-	if logf == nil {
-		panic("nil logf")
+	if inner == nil {
+		panic("nil inner logger")
 	}
 	dir := filepath.Join(os.Getenv("ProgramData"), "Tailscale", "Logs")
 
 	if err := os.MkdirAll(dir, 0700); err != nil {
-		log.Printf("failed to create local log directory; not writing logs to disk: %v", err)
-		return logf
+		inner.Printf("failed to create local log directory; not writing logs to disk: %v", err)
+		return inner.Writer()
 	}
-	logf("local disk logdir: %v", dir)
+	inner.Printf("local disk logdir: %v", dir)
 	lfw := &logFileWriter{
 		fileBasePrefix: fileBasePrefix,
 		logID:          logID,
 		dir:            dir,
-		wrappedLogf:    logf,
+		wrappedLogf:    inner.Printf,
 	}
-	return lfw.Logf
+	return logger.FuncWriter(lfw.Logf)
 }
 
 // logFileWriter is the state for the log writer & rotator.
@@ -106,6 +107,7 @@ func (w *logFileWriter) appendToFileLocked(out []byte) {
 	if w.fday != day {
 		w.startNewFileLocked()
 	}
+	out = removeDatePrefix(out)
 	if w.f != nil {
 		// RFC3339Nano but with a fixed number (3) of nanosecond digits:
 		const formatPre = "2006-01-02T15:04:05"
@@ -118,6 +120,30 @@ func (w *logFileWriter) appendToFileLocked(out []byte) {
 	}
 }
 
+func isNum(b byte) bool { return '0' <= b && b <= '9' }
+
+// removeDatePrefix returns a subslice of v with the log package's
+// standard datetime prefix format removed, if present.
+func removeDatePrefix(v []byte) []byte {
+	const format = "2009/01/23 01:23:23 "
+	if len(v) < len(format) {
+		return v
+	}
+	for i, b := range v[:len(format)] {
+		fb := format[i]
+		if isNum(fb) {
+			if !isNum(b) {
+				return v
+			}
+			continue
+		}
+		if b != fb {
+			return v
+		}
+	}
+	return v[len(format):]
+}
+
 // startNewFileLocked opens a new log file for writing
 // and also cleans up any old files.
 //

log/filelogger/log_test.go

@@ -0,0 +1,28 @@
+// Copyright (c) 2021 Tailscale Inc & AUTHORS All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package filelogger
+
+import "testing"
+
+func TestRemoveDatePrefix(t *testing.T) {
+	tests := []struct {
+		in, want string
+	}{
+		{"", ""},
+		{"\n", "\n"},
+		{"2009/01/23 01:23:23", "2009/01/23 01:23:23"},
+		{"2009/01/23 01:23:23 \n", "\n"},
+		{"2009/01/23 01:23:23 foo\n", "foo\n"},
+		{"9999/01/23 01:23:23 foo\n", "foo\n"},
+		{"2009_01/23 01:23:23 had an underscore\n", "2009_01/23 01:23:23 had an underscore\n"},
+	}
+	for i, tt := range tests {
+		got := removeDatePrefix([]byte(tt.in))
+		if string(got) != tt.want {
+			t.Logf("[%d] removeDatePrefix(%q) = %q; want %q", i, tt.in, got, tt.want)
+		}
+	}
+
+}

logpolicy/logpolicy.go

@@ -31,6 +31,8 @@ import (
 	"tailscale.com/atomicfile"
 	"tailscale.com/logtail"
 	"tailscale.com/logtail/filch"
+	"tailscale.com/net/dnscache"
+	"tailscale.com/net/dnsfallback"
 	"tailscale.com/net/netknob"
 	"tailscale.com/net/netns"
 	"tailscale.com/net/tlsdial"
@@ -38,6 +40,7 @@ import (
 	"tailscale.com/paths"
 	"tailscale.com/smallzstd"
 	"tailscale.com/types/logger"
+	"tailscale.com/util/clientmetric"
 	"tailscale.com/util/racebuild"
 	"tailscale.com/util/winutil"
 	"tailscale.com/version"
@@ -498,14 +501,17 @@ func New(collection string) *Policy {
 			}
 			return w
 		},
-		HTTPC: &http.Client{Transport: newLogtailTransport(logtail.DefaultHost)},
+		HTTPC: &http.Client{Transport: NewLogtailTransport(logtail.DefaultHost)},
+	}
+	if collection == logtail.CollectionNode {
+		c.MetricsDelta = clientmetric.EncodeLogTailMetricsDelta
 	}
 
 	if val := getLogTarget(); val != "" {
 		log.Println("You have enabled a non-default log target. Doing without being told to by Tailscale staff or your network administrator will make getting support difficult.")
 		c.BaseURL = val
 		u, _ := url.Parse(val)
-		c.HTTPC = &http.Client{Transport: newLogtailTransport(u.Host)}
+		c.HTTPC = &http.Client{Transport: NewLogtailTransport(u.Host)}
 	}
 
 	filchBuf, filchErr := filch.New(filepath.Join(dir, cmdName), filch.Options{
@@ -519,7 +525,7 @@ func New(collection string) *Policy {
 	}
 	lw := logtail.NewLogger(c, log.Printf)
 	log.SetFlags(0) // other logflags are set on console, not here
-	log.SetOutput(lw)
+	log.SetOutput(maybeWrapForPlatform(lw, cmdName, newc.PublicID.String()))
 
 	log.Printf("Program starting: v%v, Go %v: %#v",
 		version.Long,
@@ -565,9 +571,12 @@ func (p *Policy) Shutdown(ctx context.Context) error {
 	return nil
 }
 
-// newLogtailTransport returns the HTTP Transport we use for uploading
-// logs to the given host name.
-func newLogtailTransport(host string) *http.Transport {
+// NewLogtailTransport returns an HTTP Transport particularly suited to uploading
+// logs to the given host name. This includes:
+//   - If DNS lookup fails, consult the bootstrap DNS list of Tailscale hostnames.
+//   - If TLS connection fails, try again using LetsEncrypt's built-in root certificate,
+//     for the benefit of older OS platforms which might not include it.
+func NewLogtailTransport(host string) *http.Transport {
 	// Start with a copy of http.DefaultTransport and tweak it a bit.
 	tr := http.DefaultTransport.(*http.Transport).Clone()
 
@@ -581,17 +590,29 @@ func newLogtailTransport(host string) *http.Transport {
 
 	// Log whenever we dial:
 	tr.DialContext = func(ctx context.Context, netw, addr string) (net.Conn, error) {
-		nd := netns.FromDialer(&net.Dialer{
+		nd := netns.FromDialer(log.Printf, &net.Dialer{
 			Timeout:   30 * time.Second,
 			KeepAlive: netknob.PlatformTCPKeepAlive(),
 		})
 		t0 := time.Now()
 		c, err := nd.DialContext(ctx, netw, addr)
 		d := time.Since(t0).Round(time.Millisecond)
-		if err != nil {
-			log.Printf("logtail: dial %q failed: %v (in %v)", addr, err, d)
-		} else {
+		if err == nil {
 			log.Printf("logtail: dialed %q in %v", addr, d)
+			return c, nil
+		}
+
+		// If we failed to dial, try again with bootstrap DNS.
+		log.Printf("logtail: dial %q failed: %v (in %v), trying bootstrap...", addr, err, d)
+		dnsCache := &dnscache.Resolver{
+			Forward:          dnscache.Get().Forward, // use default cache's forwarder
+			UseLastGood:      true,
+			LookupIPFallback: dnsfallback.Lookup,
+		}
+		dialer := dnscache.Dialer(nd.DialContext, dnsCache)
+		c, err = dialer(ctx, netw, addr)
+		if err == nil {
+			log.Printf("logtail: bootstrap dial succeeded")
 		}
 		return c, err
 	}

logpolicy/logpolicy_notwindows.go

@@ -0,0 +1,16 @@
+// Copyright (c) 2020 Tailscale Inc & AUTHORS All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+//go:build !windows
+// +build !windows
+
+package logpolicy
+
+import (
+	"io"
+)
+
+func maybeWrapForPlatform(lw io.Writer, cmdName, logID string) io.Writer {
+	return lw
+}

logpolicy/logpolicy_windows.go

@@ -0,0 +1,26 @@
+// Copyright (c) 2020 Tailscale Inc & AUTHORS All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package logpolicy
+
+import (
+	"io"
+	"log"
+
+	"golang.org/x/sys/windows/svc"
+	"tailscale.com/log/filelogger"
+)
+
+func maybeWrapForPlatform(lw io.Writer, cmdName, logID string) io.Writer {
+	if cmdName != "tailscaled" {
+		return lw
+	}
+
+	isSvc, err := svc.IsWindowsService()
+	if err != nil || !isSvc {
+		return lw
+	}
+
+	return filelogger.New("tailscale-service", logID, log.New(lw, "", 0))
+}

logtail/logtail.go

@@ -28,6 +28,12 @@ import (
 // Config.BaseURL isn't provided.
 const DefaultHost = "log.tailscale.io"
 
+const (
+	// CollectionNode is the name of a logtail Config.Collection
+	// for tailscaled (or equivalent: IPNExtension, Android app).
+	CollectionNode = "tailnode.log.tailscale.io"
+)
+
 type Encoder interface {
 	EncodeAll(src, dst []byte) []byte
 	Close() error
@@ -46,6 +52,12 @@ type Config struct {
 	Buffer         Buffer           // temp storage, if nil a MemoryBuffer
 	NewZstdEncoder func() Encoder   // if set, used to compress logs for transmission
 
+	// MetricsDelta, if non-nil, is a func that returns an encoding
+	// delta in clientmetrics to upload alongside existing logs.
+	// It can return either an empty string (for nothing) or a string
+	// that's safe to embed in a JSON string literal without further escaping.
+	MetricsDelta func() string
+
 	// DrainLogs, if non-nil, disables automatic uploading of new logs,
 	// so that logs are only uploaded when a token is sent to DrainLogs.
 	DrainLogs <-chan struct{}
@@ -84,6 +96,7 @@ func NewLogger(cfg Config, logf tslogger.Logf) *Logger {
 		drainLogs:      cfg.DrainLogs,
 		timeNow:        cfg.TimeNow,
 		bo:             backoff.NewBackoff("logtail", logf, 30*time.Second),
+		metricsDelta:   cfg.MetricsDelta,
 
 		shutdownStart: make(chan struct{}),
 		shutdownDone:  make(chan struct{}),
@@ -119,6 +132,7 @@ type Logger struct {
 	zstdEncoder    Encoder
 	uploadCancel   func()
 	explainedRaw   bool
+	metricsDelta   func() string // or nil
 
 	shutdownStart chan struct{} // closed when shutdown begins
 	shutdownDone  chan struct{} // closed when shutdown complete
@@ -426,6 +440,14 @@ func (l *Logger) encodeText(buf []byte, skipClientTime bool) []byte {
 		b = append(b, "\"}, "...)
 	}
 
+	if l.metricsDelta != nil {
+		if d := l.metricsDelta(); d != "" {
+			b = append(b, `"metrics": "`...)
+			b = append(b, d...)
+			b = append(b, `",`...)
+		}
+	}
+
 	b = append(b, "\"text\": \""...)
 	for i, c := range buf {
 		switch c {

net/dns/direct.go

@@ -7,6 +7,7 @@ package dns
 import (
 	"bufio"
 	"bytes"
+	"context"
 	"crypto/rand"
 	"fmt"
 	"io"
@@ -16,6 +17,7 @@ import (
 	"path/filepath"
 	"runtime"
 	"strings"
+	"time"
 
 	"inet.af/netaddr"
 	"tailscale.com/types/logger"
@@ -50,10 +52,17 @@ func readResolv(r io.Reader) (config OSConfig, err error) {
 	scanner := bufio.NewScanner(r)
 	for scanner.Scan() {
 		line := strings.TrimSpace(scanner.Text())
+		i := strings.IndexByte(line, '#')
+		if i >= 0 {
+			line = line[:i]
+		}
 
 		if strings.HasPrefix(line, "nameserver") {
-			nameserver := strings.TrimPrefix(line, "nameserver")
-			nameserver = strings.TrimSpace(nameserver)
+			s := strings.TrimPrefix(line, "nameserver")
+			nameserver := strings.TrimSpace(s)
+			if len(nameserver) == len(s) {
+				return OSConfig{}, fmt.Errorf("missing space after \"nameserver\" in %q", line)
+			}
 			ip, err := netaddr.ParseIP(nameserver)
 			if err != nil {
 				return OSConfig{}, err
@@ -63,8 +72,12 @@ func readResolv(r io.Reader) (config OSConfig, err error) {
 		}
 
 		if strings.HasPrefix(line, "search") {
-			domain := strings.TrimPrefix(line, "search")
-			domain = strings.TrimSpace(domain)
+			s := strings.TrimPrefix(line, "search")
+			domain := strings.TrimSpace(s)
+			if len(domain) == len(s) {
+				// No leading space?!
+				return OSConfig{}, fmt.Errorf("missing space after \"domain\" in %q", line)
+			}
 			fqdn, err := dnsname.ToFQDN(domain)
 			if err != nil {
 				return OSConfig{}, fmt.Errorf("parsing search domains %q: %w", line, err)
@@ -121,12 +134,20 @@ func isResolvedRunning() bool {
 		return false
 	}
 
-	// is-active exits with code 3 if the service is not active.
-	err = exec.Command("systemctl", "is-active", "systemd-resolved.service").Run()
+	ctx, cancel := context.WithTimeout(context.Background(), 2*time.Second)
+	defer cancel()
+	err = exec.CommandContext(ctx, "systemctl", "is-active", "systemd-resolved.service").Run()
 
+	// is-active exits with code 3 if the service is not active.
 	return err == nil
 }
 
+func restartResolved() error {
+	ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
+	defer cancel()
+	return exec.CommandContext(ctx, "systemctl", "restart", "systemd-resolved.service").Run()
+}
+
 // directManager is an OSConfigurator which replaces /etc/resolv.conf with a file
 // generated from the given configuration, creating a backup of its old state.
 //
@@ -383,7 +404,12 @@ func (m *directManager) Close() error {
 	}
 
 	if isResolvedRunning() && !runningAsGUIDesktopUser() {
-		exec.Command("systemctl", "restart", "systemd-resolved.service").Run() // Best-effort.
+		m.logf("restarting systemd-resolved...")
+		if err := restartResolved(); err != nil {
+			m.logf("restart of systemd-resolved failed: %v", err)
+		} else {
+			m.logf("restarted systemd-resolved")
+		}
 	}
 
 	return nil

net/dns/direct_test.go

@@ -14,6 +14,7 @@ import (
 	"syscall"
 	"testing"
 
+	qt "github.com/frankban/quicktest"
 	"inet.af/netaddr"
 	"tailscale.com/util/dnsname"
 )
@@ -138,3 +139,61 @@ func TestDirectBrokenRemove(t *testing.T) {
 	}
 	testDirect(t, brokenRemoveFS{directFS{prefix: tmp}})
 }
+
+func TestReadResolve(t *testing.T) {
+	c := qt.New(t)
+	tests := []struct {
+		in      string
+		want    OSConfig
+		wantErr bool
+	}{
+		{in: `nameserver 192.168.0.100`,
+			want: OSConfig{
+				Nameservers: []netaddr.IP{
+					netaddr.MustParseIP("192.168.0.100"),
+				},
+			},
+		},
+		{in: `nameserver 192.168.0.100 # comment`,
+			want: OSConfig{
+				Nameservers: []netaddr.IP{
+					netaddr.MustParseIP("192.168.0.100"),
+				},
+			},
+		},
+		{in: `nameserver 192.168.0.100#`,
+			want: OSConfig{
+				Nameservers: []netaddr.IP{
+					netaddr.MustParseIP("192.168.0.100"),
+				},
+			},
+		},
+		{in: `nameserver #192.168.0.100`, wantErr: true},
+		{in: `nameserver`, wantErr: true},
+		{in: `# nameserver 192.168.0.100`, want: OSConfig{}},
+		{in: `nameserver192.168.0.100`, wantErr: true},
+
+		{in: `search tailsacle.com`,
+			want: OSConfig{
+				SearchDomains: []dnsname.FQDN{"tailsacle.com."},
+			},
+		},
+		{in: `search tailsacle.com # typo`,
+			want: OSConfig{
+				SearchDomains: []dnsname.FQDN{"tailsacle.com."},
+			},
+		},
+		{in: `searchtailsacle.com`, wantErr: true},
+		{in: `search`, wantErr: true},
+	}
+
+	for _, test := range tests {
+		cfg, err := readResolv(strings.NewReader(test.in))
+		if test.wantErr {
+			c.Assert(err, qt.IsNotNil)
+		} else {
+			c.Assert(err, qt.IsNil)
+		}
+		c.Assert(cfg, qt.DeepEquals, test.want)
+	}
+}

net/dns/ini_test.go

@@ -26,8 +26,8 @@ func TestParseIni(t *testing.T) {
 [network] # trailing comment
 generateResolvConf = false  # trailing comment`,
 			want: map[string]map[string]string{
-				"automount": map[string]string{"enabled": "true", "root": "/mnt/"},
-				"network":   map[string]string{"generateResolvConf": "false"},
+				"automount": {"enabled": "true", "root": "/mnt/"},
+				"network":   {"generateResolvConf": "false"},
 			},
 		},
 	}

net/dns/manager.go

@@ -10,8 +10,10 @@ import (
 	"time"
 
 	"inet.af/netaddr"
+	"tailscale.com/health"
 	"tailscale.com/net/dns/resolver"
 	"tailscale.com/net/tsaddr"
+	"tailscale.com/net/tsdial"
 	"tailscale.com/types/dnstype"
 	"tailscale.com/types/logger"
 	"tailscale.com/util/dnsname"
@@ -35,22 +37,26 @@ type Manager struct {
 
 	resolver *resolver.Resolver
 	os       OSConfigurator
-
-	config Config
 }
 
 // NewManagers created a new manager from the given config.
-func NewManager(logf logger.Logf, oscfg OSConfigurator, linkMon *monitor.Mon, linkSel resolver.ForwardLinkSelector) *Manager {
+func NewManager(logf logger.Logf, oscfg OSConfigurator, linkMon *monitor.Mon, dialer *tsdial.Dialer, linkSel resolver.ForwardLinkSelector) *Manager {
+	if dialer == nil {
+		panic("nil Dialer")
+	}
 	logf = logger.WithPrefix(logf, "dns: ")
 	m := &Manager{
 		logf:     logf,
-		resolver: resolver.New(logf, linkMon, linkSel),
+		resolver: resolver.New(logf, linkMon, linkSel, dialer),
 		os:       oscfg,
 	}
 	m.logf("using %T", m.os)
 	return m
 }
 
+// Resolver returns the Manager's DNS Resolver.
+func (m *Manager) Resolver() *resolver.Resolver { return m.resolver }
+
 func (m *Manager) Set(cfg Config) error {
 	m.logf("Set: %v", logger.ArgWriter(func(w *bufio.Writer) {
 		cfg.WriteToBufioWriter(w)
@@ -70,8 +76,10 @@ func (m *Manager) Set(cfg Config) error {
 		return err
 	}
 	if err := m.os.SetDNS(ocfg); err != nil {
+		health.SetDNSOSHealth(err)
 		return err
 	}
+	health.SetDNSOSHealth(nil)
 
 	return nil
 }
@@ -161,6 +169,7 @@ func (m *Manager) compileConfig(cfg Config) (rcfg resolver.Config, ocfg OSConfig
 	if !m.os.SupportsSplitDNS() || isWindows {
 		bcfg, err := m.os.GetBaseConfig()
 		if err != nil {
+			health.SetDNSOSHealth(err)
 			return resolver.Config{}, OSConfig{}, err
 		}
 		var defaultRoutes []dnstype.Resolver
@@ -225,7 +234,7 @@ func Cleanup(logf logger.Logf, interfaceName string) {
 		logf("creating dns cleanup: %v", err)
 		return
 	}
-	dns := NewManager(logf, oscfg, nil, nil)
+	dns := NewManager(logf, oscfg, nil, new(tsdial.Dialer), nil)
 	if err := dns.Down(); err != nil {
 		logf("dns down: %v", err)
 	}

net/dns/manager_linux.go

@@ -180,20 +180,55 @@ func dnsMode(logf logger.Logf, env newOSConfigEnv) (ret string, err error) {
 			return "direct", nil
 		}
 	case "NetworkManager":
-		// You'd think we would use newNMManager somewhere in
-		// here. However, as explained in
-		// https://github.com/tailscale/tailscale/issues/1699 , using
-		// NetworkManager for DNS configuration carries with it the
-		// cost of losing IPv6 configuration on the Tailscale network
-		// interface. So, when we can avoid it, we bypass
-		// NetworkManager by replacing resolv.conf directly.
-		//
-		// If you ever try to put NMManager back here, keep in mind
-		// that versions >=1.26.6 will ignore DNS configuration
-		// anyway, so you still need a fallback path that uses
-		// directManager.
 		dbg("rc", "nm")
-		return "direct", nil
+		// Sometimes, NetworkManager owns the configuration but points
+		// it at systemd-resolved.
+		if err := resolvedIsActuallyResolver(bs); err != nil {
+			dbg("resolved", "not-in-use")
+			// You'd think we would use newNMManager here. However, as
+			// explained in
+			// https://github.com/tailscale/tailscale/issues/1699 ,
+			// using NetworkManager for DNS configuration carries with
+			// it the cost of losing IPv6 configuration on the
+			// Tailscale network interface. So, when we can avoid it,
+			// we bypass NetworkManager by replacing resolv.conf
+			// directly.
+			//
+			// If you ever try to put NMManager back here, keep in mind
+			// that versions >=1.26.6 will ignore DNS configuration
+			// anyway, so you still need a fallback path that uses
+			// directManager.
+			return "direct", nil
+		}
+		dbg("nm-resolved", "yes")
+
+		if err := env.dbusPing("org.freedesktop.resolve1", "/org/freedesktop/resolve1"); err != nil {
+			dbg("resolved", "no")
+			return "direct", nil
+		}
+
+		// See large comment above for reasons we'd use NM rather than
+		// resolved. systemd-resolved is actually in charge of DNS
+		// configuration, but in some cases we might need to configure
+		// it via NetworkManager. All the logic below is probing for
+		// that case: is NetworkManager running? If so, is it one of
+		// the versions that requires direct interaction with it?
+		if err := env.dbusPing("org.freedesktop.NetworkManager", "/org/freedesktop/NetworkManager/DnsManager"); err != nil {
+			dbg("nm", "no")
+			return "systemd-resolved", nil
+		}
+		safe, err := env.nmVersionBetween("1.26.0", "1.26.5")
+		if err != nil {
+			// Failed to figure out NM's version, can't make a correct
+			// decision.
+			return "", fmt.Errorf("checking NetworkManager version: %v", err)
+		}
+		if safe {
+			dbg("nm-safe", "yes")
+			return "network-manager", nil
+		}
+		dbg("nm-safe", "no")
+		return "systemd-resolved", nil
 	default:
 		dbg("rc", "unknown")
 		return "direct", nil
@@ -244,6 +279,13 @@ func nmIsUsingResolved() error {
 	return nil
 }
 
+// resolvedIsActuallyResolver reports whether the given resolv.conf
+// bytes describe a configuration where systemd-resolved (127.0.0.53)
+// is the only configured nameserver.
+//
+// Returns an error if the configuration is something other than
+// exclusively systemd-resolved, or nil if the config is only
+// systemd-resolved.
 func resolvedIsActuallyResolver(bs []byte) error {
 	cfg, err := readResolv(bytes.NewBuffer(bs))
 	if err != nil {

net/dns/manager_linux_test.go

@@ -34,7 +34,7 @@ func TestLinuxDNSMode(t *testing.T) {
 				resolvDotConf(
 					"# Managed by NetworkManager",
 					"nameserver 10.0.0.1")),
-			wantLog: "dns: [rc=nm ret=direct]",
+			wantLog: "dns: [rc=nm resolved=not-in-use ret=direct]",
 			want:    "direct",
 		},
 		{
@@ -172,6 +172,52 @@ func TestLinuxDNSMode(t *testing.T) {
 			wantLog: "dns: [rc=resolved resolved=no ret=direct]",
 			want:    "direct",
 		},
+		{
+			// regression test for https://github.com/tailscale/tailscale/issues/3304
+			name: "networkmanager_but_pointing_at_systemd-resolved",
+			env: env(resolvDotConf(
+				"# Generated by NetworkManager",
+				"nameserver 127.0.0.53",
+				"options edns0 trust-ad"),
+				resolvedRunning(),
+				nmRunning("1.32.12", true)),
+			wantLog: "dns: [rc=nm nm-resolved=yes nm-safe=no ret=systemd-resolved]",
+			want:    "systemd-resolved",
+		},
+		{
+			// regression test for https://github.com/tailscale/tailscale/issues/3304
+			name: "networkmanager_but_pointing_at_systemd-resolved_but_no_resolved",
+			env: env(resolvDotConf(
+				"# Generated by NetworkManager",
+				"nameserver 127.0.0.53",
+				"options edns0 trust-ad"),
+				nmRunning("1.32.12", true)),
+			wantLog: "dns: [rc=nm nm-resolved=yes resolved=no ret=direct]",
+			want:    "direct",
+		},
+		{
+			// regression test for https://github.com/tailscale/tailscale/issues/3304
+			name: "networkmanager_but_pointing_at_systemd-resolved_and_safe_nm",
+			env: env(resolvDotConf(
+				"# Generated by NetworkManager",
+				"nameserver 127.0.0.53",
+				"options edns0 trust-ad"),
+				resolvedRunning(),
+				nmRunning("1.26.3", true)),
+			wantLog: "dns: [rc=nm nm-resolved=yes nm-safe=yes ret=network-manager]",
+			want:    "network-manager",
+		},
+		{
+			// regression test for https://github.com/tailscale/tailscale/issues/3304
+			name: "networkmanager_but_pointing_at_systemd-resolved_and_no_networkmanager",
+			env: env(resolvDotConf(
+				"# Generated by NetworkManager",
+				"nameserver 127.0.0.53",
+				"options edns0 trust-ad"),
+				resolvedRunning()),
+			wantLog: "dns: [rc=nm nm-resolved=yes nm=no ret=systemd-resolved]",
+			want:    "systemd-resolved",
+		},
 	}
 	for _, tt := range tests {
 		t.Run(tt.name, func(t *testing.T) {

net/dns/manager_test.go

@@ -13,6 +13,7 @@ import (
 	"github.com/google/go-cmp/cmp/cmpopts"
 	"inet.af/netaddr"
 	"tailscale.com/net/dns/resolver"
+	"tailscale.com/net/tsdial"
 	"tailscale.com/types/dnstype"
 	"tailscale.com/util/dnsname"
 )
@@ -398,7 +399,7 @@ func TestManager(t *testing.T) {
 				SplitDNS:   test.split,
 				BaseConfig: test.bs,
 			}
-			m := NewManager(t.Logf, &f, nil, nil)
+			m := NewManager(t.Logf, &f, nil, new(tsdial.Dialer), nil)
 			m.resolver.TestOnlySetHook(f.SetResolver)
 
 			if err := m.Set(test.in); err != nil {

net/dns/resolved.go

@@ -12,10 +12,13 @@ import (
 	"errors"
 	"fmt"
 	"net"
+	"strings"
+	"sync"
 
 	"github.com/godbus/dbus/v5"
 	"golang.org/x/sys/unix"
 	"inet.af/netaddr"
+	"tailscale.com/health"
 	"tailscale.com/types/logger"
 	"tailscale.com/util/dnsname"
 )
@@ -39,6 +42,28 @@ var resolvedListenAddr = netaddr.IPv4(127, 0, 0, 53)
 
 var errNotReady = errors.New("interface not ready")
 
+// DBus entities we talk to.
+//
+// DBus is an RPC bus. In particular, the bus we're talking to is the
+// system-wide bus (there is also a per-user session bus for
+// user-specific applications).
+//
+// Daemons connect to the bus, and advertise themselves under a
+// well-known object name. That object exposes paths, and each path
+// implements one or more interfaces that contain methods, properties,
+// and signals.
+//
+// Clients connect to the bus and walk that same hierarchy to invoke
+// RPCs, get/set properties, or listen for signals.
+const (
+	dbusResolvedObject                    = "org.freedesktop.resolve1"
+	dbusResolvedPath      dbus.ObjectPath = "/org/freedesktop/resolve1"
+	dbusResolvedInterface                 = "org.freedesktop.resolve1.Manager"
+	dbusPath              dbus.ObjectPath = "/org/freedesktop/DBus"
+	dbusInterface                         = "org.freedesktop.DBus"
+	dbusOwnerSignal                       = "NameOwnerChanged" // broadcast when a well-known name's owning process changes.
+)
+
 type resolvedLinkNameserver struct {
 	Family  int32
 	Address []byte
@@ -83,9 +108,15 @@ func isResolvedActive() bool {
 
 // resolvedManager is an OSConfigurator which uses the systemd-resolved DBus API.
 type resolvedManager struct {
-	logf     logger.Logf
-	ifidx    int
-	resolved dbus.BusObject
+	logf  logger.Logf
+	ifidx int
+
+	cancelSyncer context.CancelFunc // run to shut down syncer goroutine
+	syncerDone   chan struct{}      // closed when syncer is stopped
+	resolved     dbus.BusObject
+
+	mu     sync.Mutex // guards RPCs made by syncLocked, and the following
+	config OSConfig   // last SetDNS config
 }
 
 func newResolvedManager(logf logger.Logf, interfaceName string) (*resolvedManager, error) {
@@ -99,19 +130,88 @@ func newResolvedManager(logf logger.Logf, interfaceName string) (*resolvedManage
 		return nil, err
 	}
 
-	return &resolvedManager{
-		logf:     logf,
-		ifidx:    iface.Index,
-		resolved: conn.Object("org.freedesktop.resolve1", dbus.ObjectPath("/org/freedesktop/resolve1")),
-	}, nil
+	ctx, cancel := context.WithCancel(context.Background())
+
+	ret := &resolvedManager{
+		logf:         logf,
+		ifidx:        iface.Index,
+		cancelSyncer: cancel,
+		syncerDone:   make(chan struct{}),
+		resolved:     conn.Object(dbusResolvedObject, dbus.ObjectPath(dbusResolvedPath)),
+	}
+	signals := make(chan *dbus.Signal, 16)
+	go ret.resync(ctx, signals)
+	// Only receive the DBus signals we need to resync our config on
+	// resolved restart. Failure to set filters isn't a fatal error,
+	// we'll just receive all broadcast signals and have to ignore
+	// them on our end.
+	if err := conn.AddMatchSignal(dbus.WithMatchObjectPath(dbusPath), dbus.WithMatchInterface(dbusInterface), dbus.WithMatchMember(dbusOwnerSignal), dbus.WithMatchArg(0, dbusResolvedObject)); err != nil {
+		logf("[v1] Setting DBus signal filter failed: %v", err)
+	}
+	conn.Signal(signals)
+	return ret, nil
 }
 
 func (m *resolvedManager) SetDNS(config OSConfig) error {
-	ctx, cancel := context.WithTimeout(context.Background(), reconfigTimeout)
+	m.mu.Lock()
+	defer m.mu.Unlock()
+
+	m.config = config
+	return m.syncLocked(context.TODO()) // would be nice to plumb context through from SetDNS
+}
+
+func (m *resolvedManager) resync(ctx context.Context, signals chan *dbus.Signal) {
+	defer close(m.syncerDone)
+	for {
+		select {
+		case <-ctx.Done():
+			return
+		case signal := <-signals:
+			// In theory the signal was filtered by DBus, but if
+			// AddMatchSignal in the constructor failed, we may be
+			// getting other spam.
+			if signal.Path != dbusPath || signal.Name != dbusInterface+"."+dbusOwnerSignal {
+				continue
+			}
+			// signal.Body is a []interface{} of 3 strings: bus name, previous owner, new owner.
+			if len(signal.Body) != 3 {
+				m.logf("[unexpectected] DBus NameOwnerChanged len(Body) = %d, want 3")
+			}
+			if name, ok := signal.Body[0].(string); !ok || name != dbusResolvedObject {
+				continue
+			}
+			newOwner, ok := signal.Body[2].(string)
+			if !ok {
+				m.logf("[unexpected] DBus NameOwnerChanged.new_owner is a %T, not a string", signal.Body[2])
+			}
+			if newOwner == "" {
+				// systemd-resolved left the bus, no current owner,
+				// nothing to do.
+				continue
+			}
+			// The resolved bus name has a new owner, meaning resolved
+			// restarted. Reprogram current config.
+			m.logf("systemd-resolved restarted, syncing DNS config")
+			m.mu.Lock()
+			err := m.syncLocked(ctx)
+			// Set health while holding the lock, because this will
+			// graciously serialize the resync's health outcome with a
+			// concurrent SetDNS call.
+			health.SetDNSOSHealth(err)
+			m.mu.Unlock()
+			if err != nil {
+				m.logf("failed to configure systemd-resolved: %v", err)
+			}
+		}
+	}
+}
+
+func (m *resolvedManager) syncLocked(ctx context.Context) error {
+	ctx, cancel := context.WithTimeout(ctx, reconfigTimeout)
 	defer cancel()
 
-	var linkNameservers = make([]resolvedLinkNameserver, len(config.Nameservers))
-	for i, server := range config.Nameservers {
+	var linkNameservers = make([]resolvedLinkNameserver, len(m.config.Nameservers))
+	for i, server := range m.config.Nameservers {
 		ip := server.As16()
 		if server.Is4() {
 			linkNameservers[i] = resolvedLinkNameserver{
@@ -127,16 +227,16 @@ func (m *resolvedManager) SetDNS(config OSConfig) error {
 	}
 
 	err := m.resolved.CallWithContext(
-		ctx, "org.freedesktop.resolve1.Manager.SetLinkDNS", 0,
+		ctx, dbusResolvedInterface+".SetLinkDNS", 0,
 		m.ifidx, linkNameservers,
 	).Store()
 	if err != nil {
 		return fmt.Errorf("setLinkDNS: %w", err)
 	}
 
-	linkDomains := make([]resolvedLinkDomain, 0, len(config.SearchDomains)+len(config.MatchDomains))
+	linkDomains := make([]resolvedLinkDomain, 0, len(m.config.SearchDomains)+len(m.config.MatchDomains))
 	seenDomains := map[dnsname.FQDN]bool{}
-	for _, domain := range config.SearchDomains {
+	for _, domain := range m.config.SearchDomains {
 		if seenDomains[domain] {
 			continue
 		}
@@ -146,7 +246,7 @@ func (m *resolvedManager) SetDNS(config OSConfig) error {
 			RoutingOnly: false,
 		})
 	}
-	for _, domain := range config.MatchDomains {
+	for _, domain := range m.config.MatchDomains {
 		if seenDomains[domain] {
 			// Search domains act as both search and match in
 			// resolved, so it's correct to skip.
@@ -158,7 +258,7 @@ func (m *resolvedManager) SetDNS(config OSConfig) error {
 			RoutingOnly: true,
 		})
 	}
-	if len(config.MatchDomains) == 0 && len(config.Nameservers) > 0 {
+	if len(m.config.MatchDomains) == 0 && len(m.config.Nameservers) > 0 {
 		// Caller requested full DNS interception, install a
 		// routing-only root domain.
 		linkDomains = append(linkDomains, resolvedLinkDomain{
@@ -168,14 +268,22 @@ func (m *resolvedManager) SetDNS(config OSConfig) error {
 	}
 
 	err = m.resolved.CallWithContext(
-		ctx, "org.freedesktop.resolve1.Manager.SetLinkDomains", 0,
+		ctx, dbusResolvedInterface+".SetLinkDomains", 0,
 		m.ifidx, linkDomains,
 	).Store()
+	if err != nil && err.Error() == "Argument list too long" { // TODO: better error match
+		// Issue 3188: older systemd-resolved had argument length limits.
+		// Trim out the *.arpa. entries and try again.
+		err = m.resolved.CallWithContext(
+			ctx, dbusResolvedInterface+".SetLinkDomains", 0,
+			m.ifidx, linkDomainsWithoutReverseDNS(linkDomains),
+		).Store()
+	}
 	if err != nil {
 		return fmt.Errorf("setLinkDomains: %w", err)
 	}
 
-	if call := m.resolved.CallWithContext(ctx, "org.freedesktop.resolve1.Manager.SetLinkDefaultRoute", 0, m.ifidx, len(config.MatchDomains) == 0); call.Err != nil {
+	if call := m.resolved.CallWithContext(ctx, dbusResolvedInterface+".SetLinkDefaultRoute", 0, m.ifidx, len(m.config.MatchDomains) == 0); call.Err != nil {
 		if dbusErr, ok := call.Err.(dbus.Error); ok && dbusErr.Name == dbus.ErrMsgUnknownMethod.Name {
 			// on some older systems like Kubuntu 18.04.6 with systemd 237 method SetLinkDefaultRoute is absent,
 			// but otherwise it's working good
@@ -190,26 +298,26 @@ func (m *resolvedManager) SetDNS(config OSConfig) error {
 	// or something).
 
 	// Disable LLMNR, we don't do multicast.
-	if call := m.resolved.CallWithContext(ctx, "org.freedesktop.resolve1.Manager.SetLinkLLMNR", 0, m.ifidx, "no"); call.Err != nil {
+	if call := m.resolved.CallWithContext(ctx, dbusResolvedInterface+".SetLinkLLMNR", 0, m.ifidx, "no"); call.Err != nil {
 		m.logf("[v1] failed to disable LLMNR: %v", call.Err)
 	}
 
 	// Disable mdns.
-	if call := m.resolved.CallWithContext(ctx, "org.freedesktop.resolve1.Manager.SetLinkMulticastDNS", 0, m.ifidx, "no"); call.Err != nil {
+	if call := m.resolved.CallWithContext(ctx, dbusResolvedInterface+".SetLinkMulticastDNS", 0, m.ifidx, "no"); call.Err != nil {
 		m.logf("[v1] failed to disable mdns: %v", call.Err)
 	}
 
 	// We don't support dnssec consistently right now, force it off to
 	// avoid partial failures when we split DNS internally.
-	if call := m.resolved.CallWithContext(ctx, "org.freedesktop.resolve1.Manager.SetLinkDNSSEC", 0, m.ifidx, "no"); call.Err != nil {
+	if call := m.resolved.CallWithContext(ctx, dbusResolvedInterface+".SetLinkDNSSEC", 0, m.ifidx, "no"); call.Err != nil {
 		m.logf("[v1] failed to disable DNSSEC: %v", call.Err)
 	}
 
-	if call := m.resolved.CallWithContext(ctx, "org.freedesktop.resolve1.Manager.SetLinkDNSOverTLS", 0, m.ifidx, "no"); call.Err != nil {
+	if call := m.resolved.CallWithContext(ctx, dbusResolvedInterface+".SetLinkDNSOverTLS", 0, m.ifidx, "no"); call.Err != nil {
 		m.logf("[v1] failed to disable DoT: %v", call.Err)
 	}
 
-	if call := m.resolved.CallWithContext(ctx, "org.freedesktop.resolve1.Manager.FlushCaches", 0); call.Err != nil {
+	if call := m.resolved.CallWithContext(ctx, dbusResolvedInterface+".FlushCaches", 0); call.Err != nil {
 		m.logf("failed to flush resolved DNS cache: %v", call.Err)
 	}
 
@@ -225,12 +333,32 @@ func (m *resolvedManager) GetBaseConfig() (OSConfig, error) {
 }
 
 func (m *resolvedManager) Close() error {
+	m.cancelSyncer()
+
 	ctx, cancel := context.WithTimeout(context.Background(), reconfigTimeout)
 	defer cancel()
-
-	if call := m.resolved.CallWithContext(ctx, "org.freedesktop.resolve1.Manager.RevertLink", 0, m.ifidx); call.Err != nil {
+	if call := m.resolved.CallWithContext(ctx, dbusResolvedInterface+".RevertLink", 0, m.ifidx); call.Err != nil {
 		return fmt.Errorf("RevertLink: %w", call.Err)
 	}
 
+	select {
+	case <-m.syncerDone:
+	case <-ctx.Done():
+		m.logf("timeout in systemd-resolved syncer shutdown")
+	}
+
 	return nil
 }
+
+// linkDomainsWithoutReverseDNS returns a copy of v without
+// *.arpa. entries.
+func linkDomainsWithoutReverseDNS(v []resolvedLinkDomain) (ret []resolvedLinkDomain) {
+	for _, d := range v {
+		if strings.HasSuffix(d.Domain, ".arpa.") {
+			// Oh well. At least the rest will work.
+			continue
+		}
+		ret = append(ret, d)
+	}
+	return ret
+}

net/dns/resolver/forwarder.go

@@ -26,6 +26,7 @@ import (
 	"inet.af/netaddr"
 	"tailscale.com/hostinfo"
 	"tailscale.com/net/netns"
+	"tailscale.com/net/tsdial"
 	"tailscale.com/types/dnstype"
 	"tailscale.com/types/logger"
 	"tailscale.com/util/dnsname"
@@ -159,7 +160,8 @@ type resolverAndDelay struct {
 type forwarder struct {
 	logf    logger.Logf
 	linkMon *monitor.Mon
-	linkSel ForwardLinkSelector
+	linkSel ForwardLinkSelector // TODO(bradfitz): remove this when tsdial.Dialer absords it
+	dialer  *tsdial.Dialer
 	dohSem  chan struct{}
 
 	ctx       context.Context    // good until Close
@@ -205,11 +207,12 @@ func maxDoHInFlight(goos string) int {
 	return 1000
 }
 
-func newForwarder(logf logger.Logf, responses chan packet, linkMon *monitor.Mon, linkSel ForwardLinkSelector) *forwarder {
+func newForwarder(logf logger.Logf, responses chan packet, linkMon *monitor.Mon, linkSel ForwardLinkSelector, dialer *tsdial.Dialer) *forwarder {
 	f := &forwarder{
 		logf:      logger.WithPrefix(logf, "forward: "),
 		linkMon:   linkMon,
 		linkSel:   linkSel,
+		dialer:    dialer,
 		responses: responses,
 		dohSem:    make(chan struct{}, maxDoHInFlight(runtime.GOOS)),
 	}
@@ -342,7 +345,7 @@ func (f *forwarder) getKnownDoHClient(ip netaddr.IP) (urlBase string, c *http.Cl
 	if f.dohClient == nil {
 		f.dohClient = map[string]*http.Client{}
 	}
-	nsDialer := netns.NewDialer()
+	nsDialer := netns.NewDialer(f.logf)
 	c = &http.Client{
 		Transport: &http.Transport{
 			IdleConnTimeout: dohTransportTimeout,
@@ -385,6 +388,7 @@ func (f *forwarder) sendDoH(ctx context.Context, urlBase string, c *http.Client,
 	}
 	defer f.releaseDoHSem()
 
+	metricDNSFwdDoH.Add(1)
 	req, err := http.NewRequestWithContext(ctx, "POST", urlBase, bytes.NewReader(packet))
 	if err != nil {
 		return nil, err
@@ -398,16 +402,23 @@ func (f *forwarder) sendDoH(ctx context.Context, urlBase string, c *http.Client,
 
 	hres, err := c.Do(req)
 	if err != nil {
+		metricDNSFwdDoHErrorTransport.Add(1)
 		return nil, err
 	}
 	defer hres.Body.Close()
 	if hres.StatusCode != 200 {
+		metricDNSFwdDoHErrorStatus.Add(1)
 		return nil, errors.New(hres.Status)
 	}
 	if ct := hres.Header.Get("Content-Type"); ct != dohType {
+		metricDNSFwdDoHErrorCT.Add(1)
 		return nil, fmt.Errorf("unexpected response Content-Type %q", ct)
 	}
-	return ioutil.ReadAll(hres.Body)
+	res, err := ioutil.ReadAll(hres.Body)
+	if err != nil {
+		metricDNSFwdDoHErrorBody.Add(1)
+	}
+	return res, err
 }
 
 // send sends packet to dst. It is best effort.
@@ -415,12 +426,14 @@ func (f *forwarder) sendDoH(ctx context.Context, urlBase string, c *http.Client,
 // send expects the reply to have the same txid as txidOut.
 func (f *forwarder) send(ctx context.Context, fq *forwardQuery, rr resolverAndDelay) ([]byte, error) {
 	if strings.HasPrefix(rr.name.Addr, "http://") {
-		return nil, fmt.Errorf("http:// resolvers not supported yet")
+		return f.sendDoH(ctx, rr.name.Addr, f.dialer.PeerAPIHTTPClient(), fq.packet)
 	}
 	if strings.HasPrefix(rr.name.Addr, "https://") {
+		metricDNSFwdErrorType.Add(1)
 		return nil, fmt.Errorf("https:// resolvers not supported yet")
 	}
 	if strings.HasPrefix(rr.name.Addr, "tls://") {
+		metricDNSFwdErrorType.Add(1)
 		return nil, fmt.Errorf("tls:// resolvers not supported yet")
 	}
 	ipp, err := netaddr.ParseIPPort(rr.name.Addr)
@@ -438,6 +451,7 @@ func (f *forwarder) send(ctx context.Context, fq *forwardQuery, rr resolverAndDe
 		f.logf("DoH error from %v: %v", ipp.IP(), err)
 	}
 
+	metricDNSFwdUDP.Add(1)
 	ln, err := f.packetListener(ipp.IP())
 	if err != nil {
 		return nil, err
@@ -453,11 +467,13 @@ func (f *forwarder) send(ctx context.Context, fq *forwardQuery, rr resolverAndDe
 	defer fq.closeOnCtxDone.Remove(conn)
 
 	if _, err := conn.WriteTo(fq.packet, ipp.UDPAddr()); err != nil {
+		metricDNSFwdUDPErrorWrite.Add(1)
 		if err := ctx.Err(); err != nil {
 			return nil, err
 		}
 		return nil, err
 	}
+	metricDNSFwdUDPWrote.Add(1)
 
 	// The 1 extra byte is to detect packet truncation.
 	out := make([]byte, maxResponseBytes+1)
@@ -469,6 +485,7 @@ func (f *forwarder) send(ctx context.Context, fq *forwardQuery, rr resolverAndDe
 		if packetWasTruncated(err) {
 			err = nil
 		} else {
+			metricDNSFwdUDPErrorRead.Add(1)
 			return nil, err
 		}
 	}
@@ -482,12 +499,14 @@ func (f *forwarder) send(ctx context.Context, fq *forwardQuery, rr resolverAndDe
 	out = out[:n]
 	txid := getTxID(out)
 	if txid != fq.txid {
+		metricDNSFwdUDPErrorTxID.Add(1)
 		return nil, errors.New("txid doesn't match")
 	}
 	rcode := getRCode(out)
 	// don't forward transient errors back to the client when the server fails
 	if rcode == dns.RCodeServerFailure {
 		f.logf("recv: response code indicating server failure: %d", rcode)
+		metricDNSFwdUDPErrorServer.Add(1)
 		return nil, errors.New("response code indicates server issue")
 	}
 
@@ -505,7 +524,7 @@ func (f *forwarder) send(ctx context.Context, fq *forwardQuery, rr resolverAndDe
 	}
 
 	clampEDNSSize(out, maxResponseBytes)
-
+	metricDNSFwdUDPSuccess.Add(1)
 	return out, nil
 }
 
@@ -546,10 +565,30 @@ type forwardQuery struct {
 	// ...
 }
 
-// forward forwards the query to all upstream nameservers and returns the first response.
+// forward forwards the query to all upstream nameservers and waits for
+// the first response.
+//
+// It either sends to f.responses and returns nil, or returns a
+// non-nil error (without sending to the channel).
 func (f *forwarder) forward(query packet) error {
+	ctx, cancel := context.WithTimeout(f.ctx, responseTimeout)
+	defer cancel()
+	return f.forwardWithDestChan(ctx, query, f.responses)
+}
+
+// forward forwards the query to all upstream nameservers and waits
+// for the first response.
+//
+// It either sends to responseChan and returns nil, or returns a
+// non-nil error (without sending to the channel).
+//
+// If resolvers is non-empty, it's used explicitly (notably, for exit
+// node DNS proxy queries), otherwise f.resolvers is used.
+func (f *forwarder) forwardWithDestChan(ctx context.Context, query packet, responseChan chan<- packet, resolvers ...resolverAndDelay) error {
+	metricDNSFwd.Add(1)
 	domain, err := nameFromQuery(query.bs)
 	if err != nil {
+		metricDNSFwdErrorName.Add(1)
 		return err
 	}
 
@@ -558,14 +597,18 @@ func (f *forwarder) forward(query packet) error {
 	// when browsing for LAN devices.  But even when filtering this
 	// out, playing on Sonos still works.
 	if hasRDNSBonjourPrefix(domain) {
+		metricDNSFwdDropBonjour.Add(1)
 		return nil
 	}
 
 	clampEDNSSize(query.bs, maxResponseBytes)
 
-	resolvers := f.resolvers(domain)
 	if len(resolvers) == 0 {
-		return errNoUpstreams
+		resolvers = f.resolvers(domain)
+		if len(resolvers) == 0 {
+			metricDNSFwdErrorNoUpstream.Add(1)
+			return errNoUpstreams
+		}
 	}
 
 	fq := &forwardQuery{
@@ -575,9 +618,6 @@ func (f *forwarder) forward(query packet) error {
 	}
 	defer fq.closeOnCtxDone.Close()
 
-	ctx, cancel := context.WithTimeout(f.ctx, responseTimeout)
-	defer cancel()
-
 	resc := make(chan []byte, 1)
 	var (
 		mu       sync.Mutex
@@ -615,14 +655,18 @@ func (f *forwarder) forward(query packet) error {
 	case v := <-resc:
 		select {
 		case <-ctx.Done():
+			metricDNSFwdErrorContext.Add(1)
 			return ctx.Err()
-		case f.responses <- packet{v, query.addr}:
+		case responseChan <- packet{v, query.addr}:
+			metricDNSFwdSuccess.Add(1)
 			return nil
 		}
 	case <-ctx.Done():
 		mu.Lock()
 		defer mu.Unlock()
+		metricDNSFwdErrorContext.Add(1)
 		if firstErr != nil {
+			metricDNSFwdErrorContextGotError.Add(1)
 			return firstErr
 		}
 		return ctx.Err()

net/dns/resolver/tsdns.go

@@ -8,10 +8,14 @@ package resolver
 
 import (
 	"bufio"
+	"bytes"
+	"context"
 	"encoding/hex"
 	"errors"
 	"fmt"
 	"io"
+	"net"
+	"os"
 	"runtime"
 	"sort"
 	"strings"
@@ -19,11 +23,16 @@ import (
 	"sync/atomic"
 	"time"
 
+	"go4.org/mem"
 	dns "golang.org/x/net/dns/dnsmessage"
 	"inet.af/netaddr"
+	"tailscale.com/net/tsaddr"
+	"tailscale.com/net/tsdial"
 	"tailscale.com/types/dnstype"
 	"tailscale.com/types/logger"
+	"tailscale.com/util/clientmetric"
 	"tailscale.com/util/dnsname"
+	"tailscale.com/util/lineread"
 	"tailscale.com/wgengine/monitor"
 )
 
@@ -184,6 +193,7 @@ func WriteRoutes(w *bufio.Writer, routes map[dnsname.FQDN][]dnstype.Resolver) {
 type Resolver struct {
 	logf               logger.Logf
 	linkMon            *monitor.Mon     // or nil
+	dialer             *tsdial.Dialer   // non-nil
 	saveConfigForTests func(cfg Config) // used in tests to capture resolver config
 	// forwarder forwards requests to upstream nameservers.
 	forwarder *forwarder
@@ -215,7 +225,10 @@ type ForwardLinkSelector interface {
 
 // New returns a new resolver.
 // linkMon optionally specifies a link monitor to use for socket rebinding.
-func New(logf logger.Logf, linkMon *monitor.Mon, linkSel ForwardLinkSelector) *Resolver {
+func New(logf logger.Logf, linkMon *monitor.Mon, linkSel ForwardLinkSelector, dialer *tsdial.Dialer) *Resolver {
+	if dialer == nil {
+		panic("nil Dialer")
+	}
 	r := &Resolver{
 		logf:      logger.WithPrefix(logf, "resolver: "),
 		linkMon:   linkMon,
@@ -224,8 +237,9 @@ func New(logf logger.Logf, linkMon *monitor.Mon, linkSel ForwardLinkSelector) *R
 		closed:    make(chan struct{}),
 		hostToIP:  map[dnsname.FQDN][]netaddr.IP{},
 		ipToHost:  map[netaddr.IP]dnsname.FQDN{},
+		dialer:    dialer,
 	}
-	r.forwarder = newForwarder(r.logf, r.responses, linkMon, linkSel)
+	r.forwarder = newForwarder(r.logf, r.responses, linkMon, linkSel, dialer)
 	return r
 }
 
@@ -272,13 +286,16 @@ func (r *Resolver) Close() {
 // It takes ownership of the payload and does not block.
 // If the queue is full, the request will be dropped and an error will be returned.
 func (r *Resolver) EnqueueRequest(bs []byte, from netaddr.IPPort) error {
+	metricDNSQueryLocal.Add(1)
 	select {
 	case <-r.closed:
+		metricDNSQueryErrorClosed.Add(1)
 		return ErrClosed
 	default:
 	}
 	if n := atomic.AddInt32(&r.activeQueriesAtomic, 1); n > maxActiveQueries() {
 		atomic.AddInt32(&r.activeQueriesAtomic, -1)
+		metricDNSQueryErrorQueue.Add(1)
 		return errFullQueue
 	}
 	go r.handleQuery(packet{bs, from})
@@ -298,14 +315,259 @@ func (r *Resolver) NextResponse() (packet []byte, to netaddr.IPPort, err error)
 	}
 }
 
+// parseExitNodeQuery parses a DNS request packet.
+// It returns nil if it's malformed or lacking a question.
+func parseExitNodeQuery(q []byte) *response {
+	p := dnsParserPool.Get().(*dnsParser)
+	defer dnsParserPool.Put(p)
+	p.zeroParser()
+	defer p.zeroParser()
+	if err := p.parseQuery(q); err != nil {
+		return nil
+	}
+	return p.response()
+}
+
+// HandleExitNodeDNSQuery handles a DNS query that arrived from a peer
+// via the peerapi's DoH server. This is only used when the local
+// node is being an exit node.
+//
+// The provided allowName callback is whether a DNS query for a name
+// (as found by parsing q) is allowed.
+//
+// In most (all?) cases, err will be nil. A bogus DNS query q will
+// still result in a response DNS packet (saying there's a failure)
+// and a nil error.
+// TODO: figure out if we even need an error result.
+func (r *Resolver) HandleExitNodeDNSQuery(ctx context.Context, q []byte, from netaddr.IPPort, allowName func(name string) bool) (res []byte, err error) {
+	metricDNSExitProxyQuery.Add(1)
+	ch := make(chan packet, 1)
+
+	resp := parseExitNodeQuery(q)
+	if resp == nil {
+		return nil, errors.New("bad query")
+	}
+	name := resp.Question.Name.String()
+	if !allowName(name) {
+		metricDNSExitProxyErrorName.Add(1)
+		resp.Header.RCode = dns.RCodeRefused
+		return marshalResponse(resp)
+	}
+
+	switch runtime.GOOS {
+	default:
+		return nil, errors.New("unsupported exit node OS")
+	case "windows":
+		// TODO: use DnsQueryEx and write to ch.
+		// See https://docs.microsoft.com/en-us/windows/win32/api/windns/nf-windns-dnsqueryex.
+		// For now just use the net package:
+		return handleExitNodeDNSQueryWithNetPkg(ctx, nil, resp)
+	case "darwin":
+		// /etc/resolv.conf is a lie and only says one upstream DNS
+		// but for now that's probably good enough. Later we'll
+		// want to blend in everything from scutil --dns.
+		fallthrough
+	case "linux", "freebsd", "openbsd", "illumos":
+		nameserver, err := stubResolverForOS()
+		if err != nil {
+			r.logf("stubResolverForOS: %v", err)
+			metricDNSExitProxyErrorResolvConf.Add(1)
+			return nil, err
+		}
+		// TODO: more than 1 resolver from /etc/resolv.conf?
+
+		var resolvers []resolverAndDelay
+		if nameserver == tsaddr.TailscaleServiceIP() {
+			// If resolv.conf says 100.100.100.100, it's coming right back to us anyway
+			// so avoid the loop through the kernel and just do what we
+			// would've done anyway. By not passing any resolvers, the forwarder
+			// will use its default ones from our DNS config.
+		} else {
+			resolvers = []resolverAndDelay{{
+				name: dnstype.Resolver{Addr: net.JoinHostPort(nameserver.String(), "53")},
+			}}
+		}
+
+		err = r.forwarder.forwardWithDestChan(ctx, packet{q, from}, ch, resolvers...)
+		if err != nil {
+			metricDNSExitProxyErrorForward.Add(1)
+			return nil, err
+		}
+	}
+	select {
+	case p, ok := <-ch:
+		if ok {
+			return p.bs, nil
+		}
+		panic("unexpected close chan")
+	default:
+		panic("unexpected unreadable chan")
+	}
+}
+
+// handleExitNodeDNSQueryWithNetPkg takes a DNS query message in q and
+// return a reply (for the ExitDNS DoH service) using the net package's
+// native APIs. This is only used on Windows for now.
+//
+// If resolver is nil, the net.Resolver zero value is used.
+//
+// response contains the pre-serialized response, which notably
+// includes the original question and its header.
+func handleExitNodeDNSQueryWithNetPkg(ctx context.Context, resolver *net.Resolver, resp *response) (res []byte, err error) {
+	if resp.Question.Class != dns.ClassINET {
+		return nil, errors.New("unsupported class")
+	}
+
+	r := resolver
+	if r == nil {
+		r = new(net.Resolver)
+	}
+	name := resp.Question.Name.String()
+
+	handleError := func(err error) (res []byte, _ error) {
+		if isGoNoSuchHostError(err) {
+			resp.Header.RCode = dns.RCodeNameError
+			return marshalResponse(resp)
+		}
+		// TODO: map other errors to RCodeServerFailure?
+		// Or I guess our caller should do that?
+		return nil, err
+	}
+
+	resp.Header.RCode = dns.RCodeSuccess // unless changed below
+
+	switch resp.Question.Type {
+	case dns.TypeA, dns.TypeAAAA:
+		network := "ip4"
+		if resp.Question.Type == dns.TypeAAAA {
+			network = "ip6"
+		}
+		ips, err := r.LookupIP(ctx, network, name)
+		if err != nil {
+			return handleError(err)
+		}
+		for _, stdIP := range ips {
+			if ip, ok := netaddr.FromStdIP(stdIP); ok {
+				resp.IPs = append(resp.IPs, ip)
+			}
+		}
+	case dns.TypeTXT:
+		strs, err := r.LookupTXT(ctx, name)
+		if err != nil {
+			return handleError(err)
+		}
+		resp.TXT = strs
+	case dns.TypePTR:
+		ipStr, ok := unARPA(name)
+		if !ok {
+			// TODO: is this RCodeFormatError?
+			return nil, errors.New("bogus PTR name")
+		}
+		addrs, err := r.LookupAddr(ctx, ipStr)
+		if err != nil {
+			return handleError(err)
+		}
+		if len(addrs) > 0 {
+			resp.Name, _ = dnsname.ToFQDN(addrs[0])
+		}
+	case dns.TypeCNAME:
+		cname, err := r.LookupCNAME(ctx, name)
+		if err != nil {
+			return handleError(err)
+		}
+		resp.CNAME = cname
+	case dns.TypeSRV:
+		// Thanks, Go: "To accommodate services publishing SRV
+		// records under non-standard names, if both service
+		// and proto are empty strings, LookupSRV looks up
+		// name directly."
+		_, srvs, err := r.LookupSRV(ctx, "", "", name)
+		if err != nil {
+			return handleError(err)
+		}
+		resp.SRVs = srvs
+	case dns.TypeNS:
+		nss, err := r.LookupNS(ctx, name)
+		if err != nil {
+			return handleError(err)
+		}
+		resp.NSs = nss
+	default:
+		return nil, fmt.Errorf("unsupported record type %v", resp.Question.Type)
+	}
+	return marshalResponse(resp)
+}
+
+func isGoNoSuchHostError(err error) bool {
+	if de, ok := err.(*net.DNSError); ok {
+		return de.IsNotFound
+	}
+	return false
+}
+
+type resolvConfCache struct {
+	mod  time.Time
+	size int64
+	ip   netaddr.IP
+	// TODO: inode/dev?
+}
+
+// resolvConfCacheValue contains the most recent stat metadata and parsed
+// version of /etc/resolv.conf.
+var resolvConfCacheValue atomic.Value // of resolvConfCache
+
+var errEmptyResolvConf = errors.New("resolv.conf has no nameservers")
+
+// stubResolverForOS returns the IP address of the first nameserver in
+// /etc/resolv.conf.
+func stubResolverForOS() (ip netaddr.IP, err error) {
+	fi, err := os.Stat("/etc/resolv.conf")
+	if err != nil {
+		return netaddr.IP{}, err
+	}
+	cur := resolvConfCache{
+		mod:  fi.ModTime(),
+		size: fi.Size(),
+	}
+	if c, ok := resolvConfCacheValue.Load().(resolvConfCache); ok && c.mod == cur.mod && c.size == cur.size {
+		return c.ip, nil
+	}
+	err = lineread.File("/etc/resolv.conf", func(line []byte) error {
+		if !ip.IsZero() {
+			return nil
+		}
+		line = bytes.TrimSpace(line)
+		if len(line) == 0 || line[0] == '#' {
+			return nil
+		}
+		if mem.HasPrefix(mem.B(line), mem.S("nameserver ")) {
+			s := strings.TrimSpace(strings.TrimPrefix(string(line), "nameserver "))
+			ip, err = netaddr.ParseIP(s)
+			return err
+		}
+		return nil
+	})
+	if err != nil {
+		return netaddr.IP{}, err
+	}
+	if !ip.IsValid() {
+		return netaddr.IP{}, errEmptyResolvConf
+	}
+	cur.ip = ip
+	resolvConfCacheValue.Store(cur)
+	return ip, nil
+}
+
 // resolveLocal returns an IP for the given domain, if domain is in
 // the local hosts map and has an IP corresponding to the requested
 // typ (A, AAAA, ALL).
 // Returns dns.RCodeRefused to indicate that the local map is not
 // authoritative for domain.
 func (r *Resolver) resolveLocal(domain dnsname.FQDN, typ dns.Type) (netaddr.IP, dns.RCode) {
+	metricDNSResolveLocal.Add(1)
 	// Reject .onion domains per RFC 7686.
 	if dnsname.HasSuffix(domain.WithoutTrailingDot(), ".onion") {
+		metricDNSResolveLocalErrorOnion.Add(1)
 		return netaddr.IP{}, dns.RCodeNameError
 	}
 
@@ -319,6 +581,7 @@ func (r *Resolver) resolveLocal(domain dnsname.FQDN, typ dns.Type) (netaddr.IP,
 		for _, suffix := range localDomains {
 			if suffix.Contains(domain) {
 				// We are authoritative for the queried domain.
+				metricDNSResolveLocalErrorMissing.Add(1)
 				return netaddr.IP{}, dns.RCodeNameError
 			}
 		}
@@ -336,30 +599,37 @@ func (r *Resolver) resolveLocal(domain dnsname.FQDN, typ dns.Type) (netaddr.IP,
 	case dns.TypeA:
 		for _, ip := range addrs {
 			if ip.Is4() {
+				metricDNSResolveLocalOKA.Add(1)
 				return ip, dns.RCodeSuccess
 			}
 		}
+		metricDNSResolveLocalNoA.Add(1)
 		return netaddr.IP{}, dns.RCodeSuccess
 	case dns.TypeAAAA:
 		for _, ip := range addrs {
 			if ip.Is6() {
+				metricDNSResolveLocalOKAAAA.Add(1)
 				return ip, dns.RCodeSuccess
 			}
 		}
+		metricDNSResolveLocalNoAAAA.Add(1)
 		return netaddr.IP{}, dns.RCodeSuccess
 	case dns.TypeALL:
 		// Answer with whatever we've got.
 		// It could be IPv4, IPv6, or a zero addr.
 		// TODO: Return all available resolutions (A and AAAA, if we have them).
 		if len(addrs) == 0 {
+			metricDNSResolveLocalNoAll.Add(1)
 			return netaddr.IP{}, dns.RCodeSuccess
 		}
+		metricDNSResolveLocalOKAll.Add(1)
 		return addrs[0], dns.RCodeSuccess
 
 	// Leave some some record types explicitly unimplemented.
 	// These types relate to recursive resolution or special
 	// DNS semantics and might be implemented in the future.
 	case dns.TypeNS, dns.TypeSOA, dns.TypeAXFR, dns.TypeHINFO:
+		metricDNSResolveNotImplType.Add(1)
 		return netaddr.IP{}, dns.RCodeNotImplemented
 
 	// For everything except for the few types above that are explicitly not implemented, return no records.
@@ -369,6 +639,7 @@ func (r *Resolver) resolveLocal(domain dnsname.FQDN, typ dns.Type) (netaddr.IP,
 	//   dig -t TYPE9824 example.com
 	// and note that NOERROR is returned, despite that record type being made up.
 	default:
+		metricDNSResolveNoRecordType.Add(1)
 		// The name exists, but no records exist of the requested type.
 		return netaddr.IP{}, dns.RCodeSuccess
 	}
@@ -434,10 +705,27 @@ func (r *Resolver) handleQuery(pkt packet) {
 type response struct {
 	Header   dns.Header
 	Question dns.Question
+
 	// Name is the response to a PTR query.
 	Name dnsname.FQDN
-	// IP is the response to an A, AAAA, or ALL query.
-	IP netaddr.IP
+
+	// IP and IPs are the responses to an A, AAAA, or ALL query.
+	// Either/both/neither can be populated.
+	IP  netaddr.IP
+	IPs []netaddr.IP
+
+	// TXT is the response to a TXT query.
+	// Each one is its own RR with one string.
+	TXT []string
+
+	// CNAME is the response to a CNAME query.
+	CNAME string
+
+	// SRVs are the responses to a SRV query.
+	SRVs []*net.SRV
+
+	// NSs are the responses to an NS query.
+	NSs []*net.NS
 }
 
 var dnsParserPool = &sync.Pool{
@@ -468,6 +756,7 @@ func (p *dnsParser) zeroParser() { p.parser = dns.Parser{} }
 // p.Question.
 func (p *dnsParser) parseQuery(query []byte) error {
 	defer p.zeroParser()
+	p.zeroParser()
 	var err error
 	p.Header, err = p.parser.Start(query)
 	if err != nil {
@@ -512,6 +801,16 @@ func marshalAAAARecord(name dns.Name, ip netaddr.IP, builder *dns.Builder) error
 	return builder.AAAAResource(answerHeader, answer)
 }
 
+func marshalIP(name dns.Name, ip netaddr.IP, builder *dns.Builder) error {
+	if ip.Is4() {
+		return marshalARecord(name, ip, builder)
+	}
+	if ip.Is6() {
+		return marshalAAAARecord(name, ip, builder)
+	}
+	return nil
+}
+
 // marshalPTRRecord serializes a PTR record into an active builder.
 // The caller may continue using the builder following the call.
 func marshalPTRRecord(queryName dns.Name, name dnsname.FQDN, builder *dns.Builder) error {
@@ -531,6 +830,83 @@ func marshalPTRRecord(queryName dns.Name, name dnsname.FQDN, builder *dns.Builde
 	return builder.PTRResource(answerHeader, answer)
 }
 
+func marshalTXT(queryName dns.Name, txts []string, builder *dns.Builder) error {
+	for _, txt := range txts {
+		if err := builder.TXTResource(dns.ResourceHeader{
+			Name:  queryName,
+			Type:  dns.TypeTXT,
+			Class: dns.ClassINET,
+			TTL:   uint32(defaultTTL / time.Second),
+		}, dns.TXTResource{
+			TXT: []string{txt},
+		}); err != nil {
+			return err
+		}
+	}
+	return nil
+}
+
+func marshalCNAME(queryName dns.Name, cname string, builder *dns.Builder) error {
+	if cname == "" {
+		return nil
+	}
+	name, err := dns.NewName(cname)
+	if err != nil {
+		return err
+	}
+	return builder.CNAMEResource(dns.ResourceHeader{
+		Name:  queryName,
+		Type:  dns.TypeCNAME,
+		Class: dns.ClassINET,
+		TTL:   uint32(defaultTTL / time.Second),
+	}, dns.CNAMEResource{
+		CNAME: name,
+	})
+}
+
+func marshalNS(queryName dns.Name, nss []*net.NS, builder *dns.Builder) error {
+	for _, ns := range nss {
+		name, err := dns.NewName(ns.Host)
+		if err != nil {
+			return err
+		}
+		err = builder.NSResource(dns.ResourceHeader{
+			Name:  queryName,
+			Type:  dns.TypeNS,
+			Class: dns.ClassINET,
+			TTL:   uint32(defaultTTL / time.Second),
+		}, dns.NSResource{NS: name})
+		if err != nil {
+			return err
+		}
+	}
+	return nil
+}
+
+func marshalSRV(queryName dns.Name, srvs []*net.SRV, builder *dns.Builder) error {
+	for _, s := range srvs {
+		srvName, err := dns.NewName(s.Target)
+		if err != nil {
+			return err
+		}
+		err = builder.SRVResource(dns.ResourceHeader{
+			Name:  queryName,
+			Type:  dns.TypeSRV,
+			Class: dns.ClassINET,
+			TTL:   uint32(defaultTTL / time.Second),
+		}, dns.SRVResource{
+			Target:   srvName,
+			Priority: s.Priority,
+			Port:     s.Port,
+			Weight:   s.Weight,
+		})
+		if err != nil {
+			return err
+		}
+	}
+	return nil
+}
+
 // marshalResponse serializes the DNS response into a new buffer.
 func marshalResponse(resp *response) ([]byte, error) {
 	resp.Header.Response = true
@@ -541,6 +917,14 @@ func marshalResponse(resp *response) ([]byte, error) {
 
 	builder := dns.NewBuilder(nil, resp.Header)
 
+	// TODO(bradfitz): I'm not sure why this wasn't enabled
+	// before, but for now (2021-12-09) enable it at least when
+	// there's more than 1 record (which was never the case
+	// before), where it really helps.
+	if len(resp.IPs) > 1 {
+		builder.EnableCompression()
+	}
+
 	isSuccess := resp.Header.RCode == dns.RCodeSuccess
 
 	if resp.Question.Type != 0 || isSuccess {
@@ -567,13 +951,24 @@ func marshalResponse(resp *response) ([]byte, error) {
 
 	switch resp.Question.Type {
 	case dns.TypeA, dns.TypeAAAA, dns.TypeALL:
-		if resp.IP.Is4() {
-			err = marshalARecord(resp.Question.Name, resp.IP, &builder)
-		} else if resp.IP.Is6() {
-			err = marshalAAAARecord(resp.Question.Name, resp.IP, &builder)
+		if err := marshalIP(resp.Question.Name, resp.IP, &builder); err != nil {
+			return nil, err
+		}
+		for _, ip := range resp.IPs {
+			if err := marshalIP(resp.Question.Name, ip, &builder); err != nil {
+				return nil, err
+			}
 		}
 	case dns.TypePTR:
 		err = marshalPTRRecord(resp.Question.Name, resp.Name, &builder)
+	case dns.TypeTXT:
+		err = marshalTXT(resp.Question.Name, resp.TXT, &builder)
+	case dns.TypeCNAME:
+		err = marshalCNAME(resp.Question.Name, resp.CNAME, &builder)
+	case dns.TypeSRV:
+		err = marshalSRV(resp.Question.Name, resp.SRVs, &builder)
+	case dns.TypeNS:
+		err = marshalNS(resp.Question.Name, resp.NSs, &builder)
 	}
 	if err != nil {
 		return nil, err
@@ -700,6 +1095,7 @@ func (r *Resolver) respondReverse(query []byte, name dnsname.FQDN, resp *respons
 		return nil, errNotOurName
 	}
 
+	metricDNSMagicDNSSuccessReverse.Add(1)
 	return marshalResponse(resp)
 }
 
@@ -716,8 +1112,10 @@ func (r *Resolver) respond(query []byte) ([]byte, error) {
 	// We will not return this error: it is the sender's fault.
 	if err != nil {
 		if errors.Is(err, dns.ErrSectionDone) {
+			metricDNSErrorParseNoQ.Add(1)
 			r.logf("parseQuery(%02x): no DNS questions", query)
 		} else {
+			metricDNSErrorParseQuery.Add(1)
 			r.logf("parseQuery(%02x): %v", query, err)
 		}
 		resp := parser.response()
@@ -727,6 +1125,7 @@ func (r *Resolver) respond(query []byte) ([]byte, error) {
 	rawName := parser.Question.Name.Data[:parser.Question.Name.Length]
 	name, err := dnsname.ToFQDN(rawNameToLower(rawName))
 	if err != nil {
+		metricDNSErrorNotFQDN.Add(1)
 		// DNS packet unexpectedly contains an invalid FQDN.
 		resp := parser.response()
 		resp.Header.RCode = dns.RCodeFormatError
@@ -750,3 +1149,90 @@ func (r *Resolver) respond(query []byte) ([]byte, error) {
 	resp.IP = ip
 	return marshalResponse(resp)
 }
+
+// unARPA maps from "4.4.8.8.in-addr.arpa." to "8.8.4.4", etc.
+func unARPA(a string) (ipStr string, ok bool) {
+	const suf4 = ".in-addr.arpa."
+	if strings.HasSuffix(a, suf4) {
+		s := strings.TrimSuffix(a, suf4)
+		// Parse and reverse octets.
+		ip, err := netaddr.ParseIP(s)
+		if err != nil || !ip.Is4() {
+			return "", false
+		}
+		a4 := ip.As4()
+		return netaddr.IPv4(a4[3], a4[2], a4[1], a4[0]).String(), true
+	}
+	const suf6 = ".ip6.arpa."
+	if len(a) == len("e.0.0.2.0.0.0.0.0.0.0.0.0.0.0.0.b.0.8.0.a.0.0.4.0.b.8.f.7.0.6.2.ip6.arpa.") &&
+		strings.HasSuffix(a, suf6) {
+		var hx [32]byte
+		var a16 [16]byte
+		for i := range hx {
+			hx[31-i] = a[i*2]
+			if a[i*2+1] != '.' {
+				return "", false
+			}
+		}
+		hex.Decode(a16[:], hx[:])
+		return netaddr.IPFrom16(a16).String(), true
+	}
+	return "", false
+
+}
+
+var (
+	metricDNSQueryLocal       = clientmetric.NewCounter("dns_query_local")
+	metricDNSQueryErrorClosed = clientmetric.NewCounter("dns_query_local_error_closed")
+	metricDNSQueryErrorQueue  = clientmetric.NewCounter("dns_query_local_error_queue")
+
+	metricDNSErrorParseNoQ   = clientmetric.NewCounter("dns_query_respond_error_no_question")
+	metricDNSErrorParseQuery = clientmetric.NewCounter("dns_query_respond_error_parse")
+	metricDNSErrorNotFQDN    = clientmetric.NewCounter("dns_query_respond_error_not_fqdn")
+
+	metricDNSMagicDNSSuccessName    = clientmetric.NewCounter("dns_query_magic_success_name")
+	metricDNSMagicDNSSuccessReverse = clientmetric.NewCounter("dns_query_magic_success_reverse")
+
+	metricDNSExitProxyQuery           = clientmetric.NewCounter("dns_exit_node_query")
+	metricDNSExitProxyErrorName       = clientmetric.NewCounter("dns_exit_node_error_name")
+	metricDNSExitProxyErrorForward    = clientmetric.NewCounter("dns_exit_node_error_forward")
+	metricDNSExitProxyErrorResolvConf = clientmetric.NewCounter("dns_exit_node_error_resolvconf")
+
+	metricDNSFwd                     = clientmetric.NewCounter("dns_query_fwd")
+	metricDNSFwdDropBonjour          = clientmetric.NewCounter("dns_query_fwd_drop_bonjour")
+	metricDNSFwdErrorName            = clientmetric.NewCounter("dns_query_fwd_error_name")
+	metricDNSFwdErrorNoUpstream      = clientmetric.NewCounter("dns_query_fwd_error_no_upstream")
+	metricDNSFwdSuccess              = clientmetric.NewCounter("dns_query_fwd_success")
+	metricDNSFwdErrorContext         = clientmetric.NewCounter("dns_query_fwd_error_context")
+	metricDNSFwdErrorContextGotError = clientmetric.NewCounter("dns_query_fwd_error_context_got_error")
+
+	metricDNSFwdErrorType      = clientmetric.NewCounter("dns_query_fwd_error_type")
+	metricDNSFwdErrorParseAddr = clientmetric.NewCounter("dns_query_fwd_error_parse_addr")
+
+	metricDNSFwdUDP            = clientmetric.NewCounter("dns_query_fwd_udp")       // on entry
+	metricDNSFwdUDPWrote       = clientmetric.NewCounter("dns_query_fwd_udp_wrote") // sent UDP packet
+	metricDNSFwdUDPErrorWrite  = clientmetric.NewCounter("dns_query_fwd_udp_error_write")
+	metricDNSFwdUDPErrorServer = clientmetric.NewCounter("dns_query_fwd_udp_error_server")
+	metricDNSFwdUDPErrorTxID   = clientmetric.NewCounter("dns_query_fwd_udp_error_txid")
+	metricDNSFwdUDPErrorRead   = clientmetric.NewCounter("dns_query_fwd_udp_error_read")
+	metricDNSFwdUDPSuccess     = clientmetric.NewCounter("dns_query_fwd_udp_success")
+
+	metricDNSFwdDoH               = clientmetric.NewCounter("dns_query_fwd_doh")
+	metricDNSFwdDoHErrorStatus    = clientmetric.NewCounter("dns_query_fwd_doh_error_status")
+	metricDNSFwdDoHErrorCT        = clientmetric.NewCounter("dns_query_fwd_doh_error_content_type")
+	metricDNSFwdDoHErrorTransport = clientmetric.NewCounter("dns_query_fwd_doh_error_transport")
+	metricDNSFwdDoHErrorBody      = clientmetric.NewCounter("dns_query_fwd_doh_error_body")
+
+	metricDNSResolveLocal             = clientmetric.NewCounter("dns_resolve_local")
+	metricDNSResolveLocalErrorOnion   = clientmetric.NewCounter("dns_resolve_local_error_onion")
+	metricDNSResolveLocalErrorMissing = clientmetric.NewCounter("dns_resolve_local_error_missing")
+	metricDNSResolveLocalErrorRefused = clientmetric.NewCounter("dns_resolve_local_error_refused")
+	metricDNSResolveLocalOKA          = clientmetric.NewCounter("dns_resolve_local_ok_a")
+	metricDNSResolveLocalOKAAAA       = clientmetric.NewCounter("dns_resolve_local_ok_aaaa")
+	metricDNSResolveLocalOKAll        = clientmetric.NewCounter("dns_resolve_local_ok_all")
+	metricDNSResolveLocalNoA          = clientmetric.NewCounter("dns_resolve_local_no_a")
+	metricDNSResolveLocalNoAAAA       = clientmetric.NewCounter("dns_resolve_local_no_aaaa")
+	metricDNSResolveLocalNoAll        = clientmetric.NewCounter("dns_resolve_local_no_all")
+	metricDNSResolveNotImplType       = clientmetric.NewCounter("dns_resolve_local_not_impl_type")
+	metricDNSResolveNoRecordType      = clientmetric.NewCounter("dns_resolve_local_no_record_type")
+)

net/dns/resolver/tsdns_server_test.go

@@ -6,6 +6,7 @@ package resolver
 
 import (
 	"fmt"
+	"net"
 	"strings"
 	"testing"
 
@@ -179,6 +180,129 @@ var resolveToNXDOMAIN = dns.HandlerFunc(func(w dns.ResponseWriter, req *dns.Msg)
 	w.WriteMsg(m)
 })
 
+// weirdoGoCNAMEHandler returns a DNS handler that satisfies
+// Go's weird Resolver.LookupCNAME (read its godoc carefully!).
+//
+// This doesn't even return a CNAME record, because that's not
+// what Go looks for.
+func weirdoGoCNAMEHandler(target string) dns.HandlerFunc {
+	return func(w dns.ResponseWriter, req *dns.Msg) {
+		m := new(dns.Msg)
+		m.SetReply(req)
+		question := req.Question[0]
+
+		switch question.Qtype {
+		case dns.TypeA:
+			m.Answer = append(m.Answer, &dns.CNAME{
+				Hdr: dns.RR_Header{
+					Name:   target,
+					Rrtype: dns.TypeCNAME,
+					Class:  dns.ClassINET,
+					Ttl:    600,
+				},
+				Target: target,
+			})
+		case dns.TypeAAAA:
+			m.Answer = append(m.Answer, &dns.AAAA{
+				Hdr: dns.RR_Header{
+					Name:   target,
+					Rrtype: dns.TypeAAAA,
+					Class:  dns.ClassINET,
+					Ttl:    600,
+				},
+				AAAA: net.ParseIP("1::2"),
+			})
+		}
+		w.WriteMsg(m)
+	}
+}
+
+// dnsHandler returns a handler that replies with the answers/options
+// provided.
+//
+// Types supported: netaddr.IP.
+func dnsHandler(answers ...interface{}) dns.HandlerFunc {
+	return func(w dns.ResponseWriter, req *dns.Msg) {
+		m := new(dns.Msg)
+		m.SetReply(req)
+		if len(req.Question) != 1 {
+			panic("not a single-question request")
+		}
+		m.RecursionAvailable = true // to stop net package's errLameReferral on empty replies
+
+		question := req.Question[0]
+		for _, a := range answers {
+			switch a := a.(type) {
+			default:
+				panic(fmt.Sprintf("unsupported dnsHandler arg %T", a))
+			case netaddr.IP:
+				ip := a
+				if ip.Is4() {
+					m.Answer = append(m.Answer, &dns.A{
+						Hdr: dns.RR_Header{
+							Name:   question.Name,
+							Rrtype: dns.TypeA,
+							Class:  dns.ClassINET,
+						},
+						A: ip.IPAddr().IP,
+					})
+				} else if ip.Is6() {
+					m.Answer = append(m.Answer, &dns.AAAA{
+						Hdr: dns.RR_Header{
+							Name:   question.Name,
+							Rrtype: dns.TypeAAAA,
+							Class:  dns.ClassINET,
+						},
+						AAAA: ip.IPAddr().IP,
+					})
+				}
+			case dns.PTR:
+				ptr := a
+				ptr.Hdr = dns.RR_Header{
+					Name:   question.Name,
+					Rrtype: dns.TypePTR,
+					Class:  dns.ClassINET,
+				}
+				m.Answer = append(m.Answer, &ptr)
+			case dns.CNAME:
+				c := a
+				c.Hdr = dns.RR_Header{
+					Name:   question.Name,
+					Rrtype: dns.TypeCNAME,
+					Class:  dns.ClassINET,
+					Ttl:    600,
+				}
+				m.Answer = append(m.Answer, &c)
+			case dns.TXT:
+				txt := a
+				txt.Hdr = dns.RR_Header{
+					Name:   question.Name,
+					Rrtype: dns.TypeTXT,
+					Class:  dns.ClassINET,
+				}
+				m.Answer = append(m.Answer, &txt)
+			case dns.SRV:
+				srv := a
+				srv.Hdr = dns.RR_Header{
+					Name:   question.Name,
+					Rrtype: dns.TypeSRV,
+					Class:  dns.ClassINET,
+				}
+				m.Answer = append(m.Answer, &srv)
+			case dns.NS:
+				rr := a
+				rr.Hdr = dns.RR_Header{
+					Name:   question.Name,
+					Rrtype: dns.TypeNS,
+					Class:  dns.ClassINET,
+				}
+				m.Answer = append(m.Answer, &rr)
+			}
+		}
+		w.WriteMsg(m)
+	}
+}
+
 func serveDNS(tb testing.TB, addr string, records ...interface{}) *dns.Server {
 	if len(records)%2 != 0 {
 		panic("must have an even number of record values")

net/dns/resolver/tsdns_test.go

@@ -6,18 +6,25 @@ package resolver
 
 import (
 	"bytes"
+	"context"
 	"encoding/hex"
+	"encoding/json"
 	"errors"
 	"fmt"
 	"math/rand"
 	"net"
+	"reflect"
 	"runtime"
 	"strconv"
 	"strings"
 	"testing"
+	"time"
 
+	miekdns "github.com/miekg/dns"
+	"golang.org/x/net/dns/dnsmessage"
 	dns "golang.org/x/net/dns/dnsmessage"
 	"inet.af/netaddr"
+	"tailscale.com/net/tsdial"
 	"tailscale.com/tstest"
 	"tailscale.com/types/dnstype"
 	"tailscale.com/util/dnsname"
@@ -34,14 +41,16 @@ var (
 
 var dnsCfg = Config{
 	Hosts: map[dnsname.FQDN][]netaddr.IP{
-		"test1.ipn.dev.": []netaddr.IP{testipv4},
-		"test2.ipn.dev.": []netaddr.IP{testipv6},
+		"test1.ipn.dev.": {testipv4},
+		"test2.ipn.dev.": {testipv6},
 	},
 	LocalDomains: []dnsname.FQDN{"ipn.dev.", "3.2.1.in-addr.arpa.", "1.0.0.0.ip6.arpa."},
 }
 
 const noEdns = 0
 
+const dnsHeaderLen = 12
+
 func dnspacket(domain dnsname.FQDN, tp dns.Type, ednsSize uint16) []byte {
 	var dnsHeader dns.Header
 	question := dns.Question{
@@ -308,7 +317,7 @@ func TestRDNSNameToIPv6(t *testing.T) {
 }
 
 func newResolver(t testing.TB) *Resolver {
-	return New(t.Logf, nil /* no link monitor */, nil /* no link selector */)
+	return New(t.Logf, nil /* no link monitor */, nil /* no link selector */, new(tsdial.Dialer))
 }
 
 func TestResolveLocal(t *testing.T) {
@@ -1062,7 +1071,7 @@ func TestForwardLinkSelection(t *testing.T) {
 			return "special"
 		}
 		return ""
-	}))
+	}), new(tsdial.Dialer))
 
 	// Test non-special IP.
 	if got, err := fwd.packetListener(netaddr.IP{}); err != nil {
@@ -1092,3 +1101,383 @@ func TestForwardLinkSelection(t *testing.T) {
 type linkSelFunc func(ip netaddr.IP) string
 
 func (f linkSelFunc) PickLink(ip netaddr.IP) string { return f(ip) }
+
+func TestHandleExitNodeDNSQueryWithNetPkg(t *testing.T) {
+	if runtime.GOOS == "windows" {
+		t.Skip("skipping test on Windows; waiting for golang.org/issue/33097")
+	}
+
+	records := []interface{}{
+		"no-records.test.",
+		dnsHandler(),
+
+		"one-a.test.",
+		dnsHandler(netaddr.MustParseIP("1.2.3.4")),
+
+		"two-a.test.",
+		dnsHandler(netaddr.MustParseIP("1.2.3.4"), netaddr.MustParseIP("5.6.7.8")),
+
+		"one-aaaa.test.",
+		dnsHandler(netaddr.MustParseIP("1::2")),
+
+		"two-aaaa.test.",
+		dnsHandler(netaddr.MustParseIP("1::2"), netaddr.MustParseIP("3::4")),
+
+		"nx-domain.test.",
+		resolveToNXDOMAIN,
+
+		"4.3.2.1.in-addr.arpa.",
+		dnsHandler(miekdns.PTR{Ptr: "foo.com."}),
+
+		"cname.test.",
+		weirdoGoCNAMEHandler("the-target.foo."),
+
+		"txt.test.",
+		dnsHandler(
+			miekdns.TXT{Txt: []string{"txt1=one"}},
+			miekdns.TXT{Txt: []string{"txt2=two"}},
+			miekdns.TXT{Txt: []string{"txt3=three"}},
+		),
+
+		"srv.test.",
+		dnsHandler(
+			miekdns.SRV{
+				Priority: 1,
+				Weight:   2,
+				Port:     3,
+				Target:   "foo.com.",
+			},
+			miekdns.SRV{
+				Priority: 4,
+				Weight:   5,
+				Port:     6,
+				Target:   "bar.com.",
+			},
+		),
+
+		"ns.test.",
+		dnsHandler(miekdns.NS{Ns: "ns1.foo."}, miekdns.NS{Ns: "ns2.bar."}),
+	}
+	v4server := serveDNS(t, "127.0.0.1:0", records...)
+	defer v4server.Shutdown()
+
+	// backendResolver is the resolver between
+	// handleExitNodeDNSQueryWithNetPkg and its upstream resolver,
+	// which in this test's case is the miekg/dns test DNS server
+	// (v4server).
+	backResolver := &net.Resolver{
+		PreferGo: true,
+		Dial: func(ctx context.Context, network, addr string) (net.Conn, error) {
+			var d net.Dialer
+			return d.DialContext(ctx, "udp", v4server.PacketConn.LocalAddr().String())
+		},
+	}
+
+	t.Run("no_such_host", func(t *testing.T) {
+		res, err := handleExitNodeDNSQueryWithNetPkg(context.Background(), backResolver, &response{
+			Header: dnsmessage.Header{
+				ID:       123,
+				Response: true,
+				OpCode:   0, // query
+			},
+			Question: dnsmessage.Question{
+				Name:  dnsmessage.MustNewName("nx-domain.test."),
+				Type:  dnsmessage.TypeA,
+				Class: dnsmessage.ClassINET,
+			},
+		})
+		if err != nil {
+			t.Fatal(err)
+		}
+		if len(res) < dnsHeaderLen {
+			t.Fatal("short reply")
+		}
+		rcode := dns.RCode(res[3] & 0x0f)
+		if rcode != dns.RCodeNameError {
+			t.Errorf("RCode = %v; want dns.RCodeNameError", rcode)
+			t.Logf("Response was: %q", res)
+		}
+	})
+
+	matchPacked := func(want string) func(t testing.TB, got []byte) {
+		return func(t testing.TB, got []byte) {
+			if string(got) == want {
+				return
+			}
+			t.Errorf("unexpected reply.\n got: %q\nwant: %q\n", got, want)
+			t.Errorf("\nin hex:\n got: % 2x\nwant: % 2x\n", got, want)
+		}
+	}
+
+	tests := []struct {
+		Type  dnsmessage.Type
+		Name  string
+		Check func(t testing.TB, got []byte)
+	}{
+		{
+			Type:  dnsmessage.TypeA,
+			Name:  "one-a.test.",
+			Check: matchPacked("\x00{\x84\x00\x00\x01\x00\x01\x00\x00\x00\x00\x05one-a\x04test\x00\x00\x01\x00\x01\x05one-a\x04test\x00\x00\x01\x00\x01\x00\x00\x02X\x00\x04\x01\x02\x03\x04"),
+		},
+		{
+			Type:  dnsmessage.TypeA,
+			Name:  "two-a.test.",
+			Check: matchPacked("\x00{\x84\x00\x00\x01\x00\x02\x00\x00\x00\x00\x05two-a\x04test\x00\x00\x01\x00\x01\xc0\f\x00\x01\x00\x01\x00\x00\x02X\x00\x04\x01\x02\x03\x04\xc0\f\x00\x01\x00\x01\x00\x00\x02X\x00\x04\x05\x06\a\b"),
+		},
+		{
+			Type:  dnsmessage.TypeAAAA,
+			Name:  "one-aaaa.test.",
+			Check: matchPacked("\x00{\x84\x00\x00\x01\x00\x01\x00\x00\x00\x00\bone-aaaa\x04test\x00\x00\x1c\x00\x01\bone-aaaa\x04test\x00\x00\x1c\x00\x01\x00\x00\x02X\x00\x10\x00\x01\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x02"),
+		},
+		{
+			Type:  dnsmessage.TypeAAAA,
+			Name:  "two-aaaa.test.",
+			Check: matchPacked("\x00{\x84\x00\x00\x01\x00\x02\x00\x00\x00\x00\btwo-aaaa\x04test\x00\x00\x1c\x00\x01\xc0\f\x00\x1c\x00\x01\x00\x00\x02X\x00\x10\x00\x01\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x02\xc0\f\x00\x1c\x00\x01\x00\x00\x02X\x00\x10\x00\x03\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x04"),
+		},
+		{
+			Type:  dnsmessage.TypePTR,
+			Name:  "4.3.2.1.in-addr.arpa.",
+			Check: matchPacked("\x00{\x84\x00\x00\x01\x00\x01\x00\x00\x00\x00\x014\x013\x012\x011\ain-addr\x04arpa\x00\x00\f\x00\x01\x014\x013\x012\x011\ain-addr\x04arpa\x00\x00\f\x00\x01\x00\x00\x02X\x00\t\x03foo\x03com\x00"),
+		},
+		{
+			Type:  dnsmessage.TypeCNAME,
+			Name:  "cname.test.",
+			Check: matchPacked("\x00{\x84\x00\x00\x01\x00\x01\x00\x00\x00\x00\x05cname\x04test\x00\x00\x05\x00\x01\x05cname\x04test\x00\x00\x05\x00\x01\x00\x00\x02X\x00\x10\nthe-target\x03foo\x00"),
+		},
+
+		// No records of various types
+		{
+			Type:  dnsmessage.TypeA,
+			Name:  "no-records.test.",
+			Check: matchPacked("\x00{\x84\x03\x00\x01\x00\x00\x00\x00\x00\x00\nno-records\x04test\x00\x00\x01\x00\x01"),
+		},
+		{
+			Type:  dnsmessage.TypeAAAA,
+			Name:  "no-records.test.",
+			Check: matchPacked("\x00{\x84\x03\x00\x01\x00\x00\x00\x00\x00\x00\nno-records\x04test\x00\x00\x1c\x00\x01"),
+		},
+		{
+			Type:  dnsmessage.TypeCNAME,
+			Name:  "no-records.test.",
+			Check: matchPacked("\x00{\x84\x03\x00\x01\x00\x00\x00\x00\x00\x00\nno-records\x04test\x00\x00\x05\x00\x01"),
+		},
+		{
+			Type:  dnsmessage.TypeSRV,
+			Name:  "no-records.test.",
+			Check: matchPacked("\x00{\x84\x03\x00\x01\x00\x00\x00\x00\x00\x00\nno-records\x04test\x00\x00!\x00\x01"),
+		},
+		{
+			Type:  dnsmessage.TypeTXT,
+			Name:  "txt.test.",
+			Check: matchPacked("\x00{\x84\x00\x00\x01\x00\x03\x00\x00\x00\x00\x03txt\x04test\x00\x00\x10\x00\x01\x03txt\x04test\x00\x00\x10\x00\x01\x00\x00\x02X\x00\t\btxt1=one\x03txt\x04test\x00\x00\x10\x00\x01\x00\x00\x02X\x00\t\btxt2=two\x03txt\x04test\x00\x00\x10\x00\x01\x00\x00\x02X\x00\v\ntxt3=three"),
+		},
+		{
+			Type:  dnsmessage.TypeSRV,
+			Name:  "srv.test.",
+			Check: matchPacked("\x00{\x84\x00\x00\x01\x00\x02\x00\x00\x00\x00\x03srv\x04test\x00\x00!\x00\x01\x03srv\x04test\x00\x00!\x00\x01\x00\x00\x02X\x00\x0f\x00\x01\x00\x02\x00\x03\x03foo\x03com\x00\x03srv\x04test\x00\x00!\x00\x01\x00\x00\x02X\x00\x0f\x00\x04\x00\x05\x00\x06\x03bar\x03com\x00"),
+		},
+		{
+			Type:  dnsmessage.TypeNS,
+			Name:  "ns.test.",
+			Check: matchPacked("\x00{\x84\x00\x00\x01\x00\x02\x00\x00\x00\x00\x02ns\x04test\x00\x00\x02\x00\x01\x02ns\x04test\x00\x00\x02\x00\x01\x00\x00\x02X\x00\t\x03ns1\x03foo\x00\x02ns\x04test\x00\x00\x02\x00\x01\x00\x00\x02X\x00\t\x03ns2\x03bar\x00"),
+		},
+	}
+
+	for _, tt := range tests {
+		t.Run(fmt.Sprintf("%v_%v", tt.Type, strings.Trim(tt.Name, ".")), func(t *testing.T) {
+			got, err := handleExitNodeDNSQueryWithNetPkg(context.Background(), backResolver, &response{
+				Header: dnsmessage.Header{
+					ID:       123,
+					Response: true,
+					OpCode:   0, // query
+				},
+				Question: dnsmessage.Question{
+					Name:  dnsmessage.MustNewName(tt.Name),
+					Type:  tt.Type,
+					Class: dnsmessage.ClassINET,
+				},
+			})
+			if err != nil {
+				t.Fatal(err)
+			}
+			if len(got) < dnsHeaderLen {
+				t.Errorf("short record")
+			}
+			if tt.Check != nil {
+				tt.Check(t, got)
+				if t.Failed() {
+					t.Errorf("Got: %q\nIn hex: % 02x", got, got)
+				}
+			}
+		})
+	}
+
+	wrapRes := newWrapResolver(backResolver)
+	ctx := context.Background()
+
+	t.Run("wrap_ip_a", func(t *testing.T) {
+		ips, err := wrapRes.LookupIP(ctx, "ip", "two-a.test.")
+		if err != nil {
+			t.Fatal(err)
+		}
+		if got, want := ips, []net.IP{
+			net.ParseIP("1.2.3.4").To4(),
+			net.ParseIP("5.6.7.8").To4(),
+		}; !reflect.DeepEqual(got, want) {
+			t.Errorf("LookupIP = %v; want %v", got, want)
+		}
+	})
+
+	t.Run("wrap_ip_aaaa", func(t *testing.T) {
+		ips, err := wrapRes.LookupIP(ctx, "ip", "two-aaaa.test.")
+		if err != nil {
+			t.Fatal(err)
+		}
+		if got, want := ips, []net.IP{
+			net.ParseIP("1::2"),
+			net.ParseIP("3::4"),
+		}; !reflect.DeepEqual(got, want) {
+			t.Errorf("LookupIP(v6) = %v; want %v", got, want)
+		}
+	})
+
+	t.Run("wrap_ip_nx", func(t *testing.T) {
+		ips, err := wrapRes.LookupIP(ctx, "ip", "nx-domain.test.")
+		if !isGoNoSuchHostError(err) {
+			t.Errorf("no NX domain = (%v, %v); want no host error", ips, err)
+		}
+	})
+
+	t.Run("wrap_srv", func(t *testing.T) {
+		_, srvs, err := wrapRes.LookupSRV(ctx, "", "", "srv.test.")
+		if err != nil {
+			t.Fatal(err)
+		}
+		if got, want := srvs, []*net.SRV{
+			{
+				Target:   "foo.com.",
+				Priority: 1,
+				Weight:   2,
+				Port:     3,
+			},
+			{
+				Target:   "bar.com.",
+				Priority: 4,
+				Weight:   5,
+				Port:     6,
+			},
+		}; !reflect.DeepEqual(got, want) {
+			jgot, _ := json.Marshal(got)
+			jwant, _ := json.Marshal(want)
+			t.Errorf("SRV = %s; want %s", jgot, jwant)
+		}
+	})
+
+	t.Run("wrap_txt", func(t *testing.T) {
+		txts, err := wrapRes.LookupTXT(ctx, "txt.test.")
+		if err != nil {
+			t.Fatal(err)
+		}
+		if got, want := txts, []string{"txt1=one", "txt2=two", "txt3=three"}; !reflect.DeepEqual(got, want) {
+			t.Errorf("TXT = %q; want %q", got, want)
+		}
+	})
+
+	t.Run("wrap_ns", func(t *testing.T) {
+		nss, err := wrapRes.LookupNS(ctx, "ns.test.")
+		if err != nil {
+			t.Fatal(err)
+		}
+		if got, want := nss, []*net.NS{
+			{Host: "ns1.foo."},
+			{Host: "ns2.bar."},
+		}; !reflect.DeepEqual(got, want) {
+			jgot, _ := json.Marshal(got)
+			jwant, _ := json.Marshal(want)
+			t.Errorf("NS = %s; want %s", jgot, jwant)
+		}
+	})
+}
+
+// newWrapResolver returns a resolver that uses r (via handleExitNodeDNSQueryWithNetPkg)
+// to make DNS requests.
+func newWrapResolver(r *net.Resolver) *net.Resolver {
+	if runtime.GOOS == "windows" {
+		panic("doesn't work on Windows") // golang.org/issue/33097
+	}
+	return &net.Resolver{
+		PreferGo: true,
+		Dial: func(ctx context.Context, network, addr string) (net.Conn, error) {
+			return &wrapResolverConn{ctx: ctx, r: r}, nil
+		},
+	}
+}
+
+type wrapResolverConn struct {
+	ctx context.Context
+	r   *net.Resolver
+	buf bytes.Buffer
+}
+
+var _ net.PacketConn = (*wrapResolverConn)(nil)
+
+func (*wrapResolverConn) Close() error                       { return nil }
+func (*wrapResolverConn) LocalAddr() net.Addr                { return fakeAddr{} }
+func (*wrapResolverConn) RemoteAddr() net.Addr               { return fakeAddr{} }
+func (*wrapResolverConn) SetDeadline(t time.Time) error      { return nil }
+func (*wrapResolverConn) SetReadDeadline(t time.Time) error  { return nil }
+func (*wrapResolverConn) SetWriteDeadline(t time.Time) error { return nil }
+
+func (a *wrapResolverConn) Read(p []byte) (n int, err error) {
+	n, _, err = a.ReadFrom(p)
+	return
+}
+
+func (a *wrapResolverConn) ReadFrom(p []byte) (n int, addr net.Addr, err error) {
+	n, err = a.buf.Read(p)
+	return n, fakeAddr{}, err
+}
+
+func (a *wrapResolverConn) Write(packet []byte) (n int, err error) {
+	return a.WriteTo(packet, fakeAddr{})
+}
+
+func (a *wrapResolverConn) WriteTo(q []byte, _ net.Addr) (n int, err error) {
+	resp := parseExitNodeQuery(q)
+	if resp == nil {
+		return 0, errors.New("bad query")
+	}
+	res, err := handleExitNodeDNSQueryWithNetPkg(context.Background(), a.r, resp)
+	if err != nil {
+		return 0, err
+	}
+	a.buf.Write(res)
+	return len(q), nil
+}
+
+type fakeAddr struct{}
+
+func (fakeAddr) Network() string { return "unused" }
+func (fakeAddr) String() string  { return "unused-todoAddr" }
+
+func TestUnARPA(t *testing.T) {
+	tests := []struct {
+		in, want string
+	}{
+		{"", ""},
+		{"bad", ""},
+		{"4.4.8.8.in-addr.arpa.", "8.8.4.4"},
+		{".in-addr.arpa.", ""},
+		{"e.0.0.2.0.0.0.0.0.0.0.0.0.0.0.0.b.0.8.0.a.0.0.4.0.b.8.f.7.0.6.2.ip6.arpa.", "2607:f8b0:400a:80b::200e"},
+		{".ip6.arpa.", ""},
+	}
+	for _, tt := range tests {
+		got, ok := unARPA(tt.in)
+		if ok != (got != "") {
+			t.Errorf("inconsistent results for %q: (%q, %v)", tt.in, got, ok)
+		}
+		if got != tt.want {
+			t.Errorf("unARPA(%q) = %q; want %q", tt.in, got, tt.want)
+		}
+	}
+}

net/dns/utf.go

@@ -0,0 +1,56 @@
+// Copyright (c) 2021 Tailscale Inc & AUTHORS All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package dns
+
+// This code is only used in Windows builds, but is in an
+// OS-independent file so tests can run all the time.
+
+import (
+	"bytes"
+	"encoding/binary"
+	"unicode/utf16"
+)
+
+// maybeUnUTF16 tries to detect whether bs contains UTF-16, and if so
+// translates it to regular UTF-8.
+//
+// Some of wsl.exe's output get printed as UTF-16, which breaks a
+// bunch of things. Try to detect this by looking for a zero byte in
+// the first few bytes of output (which will appear if any of those
+// codepoints are basic ASCII - very likely). From that we can infer
+// that UTF-16 is being printed, and the byte order in use, and we
+// decode that back to UTF-8.
+//
+// https://github.com/microsoft/WSL/issues/4607
+func maybeUnUTF16(bs []byte) []byte {
+	if len(bs)%2 != 0 {
+		// Can't be complete UTF-16.
+		return bs
+	}
+	checkLen := 20
+	if len(bs) < checkLen {
+		checkLen = len(bs)
+	}
+	zeroOff := bytes.IndexByte(bs[:checkLen], 0)
+	if zeroOff == -1 {
+		return bs
+	}
+
+	// We assume wsl.exe is trying to print an ASCII codepoint,
+	// meaning the zero byte is in the upper 8 bits of the
+	// codepoint. That means we can use the zero's byte offset to
+	// work out if we're seeing little-endian or big-endian
+	// UTF-16.
+	var endian binary.ByteOrder = binary.LittleEndian
+	if zeroOff%2 == 0 {
+		endian = binary.BigEndian
+	}
+
+	var u16 []uint16
+	for i := 0; i < len(bs); i += 2 {
+		u16 = append(u16, endian.Uint16(bs[i:]))
+	}
+	return []byte(string(utf16.Decode(u16)))
+}

net/dns/utf_test.go

@@ -0,0 +1,25 @@
+// Copyright (c) 2021 Tailscale Inc & AUTHORS All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package dns
+
+import "testing"
+
+func TestMaybeUnUTF16(t *testing.T) {
+	tests := []struct {
+		in   string
+		want string
+	}{
+		{"abc", "abc"},             // UTF-8
+		{"a\x00b\x00c\x00", "abc"}, // UTF-16-LE
+		{"\x00a\x00b\x00c", "abc"}, // UTF-16-BE
+	}
+
+	for _, test := range tests {
+		got := string(maybeUnUTF16([]byte(test.in)))
+		if got != test.want {
+			t.Errorf("maybeUnUTF16(%q) = %q, want %q", test.in, got, test.want)
+		}
+	}
+}

net/dns/wsl_windows.go

@@ -6,13 +6,13 @@ package dns
 
 import (
 	"bytes"
+	"errors"
 	"fmt"
 	"os"
 	"os/exec"
 	"os/user"
 	"strings"
 	"syscall"
-	"unicode/utf16"
 
 	"golang.org/x/sys/windows"
 	"tailscale.com/types/logger"
@@ -26,29 +26,7 @@ func wslDistros() ([]string, error) {
 		return nil, fmt.Errorf("%v: %q", err, string(b))
 	}
 
-	// The first line of output is a WSL header. E.g.
-	//
-	//	C:\tsdev>wsl.exe -l
-	//	Windows Subsystem for Linux Distributions:
-	//	Ubuntu-20.04 (Default)
-	//
-	// We can skip it by passing '-q', but here we put it to work.
-	// It turns out wsl.exe -l is broken, and outputs UTF-16 names
-	// that nothing can read. (Try `wsl.exe -l | more`.)
-	// So we look at the header to see if it's UTF-16.
-	// If so, we run the rest through a UTF-16 parser.
-	//
-	// https://github.com/microsoft/WSL/issues/4607
-	var output string
-	if bytes.HasPrefix(b, []byte("W\x00i\x00n\x00d\x00o\x00w\x00s\x00")) {
-		output, err = decodeUTF16(b)
-		if err != nil {
-			return nil, fmt.Errorf("failed to decode wsl.exe -l output %q: %v", b, err)
-		}
-	} else {
-		output = string(b)
-	}
-	lines := strings.Split(output, "\n")
+	lines := strings.Split(string(b), "\n")
 	if len(lines) < 1 {
 		return nil, nil
 	}
@@ -66,19 +44,6 @@ func wslDistros() ([]string, error) {
 	return distros, nil
 }
 
-func decodeUTF16(b []byte) (string, error) {
-	if len(b) == 0 {
-		return "", nil
-	} else if len(b)%2 != 0 {
-		return "", fmt.Errorf("decodeUTF16: invalid length %d", len(b))
-	}
-	var u16 []uint16
-	for i := 0; i < len(b); i += 2 {
-		u16 = append(u16, uint16(b[i])+(uint16(b[i+1])<<8))
-	}
-	return string(utf16.Decode(u16)), nil
-}
-
 // wslManager is a DNS manager for WSL2 linux distributions.
 // It configures /etc/wsl.conf and /etc/resolv.conf.
 type wslManager struct {
@@ -193,7 +158,8 @@ func (fs wslFS) Truncate(name string) error { return fs.WriteFile(name, nil, 064
 
 func (fs wslFS) ReadFile(name string) ([]byte, error) {
 	b, err := wslCombinedOutput(fs.cmd("cat", "--", name))
-	if ee, _ := err.(*exec.ExitError); ee != nil && ee.ExitCode() == 1 {
+	var ee *exec.ExitError
+	if errors.As(err, &ee) && ee.ExitCode() == 1 {
 		return nil, os.ErrNotExist
 	}
 	return b, err
@@ -225,7 +191,10 @@ func wslCombinedOutput(cmd *exec.Cmd) ([]byte, error) {
 	cmd.Stdout = buf
 	cmd.Stderr = buf
 	err := wslRun(cmd)
-	return buf.Bytes(), err
+	if err != nil {
+		return nil, err
+	}
+	return maybeUnUTF16(buf.Bytes()), nil
 }
 
 func wslRun(cmd *exec.Cmd) (err error) {

net/dnscache/messagecache.go

@@ -0,0 +1,314 @@
+// Copyright (c) 2021 Tailscale Inc & AUTHORS All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package dnscache
+
+import (
+	"encoding/binary"
+	"errors"
+	"fmt"
+	"io"
+	"sync"
+	"time"
+
+	"github.com/golang/groupcache/lru"
+	"golang.org/x/net/dns/dnsmessage"
+)
+
+// MessageCache is a cache that works at the DNS message layer,
+// with its cache keyed on a DNS wire-level question, and capable
+// of replying to DNS messages.
+//
+// Its zero value is ready for use with a default cache size.
+// Use SetMaxCacheSize to specify the cache size.
+//
+// It's safe for concurrent use.
+type MessageCache struct {
+	// Clock is a clock, for testing.
+	// If nil, time.Now is used.
+	Clock func() time.Time
+
+	mu           sync.Mutex
+	cacheSizeSet int       // 0 means default
+	cache        lru.Cache // msgQ => *msgCacheValue
+}
+
+func (c *MessageCache) now() time.Time {
+	if c.Clock != nil {
+		return c.Clock()
+	}
+	return time.Now()
+}
+
+// SetMaxCacheSize sets the maximum number of DNS cache entries that
+// can be stored.
+func (c *MessageCache) SetMaxCacheSize(n int) {
+	c.mu.Lock()
+	defer c.mu.Unlock()
+	c.cacheSizeSet = n
+	c.pruneLocked()
+}
+
+// Flush clears the cache.
+func (c *MessageCache) Flush() {
+	c.mu.Lock()
+	defer c.mu.Unlock()
+	c.cache.Clear()
+}
+
+// pruneLocked prunes down the cache size to the configured (or
+// default) max size.
+func (c *MessageCache) pruneLocked() {
+	max := c.cacheSizeSet
+	if max == 0 {
+		max = 500
+	}
+	for c.cache.Len() > max {
+		c.cache.RemoveOldest()
+	}
+}
+
+// msgQ is the MessageCache cache key.
+//
+// It's basically a golang.org/x/net/dns/dnsmessage#Question but the
+// Class is omitted (we only cache ClassINET) and we store a Go string
+// instead of a 256 byte dnsmessage.Name array.
+type msgQ struct {
+	Name string
+	Type dnsmessage.Type // A, AAAA, MX, etc
+}
+
+// A *msgCacheValue is the cached value for a msgQ (question) key.
+//
+// Despite using pointers for storage and methods, the value is
+// immutable once placed in the cache.
+type msgCacheValue struct {
+	Expires time.Time
+
+	// Answers are the minimum data to reconstruct a DNS response
+	// message. TTLs are added later when converting to a
+	// dnsmessage.Resource.
+	Answers []msgResource
+}
+
+type msgResource struct {
+	Name string
+	Type dnsmessage.Type // dnsmessage.UnknownResource.Type
+	Data []byte          // dnsmessage.UnknownResource.Data
+}
+
+// ErrCacheMiss is a sentinel error returned by MessageCache.ReplyFromCache
+// when the request can not be satisified from cache.
+var ErrCacheMiss = errors.New("cache miss")
+
+var parserPool = &sync.Pool{
+	New: func() interface{} { return new(dnsmessage.Parser) },
+}
+
+// ReplyFromCache writes a DNS reply to w for the provided DNS query message,
+// which must begin with the two ID bytes of a DNS message.
+//
+// If there's a cache miss, the message is invalid or unexpected,
+// ErrCacheMiss is returned. On cache hit, either nil or an error from
+// a w.Write call is returned.
+func (c *MessageCache) ReplyFromCache(w io.Writer, dnsQueryMessage []byte) error {
+	cacheKey, txID, ok := getDNSQueryCacheKey(dnsQueryMessage)
+	if !ok {
+		return ErrCacheMiss
+	}
+	now := c.now()
+
+	c.mu.Lock()
+	cacheEntI, _ := c.cache.Get(cacheKey)
+	v, ok := cacheEntI.(*msgCacheValue)
+	if ok && now.After(v.Expires) {
+		c.cache.Remove(cacheKey)
+		ok = false
+	}
+	c.mu.Unlock()
+
+	if !ok {
+		return ErrCacheMiss
+	}
+
+	ttl := uint32(v.Expires.Sub(now).Seconds())
+
+	packedRes, err := packDNSResponse(cacheKey, txID, ttl, v.Answers)
+	if err != nil {
+		return ErrCacheMiss
+	}
+	_, err = w.Write(packedRes)
+	return err
+}
+
+var (
+	errNotCacheable = errors.New("question not cacheable")
+)
+
+// AddCacheEntry adds a cache entry to the cache.
+// It returns an error if the entry could not be cached.
+func (c *MessageCache) AddCacheEntry(qPacket, res []byte) error {
+	cacheKey, qID, ok := getDNSQueryCacheKey(qPacket)
+	if !ok {
+		return errNotCacheable
+	}
+	now := c.now()
+	v := &msgCacheValue{}
+
+	p := parserPool.Get().(*dnsmessage.Parser)
+	defer parserPool.Put(p)
+
+	resh, err := p.Start(res)
+	if err != nil {
+		return fmt.Errorf("reading header in response: %w", err)
+	}
+	if resh.ID != qID {
+		return fmt.Errorf("response ID doesn't match query ID")
+	}
+	q, err := p.Question()
+	if err != nil {
+		return fmt.Errorf("reading 1st question in response: %w", err)
+	}
+	if _, err := p.Question(); err != dnsmessage.ErrSectionDone {
+		if err == nil {
+			return errors.New("unexpected 2nd question in response")
+		}
+		return fmt.Errorf("after reading 1st question in response: %w", err)
+	}
+	if resName := asciiLowerName(q.Name).String(); resName != cacheKey.Name {
+		return fmt.Errorf("response question name %q != question name %q", resName, cacheKey.Name)
+	}
+	for {
+		rh, err := p.AnswerHeader()
+		if err == dnsmessage.ErrSectionDone {
+			break
+		}
+		if err != nil {
+			return fmt.Errorf("reading answer: %w", err)
+		}
+		res, err := p.UnknownResource()
+		if err != nil {
+			return fmt.Errorf("reading resource: %w", err)
+		}
+		if rh.Class != dnsmessage.ClassINET {
+			continue
+		}
+
+		// Set the cache entry's expiration to the soonest
+		// we've seen. (They should all be the same, though)
+		expires := now.Add(time.Duration(rh.TTL) * time.Second)
+		if v.Expires.IsZero() || expires.Before(v.Expires) {
+			v.Expires = expires
+		}
+		v.Answers = append(v.Answers, msgResource{
+			Name: rh.Name.String(),
+			Type: rh.Type,
+			Data: res.Data, // doesn't alias; a copy from dnsmessage.unpackUnknownResource
+		})
+	}
+	c.addCacheValue(cacheKey, v)
+	return nil
+}
+
+func (c *MessageCache) addCacheValue(cacheKey msgQ, v *msgCacheValue) {
+	c.mu.Lock()
+	defer c.mu.Unlock()
+	c.cache.Add(cacheKey, v)
+	c.pruneLocked()
+}
+
+func getDNSQueryCacheKey(msg []byte) (cacheKey msgQ, txID uint16, ok bool) {
+	p := parserPool.Get().(*dnsmessage.Parser)
+	defer parserPool.Put(p)
+	h, err := p.Start(msg)
+	const dnsHeaderSize = 12
+	if err != nil || h.OpCode != 0 || h.Response || h.Truncated ||
+		len(msg) < dnsHeaderSize { // p.Start checks this anyway, but to be explicit for slicing below
+		return cacheKey, 0, false
+	}
+	var (
+		numQ    = binary.BigEndian.Uint16(msg[4:6])
+		numAns  = binary.BigEndian.Uint16(msg[6:8])
+		numAuth = binary.BigEndian.Uint16(msg[8:10])
+		numAddn = binary.BigEndian.Uint16(msg[10:12])
+	)
+	_ = numAddn // ignore this for now; do client OSes send EDNS additional? assume so, ignore.
+	if !(numQ == 1 && numAns == 0 && numAuth == 0) {
+		// Something weird. We don't want to deal with it.
+		return cacheKey, 0, false
+	}
+	q, err := p.Question()
+	if err != nil {
+		// Already verified numQ == 1 so shouldn't happen, but:
+		return cacheKey, 0, false
+	}
+	if q.Class != dnsmessage.ClassINET {
+		// We only cache the Internet class.
+		return cacheKey, 0, false
+	}
+	return msgQ{Name: asciiLowerName(q.Name).String(), Type: q.Type}, h.ID, true
+}
+
+func asciiLowerName(n dnsmessage.Name) dnsmessage.Name {
+	nb := n.Data[:]
+	if int(n.Length) < len(n.Data) {
+		nb = nb[:n.Length]
+	}
+	for i, b := range nb {
+		if 'A' <= b && b <= 'Z' {
+			n.Data[i] += 0x20
+		}
+	}
+	return n
+}
+
+// packDNSResponse builds a DNS response for the given question and
+// transaction ID. The response resource records will have have the
+// same provided TTL.
+func packDNSResponse(q msgQ, txID uint16, ttl uint32, answers []msgResource) ([]byte, error) {
+	var baseMem []byte // TODO: guess a max size based on looping over answers?
+	b := dnsmessage.NewBuilder(baseMem, dnsmessage.Header{
+		ID:            txID,
+		Response:      true,
+		OpCode:        0,
+		Authoritative: false,
+		Truncated:     false,
+		RCode:         dnsmessage.RCodeSuccess,
+	})
+	name, err := dnsmessage.NewName(q.Name)
+	if err != nil {
+		return nil, err
+	}
+	if err := b.StartQuestions(); err != nil {
+		return nil, err
+	}
+	if err := b.Question(dnsmessage.Question{
+		Name:  name,
+		Type:  q.Type,
+		Class: dnsmessage.ClassINET,
+	}); err != nil {
+		return nil, err
+	}
+	if err := b.StartAnswers(); err != nil {
+		return nil, err
+	}
+	for _, r := range answers {
+		name, err := dnsmessage.NewName(r.Name)
+		if err != nil {
+			return nil, err
+		}
+		if err := b.UnknownResource(dnsmessage.ResourceHeader{
+			Name:  name,
+			Type:  r.Type,
+			Class: dnsmessage.ClassINET,
+			TTL:   ttl,
+		}, dnsmessage.UnknownResource{
+			Type: r.Type,
+			Data: r.Data,
+		}); err != nil {
+			return nil, err
+		}
+	}
+	return b.Finish()
+}

net/dnscache/messagecache_test.go

@@ -0,0 +1,292 @@
+// Copyright (c) 2021 Tailscale Inc & AUTHORS All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package dnscache
+
+import (
+	"bytes"
+	"context"
+	"errors"
+	"fmt"
+	"net"
+	"runtime"
+	"testing"
+	"time"
+
+	"golang.org/x/net/dns/dnsmessage"
+	"tailscale.com/tstest"
+)
+
+func TestMessageCache(t *testing.T) {
+	clock := &tstest.Clock{
+		Start: time.Date(1987, 11, 1, 0, 0, 0, 0, time.UTC),
+	}
+	mc := &MessageCache{Clock: clock.Now}
+	mc.SetMaxCacheSize(2)
+	clock.Advance(time.Second)
+
+	var out bytes.Buffer
+	if err := mc.ReplyFromCache(&out, makeQ(1, "foo.com.")); err != ErrCacheMiss {
+		t.Fatalf("unexpected error: %v", err)
+	}
+
+	if err := mc.AddCacheEntry(
+		makeQ(2, "foo.com."),
+		makeRes(2, "FOO.COM.", ttlOpt(10),
+			&dnsmessage.AResource{A: [4]byte{127, 0, 0, 1}},
+			&dnsmessage.AResource{A: [4]byte{127, 0, 0, 2}})); err != nil {
+		t.Fatal(err)
+	}
+
+	// Expect cache hit, with 10 seconds remaining.
+	out.Reset()
+	if err := mc.ReplyFromCache(&out, makeQ(3, "foo.com.")); err != nil {
+		t.Fatalf("expected cache hit; got: %v", err)
+	}
+	if p := mustParseResponse(t, out.Bytes()); p.TxID != 3 {
+		t.Errorf("TxID = %v; want %v", p.TxID, 3)
+	} else if p.TTL != 10 {
+		t.Errorf("TTL = %v; want 10", p.TTL)
+	}
+
+	// One second elapses, expect a cache hit, with 9 seconds
+	// remaining.
+	clock.Advance(time.Second)
+	out.Reset()
+	if err := mc.ReplyFromCache(&out, makeQ(4, "foo.com.")); err != nil {
+		t.Fatalf("expected cache hit; got: %v", err)
+	}
+	if p := mustParseResponse(t, out.Bytes()); p.TxID != 4 {
+		t.Errorf("TxID = %v; want %v", p.TxID, 4)
+	} else if p.TTL != 9 {
+		t.Errorf("TTL = %v; want 9", p.TTL)
+	}
+
+	// Expect cache miss on MX record.
+	if err := mc.ReplyFromCache(&out, makeQ(4, "foo.com.", dnsmessage.TypeMX)); err != ErrCacheMiss {
+		t.Fatalf("expected cache miss on MX; got: %v", err)
+	}
+	// Expect cache miss on CHAOS class.
+	if err := mc.ReplyFromCache(&out, makeQ(4, "foo.com.", dnsmessage.ClassCHAOS)); err != ErrCacheMiss {
+		t.Fatalf("expected cache miss on CHAOS; got: %v", err)
+	}
+
+	// Ten seconds elapses; expect a cache miss.
+	clock.Advance(10 * time.Second)
+	if err := mc.ReplyFromCache(&out, makeQ(5, "foo.com.")); err != ErrCacheMiss {
+		t.Fatalf("expected cache miss, got: %v", err)
+	}
+}
+
+type parsedMeta struct {
+	TxID uint16
+	TTL  uint32
+}
+
+func mustParseResponse(t testing.TB, r []byte) (ret parsedMeta) {
+	t.Helper()
+	var p dnsmessage.Parser
+	h, err := p.Start(r)
+	if err != nil {
+		t.Fatal(err)
+	}
+	ret.TxID = h.ID
+	qq, err := p.AllQuestions()
+	if err != nil {
+		t.Fatalf("AllQuestions: %v", err)
+	}
+	if len(qq) != 1 {
+		t.Fatalf("num questions = %v; want 1", len(qq))
+	}
+	aa, err := p.AllAnswers()
+	if err != nil {
+		t.Fatalf("AllAnswers: %v", err)
+	}
+	for _, r := range aa {
+		if ret.TTL == 0 {
+			ret.TTL = r.Header.TTL
+		}
+		if ret.TTL != r.Header.TTL {
+			t.Fatal("mixed TTLs")
+		}
+	}
+	return ret
+}
+
+type responseOpt bool
+
+type ttlOpt uint32
+
+func makeQ(txID uint16, name string, opt ...interface{}) []byte {
+	opt = append(opt, responseOpt(false))
+	return makeDNSPkt(txID, name, opt...)
+}
+
+func makeRes(txID uint16, name string, opt ...interface{}) []byte {
+	opt = append(opt, responseOpt(true))
+	return makeDNSPkt(txID, name, opt...)
+}
+
+func makeDNSPkt(txID uint16, name string, opt ...interface{}) []byte {
+	typ := dnsmessage.TypeA
+	class := dnsmessage.ClassINET
+	var response bool
+	var answers []dnsmessage.ResourceBody
+	var ttl uint32 = 1 // one second by default
+	for _, o := range opt {
+		switch o := o.(type) {
+		case dnsmessage.Type:
+			typ = o
+		case dnsmessage.Class:
+			class = o
+		case responseOpt:
+			response = bool(o)
+		case dnsmessage.ResourceBody:
+			answers = append(answers, o)
+		case ttlOpt:
+			ttl = uint32(o)
+		default:
+			panic(fmt.Sprintf("unknown opt type %T", o))
+		}
+	}
+	qname := dnsmessage.MustNewName(name)
+	msg := dnsmessage.Message{
+		Header: dnsmessage.Header{ID: txID, Response: response},
+		Questions: []dnsmessage.Question{
+			{
+				Name:  qname,
+				Type:  typ,
+				Class: class,
+			},
+		},
+	}
+	for _, rb := range answers {
+		msg.Answers = append(msg.Answers, dnsmessage.Resource{
+			Header: dnsmessage.ResourceHeader{
+				Name:  qname,
+				Type:  typ,
+				Class: class,
+				TTL:   ttl,
+			},
+			Body: rb,
+		})
+	}
+	buf, err := msg.Pack()
+	if err != nil {
+		panic(err)
+	}
+	return buf
+}
+
+func TestASCIILowerName(t *testing.T) {
+	n := asciiLowerName(dnsmessage.MustNewName("Foo.COM."))
+	if got, want := n.String(), "foo.com."; got != want {
+		t.Errorf("got = %q; want %q", got, want)
+	}
+}
+
+func TestGetDNSQueryCacheKey(t *testing.T) {
+	tests := []struct {
+		name  string
+		pkt   []byte
+		want  msgQ
+		txID  uint16
+		anyTX bool
+	}{
+		{
+			name: "empty",
+		},
+		{
+			name: "a",
+			pkt:  makeQ(123, "foo.com."),
+			want: msgQ{"foo.com.", dnsmessage.TypeA},
+			txID: 123,
+		},
+		{
+			name: "aaaa",
+			pkt:  makeQ(6, "foo.com.", dnsmessage.TypeAAAA),
+			want: msgQ{"foo.com.", dnsmessage.TypeAAAA},
+			txID: 6,
+		},
+		{
+			name: "normalize_case",
+			pkt:  makeQ(123, "FoO.CoM."),
+			want: msgQ{"foo.com.", dnsmessage.TypeA},
+			txID: 123,
+		},
+		{
+			name: "ignore_response",
+			pkt:  makeRes(123, "foo.com."),
+		},
+		{
+			name: "ignore_question_with_answers",
+			pkt:  makeQ(2, "foo.com.", &dnsmessage.AResource{A: [4]byte{127, 0, 0, 1}}),
+		},
+		{
+			name:  "whatever_go_generates", // in case Go's net package grows functionality we don't handle
+			pkt:   getGoNetPacketDNSQuery("from-go.foo."),
+			want:  msgQ{"from-go.foo.", dnsmessage.TypeA},
+			anyTX: true,
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			got, gotTX, ok := getDNSQueryCacheKey(tt.pkt)
+			if !ok {
+				if tt.txID == 0 && got == (msgQ{}) {
+					return
+				}
+				t.Fatal("failed")
+			}
+			if got != tt.want {
+				t.Errorf("got %+v, want %+v", got, tt.want)
+			}
+			if gotTX != tt.txID && !tt.anyTX {
+				t.Errorf("got tx %v, want %v", gotTX, tt.txID)
+			}
+		})
+	}
+}
+
+func getGoNetPacketDNSQuery(name string) []byte {
+	if runtime.GOOS == "windows" {
+		// On Windows, Go's net.Resolver doesn't use the DNS client.
+		// See https://github.com/golang/go/issues/33097 which
+		// was approved but not yet implemented.
+		// For now just pretend it's implemented to make this test
+		// pass on Windows with complicated the caller.
+		return makeQ(123, name)
+	}
+	res := make(chan []byte, 1)
+	r := &net.Resolver{
+		PreferGo: true,
+		Dial: func(ctx context.Context, network, address string) (net.Conn, error) {
+			return goResolverConn(res), nil
+		},
+	}
+	r.LookupIP(context.Background(), "ip4", name)
+	return <-res
+}
+
+type goResolverConn chan<- []byte
+
+func (goResolverConn) Close() error                       { return nil }
+func (goResolverConn) LocalAddr() net.Addr                { return todoAddr{} }
+func (goResolverConn) RemoteAddr() net.Addr               { return todoAddr{} }
+func (goResolverConn) SetDeadline(t time.Time) error      { return nil }
+func (goResolverConn) SetReadDeadline(t time.Time) error  { return nil }
+func (goResolverConn) SetWriteDeadline(t time.Time) error { return nil }
+func (goResolverConn) Read([]byte) (int, error)           { return 0, errors.New("boom") }
+func (c goResolverConn) Write(p []byte) (int, error) {
+	select {
+	case c <- p[2:]: // skip 2 byte length for TCP mode DNS query
+	default:
+	}
+	return 0, errors.New("boom")
+}
+
+type todoAddr struct{}
+
+func (todoAddr) Network() string { return "unused" }
+func (todoAddr) String() string  { return "unused-todoAddr" }

net/dnsfallback/dnsfallback.go

@@ -90,7 +90,7 @@ func Lookup(ctx context.Context, host string) ([]netaddr.IP, error) {
 // serverName and serverIP of are, say, "derpN.tailscale.com".
 // queryName is the name being sought (e.g. "controlplane.tailscale.com"), passed as hint.
 func bootstrapDNSMap(ctx context.Context, serverName string, serverIP netaddr.IP, queryName string) (dnsMap, error) {
-	dialer := netns.NewDialer()
+	dialer := netns.NewDialer(log.Printf)
 	tr := http.DefaultTransport.(*http.Transport).Clone()
 	tr.Proxy = tshttpproxy.ProxyFromEnvironment
 	tr.DialContext = func(ctx context.Context, netw, addr string) (net.Conn, error) {

net/interfaces/interfaces.go

@@ -345,9 +345,18 @@ func (s *State) String() string {
 	return sb.String()
 }
 
+// An InterfaceFilter indicates whether EqualFiltered should use i when deciding whether two States are equal.
+// ips are all the IPPrefixes associated with i.
+type InterfaceFilter func(i Interface, ips []netaddr.IPPrefix) bool
+
+// An IPFilter indicates whether EqualFiltered should use ip when deciding whether two States are equal.
+// ip is an ip address associated with some interface under consideration.
+type IPFilter func(ip netaddr.IP) bool
+
 // EqualFiltered reports whether s and s2 are equal,
-// considering only interfaces in s for which filter returns true.
-func (s *State) EqualFiltered(s2 *State, filter func(i Interface, ips []netaddr.IPPrefix) bool) bool {
+// considering only interfaces in s for which filter returns true,
+// and considering only IPs for those interfaces for which filterIP returns true.
+func (s *State) EqualFiltered(s2 *State, useInterface InterfaceFilter, useIP IPFilter) bool {
 	if s == nil && s2 == nil {
 		return true
 	}
@@ -364,7 +373,7 @@ func (s *State) EqualFiltered(s2 *State, filter func(i Interface, ips []netaddr.
 	}
 	for iname, i := range s.Interface {
 		ips := s.InterfaceIPs[iname]
-		if !filter(i, ips) {
+		if !useInterface(i, ips) {
 			continue
 		}
 		i2, ok := s2.Interface[iname]
@@ -375,7 +384,7 @@ func (s *State) EqualFiltered(s2 *State, filter func(i Interface, ips []netaddr.
 		if !ok {
 			return false
 		}
-		if !interfacesEqual(i, i2) || !prefixesEqual(ips, ips2) {
+		if !interfacesEqual(i, i2) || !prefixesEqualFiltered(ips, ips2, useIP) {
 			return false
 		}
 	}
@@ -390,6 +399,21 @@ func interfacesEqual(a, b Interface) bool {
 		bytes.Equal([]byte(a.HardwareAddr), []byte(b.HardwareAddr))
 }
 
+func filteredIPPs(ipps []netaddr.IPPrefix, useIP IPFilter) []netaddr.IPPrefix {
+	// TODO: rewrite prefixesEqualFiltered to avoid making copies
+	x := make([]netaddr.IPPrefix, 0, len(ipps))
+	for _, ipp := range ipps {
+		if useIP(ipp.IP()) {
+			x = append(x, ipp)
+		}
+	}
+	return x
+}
+
+func prefixesEqualFiltered(a, b []netaddr.IPPrefix, useIP IPFilter) bool {
+	return prefixesEqual(filteredIPPs(a, useIP), filteredIPPs(b, useIP))
+}
+
 func prefixesEqual(a, b []netaddr.IPPrefix) bool {
 	if len(a) != len(b) {
 		return false
@@ -402,13 +426,24 @@ func prefixesEqual(a, b []netaddr.IPPrefix) bool {
 	return true
 }
 
-// FilterInteresting reports whether i is an interesting non-Tailscale interface.
-func FilterInteresting(i Interface, ips []netaddr.IPPrefix) bool {
+// UseInterestingInterfaces is an InterfaceFilter that reports whether i is an interesting interface.
+// An interesting interface if it is (a) not owned by Tailscale and (b) routes interesting IP addresses.
+// See UseInterestingIPs for the defition of an interesting IP address.
+func UseInterestingInterfaces(i Interface, ips []netaddr.IPPrefix) bool {
 	return !isTailscaleInterface(i.Name, ips) && anyInterestingIP(ips)
 }
 
-// FilterAll always returns true, to use EqualFiltered against all interfaces.
-func FilterAll(i Interface, ips []netaddr.IPPrefix) bool { return true }
+// UseInterestingIPs is an IPFilter that reports whether ip is an interesting IP address.
+// An IP address is interesting if it is neither a lopback not a link local unicast IP address.
+func UseInterestingIPs(ip netaddr.IP) bool {
+	return isInterestingIP(ip)
+}
+
+// UseAllInterfaces is an InterfaceFilter that includes all interfaces.
+func UseAllInterfaces(i Interface, ips []netaddr.IPPrefix) bool { return true }
+
+// UseAllIPs is an IPFilter that includes all all IPs.
+func UseAllIPs(ips netaddr.IP) bool { return true }
 
 func (s *State) HasPAC() bool { return s != nil && s.PAC != "" }
 
@@ -594,10 +629,7 @@ func anyInterestingIP(pfxs []netaddr.IPPrefix) bool {
 // should log in interfaces.State logging. We don't need to show
 // localhost or link-local addresses.
 func isInterestingIP(ip netaddr.IP) bool {
-	if ip.IsLoopback() || ip.IsLinkLocalUnicast() {
-		return false
-	}
-	return true
+	return !ip.IsLoopback() && !ip.IsLinkLocalUnicast()
 }
 
 var altNetInterfaces func() ([]Interface, error)

net/interfaces/interfaces_test.go

@@ -6,6 +6,7 @@ package interfaces
 
 import (
 	"encoding/json"
+	"net"
 	"testing"
 
 	"inet.af/netaddr"
@@ -28,7 +29,7 @@ func TestGetState(t *testing.T) {
 		t.Fatal(err)
 	}
 
-	if !st.EqualFiltered(st2, FilterAll) {
+	if !st.EqualFiltered(st2, UseAllInterfaces, UseAllIPs) {
 		// let's assume nobody was changing the system network interfaces between
 		// the two GetState calls.
 		t.Fatal("two States back-to-back were not equal")
@@ -68,3 +69,38 @@ func TestIsUsableV6(t *testing.T) {
 		}
 	}
 }
+
+func TestStateEqualFilteredIPFilter(t *testing.T) {
+	// s1 and s2 are identical, except that an "interesting" interface
+	// has gained an "uninteresting" IP address.
+
+	s1 := &State{
+		InterfaceIPs: map[string][]netaddr.IPPrefix{"x": {
+			netaddr.MustParseIPPrefix("42.0.0.0/8"),
+			netaddr.MustParseIPPrefix("169.254.0.0/16"), // link local unicast
+		}},
+		Interface: map[string]Interface{"x": {Interface: &net.Interface{Name: "x"}}},
+	}
+
+	s2 := &State{
+		InterfaceIPs: map[string][]netaddr.IPPrefix{"x": {
+			netaddr.MustParseIPPrefix("42.0.0.0/8"),
+			netaddr.MustParseIPPrefix("169.254.0.0/16"), // link local unicast
+			netaddr.MustParseIPPrefix("127.0.0.0/8"),    // loopback (added)
+		}},
+		Interface: map[string]Interface{"x": {Interface: &net.Interface{Name: "x"}}},
+	}
+
+	// s1 and s2 are different...
+	if s1.EqualFiltered(s2, UseAllInterfaces, UseAllIPs) {
+		t.Errorf("%+v != %+v", s1, s2)
+	}
+	// ...and they look different if you only restrict to interesting interfaces...
+	if s1.EqualFiltered(s2, UseInterestingInterfaces, UseAllIPs) {
+		t.Errorf("%+v != %+v when restricting to interesting interfaces _but not_ IPs", s1, s2)
+	}
+	// ...but because the additional IP address is uninteresting, we should treat them as the same.
+	if !s1.EqualFiltered(s2, UseInterestingInterfaces, UseInterestingIPs) {
+		t.Errorf("%+v == %+v when restricting to interesting interfaces and IPs", s1, s2)
+	}
+}

net/interfaces/interfaces_windows.go

@@ -116,8 +116,12 @@ func notTailscaleInterface(iface *winipcfg.IPAdapterAddresses) bool {
 	// TODO(bradfitz): do this without the Description method's
 	// utf16-to-string allocation. But at least we only do it for
 	// the virtual interfaces, for which there won't be many.
-	return !(iface.IfType == winipcfg.IfTypePropVirtual &&
-		iface.Description() == tsconst.WintunInterfaceDesc)
+	if iface.IfType != winipcfg.IfTypePropVirtual {
+		return true
+	}
+	desc := iface.Description()
+	return !(strings.Contains(desc, tsconst.WintunInterfaceDesc) ||
+		strings.Contains(desc, tsconst.WintunInterfaceDesc0_14))
 }
 
 // NonTailscaleInterfaces returns a map of interface LUID to interface

net/netcheck/netcheck.go

@@ -34,6 +34,7 @@ import (
 	"tailscale.com/tailcfg"
 	"tailscale.com/types/logger"
 	"tailscale.com/types/opt"
+	"tailscale.com/util/clientmetric"
 )
 
 // Debugging and experimentation tweakables.
@@ -232,6 +233,12 @@ func (c *Client) MakeNextReportFull() {
 func (c *Client) ReceiveSTUNPacket(pkt []byte, src netaddr.IPPort) {
 	c.vlogf("received STUN packet from %s", src)
 
+	if src.IP().Is4() {
+		metricSTUNRecv4.Add(1)
+	} else if src.IP().Is6() {
+		metricSTUNRecv6.Add(1)
+	}
+
 	c.mu.Lock()
 	if c.handleHairSTUNLocked(pkt, src) {
 		c.mu.Unlock()
@@ -737,7 +744,13 @@ func (c *Client) udpBindAddr() string {
 // GetReport gets a report.
 //
 // It may not be called concurrently with itself.
-func (c *Client) GetReport(ctx context.Context, dm *tailcfg.DERPMap) (*Report, error) {
+func (c *Client) GetReport(ctx context.Context, dm *tailcfg.DERPMap) (_ *Report, reterr error) {
+	defer func() {
+		if reterr != nil {
+			metricNumGetReportError.Add(1)
+		}
+	}()
+	metricNumGetReport.Add(1)
 	// Mask user context with ours that we guarantee to cancel so
 	// we can depend on it being closed in goroutines later.
 	// (User ctx might be context.Background, etc)
@@ -769,6 +782,7 @@ func (c *Client) GetReport(ctx context.Context, dm *tailcfg.DERPMap) (*Report, e
 		last = nil // causes makeProbePlan below to do a full (initial) plan
 		c.nextFull = false
 		c.lastFull = now
+		metricNumGetReportFull.Add(1)
 	}
 	rs.incremental = last != nil
 	c.mu.Unlock()
@@ -793,7 +807,7 @@ func (c *Client) GetReport(ctx context.Context, dm *tailcfg.DERPMap) (*Report, e
 	}
 
 	// Create a UDP4 socket used for sending to our discovered IPv4 address.
-	rs.pc4Hair, err = netns.Listener().ListenPacket(ctx, "udp4", ":0")
+	rs.pc4Hair, err = netns.Listener(c.logf).ListenPacket(ctx, "udp4", ":0")
 	if err != nil {
 		c.logf("udp4: %v", err)
 		return nil, err
@@ -821,7 +835,7 @@ func (c *Client) GetReport(ctx context.Context, dm *tailcfg.DERPMap) (*Report, e
 	if f := c.GetSTUNConn4; f != nil {
 		rs.pc4 = f()
 	} else {
-		u4, err := netns.Listener().ListenPacket(ctx, "udp4", c.udpBindAddr())
+		u4, err := netns.Listener(c.logf).ListenPacket(ctx, "udp4", c.udpBindAddr())
 		if err != nil {
 			c.logf("udp4: %v", err)
 			return nil, err
@@ -834,7 +848,7 @@ func (c *Client) GetReport(ctx context.Context, dm *tailcfg.DERPMap) (*Report, e
 		if f := c.GetSTUNConn6; f != nil {
 			rs.pc6 = f()
 		} else {
-			u6, err := netns.Listener().ListenPacket(ctx, "udp6", c.udpBindAddr())
+			u6, err := netns.Listener(c.logf).ListenPacket(ctx, "udp6", c.udpBindAddr())
 			if err != nil {
 				c.logf("udp6: %v", err)
 			} else {
@@ -983,6 +997,7 @@ func (c *Client) runHTTPOnlyChecks(ctx context.Context, last *Report, rs *report
 }
 
 func (c *Client) measureHTTPSLatency(ctx context.Context, reg *tailcfg.DERPRegion) (time.Duration, netaddr.IP, error) {
+	metricHTTPSend.Add(1)
 	var result httpstat.Result
 	ctx, cancel := context.WithTimeout(httpstat.WithHTTPStat(ctx, &result), overallProbeTimeout)
 	defer cancel()
@@ -1217,6 +1232,7 @@ func (rs *reportState) runProbe(ctx context.Context, dm *tailcfg.DERPMap, probe
 
 	switch probe.proto {
 	case probeIPv4:
+		metricSTUNSend4.Add(1)
 		n, err := rs.pc4.WriteTo(req, addr)
 		if n == len(req) && err == nil {
 			rs.mu.Lock()
@@ -1224,6 +1240,7 @@ func (rs *reportState) runProbe(ctx context.Context, dm *tailcfg.DERPMap, probe
 			rs.mu.Unlock()
 		}
 	case probeIPv6:
+		metricSTUNSend6.Add(1)
 		n, err := rs.pc6.WriteTo(req, addr)
 		if n == len(req) && err == nil {
 			rs.mu.Lock()
@@ -1322,3 +1339,15 @@ func conciseOptBool(b opt.Bool, trueVal string) string {
 	}
 	return ""
 }
+
+var (
+	metricNumGetReport      = clientmetric.NewCounter("netcheck_report")
+	metricNumGetReportFull  = clientmetric.NewCounter("netcheck_report_full")
+	metricNumGetReportError = clientmetric.NewCounter("netcheck_report_error")
+
+	metricSTUNSend4 = clientmetric.NewCounter("netcheck_stun_send_ipv4")
+	metricSTUNSend6 = clientmetric.NewCounter("netcheck_stun_send_ipv6")
+	metricSTUNRecv4 = clientmetric.NewCounter("netcheck_stun_recv_ipv4")
+	metricSTUNRecv6 = clientmetric.NewCounter("netcheck_stun_recv_ipv6")
+	metricHTTPSend  = clientmetric.NewCounter("netcheck_https_measure")
+)

net/netns/netns.go

@@ -21,6 +21,7 @@ import (
 	"inet.af/netaddr"
 	"tailscale.com/net/netknob"
 	"tailscale.com/syncs"
+	"tailscale.com/types/logger"
 )
 
 var disabled syncs.AtomicBool
@@ -34,19 +35,19 @@ func SetEnabled(on bool) {
 // Listener returns a new net.Listener with its Control hook func
 // initialized as necessary to run in logical network namespace that
 // doesn't route back into Tailscale.
-func Listener() *net.ListenConfig {
+func Listener(logf logger.Logf) *net.ListenConfig {
 	if disabled.Get() {
 		return new(net.ListenConfig)
 	}
-	return &net.ListenConfig{Control: control}
+	return &net.ListenConfig{Control: control(logf)}
 }
 
 // NewDialer returns a new Dialer using a net.Dialer with its Control
 // hook func initialized as necessary to run in a logical network
 // namespace that doesn't route back into Tailscale. It also handles
 // using a SOCKS if configured in the environment with ALL_PROXY.
-func NewDialer() Dialer {
-	return FromDialer(&net.Dialer{
+func NewDialer(logf logger.Logf) Dialer {
+	return FromDialer(logf, &net.Dialer{
 		KeepAlive: netknob.PlatformTCPKeepAlive(),
 	})
 }
@@ -55,11 +56,11 @@ func NewDialer() Dialer {
 // network namespace that doesn't route back into Tailscale. It also
 // handles using a SOCKS if configured in the environment with
 // ALL_PROXY.
-func FromDialer(d *net.Dialer) Dialer {
+func FromDialer(logf logger.Logf, d *net.Dialer) Dialer {
 	if disabled.Get() {
 		return d
 	}
-	d.Control = control
+	d.Control = control(logf)
 	if wrapDialer != nil {
 		return wrapDialer(d)
 	}

net/netns/netns_android.go

@@ -11,6 +11,8 @@ import (
 	"fmt"
 	"sync"
 	"syscall"
+
+	"tailscale.com/types/logger"
 )
 
 var (
@@ -44,11 +46,15 @@ func SetAndroidProtectFunc(f func(fd int) error) {
 	androidProtectFunc = f
 }
 
-// control marks c as necessary to dial in a separate network namespace.
+func control(logger.Logf) func(network, address string, c syscall.RawConn) error {
+	return controlC
+}
+
+// controlC marks c as necessary to dial in a separate network namespace.
 //
 // It's intentionally the same signature as net.Dialer.Control
 // and net.ListenConfig.Control.
-func control(network, address string, c syscall.RawConn) error {
+func controlC(network, address string, c syscall.RawConn) error {
 	var sockErr error
 	err := c.Control(func(fd uintptr) {
 		androidProtectFuncMu.Lock()

net/netns/netns_darwin_tailscaled.go

@@ -9,26 +9,32 @@ package netns
 
 import (
 	"fmt"
-	"log"
 	"strings"
 	"syscall"
 
 	"golang.org/x/sys/unix"
 	"tailscale.com/net/interfaces"
+	"tailscale.com/types/logger"
 )
 
-// control marks c as necessary to dial in a separate network namespace.
+func control(logf logger.Logf) func(network, address string, c syscall.RawConn) error {
+	return func(network, address string, c syscall.RawConn) error {
+		return controlLogf(logf, network, address, c)
+	}
+}
+
+// controlLogf marks c as necessary to dial in a separate network namespace.
 //
 // It's intentionally the same signature as net.Dialer.Control
 // and net.ListenConfig.Control.
-func control(network, address string, c syscall.RawConn) error {
+func controlLogf(logf logger.Logf, network, address string, c syscall.RawConn) error {
 	if strings.HasPrefix(address, "127.") || address == "::1" {
 		// Don't bind to an interface for localhost connections.
 		return nil
 	}
 	idx, err := interfaces.DefaultRouteInterfaceIndex()
 	if err != nil {
-		log.Printf("netns: DefaultRouteInterfaceIndex: %v", err)
+		logf("[unexpected] netns: DefaultRouteInterfaceIndex: %v", err)
 		return nil
 	}
 	v6 := strings.Contains(address, "]:") || strings.HasSuffix(network, "6") // hacky test for v6
@@ -47,7 +53,7 @@ func control(network, address string, c syscall.RawConn) error {
 		return fmt.Errorf("RawConn.Control on %T: %w", c, err)
 	}
 	if sockErr != nil {
-		log.Printf("netns: control(%q, %q), v6=%v, index=%v: %v", network, address, v6, idx, sockErr)
+		logf("[unexpected] netns: control(%q, %q), v6=%v, index=%v: %v", network, address, v6, idx, sockErr)
 	}
 	return sockErr
 }