tsweb: add DebugHandler.FlagSet

For runtime-modifiable values.

Signed-off-by: Josh Bleecher Snyder <josh@tailscale.com>

.gitattributes

@@ -1 +1,2 @@
 go.mod filter=go-mod
+*.go  diff=golang

.github/ISSUE_TEMPLATE/bug_report.yml

@@ -41,6 +41,7 @@ body:
         - Windows
         - iOS
         - Android
+        - Synology
         - Other
     validations:
       required: false
@@ -49,7 +50,7 @@ body:
     attributes:
       label: OS version
       description: What OS version are you using?
-      placeholder: e.g., Debian 11.0, macOS Big Sur 11.6
+      placeholder: e.g., Debian 11.0, macOS Big Sur 11.6, Synology DSM 7
     validations:
       required: false
   - type: input

.github/ISSUE_TEMPLATE/config.yml

@@ -1,5 +1,8 @@
 blank_issues_enabled: true
 contact_links:
-  - name: Support requests and Troubleshooting
+  - name: Support
+    url: https://tailscale.com/contact/support/
+    about: Contact us for support
+  - name: Troubleshooting
     url: https://tailscale.com/kb/1023/troubleshooting
-    about: Troubleshoot common issues. Contact us by email at support@tailscale.com.
+    about: Troubleshoot common issues

.github/ISSUE_TEMPLATE/feature_request.yml

@@ -7,14 +7,7 @@ body:
     attributes:
       value: |
         Please check if your feature request is [already filed](https://github.com/tailscale/tailscale/issues).
-  - type: input
-    id: request
-    attributes:
-      label: Tell us about your idea!
-      description: What is your feature request?
-      placeholder: e.g., A pet pangolin
-    validations:
-      required: true
+        Tell us about your idea!
   - type: textarea
     id: problem
     attributes:

.github/dependabot.yml

@@ -2,15 +2,20 @@
 # https://docs.github.com/en/code-security/supply-chain-security/keeping-your-dependencies-updated-automatically/configuration-options-for-dependency-updates
 version: 2
 updates:
-  - package-ecosystem: "gomod"
-    directory: "/"
-    schedule:
-      interval: "daily"
-    commit-message:
-      prefix: "go.mod:"
+  ## Disabled between releases. We reenable it briefly after every
+  ## stable release, pull in all changes, and close it again so that
+  ## the tree remains more stable during development and the upstream
+  ## changes have time to soak before the next release.
+  # - package-ecosystem: "gomod"
+  #   directory: "/"
+  #   schedule:
+  #     interval: "daily"
+  #   commit-message:
+  #     prefix: "go.mod:"
+  #   open-pull-requests-limit: 100
   - package-ecosystem: "github-actions"
     directory: "/"
     schedule:
-      interval: "daily"
+      interval: "weekly"
     commit-message:
       prefix: ".github:"

.github/workflows/cifuzz.yml

@@ -0,0 +1,26 @@
+name: CIFuzz
+on: [pull_request]
+jobs:
+  Fuzzing:
+    runs-on: ubuntu-latest
+    steps:
+    - name: Build Fuzzers
+      id: build
+      uses: google/oss-fuzz/infra/cifuzz/actions/build_fuzzers@master
+      with:
+        oss-fuzz-project-name: 'tailscale'
+        dry-run: false
+        language: go
+    - name: Run Fuzzers
+      uses: google/oss-fuzz/infra/cifuzz/actions/run_fuzzers@master
+      with:
+        oss-fuzz-project-name: 'tailscale'
+        fuzz-seconds: 300
+        dry-run: false
+        language: go
+    - name: Upload Crash
+      uses: actions/upload-artifact@v2.3.1
+      if: failure() && steps.build.outcome == 'success'
+      with:
+        name: artifacts
+        path: ./out/artifacts

.github/workflows/cross-darwin.yml

@@ -17,7 +17,7 @@ jobs:
     steps:
 
     - name: Set up Go
-      uses: actions/setup-go@v2.1.4
+      uses: actions/setup-go@v2.1.5
       with:
         go-version: 1.17
       id: go
@@ -37,6 +37,12 @@ jobs:
         GOARCH: amd64
       run: for d in $(go list -f '{{if .TestGoFiles}}{{.Dir}}{{end}}' ./... ); do (echo $d; cd $d && go test -c ); done
 
+    - name: iOS build most
+      env:
+        GOOS: ios
+        GOARCH: arm64
+      run: go install ./ipn/... ./wgengine/ ./types/... ./control/controlclient
+
     - uses: k0kubun/action-slack@v2.0.0
       with:
         payload: |

.github/workflows/cross-freebsd.yml

@@ -17,7 +17,7 @@ jobs:
     steps:
 
     - name: Set up Go
-      uses: actions/setup-go@v2.1.4
+      uses: actions/setup-go@v2.1.5
       with:
         go-version: 1.17
       id: go

.github/workflows/cross-openbsd.yml

@@ -17,7 +17,7 @@ jobs:
     steps:
 
     - name: Set up Go
-      uses: actions/setup-go@v2.1.4
+      uses: actions/setup-go@v2.1.5
       with:
         go-version: 1.17
       id: go

.github/workflows/cross-windows.yml

@@ -17,7 +17,7 @@ jobs:
     steps:
 
     - name: Set up Go
-      uses: actions/setup-go@v2.1.4
+      uses: actions/setup-go@v2.1.5
       with:
         go-version: 1.17
       id: go

.github/workflows/depaware.yml

@@ -14,7 +14,7 @@ jobs:
 
     steps:
     - name: Set up Go
-      uses: actions/setup-go@v2.1.4
+      uses: actions/setup-go@v2.1.5
       with:
         go-version: 1.17
 

.github/workflows/go_generate.yml

@@ -15,7 +15,7 @@ jobs:
 
     steps:
       - name: Set up Go
-        uses: actions/setup-go@v2.1.4
+        uses: actions/setup-go@v2.1.5
         with:
           go-version: 1.17
 
@@ -30,7 +30,7 @@ jobs:
           then
             pkgs=$(go list ./... | grep -v dnsfallback)
           else
-            pkgs=$(go list ./...)
+            pkgs=$(go list ./... | grep -v dnsfallback)
           fi
           go generate $pkgs
           echo

.github/workflows/license.yml

@@ -14,7 +14,7 @@ jobs:
 
     steps:
     - name: Set up Go
-      uses: actions/setup-go@v2.1.4
+      uses: actions/setup-go@v2.1.5
       with:
         go-version: 1.17
 

.github/workflows/linux-race.yml

@@ -17,7 +17,7 @@ jobs:
     steps:
 
     - name: Set up Go
-      uses: actions/setup-go@v2.1.4
+      uses: actions/setup-go@v2.1.5
       with:
         go-version: 1.17
       id: go
@@ -31,6 +31,21 @@ jobs:
     - name: Run tests and benchmarks with -race flag on linux
       run: go test -race -bench=. -benchtime=1x ./...
 
+    - name: Check that no tracked files in the repo have been modified
+      run: git diff --no-ext-diff --name-only --exit-code || (echo "Build/test modified the files above."; exit 1)
+
+    - name: Check that no files have been added to the repo
+      run: |
+        # Note: The "error: pathspec..." you see below is normal!
+        # In the success case in which there are no new untracked files,
+        # git ls-files complains about the pathspec not matching anything.
+        # That's OK. It's not worth the effort to suppress. Please ignore it.
+        if git ls-files --others --exclude-standard --directory --no-empty-directory --error-unmatch -- ':/*'
+        then
+          echo "Build/test created untracked files in the repo (file names above)."
+          exit 1
+        fi
+
     - uses: k0kubun/action-slack@v2.0.0
       with:
         payload: |

.github/workflows/linux.yml

@@ -17,7 +17,7 @@ jobs:
     steps:
 
     - name: Set up Go
-      uses: actions/setup-go@v2.1.4
+      uses: actions/setup-go@v2.1.5
       with:
         go-version: 1.17
       id: go
@@ -28,9 +28,33 @@ jobs:
     - name: Basic build
       run: go build ./cmd/...
 
+    - name: Get QEMU
+      run: |
+        # The qemu in Ubuntu 20.04 (Focal) is too old; we need 5.x something
+        # to run Go binaries. 5.2.0 (Debian bullseye) empirically works, and
+        # use this PPA which brings in a modern qemu.
+        sudo add-apt-repository -y ppa:jacob/virtualisation
+        sudo apt-get -y update
+        sudo apt-get -y install qemu-user
+
     - name: Run tests on linux
       run: go test -bench=. -benchtime=1x ./...
 
+    - name: Check that no tracked files in the repo have been modified
+      run: git diff --no-ext-diff --name-only --exit-code || (echo "Build/test modified the files above."; exit 1)
+
+    - name: Check that no files have been added to the repo
+      run: |
+        # Note: The "error: pathspec..." you see below is normal!
+        # In the success case in which there are no new untracked files,
+        # git ls-files complains about the pathspec not matching anything.
+        # That's OK. It's not worth the effort to suppress. Please ignore it.
+        if git ls-files --others --exclude-standard --directory --no-empty-directory --error-unmatch -- ':/*'
+        then
+          echo "Build/test created untracked files in the repo (file names above)."
+          exit 1
+        fi
+
     - uses: k0kubun/action-slack@v2.0.0
       with:
         payload: |

.github/workflows/linux32.yml

@@ -17,7 +17,7 @@ jobs:
     steps:
 
     - name: Set up Go
-      uses: actions/setup-go@v2.1.4
+      uses: actions/setup-go@v2.1.5
       with:
         go-version: 1.17
       id: go
@@ -31,6 +31,21 @@ jobs:
     - name: Run tests on linux
       run: GOARCH=386 go test -bench=. -benchtime=1x ./...
 
+    - name: Check that no tracked files in the repo have been modified
+      run: git diff --no-ext-diff --name-only --exit-code || (echo "Build/test modified the files above."; exit 1)
+
+    - name: Check that no files have been added to the repo
+      run: |
+        # Note: The "error: pathspec..." you see below is normal!
+        # In the success case in which there are no new untracked files,
+        # git ls-files complains about the pathspec not matching anything.
+        # That's OK. It's not worth the effort to suppress. Please ignore it.
+        if git ls-files --others --exclude-standard --directory --no-empty-directory --error-unmatch -- ':/*'
+        then
+          echo "Build/test created untracked files in the repo (file names above)."
+          exit 1
+        fi
+
     - uses: k0kubun/action-slack@v2.0.0
       with:
         payload: |

.github/workflows/staticcheck.yml

@@ -14,7 +14,7 @@ jobs:
 
     steps:
     - name: Set up Go
-      uses: actions/setup-go@v2.1.4
+      uses: actions/setup-go@v2.1.5
       with:
         go-version: 1.17
 

.github/workflows/vm.yml

@@ -12,11 +12,13 @@ jobs:
     if: "!contains(github.event.head_commit.message, '[ci skip]')"
 
     steps:
+      - name: Set GOPATH
+        run: echo "GOPATH=$HOME/go" >> $GITHUB_ENV
+
       - name: Set up Go
-        uses: actions/setup-go@v1
+        uses: actions/setup-go@v2.1.5
         with:
           go-version: 1.17
-        id: go
 
       - name: Checkout Code
         uses: actions/checkout@v1

.github/workflows/windows-race.yml

@@ -17,7 +17,7 @@ jobs:
     steps:
 
     - name: Install Go
-      uses: actions/setup-go@v2.1.4
+      uses: actions/setup-go@v2.1.5
       with:
         go-version: 1.17.x
 

.github/workflows/windows.yml

@@ -17,7 +17,7 @@ jobs:
     steps:
 
     - name: Install Go
-      uses: actions/setup-go@v2.1.4
+      uses: actions/setup-go@v2.1.5
       with:
         go-version: 1.17.x
 

.gitignore

@@ -5,6 +5,7 @@
 *.dll
 *.so
 *.dylib
+*.spk
 
 cmd/tailscale/tailscale
 cmd/tailscaled/tailscaled

Dockerfile

@@ -17,11 +17,11 @@
 #
 # To build the Dockerfile:
 #
-#     $ docker build -t tailscale:tailscale .
+#     $ docker build -t tailscale/tailscale .
 #
 # To run the tailscaled agent:
 #
-#     $ docker run -d --name=tailscaled -v /var/lib:/var/lib -v /dev/net/tun:/dev/net/tun --network=host --privileged tailscale:tailscale tailscaled
+#     $ docker run -d --name=tailscaled -v /var/lib:/var/lib -v /dev/net/tun:/dev/net/tun --network=host --privileged tailscale/tailscale tailscaled
 #
 # To then log in:
 #
@@ -48,13 +48,13 @@ ARG VERSION_SHORT=""
 ENV VERSION_SHORT=$VERSION_SHORT
 ARG VERSION_GIT_HASH=""
 ENV VERSION_GIT_HASH=$VERSION_GIT_HASH
+ARG TARGETARCH
 
-RUN go install -tags=xversion -ldflags="\
+RUN GOARCH=$TARGETARCH go install -ldflags="\
       -X tailscale.com/version.Long=$VERSION_LONG \
       -X tailscale.com/version.Short=$VERSION_SHORT \
       -X tailscale.com/version.GitCommit=$VERSION_GIT_HASH" \
       -v ./cmd/tailscale ./cmd/tailscaled
 
-FROM alpine:3.14
-RUN apk add --no-cache ca-certificates iptables iproute2 ip6tables
+FROM ghcr.io/tailscale/alpine-base:3.14
 COPY --from=build-env /go/bin/* /usr/local/bin/

Dockerfile.base

@@ -0,0 +1,6 @@
+# Copyright (c) 2021 Tailscale Inc & AUTHORS All rights reserved.
+# Use of this source code is governed by a BSD-style
+# license that can be found in the LICENSE file.
+
+FROM alpine:3.15
+RUN apk add --no-cache ca-certificates iptables iproute2 ip6tables

Makefile

@@ -1,3 +1,7 @@
+IMAGE_REPO ?= tailscale/tailscale
+SYNO_ARCH ?= "amd64"
+SYNO_DSM ?= "7"
+
 usage:
 	echo "See Makefile"
 
@@ -21,7 +25,22 @@ build386:
 buildlinuxarm:
 	GOOS=linux GOARCH=arm go install tailscale.com/cmd/tailscale tailscale.com/cmd/tailscaled
 
+buildmultiarchimage:
+	./build_docker.sh
+
 check: staticcheck vet depaware buildwindows build386 buildlinuxarm
 
 staticcheck:
 	go run honnef.co/go/tools/cmd/staticcheck -- $$(go list ./... | grep -v tempfork)
+
+spk:
+	PATH="${PWD}/tool:${PATH}" ./tool/go run github.com/tailscale/tailscale-synology@main -o tailscale.spk --source=. --goarch=${SYNO_ARCH} --dsm-version=${SYNO_DSM}
+
+spkall:
+	mkdir -p spks
+	PATH="${PWD}/tool:${PATH}" ./tool/go run github.com/tailscale/tailscale-synology@main -o spks --source=. --goarch=all --dsm-version=all
+
+pushspk: spk
+	echo "Pushing SPK to root@${SYNO_HOST} (env var SYNO_HOST) ..."
+	scp tailscale.spk root@${SYNO_HOST}:
+	ssh root@${SYNO_HOST} /usr/syno/bin/synopkg install tailscale.spk

README.md

@@ -8,11 +8,12 @@ Private WireGuard® networks made easy
 
 This repository contains all the open source Tailscale client code and
 the `tailscaled` daemon and `tailscale` CLI tool. The `tailscaled`
-daemon runs primarily on Linux; it also works to varying degrees on
-FreeBSD, OpenBSD, Darwin, and Windows.
+daemon runs on Linux, Windows and [macOS](https://tailscale.com/kb/1065/macos-variants/), and to varying degrees on FreeBSD, OpenBSD, and Darwin. (The Tailscale iOS and Android apps use this repo's code, but this repo doesn't contain the mobile GUI code.)
 
 The Android app is at https://github.com/tailscale/tailscale-android
 
+The Synology package is at https://github.com/tailscale/tailscale-synology
+
 ## Using
 
 We serve packages for a variety of distros at

VERSION.txt

@@ -1 +1 @@
-1.17.0
+1.21.0

api.md

@@ -15,6 +15,10 @@ Currently based on {some authentication method}. Visit the [admin panel](https:/
     - [POST device routes](#device-routes-post)
   - Authorize machine
     - [POST device authorized](#device-authorized-post)
+  - Tags
+    - [POST device tags](#device-tags-post)
+  - Key
+    - [POST device key](#device-key-post)
 * **[Tailnets](#tailnet)**
   - ACLs
     - [GET tailnet ACL](#tailnet-acl-get)
@@ -23,6 +27,11 @@ Currently based on {some authentication method}. Visit the [admin panel](https:/
 	- [POST tailnet ACL validate](#tailnet-acl-validate-post): run validation tests against the tailnet's existing ACL
   - [Devices](#tailnet-devices)
     - [GET tailnet devices](#tailnet-devices-get)
+  - [Keys](#tailnet-keys)
+    - [GET tailnet keys](#tailnet-keys-get)
+    - [POST tailnet key](#tailnet-keys-post)
+    - [GET tailnet key](#tailnet-keys-key-get)
+    - [DELETE tailnet key](#tailnet-keys-key-delete)
   - [DNS](#tailnet-dns)
     - [GET tailnet DNS nameservers](#tailnet-dns-nameservers-get)
     - [POST tailnet DNS nameservers](#tailnet-dns-nameservers-post)
@@ -263,6 +272,68 @@ curl 'https://api.tailscale.com/api/v2/device/11055/authorized' \
 The response is 2xx on success. The response body is currently an empty JSON
 object.
 
+<a name=device-tags-post></a>
+
+#### `POST /api/v2/device/:deviceID/tags` - update tags on a device
+
+Updates the tags set on a device.
+
+##### Parameters
+
+###### POST Body
+
+`tags` - The new list of tags for the device.
+
+```
+{
+  "tags": ["tag:foo", "tag:bar"]
+}
+```
+
+##### Example
+
+```
+curl 'https://api.tailscale.com/api/v2/device/11055/tags' \
+-u "tskey-yourapikey123:" \
+--data-binary '{"tags": ["tag:foo", "tag:bar"]}'
+```
+
+The response is 2xx on success. The response body is currently an empty JSON
+object.
+
+<a name=device-key-post></a>
+
+#### `POST /api/v2/device/:deviceID/key` - update device key
+
+Allows for updating properties on the device key.
+
+##### Parameters
+
+###### POST Body
+
+`keyExpiryDisabled`
+
+- Provide `true` to disable the device's key expiry. The original key expiry time is still maintained. Upon re-enabling, the key will expire at that original time.
+- Provide `false` to enable the device's key expiry. Sets the key to expire at the original expiry time prior to disabling. The key may already have expired. In that case, the device must be re-authenticated.
+- Empty value will not change the key expiry.
+
+```
+{
+  "keyExpiryDisabled": true
+}
+```
+
+##### Example
+
+```
+curl 'https://api.tailscale.com/api/v2/device/11055/key' \
+-u "tskey-yourapikey123:" \
+--data-binary '{"keyExpiryDisabled": true}'
+```
+
+The response is 2xx on success. The response body is currently an empty JSON
+object.
+
 ## Tailnet 
 A tailnet is the name of your Tailscale network. 
 You can find it in the top left corner of the [Admin Panel](https://login.tailscale.com/admin) beside the Tailscale logo.
@@ -670,6 +741,159 @@ Response
 }
 ```
 
+<a name=tailnet-keys></a>
+
+### Keys
+
+<a name=tailnet-keys-get></a>
+
+#### `GET /api/v2/tailnet/:tailnet/keys` - list the keys for a tailnet
+
+Returns a list of active keys for a tailnet
+for the user who owns the API key used to perform this query.
+Supply the tailnet of interest in the path.
+
+##### Parameters
+No parameters.
+
+##### Returns
+
+Returns a JSON object with the IDs of all active keys.
+This includes both API keys and also machine authentication keys.
+In the future, this may provide more information about each key than just the ID.
+
+##### Example
+
+```
+curl 'https://api.tailscale.com/api/v2/tailnet/example.com/keys' \
+  -u "tskey-yourapikey123:"
+```
+
+Response:
+```
+{"keys": [
+	{"id": "kYKVU14CNTRL"},
+	{"id": "k68VdZ3CNTRL"},
+	{"id": "kJ9nq43CNTRL"},
+	{"id": "kkThgj1CNTRL"}
+]}
+```
+
+<a name=tailnet-keys-post></a>
+
+#### `POST /api/v2/tailnet/:tailnet/keys` - create a new key for a tailnet
+
+Create a new key in a tailnet associated
+with the user who owns the API key used to perform this request.
+Supply the tailnet in the path.
+
+##### Parameters
+
+###### POST Body
+`capabilities` - A mapping of resources to permissible actions.
+```
+{
+	"capabilities": {
+    "devices": {
+      "create": {
+        "reusable": false,
+        "ephemeral": false
+      }
+    }
+  }
+}
+```
+
+##### Returns
+
+Returns a JSON object with the provided capabilities in addition to the
+generated key. The key should be recorded and kept safe and secure as it
+wields the capabilities specified in the request. The identity of the key
+is embedded in the key itself and can be used to perform operations on
+the key (e.g., revoking it or retrieving information about it).
+The full key can no longer be retrieved by the server.
+
+##### Example
+
+```
+echo '{
+	"capabilities": {
+    "devices": {
+      "create": {
+        "reusable": false,
+        "ephemeral": false
+      }
+    }
+  }
+}' | curl -X POST --data-binary @- https://api.tailscale.com/api/v2/tailnet/example.com/keys \
+  -u "tskey-yourapikey123:" \
+  -H "Content-Type: application/json" | jsonfmt
+```
+
+Response:
+```
+{
+	"id":           "k123456CNTRL",
+	"key":          "tskey-k123456CNTRL-abcdefghijklmnopqrstuvwxyz",
+	"created":      "2021-12-09T23:22:39Z",
+	"expires":      "2022-03-09T23:22:39Z",
+	"capabilities": {"devices": {"create": {"reusable": false, "ephemeral": false}}}
+}
+```
+
+<a name=tailnet-keys-key-get></a>
+
+#### `GET /api/v2/tailnet/:tailnet/keys/:keyid` - get information for a specific key
+
+Returns a JSON object with information about specific key.
+Supply the tailnet and key ID of interest in the path.
+
+##### Parameters
+No parameters.
+
+##### Returns
+
+Returns a JSON object with information about the key such as
+when it was created and when it expires.
+It also lists the capabilities associated with the key.
+
+##### Example
+
+```
+curl 'https://api.tailscale.com/api/v2/tailnet/example.com/keys/k123456CNTRL' \
+  -u "tskey-yourapikey123:"
+```
+
+Response:
+```
+{
+	"id":           "k123456CNTRL",
+	"created":      "2021-12-09T22:13:53Z",
+	"expires":      "2022-03-09T22:13:53Z",
+	"capabilities": {"devices": {"create": {"reusable": false, "ephemeral": false}}}
+}
+```
+
+<a name=tailnet-keys-key-delete></a>
+
+#### `DELETE /api/v2/tailnet/:tailnet/keys/:keyid` - delete a specific key
+
+Deletes a specific key.
+Supply the tailnet and key ID of interest in the path.
+
+##### Parameters
+No parameters.
+
+##### Returns
+This reports status 200 upon success.
+
+##### Example
+
+```
+curl -X DELETE 'https://api.tailscale.com/api/v2/tailnet/example.com/keys/k123456CNTRL' \
+  -u "tskey-yourapikey123:"
+```
+
 <a name=tailnet-dns></a>
 
 ### DNS

build_dist.sh

@@ -30,12 +30,14 @@ else
 fi
 
 long_suffix="$change_suffix-t$short_hash"
-SHORT="$major.$minor.$patch"
+MINOR="$major.$minor"
+SHORT="$MINOR.$patch"
 LONG="${SHORT}$long_suffix"
 GIT_HASH="$git_hash"
 
 if [ "$1" = "shellvars" ]; then
 	cat <<EOF
+VERSION_MINOR="$MINOR"
 VERSION_SHORT="$SHORT"
 VERSION_LONG="$LONG"
 VERSION_GIT_HASH="$GIT_HASH"
@@ -43,4 +45,4 @@ EOF
 	exit 0
 fi
 
-exec go build -ldflags "-X tailscale.com/version.Long=${LONG} -X tailscale.com/version.Short=${SHORT} -X tailscale.com/version.GitCommit=${GIT_HASH}" "$@"
+exec ./tool/go build -ldflags "-X tailscale.com/version.Long=${LONG} -X tailscale.com/version.Short=${SHORT} -X tailscale.com/version.GitCommit=${GIT_HASH}" "$@"

build_docker.sh

@@ -19,10 +19,28 @@
 
 set -eu
 
-eval $(./build_dist.sh shellvars)
+# Use the "go" binary from the "tool" directory (which is github.com/tailscale/go)
+export PATH=$PWD/tool:$PATH
 
-docker build \
-  --build-arg VERSION_LONG=$VERSION_LONG \
-  --build-arg VERSION_SHORT=$VERSION_SHORT \
-  --build-arg VERSION_GIT_HASH=$VERSION_GIT_HASH \
-  -t tailscale:$VERSION_SHORT -t tailscale:latest .
+eval $(./build_dist.sh shellvars)
+DEFAULT_TAGS="v${VERSION_SHORT},v${VERSION_MINOR}"
+DEFAULT_REPOS="tailscale/tailscale,ghcr.io/tailscale/tailscale"
+DEFAULT_BASE="ghcr.io/tailscale/alpine-base:3.14"
+
+PUSH="${PUSH:-false}"
+REPOS="${REPOS:-${DEFAULT_REPOS}}"
+TAGS="${TAGS:-${DEFAULT_TAGS}}"
+BASE="${BASE:-${DEFAULT_BASE}}"
+
+go run github.com/tailscale/mkctr@latest \
+  --gopaths="\
+    tailscale.com/cmd/tailscale:/usr/local/bin/tailscale, \
+    tailscale.com/cmd/tailscaled:/usr/local/bin/tailscaled" \
+  --ldflags="\
+    -X tailscale.com/version.Long=${VERSION_LONG} \
+    -X tailscale.com/version.Short=${VERSION_SHORT} \
+    -X tailscale.com/version.GitCommit=${VERSION_GIT_HASH}" \
+  --base="${BASE}" \
+  --tags="${TAGS}" \
+  --repos="${REPOS}" \
+  --push="${PUSH}"

chirp/chirp.go

@@ -19,9 +19,9 @@ func New(socket string) (*BIRDClient, error) {
 	if err != nil {
 		return nil, fmt.Errorf("failed to connect to BIRD: %w", err)
 	}
-	b := &BIRDClient{socket: socket, conn: conn, bs: bufio.NewScanner(conn)}
+	b := &BIRDClient{socket: socket, conn: conn, scanner: bufio.NewScanner(conn)}
 	// Read and discard the first line as that is the welcome message.
-	if _, err := b.readLine(); err != nil {
+	if _, err := b.readResponse(); err != nil {
 		return nil, err
 	}
 	return b, nil
@@ -29,9 +29,9 @@ func New(socket string) (*BIRDClient, error) {
 
 // BIRDClient handles communication with the BIRD Internet Routing Daemon.
 type BIRDClient struct {
-	socket string
-	conn   net.Conn
-	bs     *bufio.Scanner
+	socket  string
+	conn    net.Conn
+	scanner *bufio.Scanner
 }
 
 // Close closes the underlying connection to BIRD.
@@ -39,7 +39,7 @@ func (b *BIRDClient) Close() error { return b.conn.Close() }
 
 // DisableProtocol disables the provided protocol.
 func (b *BIRDClient) DisableProtocol(protocol string) error {
-	out, err := b.exec("disable %s\n", protocol)
+	out, err := b.exec("disable %s", protocol)
 	if err != nil {
 		return err
 	}
@@ -53,7 +53,7 @@ func (b *BIRDClient) DisableProtocol(protocol string) error {
 
 // EnableProtocol enables the provided protocol.
 func (b *BIRDClient) EnableProtocol(protocol string) error {
-	out, err := b.exec("enable %s\n", protocol)
+	out, err := b.exec("enable %s", protocol)
 	if err != nil {
 		return err
 	}
@@ -65,19 +65,65 @@ func (b *BIRDClient) EnableProtocol(protocol string) error {
 	return fmt.Errorf("failed to enable %s: %v", protocol, out)
 }
 
+// BIRD CLI docs from https://bird.network.cz/?get_doc&v=20&f=prog-2.html#ss2.9
+
+// Each session of the CLI consists of a sequence of request and replies,
+// slightly resembling the FTP and SMTP protocols.
+// Requests are commands encoded as a single line of text,
+// replies are sequences of lines starting with a four-digit code
+// followed by either a space (if it's the last line of the reply) or
+// a minus sign (when the reply is going to continue with the next line),
+// the rest of the line contains a textual message semantics of which depends on the numeric code.
+// If a reply line has the same code as the previous one and it's a continuation line,
+// the whole prefix can be replaced by a single white space character.
+//
+// Reply codes starting with 0 stand for ‘action successfully completed’ messages,
+// 1 means ‘table entry’, 8 ‘runtime error’ and 9 ‘syntax error’.
+
 func (b *BIRDClient) exec(cmd string, args ...interface{}) (string, error) {
 	if _, err := fmt.Fprintf(b.conn, cmd, args...); err != nil {
 		return "", err
 	}
-	return b.readLine()
+	fmt.Fprintln(b.conn)
+	return b.readResponse()
 }
 
-func (b *BIRDClient) readLine() (string, error) {
-	if !b.bs.Scan() {
-		return "", fmt.Errorf("reading response from bird failed")
+// hasResponseCode reports whether the provided byte slice is
+// prefixed with a BIRD response code.
+// Equivalent regex: `^\d{4}[ -]`.
+func hasResponseCode(s []byte) bool {
+	if len(s) < 5 {
+		return false
 	}
-	if err := b.bs.Err(); err != nil {
-		return "", err
+	for _, b := range s[:4] {
+		if '0' <= b && b <= '9' {
+			continue
+		}
+		return false
 	}
-	return b.bs.Text(), nil
+	return s[4] == ' ' || s[4] == '-'
+}
+
+func (b *BIRDClient) readResponse() (string, error) {
+	var resp strings.Builder
+	var done bool
+	for !done {
+		if !b.scanner.Scan() {
+			return "", fmt.Errorf("reading response from bird failed: %q", resp.String())
+		}
+		if err := b.scanner.Err(); err != nil {
+			return "", err
+		}
+		out := b.scanner.Bytes()
+		if _, err := resp.Write(out); err != nil {
+			return "", err
+		}
+		if hasResponseCode(out) {
+			done = out[4] == ' '
+		}
+		if !done {
+			resp.WriteRune('\n')
+		}
+	}
+	return resp.String(), nil
 }

chirp/chirp_test.go

@@ -0,0 +1,111 @@
+// Copyright (c) 2022 Tailscale Inc & AUTHORS All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+package chirp
+
+import (
+	"bufio"
+	"errors"
+	"fmt"
+	"net"
+	"path/filepath"
+	"strings"
+	"testing"
+)
+
+type fakeBIRD struct {
+	net.Listener
+	protocolsEnabled map[string]bool
+	sock             string
+}
+
+func newFakeBIRD(t *testing.T, protocols ...string) *fakeBIRD {
+	sock := filepath.Join(t.TempDir(), "sock")
+	l, err := net.Listen("unix", sock)
+	if err != nil {
+		t.Fatal(err)
+	}
+	pe := make(map[string]bool)
+	for _, p := range protocols {
+		pe[p] = false
+	}
+	return &fakeBIRD{
+		Listener:         l,
+		protocolsEnabled: pe,
+		sock:             sock,
+	}
+}
+
+func (fb *fakeBIRD) listen() error {
+	for {
+		c, err := fb.Accept()
+		if err != nil {
+			if errors.Is(err, net.ErrClosed) {
+				return nil
+			}
+			return err
+		}
+		go fb.handle(c)
+	}
+}
+
+func (fb *fakeBIRD) handle(c net.Conn) {
+	fmt.Fprintln(c, "0001 BIRD 2.0.8 ready.")
+	sc := bufio.NewScanner(c)
+	for sc.Scan() {
+		cmd := sc.Text()
+		args := strings.Split(cmd, " ")
+		switch args[0] {
+		case "enable":
+			en, ok := fb.protocolsEnabled[args[1]]
+			if !ok {
+				fmt.Fprintln(c, "9001 syntax error, unexpected CF_SYM_UNDEFINED, expecting CF_SYM_KNOWN or TEXT or ALL")
+			} else if en {
+				fmt.Fprintf(c, "0010-%s: already enabled\n", args[1])
+			} else {
+				fmt.Fprintf(c, "0011-%s: enabled\n", args[1])
+			}
+			fmt.Fprintln(c, "0000 ")
+			fb.protocolsEnabled[args[1]] = true
+		case "disable":
+			en, ok := fb.protocolsEnabled[args[1]]
+			if !ok {
+				fmt.Fprintln(c, "9001 syntax error, unexpected CF_SYM_UNDEFINED, expecting CF_SYM_KNOWN or TEXT or ALL")
+			} else if !en {
+				fmt.Fprintf(c, "0008-%s: already disabled\n", args[1])
+			} else {
+				fmt.Fprintf(c, "0009-%s: disabled\n", args[1])
+			}
+			fmt.Fprintln(c, "0000 ")
+			fb.protocolsEnabled[args[1]] = false
+		}
+	}
+}
+
+func TestChirp(t *testing.T) {
+	fb := newFakeBIRD(t, "tailscale")
+	defer fb.Close()
+	go fb.listen()
+	c, err := New(fb.sock)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if err := c.EnableProtocol("tailscale"); err != nil {
+		t.Fatal(err)
+	}
+	if err := c.EnableProtocol("tailscale"); err != nil {
+		t.Fatal(err)
+	}
+	if err := c.DisableProtocol("tailscale"); err != nil {
+		t.Fatal(err)
+	}
+	if err := c.DisableProtocol("tailscale"); err != nil {
+		t.Fatal(err)
+	}
+	if err := c.EnableProtocol("rando"); err == nil {
+		t.Fatalf("enabling %q succeded", "rando")
+	}
+	if err := c.DisableProtocol("rando"); err == nil {
+		t.Fatalf("disabling %q succeded", "rando")
+	}
+}

client/tailscale/tailscale.go

@@ -38,6 +38,9 @@ var (
 	// TailscaledSocket is the tailscaled Unix socket. It's used by the TailscaledDialer.
 	TailscaledSocket = paths.DefaultTailscaledSocket()
 
+	// TailscaledSocketSetExplicitly reports whether the user explicitly set TailscaledSocket.
+	TailscaledSocketSetExplicitly bool
+
 	// TailscaledDialer is the DialContext func that connects to the local machine's
 	// tailscaled or equivalent.
 	TailscaledDialer = defaultDialer
@@ -47,7 +50,8 @@ func defaultDialer(ctx context.Context, network, addr string) (net.Conn, error)
 	if addr != "local-tailscaled.sock:80" {
 		return nil, fmt.Errorf("unexpected URL address %q", addr)
 	}
-	if TailscaledSocket == paths.DefaultTailscaledSocket() {
+	// TODO: make this part of a safesocket.ConnectionStrategy
+	if !TailscaledSocketSetExplicitly {
 		// On macOS, when dialing from non-sandboxed program to sandboxed GUI running
 		// a TCP server on a random port, find the random port. For HTTP connections,
 		// we don't send the token. It gets added in an HTTP Basic-Auth header.
@@ -56,7 +60,11 @@ func defaultDialer(ctx context.Context, network, addr string) (net.Conn, error)
 			return d.DialContext(ctx, "tcp", "localhost:"+strconv.Itoa(port))
 		}
 	}
-	return safesocket.Connect(TailscaledSocket, 41112)
+	s := safesocket.DefaultConnectionStrategy(TailscaledSocket)
+	// The user provided a non-default tailscaled socket address.
+	// Connect only to exactly what they provided.
+	s.UseFallback(false)
+	return safesocket.Connect(s)
 }
 
 var (
@@ -90,6 +98,31 @@ func DoLocalRequest(req *http.Request) (*http.Response, error) {
 	return tsClient.Do(req)
 }
 
+func doLocalRequestNiceError(req *http.Request) (*http.Response, error) {
+	res, err := DoLocalRequest(req)
+	if err == nil {
+		if server := res.Header.Get("Tailscale-Version"); server != "" && server != version.Long && onVersionMismatch != nil {
+			onVersionMismatch(version.Long, server)
+		}
+		if res.StatusCode == 403 {
+			all, _ := ioutil.ReadAll(res.Body)
+			return nil, &AccessDeniedError{errors.New(errorMessageFromBody(all))}
+		}
+		return res, nil
+	}
+	if ue, ok := err.(*url.Error); ok {
+		if oe, ok := ue.Err.(*net.OpError); ok && oe.Op == "dial" {
+			path := req.URL.Path
+			pathPrefix := path
+			if i := strings.Index(path, "?"); i != -1 {
+				pathPrefix = path[:i]
+			}
+			return nil, fmt.Errorf("Failed to connect to local Tailscale daemon for %s; %s Error: %w", pathPrefix, tailscaledConnectHint(), oe)
+		}
+	}
+	return nil, err
+}
+
 type errorJSON struct {
 	Error string
 }
@@ -140,32 +173,16 @@ func send(ctx context.Context, method, path string, wantStatus int, body io.Read
 	if err != nil {
 		return nil, err
 	}
-	res, err := DoLocalRequest(req)
+	res, err := doLocalRequestNiceError(req)
 	if err != nil {
-		if ue, ok := err.(*url.Error); ok {
-			if oe, ok := ue.Err.(*net.OpError); ok && oe.Op == "dial" {
-				pathPrefix := path
-				if i := strings.Index(path, "?"); i != -1 {
-					pathPrefix = path[:i]
-				}
-				return nil, fmt.Errorf("Failed to connect to local Tailscale daemon for %s; %s Error: %w", pathPrefix, tailscaledConnectHint(), oe)
-			}
-		}
 		return nil, err
 	}
 	defer res.Body.Close()
-	if server := res.Header.Get("Tailscale-Version"); server != "" && server != version.Long && onVersionMismatch != nil {
-		onVersionMismatch(version.Long, server)
-	}
 	slurp, err := ioutil.ReadAll(res.Body)
 	if err != nil {
 		return nil, err
 	}
 	if res.StatusCode != wantStatus {
-		if res.StatusCode == 403 {
-			return nil, &AccessDeniedError{errors.New(errorMessageFromBody(slurp))}
-		}
-		err := fmt.Errorf("HTTP %s: %s (expected %v)", res.Status, slurp, wantStatus)
 		return nil, bestError(err, slurp)
 	}
 	return slurp, nil
@@ -196,6 +213,12 @@ func Goroutines(ctx context.Context) ([]byte, error) {
 	return get200(ctx, "/localapi/v0/goroutines")
 }
 
+// DaemonMetrics returns the Tailscale daemon's metrics in
+// the Prometheus text exposition format.
+func DaemonMetrics(ctx context.Context) ([]byte, error) {
+	return get200(ctx, "/localapi/v0/metrics")
+}
+
 // Profile returns a pprof profile of the Tailscale daemon.
 func Profile(ctx context.Context, pprofType string, sec int) ([]byte, error) {
 	var secArg string
@@ -217,12 +240,22 @@ func BugReport(ctx context.Context, note string) (string, error) {
 	return strings.TrimSpace(string(body)), nil
 }
 
+// DebugAction invokes a debug action, such as "rebind" or "restun".
+// These are development tools and subject to change or removal over time.
+func DebugAction(ctx context.Context, action string) error {
+	body, err := send(ctx, "POST", "/localapi/v0/debug?action="+url.QueryEscape(action), 200, nil)
+	if err != nil {
+		return fmt.Errorf("error %w: %s", err, body)
+	}
+	return nil
+}
+
 // Status returns the Tailscale daemon's status.
 func Status(ctx context.Context) (*ipnstate.Status, error) {
 	return status(ctx, "")
 }
 
-// StatusWithPeers returns the Tailscale daemon's status, without the peer info.
+// StatusWithoutPeers returns the Tailscale daemon's status, without the peer info.
 func StatusWithoutPeers(ctx context.Context) (*ipnstate.Status, error) {
 	return status(ctx, "?peers=false")
 }
@@ -261,7 +294,7 @@ func GetWaitingFile(ctx context.Context, baseName string) (rc io.ReadCloser, siz
 	if err != nil {
 		return nil, 0, err
 	}
-	res, err := DoLocalRequest(req)
+	res, err := doLocalRequestNiceError(req)
 	if err != nil {
 		return nil, 0, err
 	}
@@ -289,6 +322,30 @@ func FileTargets(ctx context.Context) ([]apitype.FileTarget, error) {
 	return fts, nil
 }
 
+// PushFile sends Taildrop file r to target.
+//
+// A size of -1 means unknown.
+// The name parameter is the original filename, not escaped.
+func PushFile(ctx context.Context, target tailcfg.StableNodeID, size int64, name string, r io.Reader) error {
+	req, err := http.NewRequestWithContext(ctx, "PUT", "http://local-tailscaled.sock/localapi/v0/file-put/"+string(target)+"/"+url.PathEscape(name), r)
+	if err != nil {
+		return err
+	}
+	if size != -1 {
+		req.ContentLength = size
+	}
+	res, err := doLocalRequestNiceError(req)
+	if err != nil {
+		return err
+	}
+	if res.StatusCode == 200 {
+		io.Copy(io.Discard, res.Body)
+		return nil
+	}
+	all, _ := io.ReadAll(res.Body)
+	return bestError(fmt.Errorf("%s: %s", res.Status, all), all)
+}
+
 func CheckIPForwarding(ctx context.Context) error {
 	body, err := get200(ctx, "/localapi/v0/check-ip-forwarding")
 	if err != nil {

cmd/cloner/cloner.go

@@ -95,7 +95,20 @@ func main() {
 	}
 
 	contents := new(bytes.Buffer)
-	fmt.Fprintf(contents, header, *flagTypes, pkg.Name)
+	var flagArgs []string
+	if *flagTypes != "" {
+		flagArgs = append(flagArgs, "-type="+*flagTypes)
+	}
+	if *flagOutput != "" {
+		flagArgs = append(flagArgs, "-output="+*flagOutput)
+	}
+	if *flagBuildTags != "" {
+		flagArgs = append(flagArgs, "-tags="+*flagBuildTags)
+	}
+	if *flagCloneFunc {
+		flagArgs = append(flagArgs, "-clonefunc")
+	}
+	fmt.Fprintf(contents, header, strings.Join(flagArgs, " "), pkg.Name)
 	fmt.Fprintf(contents, "import (\n")
 	for s := range imports {
 		fmt.Fprintf(contents, "\t%q\n", s)
@@ -117,8 +130,8 @@ const header = `// Copyright (c) 2020 Tailscale Inc & AUTHORS All rights reserve
 // Use of this source code is governed by a BSD-style
 // license that can be found in the LICENSE file.
 
-// Code generated by the following command; DO NOT EDIT.
-//   tailscale.com/cmd/cloner -type %s
+// Code generated by tailscale.com/cmd/cloner; DO NOT EDIT.
+//` + `go:generate` + ` go run tailscale.com/cmd/cloner %s
 
 package %s
 

cmd/derper/bootstrap_dns.go

@@ -12,14 +12,11 @@ import (
 	"net"
 	"net/http"
 	"strings"
-	"sync"
+	"sync/atomic"
 	"time"
 )
 
-var (
-	dnsMu    sync.Mutex
-	dnsCache = map[string][]net.IP{}
-)
+var dnsCache atomic.Value // of []byte
 
 var bootstrapDNSRequests = expvar.NewInt("counter_bootstrap_dns_requests")
 
@@ -37,6 +34,7 @@ func refreshBootstrapDNS() {
 	if *bootstrapDNS == "" {
 		return
 	}
+	dnsEntries := make(map[string][]net.IP)
 	ctx, cancel := context.WithTimeout(context.Background(), time.Minute)
 	defer cancel()
 	names := strings.Split(*bootstrapDNS, ",")
@@ -47,23 +45,23 @@ func refreshBootstrapDNS() {
 			log.Printf("bootstrap DNS lookup %q: %v", name, err)
 			continue
 		}
-		dnsMu.Lock()
-		dnsCache[name] = addrs
-		dnsMu.Unlock()
+		dnsEntries[name] = addrs
 	}
+	j, err := json.MarshalIndent(dnsEntries, "", "\t")
+	if err != nil {
+		// leave the old values in place
+		return
+	}
+	dnsCache.Store(j)
 }
 
 func handleBootstrapDNS(w http.ResponseWriter, r *http.Request) {
 	bootstrapDNSRequests.Add(1)
-	dnsMu.Lock()
-	j, err := json.MarshalIndent(dnsCache, "", "\t")
-	dnsMu.Unlock()
-	if err != nil {
-		log.Printf("bootstrap DNS JSON: %v", err)
-		http.Error(w, "JSON marshal error", 500)
-		return
-	}
-
 	w.Header().Set("Content-Type", "application/json")
+	j, _ := dnsCache.Load().([]byte)
+	// Bootstrap DNS requests occur cross-regions,
+	// and are randomized per request,
+	// so keeping a connection open is pointlessly expensive.
+	w.Header().Set("Connection", "close")
 	w.Write(j)
 }

cmd/derper/bootstrap_dns_test.go

@@ -0,0 +1,35 @@
+// Copyright (c) 2022 Tailscale Inc & AUTHORS All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package main
+
+import (
+	"net/http"
+	"testing"
+)
+
+func BenchmarkHandleBootstrapDNS(b *testing.B) {
+	prev := *bootstrapDNS
+	*bootstrapDNS = "log.tailscale.io,login.tailscale.com,controlplane.tailscale.com,login.us.tailscale.com"
+	defer func() {
+		*bootstrapDNS = prev
+	}()
+	refreshBootstrapDNS()
+	w := new(bitbucketResponseWriter)
+	b.ReportAllocs()
+	b.ResetTimer()
+	b.RunParallel(func(b *testing.PB) {
+		for b.Next() {
+			handleBootstrapDNS(w, nil)
+		}
+	})
+}
+
+type bitbucketResponseWriter struct{}
+
+func (b *bitbucketResponseWriter) Header() http.Header { return make(http.Header) }
+
+func (b *bitbucketResponseWriter) Write(p []byte) (int, error) { return len(p), nil }
+
+func (b *bitbucketResponseWriter) WriteHeader(statusCode int) {}

cmd/derper/cert.go

@@ -67,8 +67,8 @@ func NewManualCertManager(certdir, hostname string) (certProvider, error) {
 	if err != nil {
 		return nil, fmt.Errorf("can not load cert: %w", err)
 	}
-	if x509Cert.VerifyHostname(hostname) != nil {
-		return nil, errors.New("refuse to load cert: hostname mismatch with key")
+	if err := x509Cert.VerifyHostname(hostname); err != nil {
+		return nil, fmt.Errorf("cert invalid for hostname %q: %w", hostname, err)
 	}
 	return &manualCertManager{cert: &cert, hostname: hostname}, nil
 }

cmd/derper/derper.go

@@ -12,9 +12,11 @@ import (
 	"errors"
 	"expvar"
 	"flag"
+	"fmt"
 	"io"
 	"io/ioutil"
 	"log"
+	"math"
 	"net"
 	"net/http"
 	"os"
@@ -23,6 +25,7 @@ import (
 	"strings"
 	"time"
 
+	"golang.org/x/time/rate"
 	"tailscale.com/atomicfile"
 	"tailscale.com/derp"
 	"tailscale.com/derp/derphttp"
@@ -31,28 +34,34 @@ import (
 	"tailscale.com/net/stun"
 	"tailscale.com/tsweb"
 	"tailscale.com/types/key"
-	"tailscale.com/types/wgkey"
 )
 
 var (
 	dev           = flag.Bool("dev", false, "run in localhost development mode")
-	addr          = flag.String("a", ":443", "server address")
+	addr          = flag.String("a", ":443", "server HTTPS listen address, in form \":port\", \"ip:port\", or for IPv6 \"[ip]:port\". If the IP is omitted, it defaults to all interfaces.")
+	httpPort      = flag.Int("http-port", 80, "The port on which to serve HTTP. Set to -1 to disable")
 	configPath    = flag.String("c", "", "config file path")
 	certMode      = flag.String("certmode", "letsencrypt", "mode for getting a cert. possible options: manual, letsencrypt")
 	certDir       = flag.String("certdir", tsweb.DefaultCertDir("derper-certs"), "directory to store LetsEncrypt certs, if addr's port is :443")
 	hostname      = flag.String("hostname", "derp.tailscale.com", "LetsEncrypt host name, if addr's port is :443")
 	logCollection = flag.String("logcollection", "", "If non-empty, logtail collection to log to")
-	runSTUN       = flag.Bool("stun", false, "also run a STUN server")
+	runSTUN       = flag.Bool("stun", true, "whether to run a STUN server. It will bind to the same IP (if any) as the --addr flag value.")
+
 	meshPSKFile   = flag.String("mesh-psk-file", defaultMeshPSKFile(), "if non-empty, path to file containing the mesh pre-shared key file. It should contain some hex string; whitespace is trimmed.")
 	meshWith      = flag.String("mesh-with", "", "optional comma-separated list of hostnames to mesh with; the server's own hostname can be in the list")
 	bootstrapDNS  = flag.String("bootstrap-dns-names", "", "optional comma-separated list of hostnames to make available at /bootstrap-dns")
 	verifyClients = flag.Bool("verify-clients", false, "verify clients to this DERP server through a local tailscaled instance.")
+
+	acceptConnLimit = flag.Float64("accept-connection-limit", math.Inf(+1), "rate limit for accepting new connection")
+	acceptConnBurst = flag.Int("accept-connection-burst", math.MaxInt, "burst limit for accepting new connection")
 )
 
 var (
-	stats           = new(metrics.Set)
-	stunDisposition = &metrics.LabelMap{Label: "disposition"}
-	stunAddrFamily  = &metrics.LabelMap{Label: "family"}
+	stats             = new(metrics.Set)
+	stunDisposition   = &metrics.LabelMap{Label: "disposition"}
+	stunAddrFamily    = &metrics.LabelMap{Label: "family"}
+	tlsRequestVersion = &metrics.LabelMap{Label: "version"}
+	tlsActiveVersion  = &metrics.LabelMap{Label: "version"}
 
 	stunReadError  = stunDisposition.Get("read_error")
 	stunNotSTUN    = stunDisposition.Get("not_stun")
@@ -67,15 +76,17 @@ func init() {
 	stats.Set("counter_requests", stunDisposition)
 	stats.Set("counter_addrfamily", stunAddrFamily)
 	expvar.Publish("stun", stats)
+	expvar.Publish("derper_tls_request_version", tlsRequestVersion)
+	expvar.Publish("gauge_derper_tls_active_version", tlsActiveVersion)
 }
 
 type config struct {
-	PrivateKey wgkey.Private
+	PrivateKey key.NodePrivate
 }
 
 func loadConfig() config {
 	if *dev {
-		return config{PrivateKey: mustNewKey()}
+		return config{PrivateKey: key.NewNode()}
 	}
 	if *configPath == "" {
 		if os.Getuid() == 0 {
@@ -101,21 +112,13 @@ func loadConfig() config {
 	}
 }
 
-func mustNewKey() wgkey.Private {
-	key, err := wgkey.NewPrivate()
-	if err != nil {
-		log.Fatal(err)
-	}
-	return key
-}
-
 func writeNewConfig() config {
-	key := mustNewKey()
+	k := key.NewNode()
 	if err := os.MkdirAll(filepath.Dir(*configPath), 0777); err != nil {
 		log.Fatal(err)
 	}
 	cfg := config{
-		PrivateKey: key,
+		PrivateKey: k,
 	}
 	b, err := json.MarshalIndent(cfg, "", "\t")
 	if err != nil {
@@ -150,9 +153,9 @@ func main() {
 
 	cfg := loadConfig()
 
-	serveTLS := tsweb.IsProd443(*addr)
+	serveTLS := tsweb.IsProd443(*addr) || *certMode == "manual"
 
-	s := derp.NewServer(key.Private(cfg.PrivateKey), log.Printf)
+	s := derp.NewServer(cfg.PrivateKey, log.Printf)
 	s.SetVerifyClient(*verifyClients)
 
 	if *meshPSKFile != "" {
@@ -173,7 +176,10 @@ func main() {
 	expvar.Publish("derp", s.ExpVar())
 
 	mux := http.NewServeMux()
-	mux.Handle("/derp", derphttp.Handler(s))
+	derpHandler := derphttp.Handler(s)
+	derpHandler = addWebSocketSupport(s, derpHandler)
+	mux.Handle("/derp", derpHandler)
+	mux.HandleFunc("/derp/probe", probeHandler)
 	go refreshBootstrapDNSLoop()
 	mux.HandleFunc("/bootstrap-dns", handleBootstrapDNS)
 	mux.Handle("/", http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
@@ -241,25 +247,61 @@ func main() {
 			cert.Certificate = append(cert.Certificate, s.MetaCert())
 			return cert, nil
 		}
-		go func() {
-			port80srv := &http.Server{
-				Addr:        net.JoinHostPort(listenHost, "80"),
-				Handler:     certManager.HTTPHandler(tsweb.Port80Handler{Main: mux}),
-				ReadTimeout: 30 * time.Second,
-				// Crank up WriteTimeout a bit more than usually
-				// necessary just so we can do long CPU profiles
-				// and not hit net/http/pprof's "profile
-				// duration exceeds server's WriteTimeout".
-				WriteTimeout: 5 * time.Minute,
-			}
-			err := port80srv.ListenAndServe()
-			if err != nil {
-				if err != http.ErrServerClosed {
-					log.Fatal(err)
+		// Disable TLS 1.0 and 1.1, which are obsolete and have security issues.
+		httpsrv.TLSConfig.MinVersion = tls.VersionTLS12
+		httpsrv.Handler = http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+			if r.TLS != nil {
+				label := "unknown"
+				switch r.TLS.Version {
+				case tls.VersionTLS10:
+					label = "1.0"
+				case tls.VersionTLS11:
+					label = "1.1"
+				case tls.VersionTLS12:
+					label = "1.2"
+				case tls.VersionTLS13:
+					label = "1.3"
 				}
+				tlsRequestVersion.Add(label, 1)
+				tlsActiveVersion.Add(label, 1)
+				defer tlsActiveVersion.Add(label, -1)
 			}
-		}()
-		err = httpsrv.ListenAndServeTLS("", "")
+
+			// Set HTTP headers to appease automated security scanners.
+			//
+			// Security automation gets cranky when HTTPS sites don't
+			// set HSTS, and when they don't specify a content
+			// security policy for XSS mitigation.
+			//
+			// DERP's HTTP interface is only ever used for debug
+			// access (for which trivial safe policies work just
+			// fine), and by DERP clients which don't obey any of
+			// these browser-centric headers anyway.
+			w.Header().Set("Strict-Transport-Security", "max-age=63072000; includeSubDomains")
+			w.Header().Set("Content-Security-Policy", "default-src 'none'; frame-ancestors 'none'; form-action 'none'; base-uri 'self'; block-all-mixed-content; plugin-types 'none'")
+			mux.ServeHTTP(w, r)
+		})
+		if *httpPort > -1 {
+			go func() {
+				port80srv := &http.Server{
+					Addr:        net.JoinHostPort(listenHost, fmt.Sprintf("%d", *httpPort)),
+					Handler:     certManager.HTTPHandler(tsweb.Port80Handler{Main: mux}),
+					ReadTimeout: 30 * time.Second,
+					// Crank up WriteTimeout a bit more than usually
+					// necessary just so we can do long CPU profiles
+					// and not hit net/http/pprof's "profile
+					// duration exceeds server's WriteTimeout".
+					WriteTimeout: 5 * time.Minute,
+				}
+				err := port80srv.ListenAndServe()
+				if err != nil {
+					if err != http.ErrServerClosed {
+						log.Fatal(err)
+					}
+				}
+			}()
+		}
+		err = rateLimitedListenAndServeTLS(httpsrv)
 	} else {
 		log.Printf("derper: serving on %s", *addr)
 		err = httpsrv.ListenAndServe()
@@ -269,8 +311,18 @@ func main() {
 	}
 }
 
-func serveSTUN(host string) {
+// probeHandler is the endpoint that js/wasm clients hit to measure
+// DERP latency, since they can't do UDP STUN queries.
+func probeHandler(w http.ResponseWriter, r *http.Request) {
+	switch r.Method {
+	case "HEAD", "GET":
+		w.Header().Set("Access-Control-Allow-Origin", "*")
+	default:
+		http.Error(w, "bogus probe method", http.StatusMethodNotAllowed)
+	}
+}
 
+func serveSTUN(host string) {
 	pc, err := net.ListenPacket("udp", net.JoinHostPort(host, "3478"))
 	if err != nil {
 		log.Fatalf("failed to open STUN listener: %v", err)
@@ -343,3 +395,63 @@ func defaultMeshPSKFile() string {
 	}
 	return ""
 }
+
+func rateLimitedListenAndServeTLS(srv *http.Server) error {
+	addr := srv.Addr
+	if addr == "" {
+		addr = ":https"
+	}
+	ln, err := net.Listen("tcp", addr)
+	if err != nil {
+		return err
+	}
+	rln := newRateLimitedListener(ln, rate.Limit(*acceptConnLimit), *acceptConnBurst)
+	expvar.Publish("tls_listener", rln.ExpVar())
+	defer rln.Close()
+	return srv.ServeTLS(rln, "", "")
+}
+
+type rateLimitedListener struct {
+	// These are at the start of the struct to ensure 64-bit alignment
+	// on 32-bit architecture regardless of what other fields may exist
+	// in this package.
+	numAccepts expvar.Int // does not include number of rejects
+	numRejects expvar.Int
+
+	net.Listener
+
+	lim *rate.Limiter
+}
+
+func newRateLimitedListener(ln net.Listener, limit rate.Limit, burst int) *rateLimitedListener {
+	return &rateLimitedListener{Listener: ln, lim: rate.NewLimiter(limit, burst)}
+}
+
+func (l *rateLimitedListener) ExpVar() expvar.Var {
+	m := new(metrics.Set)
+	m.Set("counter_accepted_connections", &l.numAccepts)
+	m.Set("counter_rejected_connections", &l.numRejects)
+	return m
+}
+
+var errLimitedConn = errors.New("cannot accept connection; rate limited")
+
+func (l *rateLimitedListener) Accept() (net.Conn, error) {
+	// Even under a rate limited situation, we accept the connection immediately
+	// and close it, rather than being slow at accepting new connections.
+	// This provides two benefits: 1) it signals to the client that something
+	// is going on on the server, and 2) it prevents new connections from
+	// piling up and occupying resources in the OS kernel.
+	// The client will retry as needing (with backoffs in place).
+	cn, err := l.Listener.Accept()
+	if err != nil {
+		return nil, err
+	}
+	if !l.lim.Allow() {
+		l.numRejects.Add(1)
+		cn.Close()
+		return nil, errLimitedConn
+	}
+	l.numAccepts.Add(1)
+	return cn, nil
+}

cmd/derper/mesh.go

@@ -69,8 +69,8 @@ func startMeshWithHost(s *derp.Server, host string) error {
 		return d.DialContext(ctx, network, addr)
 	})
 
-	add := func(k key.Public) { s.AddPacketForwarder(k, c) }
-	remove := func(k key.Public) { s.RemovePacketForwarder(k, c) }
+	add := func(k key.NodePublic) { s.AddPacketForwarder(k, c) }
+	remove := func(k key.NodePublic) { s.RemovePacketForwarder(k, c) }
 	go c.RunWatchConnectionLoop(context.Background(), s.PublicKey(), logf, add, remove)
 	return nil
 }

cmd/derper/websocket.go

@@ -0,0 +1,52 @@
+// Copyright (c) 2021 Tailscale Inc & AUTHORS All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package main
+
+import (
+	"bufio"
+	"expvar"
+	"log"
+	"net/http"
+	"strings"
+
+	"nhooyr.io/websocket"
+	"tailscale.com/derp"
+	"tailscale.com/derp/wsconn"
+)
+
+var counterWebSocketAccepts = expvar.NewInt("derp_websocket_accepts")
+
+// addWebSocketSupport returns a Handle wrapping base that adds WebSocket server support.
+func addWebSocketSupport(s *derp.Server, base http.Handler) http.Handler {
+	return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		up := strings.ToLower(r.Header.Get("Upgrade"))
+
+		// Very early versions of Tailscale set "Upgrade: WebSocket" but didn't actually
+		// speak WebSockets (they still assumed DERP's binary framining). So to distinguish
+		// clients that actually want WebSockets, look for an explicit "derp" subprotocol.
+		if up != "websocket" || !strings.Contains(r.Header.Get("Sec-Websocket-Protocol"), "derp") {
+			base.ServeHTTP(w, r)
+			return
+		}
+
+		c, err := websocket.Accept(w, r, &websocket.AcceptOptions{
+			Subprotocols:   []string{"derp"},
+			OriginPatterns: []string{"*"},
+		})
+		if err != nil {
+			log.Printf("websocket.Accept: %v", err)
+			return
+		}
+		defer c.Close(websocket.StatusInternalError, "closing")
+		if c.Subprotocol() != "derp" {
+			c.Close(websocket.StatusPolicyViolation, "client must speak the derp subprotocol")
+			return
+		}
+		counterWebSocketAccepts.Add(1)
+		wc := wsconn.New(c)
+		brw := bufio.NewReadWriter(bufio.NewReader(wc), bufio.NewWriter(wc))
+		s.Accept(wc, brw, r.RemoteAddr)
+	})
+}

cmd/derpprobe/derpprobe.go

@@ -9,7 +9,9 @@ import (
 	"bytes"
 	"context"
 	crand "crypto/rand"
+	"crypto/x509"
 	"encoding/json"
+	"errors"
 	"flag"
 	"fmt"
 	"html"
@@ -33,11 +35,21 @@ var (
 	listen     = flag.String("listen", ":8030", "HTTP listen address")
 )
 
+// certReissueAfter is the time after which we expect all certs to be
+// reissued, at minimum.
+//
+// This is currently set to the date of the LetsEncrypt ALPN revocation event of Jan 2022:
+// https://community.letsencrypt.org/t/questions-about-renewing-before-tls-alpn-01-revocations/170449
+//
+// If there's another revocation event, bump this again.
+var certReissueAfter = time.Unix(1643226768, 0)
+
 var (
 	mu            sync.Mutex
 	state         = map[nodePair]pairStatus{}
 	lastDERPMap   *tailcfg.DERPMap
 	lastDERPMapAt time.Time
+	certs         = map[string]*x509.Certificate{}
 )
 
 func main() {
@@ -46,6 +58,12 @@ func main() {
 	log.Fatal(http.ListenAndServe(*listen, http.HandlerFunc(serve)))
 }
 
+func setCert(name string, cert *x509.Certificate) {
+	mu.Lock()
+	defer mu.Unlock()
+	certs[name] = cert
+}
+
 type overallStatus struct {
 	good, bad []string
 }
@@ -93,6 +111,27 @@ func getOverallStatus() (o overallStatus) {
 			}
 		}
 	}
+
+	var subjs []string
+	for k := range certs {
+		subjs = append(subjs, k)
+	}
+	sort.Strings(subjs)
+
+	soon := time.Now().Add(14 * 24 * time.Hour) // in 2 weeks; autocert does 30 days by default
+	for _, s := range subjs {
+		cert := certs[s]
+		if cert.NotBefore.Before(certReissueAfter) {
+			o.addBadf("cert %q needs reissuing; NotBefore=%v", s, cert.NotBefore.Format(time.RFC3339))
+			continue
+		}
+		if cert.NotAfter.Before(soon) {
+			o.addBadf("cert %q expiring soon (%v); wasn't auto-refreshed", s, cert.NotAfter.Format(time.RFC3339))
+			continue
+		}
+		o.addGoodf("cert %q good %v - %v", s, cert.NotBefore.Format(time.RFC3339), cert.NotAfter.Format(time.RFC3339))
+	}
+
 	return
 }
 
@@ -344,7 +383,7 @@ func probeNodePair(ctx context.Context, dm *tailcfg.DERPMap, from, to *tailcfg.D
 }
 
 func newConn(ctx context.Context, dm *tailcfg.DERPMap, n *tailcfg.DERPNode) (*derphttp.Client, error) {
-	priv := key.NewPrivate()
+	priv := key.NewNode()
 	dc := derphttp.NewRegionClient(priv, log.Printf, func() *tailcfg.DERPRegion {
 		rid := n.RegionID
 		return &tailcfg.DERPRegion{
@@ -359,6 +398,21 @@ func newConn(ctx context.Context, dm *tailcfg.DERPMap, n *tailcfg.DERPNode) (*de
 	if err != nil {
 		return nil, err
 	}
+	cs, ok := dc.TLSConnectionState()
+	if !ok {
+		dc.Close()
+		return nil, errors.New("no TLS state")
+	}
+	if len(cs.PeerCertificates) == 0 {
+		dc.Close()
+		return nil, errors.New("no peer certificates")
+	}
+	if cs.ServerName != n.HostName {
+		dc.Close()
+		return nil, fmt.Errorf("TLS server name %q != derp hostname %q", cs.ServerName, n.HostName)
+	}
+	setCert(cs.ServerName, cs.PeerCertificates[0])
+
 	errc := make(chan error, 1)
 	go func() {
 		m, err := dc.Recv()

cmd/hello/hello.go

@@ -2,13 +2,15 @@
 // Use of this source code is governed by a BSD-style
 // license that can be found in the LICENSE file.
 
-// The hello binary runs hello.ipn.dev.
+// The hello binary runs hello.ts.net.
 package main // import "tailscale.com/cmd/hello"
 
 import (
 	"context"
+	"crypto/tls"
 	_ "embed"
 	"encoding/json"
+	"errors"
 	"flag"
 	"html/template"
 	"io/ioutil"
@@ -16,6 +18,7 @@ import (
 	"net/http"
 	"os"
 	"strings"
+	"time"
 
 	"tailscale.com/client/tailscale"
 	"tailscale.com/client/tailscale/apitype"
@@ -69,11 +72,31 @@ func main() {
 	if *httpsAddr != "" {
 		log.Printf("running HTTPS server on %s", *httpsAddr)
 		go func() {
-			errc <- http.ListenAndServeTLS(*httpsAddr,
-				"/etc/hello/hello.ipn.dev.crt",
-				"/etc/hello/hello.ipn.dev.key",
-				nil,
-			)
+			hs := &http.Server{
+				Addr: *httpsAddr,
+				TLSConfig: &tls.Config{
+					GetCertificate: func(hi *tls.ClientHelloInfo) (*tls.Certificate, error) {
+						switch hi.ServerName {
+						case "hello.ts.net":
+							return tailscale.GetCertificate(hi)
+						case "hello.ipn.dev":
+							c, err := tls.LoadX509KeyPair(
+								"/etc/hello/hello.ipn.dev.crt",
+								"/etc/hello/hello.ipn.dev.key",
+							)
+							if err != nil {
+								return nil, err
+							}
+							return &c, nil
+						}
+						return nil, errors.New("invalid SNI name")
+					},
+				},
+				IdleTimeout:       30 * time.Second,
+				ReadHeaderTimeout: 20 * time.Second,
+				MaxHeaderBytes:    10 << 10,
+			}
+			errc <- hs.ListenAndServeTLS("", "")
 		}()
 	}
 	log.Fatal(<-errc)
@@ -127,8 +150,9 @@ func tailscaleIP(who *apitype.WhoIsResponse) string {
 func root(w http.ResponseWriter, r *http.Request) {
 	if r.TLS == nil && *httpsAddr != "" {
 		host := r.Host
-		if strings.Contains(r.Host, "100.101.102.103") {
-			host = "hello.ipn.dev"
+		if strings.Contains(r.Host, "100.101.102.103") ||
+			strings.Contains(r.Host, "hello.ipn.dev") {
+			host = "hello.ts.net"
 		}
 		http.Redirect(w, r, "https://"+host, http.StatusFound)
 		return
@@ -137,6 +161,10 @@ func root(w http.ResponseWriter, r *http.Request) {
 		http.Redirect(w, r, "/", http.StatusFound)
 		return
 	}
+	if r.TLS != nil && *httpsAddr != "" && strings.Contains(r.Host, "hello.ipn.dev") {
+		http.Redirect(w, r, "https://hello.ts.net", http.StatusFound)
+		return
+	}
 	tmpl, err := getTmpl()
 	if err != nil {
 		w.Header().Set("Content-Type", "text/plain")

cmd/printdep/printdep.go

@@ -0,0 +1,46 @@
+// Copyright (c) 2021 Tailscale Inc & AUTHORS All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+// The printdep command is a build system tool for printing out information
+// about dependencies.
+package main
+
+import (
+	"flag"
+	"fmt"
+	"log"
+	"runtime"
+	"strings"
+
+	ts "tailscale.com"
+)
+
+var (
+	goToolchain    = flag.Bool("go", false, "print the supported Go toolchain git hash (a github.com/tailscale/go commit)")
+	goToolchainURL = flag.Bool("go-url", false, "print the URL to the tarball of the Tailscale Go toolchain")
+)
+
+func main() {
+	flag.Parse()
+	if *goToolchain {
+		fmt.Println(strings.TrimSpace(ts.GoToolchainRev))
+	}
+	if *goToolchainURL {
+		var suffix string
+		switch runtime.GOARCH {
+		case "amd64":
+			// None
+		case "arm64":
+			suffix = "-" + runtime.GOARCH
+		default:
+			log.Fatalf("unsupported GOARCH %q", runtime.GOARCH)
+		}
+		switch runtime.GOOS {
+		case "linux", "darwin":
+		default:
+			log.Fatalf("unsupported GOOS %q", runtime.GOOS)
+		}
+		fmt.Printf("https://github.com/tailscale/go/releases/download/build-%s/%s%s.tar.gz\n", strings.TrimSpace(ts.GoToolchainRev), runtime.GOOS, suffix)
+	}
+}

cmd/tailscale/cli/bugreport.go

@@ -7,7 +7,6 @@ package cli
 import (
 	"context"
 	"errors"
-	"fmt"
 
 	"github.com/peterbourgon/ff/v3/ffcli"
 	"tailscale.com/client/tailscale"
@@ -33,6 +32,6 @@ func runBugReport(ctx context.Context, args []string) error {
 	if err != nil {
 		return err
 	}
-	fmt.Println(logMarker)
+	outln(logMarker)
 	return nil
 }

cmd/tailscale/cli/cert.go

@@ -13,7 +13,6 @@ import (
 	"log"
 	"net/http"
 	"os"
-	"runtime"
 	"strings"
 
 	"github.com/peterbourgon/ff/v3/ffcli"
@@ -29,7 +28,7 @@ var certCmd = &ffcli.Command{
 	ShortHelp:  "get TLS certs",
 	ShortUsage: "cert [flags] <domain>",
 	FlagSet: (func() *flag.FlagSet {
-		fs := flag.NewFlagSet("cert", flag.ExitOnError)
+		fs := newFlagSet("cert")
 		fs.StringVar(&certArgs.certFile, "cert-file", "", "output cert file or \"-\" for stdout; defaults to DOMAIN.crt if --cert-file and --key-file are both unset")
 		fs.StringVar(&certArgs.keyFile, "key-file", "", "output cert file or \"-\" for stdout; defaults to DOMAIN.key if --cert-file and --key-file are both unset")
 		fs.BoolVar(&certArgs.serve, "serve-demo", false, "if true, serve on port :443 using the cert as a demo, instead of writing out the files to disk")
@@ -81,7 +80,7 @@ func runCert(ctx context.Context, args []string) error {
 	domain := args[0]
 
 	printf := func(format string, a ...interface{}) {
-		fmt.Printf(format, a...)
+		printf(format, a...)
 	}
 	if certArgs.certFile == "-" || certArgs.keyFile == "-" {
 		printf = log.Printf
@@ -92,9 +91,6 @@ func runCert(ctx context.Context, args []string) error {
 		certArgs.keyFile = domain + ".key"
 	}
 	certPEM, keyPEM, err := tailscale.CertPair(ctx, domain)
-	if tailscale.IsAccessDeniedError(err) && os.Getuid() != 0 && runtime.GOOS != "windows" {
-		return fmt.Errorf("%v\n\nUse 'sudo tailscale cert' or 'tailscale up --operator=$USER' to not require root.", err)
-	}
 	if err != nil {
 		return err
 	}
@@ -108,7 +104,7 @@ func runCert(ctx context.Context, args []string) error {
 		if version.IsMacSysExt() {
 			dir = "io.tailscale.ipn.macsys"
 		}
-		printf("Warning: the macOS CLI runs in a sandbox; this binary's filesystem writes go to $HOME/Library/Containers/%s\n", dir)
+		printf("Warning: the macOS CLI runs in a sandbox; this binary's filesystem writes go to $HOME/Library/Containers/%s/Data\n", dir)
 	}
 	if certArgs.certFile != "" {
 		certChanged, err := writeIfChanged(certArgs.certFile, certPEM, 0644)
@@ -143,7 +139,7 @@ func runCert(ctx context.Context, args []string) error {
 
 func writeIfChanged(filename string, contents []byte, mode os.FileMode) (changed bool, err error) {
 	if filename == "-" {
-		os.Stdout.Write(contents)
+		Stdout.Write(contents)
 		return false, nil
 	}
 	if old, err := os.ReadFile(filename); err == nil && bytes.Equal(contents, old) {

cmd/tailscale/cli/cli.go

@@ -29,8 +29,25 @@ import (
 	"tailscale.com/paths"
 	"tailscale.com/safesocket"
 	"tailscale.com/syncs"
+	"tailscale.com/version/distro"
 )
 
+var Stderr io.Writer = os.Stderr
+var Stdout io.Writer = os.Stdout
+
+func printf(format string, a ...interface{}) {
+	fmt.Fprintf(Stdout, format, a...)
+}
+
+// outln is like fmt.Println in the common case, except when Stdout is
+// changed (as in js/wasm).
+//
+// It's not named println because that looks like the Go built-in
+// which goes to stderr and formats slightly differently.
+func outln(a ...interface{}) {
+	fmt.Fprintln(Stdout, a...)
+}
+
 // ActLikeCLI reports whether a GUI application should act like the
 // CLI based on os.Args, GOOS, the context the process is running in
 // (pty, parent PID), etc.
@@ -77,6 +94,16 @@ func ActLikeCLI() bool {
 	return false
 }
 
+func newFlagSet(name string) *flag.FlagSet {
+	onError := flag.ExitOnError
+	if runtime.GOOS == "js" {
+		onError = flag.ContinueOnError
+	}
+	fs := flag.NewFlagSet(name, onError)
+	fs.SetOutput(Stderr)
+	return fs
+}
+
 // Run runs the CLI. The args do not include the binary name.
 func Run(args []string) error {
 	if len(args) == 1 && (args[0] == "-V" || args[0] == "--version") {
@@ -86,11 +113,11 @@ func Run(args []string) error {
 	var warnOnce sync.Once
 	tailscale.SetVersionMismatchHandler(func(clientVer, serverVer string) {
 		warnOnce.Do(func() {
-			fmt.Fprintf(os.Stderr, "Warning: client version %q != tailscaled server version %q\n", clientVer, serverVer)
+			fmt.Fprintf(Stderr, "Warning: client version %q != tailscaled server version %q\n", clientVer, serverVer)
 		})
 	})
 
-	rootfs := flag.NewFlagSet("tailscale", flag.ExitOnError)
+	rootfs := newFlagSet("tailscale")
 	rootfs.StringVar(&rootArgs.socket, "socket", paths.DefaultTailscaledSocket(), "path to tailscaled's unix socket")
 
 	rootCmd := &ffcli.Command{
@@ -129,25 +156,46 @@ change in the future.
 	if strSliceContains(args, "debug") {
 		rootCmd.Subcommands = append(rootCmd.Subcommands, debugCmd)
 	}
+	if runtime.GOOS == "linux" && distro.Get() == distro.Synology {
+		rootCmd.Subcommands = append(rootCmd.Subcommands, configureHostCmd)
+	}
 
 	if err := rootCmd.Parse(args); err != nil {
+		if errors.Is(err, flag.ErrHelp) {
+			return nil
+		}
 		return err
 	}
 
 	tailscale.TailscaledSocket = rootArgs.socket
+	rootfs.Visit(func(f *flag.Flag) {
+		if f.Name == "socket" {
+			tailscale.TailscaledSocketSetExplicitly = true
+		}
+	})
 
 	err := rootCmd.Run(context.Background())
-	if err == flag.ErrHelp {
+	if tailscale.IsAccessDeniedError(err) && os.Getuid() != 0 && runtime.GOOS != "windows" {
+		return fmt.Errorf("%v\n\nUse 'sudo tailscale %s' or 'tailscale up --operator=$USER' to not require root.", err, strings.Join(args, " "))
+	}
+	if errors.Is(err, flag.ErrHelp) {
 		return nil
 	}
 	return err
 }
 
 func fatalf(format string, a ...interface{}) {
+	if Fatalf != nil {
+		Fatalf(format, a...)
+		return
+	}
 	log.SetFlags(0)
 	log.Fatalf(format, a...)
 }
 
+// Fatalf, if non-nil, is used instead of log.Fatalf.
+var Fatalf func(format string, a ...interface{})
+
 var rootArgs struct {
 	socket string
 }
@@ -155,7 +203,8 @@ var rootArgs struct {
 var gotSignal syncs.AtomicBool
 
 func connect(ctx context.Context) (net.Conn, *ipn.BackendClient, context.Context, context.CancelFunc) {
-	c, err := safesocket.Connect(rootArgs.socket, 41112)
+	s := safesocket.DefaultConnectionStrategy(rootArgs.socket)
+	c, err := safesocket.Connect(s)
 	if err != nil {
 		if runtime.GOOS != "windows" && rootArgs.socket == "" {
 			fatalf("--socket cannot be empty")

cmd/tailscale/cli/cli_test.go

@@ -18,8 +18,10 @@ import (
 	"tailscale.com/ipn"
 	"tailscale.com/ipn/ipnstate"
 	"tailscale.com/tstest"
+	"tailscale.com/types/key"
 	"tailscale.com/types/persist"
 	"tailscale.com/types/preftype"
+	"tailscale.com/version/distro"
 )
 
 // geese is a collection of gooses. It need not be complete.
@@ -57,6 +59,7 @@ func TestCheckForAccidentalSettingReverts(t *testing.T) {
 		curExitNodeIP netaddr.IP
 		curUser       string // os.Getenv("USER") on the client side
 		goos          string // empty means "linux"
+		distro        distro.Distro
 
 		want string
 	}{
@@ -313,6 +316,7 @@ func TestCheckForAccidentalSettingReverts(t *testing.T) {
 				ControlURL:       ipn.DefaultControlURL,
 				AllowSingleHosts: true,
 				CorpDNS:          true,
+				RouteAll:         true,
 
 				// And assume this no-op accidental pre-1.8 value:
 				NoSNAT: true,
@@ -329,7 +333,7 @@ func TestCheckForAccidentalSettingReverts(t *testing.T) {
 
 				NetfilterMode: preftype.NetfilterNoDivert, // we never had this bug, but pretend it got set non-zero on Windows somehow
 			},
-			goos: "windows",
+			goos: "openbsd",
 			want: "", // not an error
 		},
 		{
@@ -405,6 +409,21 @@ func TestCheckForAccidentalSettingReverts(t *testing.T) {
 			},
 			want: accidentalUpPrefix + " --hostname=foo --exit-node=100.64.5.7",
 		},
+		{
+			name:          "error_exit_node_and_allow_lan_omit_with_id_pref", // Isue 3480
+			flags:         []string{"--hostname=foo"},
+			curExitNodeIP: netaddr.MustParseIP("100.2.3.4"),
+			curPrefs: &ipn.Prefs{
+				ControlURL:       ipn.DefaultControlURL,
+				AllowSingleHosts: true,
+				CorpDNS:          true,
+				NetfilterMode:    preftype.NetfilterOn,
+
+				ExitNodeAllowLANAccess: true,
+				ExitNodeID:             "some_stable_id",
+			},
+			want: accidentalUpPrefix + " --hostname=foo --exit-node-allow-lan-access --exit-node=100.2.3.4",
+		},
 		{
 			name:  "ignore_login_server_synonym",
 			flags: []string{"--login-server=https://controlplane.tailscale.com"},
@@ -427,6 +446,38 @@ func TestCheckForAccidentalSettingReverts(t *testing.T) {
 			},
 			want: accidentalUpPrefix + " --netfilter-mode=off --accept-dns=false",
 		},
+		{
+			// Issue 3176: on Synology, don't require --accept-routes=false because user
+			// migth've had old an install, and we don't support --accept-routes anyway.
+			name:  "synology_permit_omit_accept_routes",
+			flags: []string{"--hostname=foo"},
+			curPrefs: &ipn.Prefs{
+				ControlURL:       "https://login.tailscale.com",
+				CorpDNS:          true,
+				AllowSingleHosts: true,
+				RouteAll:         true,
+				NetfilterMode:    preftype.NetfilterOn,
+			},
+			goos:   "linux",
+			distro: distro.Synology,
+			want:   "",
+		},
+		{
+			// Same test case as "synology_permit_omit_accept_routes" above, but
+			// on non-Synology distro.
+			name:  "not_synology_dont_permit_omit_accept_routes",
+			flags: []string{"--hostname=foo"},
+			curPrefs: &ipn.Prefs{
+				ControlURL:       "https://login.tailscale.com",
+				CorpDNS:          true,
+				AllowSingleHosts: true,
+				RouteAll:         true,
+				NetfilterMode:    preftype.NetfilterOn,
+			},
+			goos:   "linux",
+			distro: "", // not Synology
+			want:   accidentalUpPrefix + " --hostname=foo --accept-routes",
+		},
 	}
 	for _, tt := range tests {
 		t.Run(tt.name, func(t *testing.T) {
@@ -447,6 +498,7 @@ func TestCheckForAccidentalSettingReverts(t *testing.T) {
 				goos:          goos,
 				flagSet:       flagSet,
 				curExitNodeIP: tt.curExitNodeIP,
+				distro:        tt.distro,
 			}); err != nil {
 				got = err.Error()
 			}
@@ -495,6 +547,7 @@ func TestPrefsFromUpArgs(t *testing.T) {
 				WantRunning:      true,
 				CorpDNS:          true,
 				AllowSingleHosts: true,
+				RouteAll:         true,
 				NetfilterMode:    preftype.NetfilterOn,
 			},
 		},
@@ -532,7 +585,7 @@ func TestPrefsFromUpArgs(t *testing.T) {
 			args: upArgsT{
 				exitNodeIP: "foo",
 			},
-			wantErr: `invalid IP address "foo" for --exit-node: ParseIP("foo"): unable to parse IP`,
+			wantErr: `invalid value "foo" for --exit-node; must be IP or unique node name`,
 		},
 		{
 			name: "error_exit_node_allow_lan_without_exit_node",
@@ -733,6 +786,33 @@ func TestUpdatePrefs(t *testing.T) {
 			wantSimpleUp:   true,
 			wantJustEditMP: &ipn.MaskedPrefs{WantRunningSet: true},
 		},
+		{
+			name:  "just_edit_reset",
+			flags: []string{"--reset"},
+			curPrefs: &ipn.Prefs{
+				ControlURL: ipn.DefaultControlURL,
+				Persist:    &persist.Persist{LoginName: "crawshaw.github"},
+			},
+			env: upCheckEnv{backendState: "Running"},
+			wantJustEditMP: &ipn.MaskedPrefs{
+				AdvertiseRoutesSet:        true,
+				AdvertiseTagsSet:          true,
+				AllowSingleHostsSet:       true,
+				ControlURLSet:             true,
+				CorpDNSSet:                true,
+				ExitNodeAllowLANAccessSet: true,
+				ExitNodeIDSet:             true,
+				ExitNodeIPSet:             true,
+				HostnameSet:               true,
+				NetfilterModeSet:          true,
+				NoSNATSet:                 true,
+				OperatorUserSet:           true,
+				RouteAllSet:               true,
+				RunSSHSet:                 true,
+				ShieldsUpSet:              true,
+				WantRunningSet:            true,
+			},
+		},
 		{
 			name:  "control_synonym",
 			flags: []string{},
@@ -759,6 +839,18 @@ func TestUpdatePrefs(t *testing.T) {
 			wantJustEditMP: &ipn.MaskedPrefs{WantRunningSet: true},
 			wantErrSubtr:   "can't change --login-server without --force-reauth",
 		},
+		{
+			name:  "change_tags",
+			flags: []string{"--advertise-tags=tag:foo"},
+			curPrefs: &ipn.Prefs{
+				ControlURL:       "https://login.tailscale.com",
+				Persist:          &persist.Persist{LoginName: "crawshaw.github"},
+				AllowSingleHosts: true,
+				CorpDNS:          true,
+				NetfilterMode:    preftype.NetfilterOn,
+			},
+			env: upCheckEnv{backendState: "Running"},
+		},
 	}
 	for _, tt := range tests {
 		t.Run(tt.name, func(t *testing.T) {
@@ -785,11 +877,148 @@ func TestUpdatePrefs(t *testing.T) {
 			if simpleUp != tt.wantSimpleUp {
 				t.Fatalf("simpleUp=%v, want %v", simpleUp, tt.wantSimpleUp)
 			}
+			var oldEditPrefs ipn.Prefs
 			if justEditMP != nil {
+				oldEditPrefs = justEditMP.Prefs
 				justEditMP.Prefs = ipn.Prefs{} // uninteresting
 			}
 			if !reflect.DeepEqual(justEditMP, tt.wantJustEditMP) {
-				t.Fatalf("justEditMP: %v", cmp.Diff(justEditMP, tt.wantJustEditMP))
+				t.Logf("justEditMP != wantJustEditMP; following diff omits the Prefs field, which was %+v", oldEditPrefs)
+				t.Fatalf("justEditMP: %v\n\n: ", cmp.Diff(justEditMP, tt.wantJustEditMP, cmpIP))
+			}
+		})
+	}
+}
+
+var cmpIP = cmp.Comparer(func(a, b netaddr.IP) bool {
+	return a == b
+})
+
+func TestExitNodeIPOfArg(t *testing.T) {
+	mustIP := netaddr.MustParseIP
+	tests := []struct {
+		name    string
+		arg     string
+		st      *ipnstate.Status
+		want    netaddr.IP
+		wantErr string
+	}{
+		{
+			name: "ip_while_stopped_okay",
+			arg:  "1.2.3.4",
+			st: &ipnstate.Status{
+				BackendState: "Stopped",
+			},
+			want: mustIP("1.2.3.4"),
+		},
+		{
+			name: "ip_not_found",
+			arg:  "1.2.3.4",
+			st: &ipnstate.Status{
+				BackendState: "Running",
+			},
+			wantErr: `no node found in netmap with IP 1.2.3.4`,
+		},
+		{
+			name: "ip_not_exit",
+			arg:  "1.2.3.4",
+			st: &ipnstate.Status{
+				BackendState: "Running",
+				Peer: map[key.NodePublic]*ipnstate.PeerStatus{
+					key.NewNode().Public(): {
+						TailscaleIPs: []netaddr.IP{mustIP("1.2.3.4")},
+					},
+				},
+			},
+			wantErr: `node 1.2.3.4 is not advertising an exit node`,
+		},
+		{
+			name: "ip",
+			arg:  "1.2.3.4",
+			st: &ipnstate.Status{
+				BackendState: "Running",
+				Peer: map[key.NodePublic]*ipnstate.PeerStatus{
+					key.NewNode().Public(): {
+						TailscaleIPs:   []netaddr.IP{mustIP("1.2.3.4")},
+						ExitNodeOption: true,
+					},
+				},
+			},
+			want: mustIP("1.2.3.4"),
+		},
+		{
+			name:    "no_match",
+			arg:     "unknown",
+			st:      &ipnstate.Status{MagicDNSSuffix: ".foo"},
+			wantErr: `invalid value "unknown" for --exit-node; must be IP or unique node name`,
+		},
+		{
+			name: "name",
+			arg:  "skippy",
+			st: &ipnstate.Status{
+				MagicDNSSuffix: ".foo",
+				Peer: map[key.NodePublic]*ipnstate.PeerStatus{
+					key.NewNode().Public(): {
+						DNSName:        "skippy.foo.",
+						TailscaleIPs:   []netaddr.IP{mustIP("1.0.0.2")},
+						ExitNodeOption: true,
+					},
+				},
+			},
+			want: mustIP("1.0.0.2"),
+		},
+		{
+			name: "name_not_exit",
+			arg:  "skippy",
+			st: &ipnstate.Status{
+				MagicDNSSuffix: ".foo",
+				Peer: map[key.NodePublic]*ipnstate.PeerStatus{
+					key.NewNode().Public(): {
+						DNSName:      "skippy.foo.",
+						TailscaleIPs: []netaddr.IP{mustIP("1.0.0.2")},
+					},
+				},
+			},
+			wantErr: `node "skippy" is not advertising an exit node`,
+		},
+		{
+			name: "ambiguous",
+			arg:  "skippy",
+			st: &ipnstate.Status{
+				MagicDNSSuffix: ".foo",
+				Peer: map[key.NodePublic]*ipnstate.PeerStatus{
+					key.NewNode().Public(): {
+						DNSName:        "skippy.foo.",
+						TailscaleIPs:   []netaddr.IP{mustIP("1.0.0.2")},
+						ExitNodeOption: true,
+					},
+					key.NewNode().Public(): {
+						DNSName:        "SKIPPY.foo.",
+						TailscaleIPs:   []netaddr.IP{mustIP("1.0.0.2")},
+						ExitNodeOption: true,
+					},
+				},
+			},
+			wantErr: `ambiguous exit node name "skippy"`,
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			got, err := exitNodeIPOfArg(tt.arg, tt.st)
+			if err != nil {
+				if err.Error() == tt.wantErr {
+					return
+				}
+				if tt.wantErr == "" {
+					t.Fatal(err)
+				}
+				t.Fatalf("error = %#q; want %#q", err, tt.wantErr)
+			}
+			if tt.wantErr != "" {
+				t.Fatalf("got %v; want error %#q", got, tt.wantErr)
+			}
+			if got != tt.want {
+				t.Fatalf("got %v; want %v", got, tt.want)
 			}
 		})
 	}

cmd/tailscale/cli/configure-host.go

@@ -0,0 +1,85 @@
+// Copyright (c) 2022 Tailscale Inc & AUTHORS All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package cli
+
+import (
+	"context"
+	"errors"
+	"flag"
+	"fmt"
+	"os"
+	"os/exec"
+	"runtime"
+	"strings"
+
+	"github.com/peterbourgon/ff/v3/ffcli"
+	"tailscale.com/hostinfo"
+	"tailscale.com/version/distro"
+)
+
+var configureHostCmd = &ffcli.Command{
+	Name:      "configure-host",
+	Exec:      runConfigureHost,
+	ShortHelp: "Configure Synology to enable more Tailscale features",
+	LongHelp: strings.TrimSpace(`
+The 'configure-host' command is intended to run at boot as root
+to create the /dev/net/tun device and give the tailscaled binary
+permission to use it.
+
+See: https://tailscale.com/kb/1152/synology-outbound/
+`),
+	FlagSet: (func() *flag.FlagSet {
+		fs := newFlagSet("configure-host")
+		return fs
+	})(),
+}
+
+var configureHostArgs struct{}
+
+func runConfigureHost(ctx context.Context, args []string) error {
+	if len(args) > 0 {
+		return errors.New("unknown arguments")
+	}
+	if runtime.GOOS != "linux" || distro.Get() != distro.Synology {
+		return errors.New("only implemented on Synology")
+	}
+	if uid := os.Getuid(); uid != 0 {
+		return fmt.Errorf("must be run as root, not %q (%v)", os.Getenv("USER"), uid)
+	}
+	osVer := hostinfo.GetOSVersion()
+	isDSM6 := strings.HasPrefix(osVer, "Synology 6")
+	isDSM7 := strings.HasPrefix(osVer, "Synology 7")
+	if !isDSM6 && !isDSM7 {
+		return fmt.Errorf("unsupported DSM version %q", osVer)
+	}
+	if _, err := os.Stat("/dev/net/tun"); os.IsNotExist(err) {
+		if err := os.MkdirAll("/dev/net", 0755); err != nil {
+			return fmt.Errorf("creating /dev/net: %v", err)
+		}
+		if out, err := exec.Command("/bin/mknod", "/dev/net/tun", "c", "10", "200").CombinedOutput(); err != nil {
+			return fmt.Errorf("creating /dev/net/tun: %v, %s", err, out)
+		}
+	}
+	if err := os.Chmod("/dev/net/tun", 0666); err != nil {
+		return err
+	}
+	if isDSM6 {
+		fmt.Printf("/dev/net/tun exists and has permissions 0666. Skipping setcap on DSM6.\n")
+		return nil
+	}
+
+	const daemonBin = "/var/packages/Tailscale/target/bin/tailscaled"
+	if _, err := os.Stat(daemonBin); err != nil {
+		if os.IsNotExist(err) {
+			return fmt.Errorf("tailscaled binary not found at %s. Is the Tailscale *.spk package installed?", daemonBin)
+		}
+		return err
+	}
+	if out, err := exec.Command("/bin/setcap", "cap_net_admin+eip", daemonBin).CombinedOutput(); err != nil {
+		return fmt.Errorf("setcap: %v, %s", err, out)
+	}
+	fmt.Printf("Done. To restart Tailscale to use the new permissions, run:\n\n  sudo synosystemctl restart pkgctl-Tailscale.service\n\n")
+	return nil
+}

cmd/tailscale/cli/debug.go

@@ -5,6 +5,8 @@
 package cli
 
 import (
+	"bufio"
+	"bytes"
 	"context"
 	"encoding/json"
 	"errors"
@@ -14,7 +16,9 @@ import (
 	"log"
 	"os"
 	"runtime"
+	"strconv"
 	"strings"
+	"time"
 
 	"github.com/peterbourgon/ff/v3/ffcli"
 	"tailscale.com/client/tailscale"
@@ -24,44 +28,91 @@ import (
 )
 
 var debugCmd = &ffcli.Command{
-	Name: "debug",
-	Exec: runDebug,
+	Name:     "debug",
+	Exec:     runDebug,
+	LongHelp: `"tailscale debug" contains misc debug facilities; it is not a stable interface.`,
 	FlagSet: (func() *flag.FlagSet {
-		fs := flag.NewFlagSet("debug", flag.ExitOnError)
-		fs.BoolVar(&debugArgs.goroutines, "daemon-goroutines", false, "If true, dump the tailscaled daemon's goroutines")
-		fs.BoolVar(&debugArgs.ipn, "ipn", false, "If true, subscribe to IPN notifications")
-		fs.BoolVar(&debugArgs.prefs, "prefs", false, "If true, dump active prefs")
-		fs.BoolVar(&debugArgs.derpMap, "derp", false, "If true, dump DERP map")
-		fs.BoolVar(&debugArgs.pretty, "pretty", false, "If true, pretty-print output (for --prefs)")
-		fs.BoolVar(&debugArgs.netMap, "netmap", true, "whether to include netmap in --ipn mode")
-		fs.BoolVar(&debugArgs.env, "env", false, "dump environment")
-		fs.BoolVar(&debugArgs.localCreds, "local-creds", false, "print how to connect to local tailscaled")
+		fs := newFlagSet("debug")
 		fs.StringVar(&debugArgs.file, "file", "", "get, delete:NAME, or NAME")
 		fs.StringVar(&debugArgs.cpuFile, "cpu-profile", "", "if non-empty, grab a CPU profile for --profile-sec seconds and write it to this file; - for stdout")
 		fs.StringVar(&debugArgs.memFile, "mem-profile", "", "if non-empty, grab a memory profile and write it to this file; - for stdout")
 		fs.IntVar(&debugArgs.cpuSec, "profile-seconds", 15, "number of seconds to run a CPU profile for, when --cpu-profile is non-empty")
 		return fs
 	})(),
+	Subcommands: []*ffcli.Command{
+		{
+			Name:      "derp-map",
+			Exec:      runDERPMap,
+			ShortHelp: "print DERP map",
+		},
+		{
+			Name:      "daemon-goroutines",
+			Exec:      runDaemonGoroutines,
+			ShortHelp: "print tailscaled's goroutines",
+		},
+		{
+			Name:      "metrics",
+			Exec:      runDaemonMetrics,
+			ShortHelp: "print tailscaled's metrics",
+			FlagSet: (func() *flag.FlagSet {
+				fs := newFlagSet("metrics")
+				fs.BoolVar(&metricsArgs.watch, "watch", false, "print JSON dump of delta values")
+				return fs
+			})(),
+		},
+		{
+			Name:      "env",
+			Exec:      runEnv,
+			ShortHelp: "print cmd/tailscale environment",
+		},
+		{
+			Name:      "local-creds",
+			Exec:      runLocalCreds,
+			ShortHelp: "print how to access Tailscale local API",
+		},
+		{
+			Name:      "restun",
+			Exec:      localAPIAction("restun"),
+			ShortHelp: "force a magicsock restun",
+		},
+		{
+			Name:      "rebind",
+			Exec:      localAPIAction("rebind"),
+			ShortHelp: "force a magicsock rebind",
+		},
+		{
+			Name:      "prefs",
+			Exec:      runPrefs,
+			ShortHelp: "print prefs",
+			FlagSet: (func() *flag.FlagSet {
+				fs := newFlagSet("prefs")
+				fs.BoolVar(&prefsArgs.pretty, "pretty", false, "If true, pretty-print output")
+				return fs
+			})(),
+		},
+		{
+			Name:      "watch-ipn",
+			Exec:      runWatchIPN,
+			ShortHelp: "subscribe to IPN message bus",
+			FlagSet: (func() *flag.FlagSet {
+				fs := newFlagSet("watch-ipn")
+				fs.BoolVar(&watchIPNArgs.netmap, "netmap", true, "include netmap in messages")
+				return fs
+			})(),
+		},
+	},
 }
 
 var debugArgs struct {
-	env        bool
-	localCreds bool
-	goroutines bool
-	ipn        bool
-	netMap     bool
-	derpMap    bool
-	file       string
-	prefs      bool
-	pretty     bool
-	cpuSec     int
-	cpuFile    string
-	memFile    string
+	file    string
+	cpuSec  int
+	cpuFile string
+	memFile string
 }
 
 func writeProfile(dst string, v []byte) error {
 	if dst == "-" {
-		_, err := os.Stdout.Write(v)
+		_, err := Stdout.Write(v)
 		return err
 	}
 	return os.WriteFile(dst, v, 0600)
@@ -81,26 +132,9 @@ func runDebug(ctx context.Context, args []string) error {
 	if len(args) > 0 {
 		return errors.New("unknown arguments")
 	}
-	if debugArgs.env {
-		for _, e := range os.Environ() {
-			fmt.Println(e)
-		}
-		return nil
-	}
-	if debugArgs.localCreds {
-		port, token, err := safesocket.LocalTCPPortAndToken()
-		if err == nil {
-			fmt.Printf("curl -u:%s http://localhost:%d/localapi/v0/status\n", token, port)
-			return nil
-		}
-		if runtime.GOOS == "windows" {
-			fmt.Printf("curl http://localhost:41112/localapi/v0/status\n")
-			return nil
-		}
-		fmt.Printf("curl --unix-socket %s http://foo/localapi/v0/status\n", paths.DefaultTailscaledSocket())
-		return nil
-	}
+	var usedFlag bool
 	if out := debugArgs.cpuFile; out != "" {
+		usedFlag = true // TODO(bradfitz): add "profile" subcommand
 		log.Printf("Capturing CPU profile for %v seconds ...", debugArgs.cpuSec)
 		if v, err := tailscale.Profile(ctx, "profile", debugArgs.cpuSec); err != nil {
 			return err
@@ -112,6 +146,7 @@ func runDebug(ctx context.Context, args []string) error {
 		}
 	}
 	if out := debugArgs.memFile; out != "" {
+		usedFlag = true // TODO(bradfitz): add "profile" subcommand
 		log.Printf("Capturing memory profile ...")
 		if v, err := tailscale.Profile(ctx, "heap", 0); err != nil {
 			return err
@@ -122,61 +157,14 @@ func runDebug(ctx context.Context, args []string) error {
 			log.Printf("Memory profile written to %s", outName(out))
 		}
 	}
-	if debugArgs.prefs {
-		prefs, err := tailscale.GetPrefs(ctx)
-		if err != nil {
-			return err
-		}
-		if debugArgs.pretty {
-			fmt.Println(prefs.Pretty())
-		} else {
-			j, _ := json.MarshalIndent(prefs, "", "\t")
-			fmt.Println(string(j))
-		}
-		return nil
-	}
-	if debugArgs.goroutines {
-		goroutines, err := tailscale.Goroutines(ctx)
-		if err != nil {
-			return err
-		}
-		os.Stdout.Write(goroutines)
-		return nil
-	}
-	if debugArgs.derpMap {
-		dm, err := tailscale.CurrentDERPMap(ctx)
-		if err != nil {
-			return fmt.Errorf(
-				"failed to get local derp map, instead `curl %s/derpmap/default`: %w", ipn.DefaultControlURL, err,
-			)
-		}
-		enc := json.NewEncoder(os.Stdout)
-		enc.SetIndent("", "\t")
-		enc.Encode(dm)
-		return nil
-	}
-	if debugArgs.ipn {
-		c, bc, ctx, cancel := connect(ctx)
-		defer cancel()
-
-		bc.SetNotifyCallback(func(n ipn.Notify) {
-			if !debugArgs.netMap {
-				n.NetMap = nil
-			}
-			j, _ := json.MarshalIndent(n, "", "\t")
-			fmt.Printf("%s\n", j)
-		})
-		bc.RequestEngineStatus()
-		pump(ctx, bc, c)
-		return errors.New("exit")
-	}
 	if debugArgs.file != "" {
+		usedFlag = true // TODO(bradfitz): add "file" subcommand
 		if debugArgs.file == "get" {
 			wfs, err := tailscale.WaitingFiles(ctx)
 			if err != nil {
-				log.Fatal(err)
+				fatalf("%v\n", err)
 			}
-			e := json.NewEncoder(os.Stdout)
+			e := json.NewEncoder(Stdout)
 			e.SetIndent("", "\t")
 			e.Encode(wfs)
 			return nil
@@ -190,8 +178,160 @@ func runDebug(ctx context.Context, args []string) error {
 			return err
 		}
 		log.Printf("Size: %v\n", size)
-		io.Copy(os.Stdout, rc)
+		io.Copy(Stdout, rc)
 		return nil
 	}
+	if usedFlag {
+		// TODO(bradfitz): delete this path when all debug flags are migrated
+		// to subcommands.
+		return nil
+	}
+	return errors.New("see 'tailscale debug --help")
+}
+
+func runLocalCreds(ctx context.Context, args []string) error {
+	port, token, err := safesocket.LocalTCPPortAndToken()
+	if err == nil {
+		printf("curl -u:%s http://localhost:%d/localapi/v0/status\n", token, port)
+		return nil
+	}
+	if runtime.GOOS == "windows" {
+		printf("curl http://localhost:%v/localapi/v0/status\n", safesocket.WindowsLocalPort)
+		return nil
+	}
+	printf("curl --unix-socket %s http://foo/localapi/v0/status\n", paths.DefaultTailscaledSocket())
+	return nil
+}
+
+var prefsArgs struct {
+	pretty bool
+}
+
+func runPrefs(ctx context.Context, args []string) error {
+	prefs, err := tailscale.GetPrefs(ctx)
+	if err != nil {
+		return err
+	}
+	if prefsArgs.pretty {
+		outln(prefs.Pretty())
+	} else {
+		j, _ := json.MarshalIndent(prefs, "", "\t")
+		outln(string(j))
+	}
 	return nil
 }
+
+var watchIPNArgs struct {
+	netmap bool
+}
+
+func runWatchIPN(ctx context.Context, args []string) error {
+	c, bc, ctx, cancel := connect(ctx)
+	defer cancel()
+
+	bc.SetNotifyCallback(func(n ipn.Notify) {
+		if !watchIPNArgs.netmap {
+			n.NetMap = nil
+		}
+		j, _ := json.MarshalIndent(n, "", "\t")
+		printf("%s\n", j)
+	})
+	bc.RequestEngineStatus()
+	pump(ctx, bc, c)
+	return errors.New("exit")
+}
+
+func runDERPMap(ctx context.Context, args []string) error {
+	dm, err := tailscale.CurrentDERPMap(ctx)
+	if err != nil {
+		return fmt.Errorf(
+			"failed to get local derp map, instead `curl %s/derpmap/default`: %w", ipn.DefaultControlURL, err,
+		)
+	}
+	enc := json.NewEncoder(Stdout)
+	enc.SetIndent("", "\t")
+	enc.Encode(dm)
+	return nil
+}
+
+func localAPIAction(action string) func(context.Context, []string) error {
+	return func(ctx context.Context, args []string) error {
+		if len(args) > 0 {
+			return errors.New("unexpected arguments")
+		}
+		return tailscale.DebugAction(ctx, action)
+	}
+}
+
+func runEnv(ctx context.Context, args []string) error {
+	for _, e := range os.Environ() {
+		outln(e)
+	}
+	return nil
+}
+
+func runDaemonGoroutines(ctx context.Context, args []string) error {
+	goroutines, err := tailscale.Goroutines(ctx)
+	if err != nil {
+		return err
+	}
+	Stdout.Write(goroutines)
+	return nil
+}
+
+var metricsArgs struct {
+	watch bool
+}
+
+func runDaemonMetrics(ctx context.Context, args []string) error {
+	last := map[string]int64{}
+	for {
+		out, err := tailscale.DaemonMetrics(ctx)
+		if err != nil {
+			return err
+		}
+		if !metricsArgs.watch {
+			Stdout.Write(out)
+			return nil
+		}
+		bs := bufio.NewScanner(bytes.NewReader(out))
+		type change struct {
+			name     string
+			from, to int64
+		}
+		var changes []change
+		var maxNameLen int
+		for bs.Scan() {
+			line := bytes.TrimSpace(bs.Bytes())
+			if len(line) == 0 || line[0] == '#' {
+				continue
+			}
+			f := strings.Fields(string(line))
+			if len(f) != 2 {
+				continue
+			}
+			name := f[0]
+			n, _ := strconv.ParseInt(f[1], 10, 64)
+			prev, ok := last[name]
+			if ok && prev == n {
+				continue
+			}
+			last[name] = n
+			if !ok {
+				continue
+			}
+			changes = append(changes, change{name, prev, n})
+			if len(name) > maxNameLen {
+				maxNameLen = len(name)
+			}
+		}
+		if len(changes) > 0 {
+			format := fmt.Sprintf("%%-%ds %%+5d => %%v\n", maxNameLen)
+			for _, c := range changes {
+				fmt.Fprintf(Stdout, format, c.name, c.to-c.from, c.to)
+			}
+			io.WriteString(Stdout, "\n")
+		}
+		time.Sleep(time.Second)
+	}
+}

cmd/tailscale/cli/diag.go

@@ -25,23 +25,23 @@ func fixTailscaledConnectError(origErr error) error {
 	if err != nil {
 		return fmt.Errorf("failed to connect to local Tailscaled process and failed to enumerate processes while looking for it")
 	}
-	found := false
+	var foundProc ps.Process
 	for _, proc := range procs {
 		base := filepath.Base(proc.Executable())
 		if base == "tailscaled" {
-			found = true
+			foundProc = proc
 			break
 		}
 		if runtime.GOOS == "darwin" && base == "IPNExtension" {
-			found = true
+			foundProc = proc
 			break
 		}
 		if runtime.GOOS == "windows" && strings.EqualFold(base, "tailscaled.exe") {
-			found = true
+			foundProc = proc
 			break
 		}
 	}
-	if !found {
+	if foundProc == nil {
 		switch runtime.GOOS {
 		case "windows":
 			return fmt.Errorf("failed to connect to local tailscaled process; is the Tailscale service running?")
@@ -52,5 +52,5 @@ func fixTailscaledConnectError(origErr error) error {
 		}
 		return fmt.Errorf("failed to connect to local tailscaled process; it doesn't appear to be running")
 	}
-	return fmt.Errorf("failed to connect to local tailscaled (which appears to be running). Got error: %w", origErr)
+	return fmt.Errorf("failed to connect to local tailscaled (which appears to be running as %v, pid %v). Got error: %w", foundProc.Executable(), foundProc.Pid(), origErr)
 }

cmd/tailscale/cli/down.go

@@ -7,8 +7,6 @@ package cli
 import (
 	"context"
 	"fmt"
-	"log"
-	"os"
 
 	"github.com/peterbourgon/ff/v3/ffcli"
 	"tailscale.com/client/tailscale"
@@ -25,7 +23,7 @@ var downCmd = &ffcli.Command{
 
 func runDown(ctx context.Context, args []string) error {
 	if len(args) > 0 {
-		log.Fatalf("too many non-flag arguments: %q", args)
+		return fmt.Errorf("too many non-flag arguments: %q", args)
 	}
 
 	st, err := tailscale.Status(ctx)
@@ -33,7 +31,7 @@ func runDown(ctx context.Context, args []string) error {
 		return fmt.Errorf("error fetching current status: %w", err)
 	}
 	if st.BackendState == "Stopped" {
-		fmt.Fprintf(os.Stderr, "Tailscale was already stopped.\n")
+		fmt.Fprintf(Stderr, "Tailscale was already stopped.\n")
 		return nil
 	}
 	_, err = tailscale.EditPrefs(ctx, &ipn.MaskedPrefs{

cmd/tailscale/cli/file.go

@@ -11,14 +11,11 @@ import (
 	"flag"
 	"fmt"
 	"io"
-	"io/ioutil"
 	"log"
 	"mime"
 	"net/http"
-	"net/url"
 	"os"
 	"path/filepath"
-	"strconv"
 	"strings"
 	"time"
 	"unicode/utf8"
@@ -28,8 +25,10 @@ import (
 	"inet.af/netaddr"
 	"tailscale.com/client/tailscale"
 	"tailscale.com/client/tailscale/apitype"
+	"tailscale.com/envknob"
 	"tailscale.com/ipn"
 	"tailscale.com/net/tsaddr"
+	"tailscale.com/tailcfg"
 	"tailscale.com/version"
 )
 
@@ -55,7 +54,7 @@ var fileCpCmd = &ffcli.Command{
 	ShortHelp:  "Copy file(s) to a host",
 	Exec:       runCp,
 	FlagSet: (func() *flag.FlagSet {
-		fs := flag.NewFlagSet("cp", flag.ExitOnError)
+		fs := newFlagSet("cp")
 		fs.StringVar(&cpArgs.name, "name", "", "alternate filename to use, especially useful when <file> is \"-\" (stdin)")
 		fs.BoolVar(&cpArgs.verbose, "verbose", false, "verbose output")
 		fs.BoolVar(&cpArgs.targets, "targets", false, "list possible file cp targets")
@@ -96,12 +95,12 @@ func runCp(ctx context.Context, args []string) error {
 		return err
 	}
 
-	peerAPIBase, isOffline, err := discoverPeerAPIBase(ctx, ip)
+	stableID, isOffline, err := getTargetStableID(ctx, ip)
 	if err != nil {
 		return fmt.Errorf("can't send to %s: %v", target, err)
 	}
 	if isOffline {
-		fmt.Fprintf(os.Stderr, "# warning: %s is offline\n", target)
+		fmt.Fprintf(Stderr, "# warning: %s is offline\n", target)
 	}
 
 	if len(files) > 1 {
@@ -149,37 +148,26 @@ func runCp(ctx context.Context, args []string) error {
 				name = filepath.Base(fileArg)
 			}
 
-			if slow, _ := strconv.ParseBool(os.Getenv("TS_DEBUG_SLOW_PUSH")); slow {
+			if envknob.Bool("TS_DEBUG_SLOW_PUSH") {
 				fileContents = &slowReader{r: fileContents}
 			}
 		}
 
-		dstURL := peerAPIBase + "/v0/put/" + url.PathEscape(name)
-		req, err := http.NewRequestWithContext(ctx, "PUT", dstURL, fileContents)
-		if err != nil {
-			return err
-		}
-		req.ContentLength = contentLength
 		if cpArgs.verbose {
-			log.Printf("sending to %v ...", dstURL)
+			log.Printf("sending %q to %v/%v/%v ...", name, target, ip, stableID)
 		}
-		res, err := http.DefaultClient.Do(req)
+		err := tailscale.PushFile(ctx, stableID, contentLength, name, fileContents)
 		if err != nil {
 			return err
 		}
-		if res.StatusCode == 200 {
-			io.Copy(ioutil.Discard, res.Body)
-			res.Body.Close()
-			continue
+		if cpArgs.verbose {
+			log.Printf("sent %q", name)
 		}
-		io.Copy(os.Stdout, res.Body)
-		res.Body.Close()
-		return errors.New(res.Status)
 	}
 	return nil
 }
 
-func discoverPeerAPIBase(ctx context.Context, ipStr string) (base string, isOffline bool, err error) {
+func getTargetStableID(ctx context.Context, ipStr string) (id tailcfg.StableNodeID, isOffline bool, err error) {
 	ip, err := netaddr.ParseIP(ipStr)
 	if err != nil {
 		return "", false, err
@@ -195,7 +183,7 @@ func discoverPeerAPIBase(ctx context.Context, ipStr string) (base string, isOffl
 				continue
 			}
 			isOffline = n.Online != nil && !*n.Online
-			return ft.PeerAPIURL, isOffline, nil
+			return n.StableID, isOffline, nil
 		}
 	}
 	return "", false, fileTargetErrorDetail(ctx, ip)
@@ -293,7 +281,7 @@ func runCpTargets(ctx context.Context, args []string) error {
 		if detail != "" {
 			detail = "\t" + detail
 		}
-		fmt.Printf("%s\t%s%s\n", n.Addresses[0].IP(), n.ComputedName, detail)
+		printf("%s\t%s%s\n", n.Addresses[0].IP(), n.ComputedName, detail)
 	}
 	return nil
 }
@@ -304,7 +292,7 @@ var fileGetCmd = &ffcli.Command{
 	ShortHelp:  "Move files out of the Tailscale file inbox",
 	Exec:       runFileGet,
 	FlagSet: (func() *flag.FlagSet {
-		fs := flag.NewFlagSet("get", flag.ExitOnError)
+		fs := newFlagSet("get")
 		fs.BoolVar(&getArgs.wait, "wait", false, "wait for a file to arrive if inbox is empty")
 		fs.BoolVar(&getArgs.verbose, "verbose", false, "verbose output")
 		return fs
@@ -336,7 +324,7 @@ func runFileGet(ctx context.Context, args []string) error {
 	for {
 		wfs, err = tailscale.WaitingFiles(ctx)
 		if err != nil {
-			return fmt.Errorf("getting WaitingFiles: %v", err)
+			return fmt.Errorf("getting WaitingFiles: %w", err)
 		}
 		if len(wfs) != 0 || !getArgs.wait {
 			break
@@ -391,7 +379,7 @@ func wipeInbox(ctx context.Context) error {
 	}
 	wfs, err := tailscale.WaitingFiles(ctx)
 	if err != nil {
-		return fmt.Errorf("getting WaitingFiles: %v", err)
+		return fmt.Errorf("getting WaitingFiles: %w", err)
 	}
 	deleted := 0
 	for _, wf := range wfs {
@@ -415,7 +403,7 @@ func waitForFile(ctx context.Context) error {
 	fileWaiting := make(chan bool, 1)
 	bc.SetNotifyCallback(func(n ipn.Notify) {
 		if n.ErrMessage != nil {
-			log.Fatal(*n.ErrMessage)
+			fatalf("Notify.ErrMessage: %v\n", *n.ErrMessage)
 		}
 		if n.FilesWaiting != nil {
 			select {

cmd/tailscale/cli/ip.go

@@ -18,12 +18,13 @@ import (
 
 var ipCmd = &ffcli.Command{
 	Name:       "ip",
-	ShortUsage: "ip [-4] [-6] [peername]",
-	ShortHelp:  "Show current Tailscale IP address(es)",
-	LongHelp:   "Shows the Tailscale IP address of the current machine without an argument. With an argument, it shows the IP of a named peer.",
+	ShortUsage: "ip [-1] [-4] [-6] [peer hostname or ip address]",
+	ShortHelp:  "Show Tailscale IP addresses",
+	LongHelp:   "Show Tailscale IP addresses for peer. Peer defaults to the current machine.",
 	Exec:       runIP,
 	FlagSet: (func() *flag.FlagSet {
-		fs := flag.NewFlagSet("ip", flag.ExitOnError)
+		fs := newFlagSet("ip")
+		fs.BoolVar(&ipArgs.want1, "1", false, "only print one IP address")
 		fs.BoolVar(&ipArgs.want4, "4", false, "only print IPv4 address")
 		fs.BoolVar(&ipArgs.want6, "6", false, "only print IPv6 address")
 		return fs
@@ -31,13 +32,14 @@ var ipCmd = &ffcli.Command{
 }
 
 var ipArgs struct {
+	want1 bool
 	want4 bool
 	want6 bool
 }
 
 func runIP(ctx context.Context, args []string) error {
 	if len(args) > 1 {
-		return errors.New("unknown arguments")
+		return errors.New("too many arguments, expected at most one peer")
 	}
 	var of string
 	if len(args) == 1 {
@@ -45,8 +47,14 @@ func runIP(ctx context.Context, args []string) error {
 	}
 
 	v4, v6 := ipArgs.want4, ipArgs.want6
-	if v4 && v6 {
-		return errors.New("tailscale ip -4 and -6 are mutually exclusive")
+	nflags := 0
+	for _, b := range []bool{ipArgs.want1, v4, v6} {
+		if b {
+			nflags++
+		}
+	}
+	if nflags > 1 {
+		return errors.New("tailscale ip -1, -4, and -6 are mutually exclusive")
 	}
 	if !v4 && !v6 {
 		v4, v6 = true, true
@@ -71,11 +79,14 @@ func runIP(ctx context.Context, args []string) error {
 		return fmt.Errorf("no current Tailscale IPs; state: %v", st.BackendState)
 	}
 
+	if ipArgs.want1 {
+		ips = ips[:1]
+	}
 	match := false
 	for _, ip := range ips {
 		if ip.Is4() && v4 || ip.Is6() && v6 {
 			match = true
-			fmt.Println(ip)
+			outln(ip)
 		}
 	}
 	if !match {

cmd/tailscale/cli/logout.go

@@ -6,7 +6,7 @@ package cli
 
 import (
 	"context"
-	"log"
+	"fmt"
 	"strings"
 
 	"github.com/peterbourgon/ff/v3/ffcli"
@@ -28,7 +28,7 @@ a reauthentication.
 
 func runLogout(ctx context.Context, args []string) error {
 	if len(args) > 0 {
-		log.Fatalf("too many non-flag arguments: %q", args)
+		return fmt.Errorf("too many non-flag arguments: %q", args)
 	}
 	return tailscale.Logout(ctx)
 }

cmd/tailscale/cli/netcheck.go

@@ -13,13 +13,13 @@ import (
 	"io/ioutil"
 	"log"
 	"net/http"
-	"os"
 	"sort"
 	"strings"
 	"time"
 
 	"github.com/peterbourgon/ff/v3/ffcli"
 	"tailscale.com/client/tailscale"
+	"tailscale.com/envknob"
 	"tailscale.com/ipn"
 	"tailscale.com/net/netcheck"
 	"tailscale.com/net/portmapper"
@@ -33,7 +33,7 @@ var netcheckCmd = &ffcli.Command{
 	ShortHelp:  "Print an analysis of local network conditions",
 	Exec:       runNetcheck,
 	FlagSet: (func() *flag.FlagSet {
-		fs := flag.NewFlagSet("netcheck", flag.ExitOnError)
+		fs := newFlagSet("netcheck")
 		fs.StringVar(&netcheckArgs.format, "format", "", `output format; empty (for human-readable), "json" or "json-line"`)
 		fs.DurationVar(&netcheckArgs.every, "every", 0, "if non-zero, do an incremental report with the given frequency")
 		fs.BoolVar(&netcheckArgs.verbose, "verbose", false, "verbose logs")
@@ -49,7 +49,7 @@ var netcheckArgs struct {
 
 func runNetcheck(ctx context.Context, args []string) error {
 	c := &netcheck.Client{
-		UDPBindAddr: os.Getenv("TS_DEBUG_NETCHECK_UDP_BIND"),
+		UDPBindAddr: envknob.String("TS_DEBUG_NETCHECK_UDP_BIND"),
 		PortMapper:  portmapper.NewClient(logger.WithPrefix(log.Printf, "portmap: "), nil),
 	}
 	if netcheckArgs.verbose {
@@ -60,7 +60,7 @@ func runNetcheck(ctx context.Context, args []string) error {
 	}
 
 	if strings.HasPrefix(netcheckArgs.format, "json") {
-		fmt.Fprintln(os.Stderr, "# Warning: this JSON format is not yet considered a stable interface")
+		fmt.Fprintln(Stderr, "# Warning: this JSON format is not yet considered a stable interface")
 	}
 
 	dm, err := tailscale.CurrentDERPMap(ctx)
@@ -82,7 +82,7 @@ func runNetcheck(ctx context.Context, args []string) error {
 			c.Logf("GetReport took %v; err=%v", d.Round(time.Millisecond), err)
 		}
 		if err != nil {
-			log.Fatalf("netcheck: %v", err)
+			return fmt.Errorf("netcheck: %w", err)
 		}
 		if err := printReport(dm, report); err != nil {
 			return err
@@ -112,36 +112,36 @@ func printReport(dm *tailcfg.DERPMap, report *netcheck.Report) error {
 	}
 	if j != nil {
 		j = append(j, '\n')
-		os.Stdout.Write(j)
+		Stdout.Write(j)
 		return nil
 	}
 
-	fmt.Printf("\nReport:\n")
-	fmt.Printf("\t* UDP: %v\n", report.UDP)
+	printf("\nReport:\n")
+	printf("\t* UDP: %v\n", report.UDP)
 	if report.GlobalV4 != "" {
-		fmt.Printf("\t* IPv4: yes, %v\n", report.GlobalV4)
+		printf("\t* IPv4: yes, %v\n", report.GlobalV4)
 	} else {
-		fmt.Printf("\t* IPv4: (no addr found)\n")
+		printf("\t* IPv4: (no addr found)\n")
 	}
 	if report.GlobalV6 != "" {
-		fmt.Printf("\t* IPv6: yes, %v\n", report.GlobalV6)
+		printf("\t* IPv6: yes, %v\n", report.GlobalV6)
 	} else if report.IPv6 {
-		fmt.Printf("\t* IPv6: (no addr found)\n")
+		printf("\t* IPv6: (no addr found)\n")
 	} else {
-		fmt.Printf("\t* IPv6: no\n")
+		printf("\t* IPv6: no\n")
 	}
-	fmt.Printf("\t* MappingVariesByDestIP: %v\n", report.MappingVariesByDestIP)
-	fmt.Printf("\t* HairPinning: %v\n", report.HairPinning)
-	fmt.Printf("\t* PortMapping: %v\n", portMapping(report))
+	printf("\t* MappingVariesByDestIP: %v\n", report.MappingVariesByDestIP)
+	printf("\t* HairPinning: %v\n", report.HairPinning)
+	printf("\t* PortMapping: %v\n", portMapping(report))
 
 	// When DERP latency checking failed,
 	// magicsock will try to pick the DERP server that
 	// most of your other nodes are also using
 	if len(report.RegionLatency) == 0 {
-		fmt.Printf("\t* Nearest DERP: unknown (no response to latency probes)\n")
+		printf("\t* Nearest DERP: unknown (no response to latency probes)\n")
 	} else {
-		fmt.Printf("\t* Nearest DERP: %v\n", dm.Regions[report.PreferredDERP].RegionName)
-		fmt.Printf("\t* DERP latency:\n")
+		printf("\t* Nearest DERP: %v\n", dm.Regions[report.PreferredDERP].RegionName)
+		printf("\t* DERP latency:\n")
 		var rids []int
 		for rid := range dm.Regions {
 			rids = append(rids, rid)
@@ -168,7 +168,7 @@ func printReport(dm *tailcfg.DERPMap, report *netcheck.Report) error {
 			if netcheckArgs.verbose {
 				derpNum = fmt.Sprintf("derp%d, ", rid)
 			}
-			fmt.Printf("\t\t- %3s: %-7s (%s%s)\n", r.RegionCode, latency, derpNum, r.RegionName)
+			printf("\t\t- %3s: %-7s (%s%s)\n", r.RegionCode, latency, derpNum, r.RegionName)
 		}
 	}
 	return nil

cmd/tailscale/cli/ping.go

@@ -11,6 +11,7 @@ import (
 	"fmt"
 	"log"
 	"net"
+	"os"
 	"strings"
 	"time"
 
@@ -45,7 +46,7 @@ relay node.
 `),
 	Exec: runPing,
 	FlagSet: (func() *flag.FlagSet {
-		fs := flag.NewFlagSet("ping", flag.ExitOnError)
+		fs := newFlagSet("ping")
 		fs.BoolVar(&pingArgs.verbose, "verbose", false, "verbose output")
 		fs.BoolVar(&pingArgs.untilDirect, "until-direct", true, "stop once a direct path is established")
 		fs.BoolVar(&pingArgs.tsmp, "tsmp", false, "do a TSMP-level ping (through IP + wireguard, but not involving host OS stack)")
@@ -64,6 +65,16 @@ var pingArgs struct {
 }
 
 func runPing(ctx context.Context, args []string) error {
+	st, err := tailscale.Status(ctx)
+	if err != nil {
+		return fixTailscaledConnectError(err)
+	}
+	description, ok := isRunningOrStarting(st)
+	if !ok {
+		printf("%s\n", description)
+		os.Exit(1)
+	}
+
 	c, bc, ctx, cancel := connect(ctx)
 	defer cancel()
 
@@ -74,7 +85,7 @@ func runPing(ctx context.Context, args []string) error {
 	prc := make(chan *ipnstate.PingResult, 1)
 	bc.SetNotifyCallback(func(n ipn.Notify) {
 		if n.ErrMessage != nil {
-			log.Fatal(*n.ErrMessage)
+			fatalf("Notify.ErrMessage: %v", *n.ErrMessage)
 		}
 		if pr := n.PingResult; pr != nil && pr.IP == ip {
 			prc <- pr
@@ -89,7 +100,7 @@ func runPing(ctx context.Context, args []string) error {
 		return err
 	}
 	if self {
-		fmt.Printf("%v is local Tailscale IP\n", ip)
+		printf("%v is local Tailscale IP\n", ip)
 		return nil
 	}
 
@@ -105,14 +116,14 @@ func runPing(ctx context.Context, args []string) error {
 		timer := time.NewTimer(pingArgs.timeout)
 		select {
 		case <-timer.C:
-			fmt.Printf("timeout waiting for ping reply\n")
+			printf("timeout waiting for ping reply\n")
 		case err := <-pumpErr:
 			return err
 		case pr := <-prc:
 			timer.Stop()
 			if pr.Err != "" {
 				if pr.IsLocalIP {
-					fmt.Println(pr.Err)
+					outln(pr.Err)
 					return nil
 				}
 				return errors.New(pr.Err)
@@ -132,7 +143,7 @@ func runPing(ctx context.Context, args []string) error {
 			if pr.PeerAPIPort != 0 {
 				extra = fmt.Sprintf(", %d", pr.PeerAPIPort)
 			}
-			fmt.Printf("pong from %s (%s%s) via %v in %v\n", pr.NodeName, pr.NodeIP, extra, via, latency)
+			printf("pong from %s (%s%s) via %v in %v\n", pr.NodeName, pr.NodeIP, extra, via, latency)
 			if pingArgs.tsmp {
 				return nil
 			}

cmd/tailscale/cli/status.go

@@ -29,9 +29,24 @@ var statusCmd = &ffcli.Command{
 	Name:       "status",
 	ShortUsage: "status [--active] [--web] [--json]",
 	ShortHelp:  "Show state of tailscaled and its connections",
-	Exec:       runStatus,
+	LongHelp: strings.TrimSpace(`
+
+JSON FORMAT
+
+Warning: this format has changed between releases and might change more
+in the future.
+
+For a description of the fields, see the "type Status" declaration at:
+
+https://github.com/tailscale/tailscale/blob/main/ipn/ipnstate/ipnstate.go
+
+(and be sure to select branch/tag that corresponds to the version
+ of Tailscale you're running)
+
+`),
+	Exec: runStatus,
 	FlagSet: (func() *flag.FlagSet {
-		fs := flag.NewFlagSet("status", flag.ExitOnError)
+		fs := newFlagSet("status")
 		fs.BoolVar(&statusArgs.json, "json", false, "output in JSON format (WARNING: format subject to change)")
 		fs.BoolVar(&statusArgs.web, "web", false, "run webserver with HTML showing status")
 		fs.BoolVar(&statusArgs.active, "active", false, "filter output to only peers with active sessions (not applicable to web mode)")
@@ -70,7 +85,7 @@ func runStatus(ctx context.Context, args []string) error {
 		if err != nil {
 			return err
 		}
-		fmt.Printf("%s", j)
+		printf("%s", j)
 		return nil
 	}
 	if statusArgs.web {
@@ -79,7 +94,7 @@ func runStatus(ctx context.Context, args []string) error {
 			return err
 		}
 		statusURL := interfaces.HTTPOfListener(ln)
-		fmt.Printf("Serving Tailscale status at %v ...\n", statusURL)
+		printf("Serving Tailscale status at %v ...\n", statusURL)
 		go func() {
 			<-ctx.Done()
 			ln.Close()
@@ -106,32 +121,18 @@ func runStatus(ctx context.Context, args []string) error {
 		return err
 	}
 
-	switch st.BackendState {
-	default:
-		fmt.Fprintf(os.Stderr, "unexpected state: %s\n", st.BackendState)
+	description, ok := isRunningOrStarting(st)
+	if !ok {
+		outln(description)
 		os.Exit(1)
-	case ipn.Stopped.String():
-		fmt.Println("Tailscale is stopped.")
-		os.Exit(1)
-	case ipn.NeedsLogin.String():
-		fmt.Println("Logged out.")
-		if st.AuthURL != "" {
-			fmt.Printf("\nLog in at: %s\n", st.AuthURL)
-		}
-		os.Exit(1)
-	case ipn.NeedsMachineAuth.String():
-		fmt.Println("Machine is not yet authorized by tailnet admin.")
-		os.Exit(1)
-	case ipn.Running.String(), ipn.Starting.String():
-		// Run below.
 	}
 
 	if len(st.Health) > 0 {
-		fmt.Printf("# Health check:\n")
+		printf("# Health check:\n")
 		for _, m := range st.Health {
-			fmt.Printf("#     - %s\n", m)
+			printf("#     - %s\n", m)
 		}
-		fmt.Println()
+		outln()
 	}
 
 	var buf bytes.Buffer
@@ -145,11 +146,19 @@ func runStatus(ctx context.Context, args []string) error {
 		)
 		relay := ps.Relay
 		anyTraffic := ps.TxBytes != 0 || ps.RxBytes != 0
+		var offline string
+		if !ps.Online {
+			offline = "; offline"
+		}
 		if !ps.Active {
 			if ps.ExitNode {
-				f("idle; exit node")
+				f("idle; exit node" + offline)
+			} else if ps.ExitNodeOption {
+				f("idle; offers exit node" + offline)
 			} else if anyTraffic {
-				f("idle")
+				f("idle" + offline)
+			} else if !ps.Online {
+				f("offline")
 			} else {
 				f("-")
 			}
@@ -157,12 +166,17 @@ func runStatus(ctx context.Context, args []string) error {
 			f("active; ")
 			if ps.ExitNode {
 				f("exit node; ")
+			} else if ps.ExitNodeOption {
+				f("offers exit node; ")
 			}
 			if relay != "" && ps.CurAddr == "" {
 				f("relay %q", relay)
 			} else if ps.CurAddr != "" {
 				f("direct %s", ps.CurAddr)
 			}
+			if !ps.Online {
+				f("; offline")
+			}
 		}
 		if anyTraffic {
 			f(", tx %d rx %d", ps.TxBytes, ps.RxBytes)
@@ -190,10 +204,31 @@ func runStatus(ctx context.Context, args []string) error {
 			printPS(ps)
 		}
 	}
-	os.Stdout.Write(buf.Bytes())
+	Stdout.Write(buf.Bytes())
 	return nil
 }
 
+// isRunningOrStarting reports whether st is in state Running or Starting.
+// It also returns a description of the status suitable to display to a user.
+func isRunningOrStarting(st *ipnstate.Status) (description string, ok bool) {
+	switch st.BackendState {
+	default:
+		return fmt.Sprintf("unexpected state: %s", st.BackendState), false
+	case ipn.Stopped.String():
+		return "Tailscale is stopped.", false
+	case ipn.NeedsLogin.String():
+		s := "Logged out."
+		if st.AuthURL != "" {
+			s += fmt.Sprintf("\nLog in at: %s", st.AuthURL)
+		}
+		return s, false
+	case ipn.NeedsMachineAuth.String():
+		return "Machine is not yet authorized by tailnet admin.", false
+	case ipn.Running.String(), ipn.Starting.String():
+		return st.BackendState, true
+	}
+}
+
 func dnsOrQuoteHostname(st *ipnstate.Status, ps *ipnstate.PeerStatus) string {
 	baseName := dnsname.TrimSuffix(ps.DNSName, st.MagicDNSSuffix)
 	if baseName != "" {

cmd/tailscale/cli/up.go

@@ -6,6 +6,8 @@ package cli
 
 import (
 	"context"
+	"encoding/base64"
+	"encoding/json"
 	"errors"
 	"flag"
 	"fmt"
@@ -22,12 +24,15 @@ import (
 	qrcode "github.com/skip2/go-qrcode"
 	"inet.af/netaddr"
 	"tailscale.com/client/tailscale"
+	"tailscale.com/envknob"
 	"tailscale.com/ipn"
 	"tailscale.com/ipn/ipnstate"
 	"tailscale.com/safesocket"
 	"tailscale.com/tailcfg"
 	"tailscale.com/types/logger"
 	"tailscale.com/types/preftype"
+	"tailscale.com/util/dnsname"
+	"tailscale.com/version"
 	"tailscale.com/version/distro"
 )
 
@@ -46,8 +51,10 @@ down").
 
 If flags are specified, the flags must be the complete set of desired
 settings. An error is returned if any setting would be changed as a
-result of an unspecified flag's default value, unless the --reset
-flag is also used.
+result of an unspecified flag's default value, unless the --reset flag
+is also used. (The flags --authkey, --force-reauth, and --qr are not
+considered settings that need to be re-specified when modifying
+settings.)
 `),
 	FlagSet: upFlagSet,
 	Exec:    runUp,
@@ -60,22 +67,41 @@ func effectiveGOOS() string {
 	return runtime.GOOS
 }
 
+// acceptRouteDefault returns the CLI's default value of --accept-routes as
+// a function of the platform it's running on.
+func acceptRouteDefault(goos string) bool {
+	switch goos {
+	case "windows":
+		return true
+	case "darwin":
+		return version.IsSandboxedMacOS()
+	default:
+		return false
+	}
+}
+
 var upFlagSet = newUpFlagSet(effectiveGOOS(), &upArgs)
 
+func inTest() bool { return flag.Lookup("test.v") != nil }
+
 func newUpFlagSet(goos string, upArgs *upArgsT) *flag.FlagSet {
-	upf := flag.NewFlagSet("up", flag.ExitOnError)
+	upf := newFlagSet("up")
 
 	upf.BoolVar(&upArgs.qr, "qr", false, "show QR code for login URLs")
+	upf.BoolVar(&upArgs.json, "json", false, "output in JSON format (WARNING: format subject to change)")
 	upf.BoolVar(&upArgs.forceReauth, "force-reauth", false, "force reauthentication")
 	upf.BoolVar(&upArgs.reset, "reset", false, "reset unspecified settings to their default values")
 
 	upf.StringVar(&upArgs.server, "login-server", ipn.DefaultControlURL, "base URL of control server")
-	upf.BoolVar(&upArgs.acceptRoutes, "accept-routes", false, "accept routes advertised by other Tailscale nodes")
+	upf.BoolVar(&upArgs.acceptRoutes, "accept-routes", acceptRouteDefault(goos), "accept routes advertised by other Tailscale nodes")
 	upf.BoolVar(&upArgs.acceptDNS, "accept-dns", true, "accept DNS configuration from the admin panel")
 	upf.BoolVar(&upArgs.singleRoutes, "host-routes", true, "install host routes to other Tailscale nodes")
-	upf.StringVar(&upArgs.exitNodeIP, "exit-node", "", "Tailscale IP of the exit node for internet traffic, or empty string to not use an exit node")
+	upf.StringVar(&upArgs.exitNodeIP, "exit-node", "", "Tailscale exit node (IP or base name) for internet traffic, or empty string to not use an exit node")
 	upf.BoolVar(&upArgs.exitNodeAllowLANAccess, "exit-node-allow-lan-access", false, "Allow direct access to the local network when routing traffic via an exit node")
 	upf.BoolVar(&upArgs.shieldsUp, "shields-up", false, "don't allow incoming connections")
+	if envknob.UseWIPCode() || inTest() {
+		upf.BoolVar(&upArgs.runSSH, "ssh", false, "run an SSH server, permitting access per tailnet admin's declared policy")
+	}
 	upf.StringVar(&upArgs.advertiseTags, "advertise-tags", "", "comma-separated ACL tags to request; each must start with \"tag:\" (e.g. \"tag:eng,tag:montreal,tag:ssh\")")
 	upf.StringVar(&upArgs.authKeyOrFile, "authkey", "", `node authorization key; if it begins with "file:", then it's a path to a file containing the authkey`)
 	upf.StringVar(&upArgs.hostname, "hostname", "", "hostname to use instead of the one provided by the OS")
@@ -111,6 +137,7 @@ type upArgsT struct {
 	exitNodeIP             string
 	exitNodeAllowLANAccess bool
 	shieldsUp              bool
+	runSSH                 bool
 	forceReauth            bool
 	forceDaemon            bool
 	advertiseRoutes        string
@@ -121,6 +148,7 @@ type upArgsT struct {
 	authKeyOrFile          string // "secret" or "file:/path/to/secret"
 	hostname               string
 	opUser                 string
+	json                   bool
 }
 
 func (a upArgsT) getAuthKey() (string, error) {
@@ -138,8 +166,35 @@ func (a upArgsT) getAuthKey() (string, error) {
 
 var upArgs upArgsT
 
+// Fields output when `tailscale up --json` is used. Two JSON blocks will be output.
+//
+// When "tailscale up" is run it first outputs a block with AuthURL and QR populated,
+// providing the link for where to authenticate this client. BackendState would be
+// valid but boring, as it will almost certainly be "NeedsLogin". Error would be
+// populated if something goes badly wrong.
+//
+// When the client is authenticated by having someone visit the AuthURL, a second
+// JSON block will be output. The AuthURL and QR fields will not be present, the
+// BackendState and Error fields will give the result of the authentication.
+// Ex:
+// {
+//    "AuthURL": "https://login.tailscale.com/a/0123456789abcdef",
+//    "QR": "data:image/png;base64,0123...cdef"
+//    "BackendState": "NeedsLogin"
+// }
+// {
+//    "BackendState": "Running"
+// }
+//
+type upOutputJSON struct {
+	AuthURL      string `json:",omitempty"` // Authentication URL of the form https://login.tailscale.com/a/0123456789
+	QR           string `json:",omitempty"` // a DataURL (base64) PNG of a QR code AuthURL
+	BackendState string `json:",omitempty"` // name of state like Running or NeedsMachineAuth
+	Error        string `json:",omitempty"` // description of an error
+}
+
 func warnf(format string, args ...interface{}) {
-	fmt.Printf("Warning: "+format+"\n", args...)
+	printf("Warning: "+format+"\n", args...)
 }
 
 var (
@@ -190,6 +245,65 @@ func calcAdvertiseRoutes(advertiseRoutes string, advertiseDefaultRoute bool) ([]
 	return routes, nil
 }
 
+// peerWithTailscaleIP returns the peer in st with the provided
+// Tailscale IP.
+func peerWithTailscaleIP(st *ipnstate.Status, ip netaddr.IP) (ps *ipnstate.PeerStatus, ok bool) {
+	for _, ps := range st.Peer {
+		for _, ip2 := range ps.TailscaleIPs {
+			if ip == ip2 {
+				return ps, true
+			}
+		}
+	}
+	return nil, false
+}
+
+// exitNodeIPOfArg maps from a user-provided CLI flag value to an IP
+// address they want to use as an exit node.
+func exitNodeIPOfArg(arg string, st *ipnstate.Status) (ip netaddr.IP, err error) {
+	if arg == "" {
+		return ip, errors.New("invalid use of exitNodeIPOfArg with empty string")
+	}
+	ip, err = netaddr.ParseIP(arg)
+	if err == nil {
+		// If we're online already and have a netmap, double check that the IP
+		// address specified is valid.
+		if st.BackendState == "Running" {
+			ps, ok := peerWithTailscaleIP(st, ip)
+			if !ok {
+				return ip, fmt.Errorf("no node found in netmap with IP %v", ip)
+			}
+			if !ps.ExitNodeOption {
+				return ip, fmt.Errorf("node %v is not advertising an exit node", ip)
+			}
+		}
+		return ip, err
+	}
+	match := 0
+	for _, ps := range st.Peer {
+		baseName := dnsname.TrimSuffix(ps.DNSName, st.MagicDNSSuffix)
+		if !strings.EqualFold(arg, baseName) {
+			continue
+		}
+		match++
+		if len(ps.TailscaleIPs) == 0 {
+			return ip, fmt.Errorf("node %q has no Tailscale IP?", arg)
+		}
+		if !ps.ExitNodeOption {
+			return ip, fmt.Errorf("node %q is not advertising an exit node", arg)
+		}
+		ip = ps.TailscaleIPs[0]
+	}
+	switch match {
+	case 0:
+		return ip, fmt.Errorf("invalid value %q for --exit-node; must be IP or unique node name", arg)
+	case 1:
+		return ip, nil
+	default:
+		return ip, fmt.Errorf("ambiguous exit node name %q", arg)
+	}
+}
+
 // prefsFromUpArgs returns the ipn.Prefs for the provided args.
 //
 // Note that the parameters upArgs and warnf are named intentionally
@@ -205,9 +319,9 @@ func prefsFromUpArgs(upArgs upArgsT, warnf logger.Logf, st *ipnstate.Status, goo
 	var exitNodeIP netaddr.IP
 	if upArgs.exitNodeIP != "" {
 		var err error
-		exitNodeIP, err = netaddr.ParseIP(upArgs.exitNodeIP)
+		exitNodeIP, err = exitNodeIPOfArg(upArgs.exitNodeIP, st)
 		if err != nil {
-			return nil, fmt.Errorf("invalid IP address %q for --exit-node: %v", upArgs.exitNodeIP, err)
+			return nil, err
 		}
 	} else if upArgs.exitNodeAllowLANAccess {
 		return nil, fmt.Errorf("--exit-node-allow-lan-access can only be used with --exit-node")
@@ -245,6 +359,7 @@ func prefsFromUpArgs(upArgs upArgsT, warnf logger.Logf, st *ipnstate.Status, goo
 	prefs.CorpDNS = upArgs.acceptDNS
 	prefs.AllowSingleHosts = upArgs.singleRoutes
 	prefs.ShieldsUp = upArgs.shieldsUp
+	prefs.RunSSH = upArgs.runSSH
 	prefs.AdvertiseRoutes = routes
 	prefs.AdvertiseTags = tags
 	prefs.Hostname = upArgs.hostname
@@ -272,12 +387,13 @@ func prefsFromUpArgs(upArgs upArgsT, warnf logger.Logf, st *ipnstate.Status, goo
 	return prefs, nil
 }
 
-// updatePrefs updates prefs based on curPrefs
+// updatePrefs returns how to edit preferences based on the
+// flag-provided 'prefs' and the currently active 'curPrefs'.
 //
 // It returns a non-nil justEditMP if we're already running and none of
 // the flags require a restart, so we can just do an EditPrefs call and
 // change the prefs at runtime (e.g. changing hostname, changing
-// advertised tags, routes, etc).
+// advertised routes, etc).
 //
 // It returns simpleUp if we're running a simple "tailscale up" to
 // transition to running from a previously-logged-in but down state,
@@ -297,6 +413,8 @@ func updatePrefs(prefs, curPrefs *ipn.Prefs, env upCheckEnv) (simpleUp bool, jus
 		return false, nil, fmt.Errorf("can't change --login-server without --force-reauth")
 	}
 
+	tagsChanged := !reflect.DeepEqual(curPrefs.AdvertiseTags, prefs.AdvertiseTags)
+
 	simpleUp = env.flagSet.NFlag() == 0 &&
 		curPrefs.Persist != nil &&
 		curPrefs.Persist.LoginName != "" &&
@@ -304,14 +422,19 @@ func updatePrefs(prefs, curPrefs *ipn.Prefs, env upCheckEnv) (simpleUp bool, jus
 
 	justEdit := env.backendState == ipn.Running.String() &&
 		!env.upArgs.forceReauth &&
-		!env.upArgs.reset &&
 		env.upArgs.authKeyOrFile == "" &&
-		!controlURLChanged
+		!controlURLChanged &&
+		!tagsChanged
+
 	if justEdit {
 		justEditMP = new(ipn.MaskedPrefs)
 		justEditMP.WantRunningSet = true
 		justEditMP.Prefs = *prefs
-		env.flagSet.Visit(func(f *flag.Flag) {
+		visitFlags := env.flagSet.Visit
+		if env.upArgs.reset {
+			visitFlags = env.flagSet.VisitAll
+		}
+		visitFlags(func(f *flag.Flag) {
 			updateMaskedPrefsFromUpFlag(justEditMP, f.Name)
 		})
 	}
@@ -377,11 +500,12 @@ func runUp(ctx context.Context, args []string) error {
 
 	env := upCheckEnv{
 		goos:          effectiveGOOS(),
+		distro:        distro.Get(),
 		user:          os.Getenv("USER"),
 		flagSet:       upFlagSet,
 		upArgs:        upArgs,
 		backendState:  st.BackendState,
-		curExitNodeIP: exitNodeIP(prefs, st),
+		curExitNodeIP: exitNodeIP(curPrefs, st),
 	}
 	simpleUp, justEditMP, err := updatePrefs(prefs, curPrefs, env)
 	if err != nil {
@@ -402,7 +526,7 @@ func runUp(ctx context.Context, args []string) error {
 	pumpErr := make(chan error, 1)
 	go func() { pumpErr <- pump(pumpCtx, bc, c) }()
 
-	printed := !simpleUp
+	var printed bool // whether we've yet printed anything to stdout or stderr
 	var loginOnce sync.Once
 	startLoginInteractive := func() { loginOnce.Do(func() { bc.StartLoginInteractive() }) }
 
@@ -428,16 +552,21 @@ func runUp(ctx context.Context, args []string) error {
 		if s := n.State; s != nil {
 			switch *s {
 			case ipn.NeedsLogin:
-				printed = true
 				startLoginInteractive()
 			case ipn.NeedsMachineAuth:
 				printed = true
-				fmt.Fprintf(os.Stderr, "\nTo authorize your machine, visit (as admin):\n\n\t%s\n\n", prefs.AdminPageURL())
+				if env.upArgs.json {
+					printUpDoneJSON(ipn.NeedsMachineAuth, "")
+				} else {
+					fmt.Fprintf(Stderr, "\nTo authorize your machine, visit (as admin):\n\n\t%s\n\n", prefs.AdminPageURL())
+				}
 			case ipn.Running:
 				// Done full authentication process
-				if printed {
+				if env.upArgs.json {
+					printUpDoneJSON(ipn.Running, "")
+				} else if printed {
 					// Only need to print an update if we printed the "please click" message earlier.
-					fmt.Fprintf(os.Stderr, "Success.\n")
+					fmt.Fprintf(Stderr, "Success.\n")
 				}
 				select {
 				case running <- true:
@@ -448,15 +577,33 @@ func runUp(ctx context.Context, args []string) error {
 		}
 		if url := n.BrowseToURL; url != nil && printAuthURL(*url) {
 			printed = true
-			fmt.Fprintf(os.Stderr, "\nTo authenticate, visit:\n\n\t%s\n\n", *url)
-			if upArgs.qr {
+			if upArgs.json {
+				js := &upOutputJSON{AuthURL: *url, BackendState: st.BackendState}
+
 				q, err := qrcode.New(*url, qrcode.Medium)
-				if err != nil {
-					log.Printf("QR code error: %v", err)
-				} else {
-					fmt.Fprintf(os.Stderr, "%s\n", q.ToString(false))
+				if err == nil {
+					png, err := q.PNG(128)
+					if err == nil {
+						js.QR = "data:image/png;base64," + base64.StdEncoding.EncodeToString(png)
+					}
 				}
 
+				data, err := json.MarshalIndent(js, "", "\t")
+				if err != nil {
+					log.Printf("upOutputJSON marshalling error: %v", err)
+				} else {
+					fmt.Println(string(data))
+				}
+			} else {
+				fmt.Fprintf(Stderr, "\nTo authenticate, visit:\n\n\t%s\n\n", *url)
+				if upArgs.qr {
+					q, err := qrcode.New(*url, qrcode.Medium)
+					if err != nil {
+						log.Printf("QR code error: %v", err)
+					} else {
+						fmt.Fprintf(Stderr, "%s\n", q.ToString(false))
+					}
+				}
 			}
 		}
 	})
@@ -543,6 +690,16 @@ func runUp(ctx context.Context, args []string) error {
 	}
 }
 
+func printUpDoneJSON(state ipn.State, errorString string) {
+	js := &upOutputJSON{BackendState: state.String(), Error: errorString}
+	data, err := json.MarshalIndent(js, "", "  ")
+	if err != nil {
+		log.Printf("printUpDoneJSON marshalling error: %v", err)
+	} else {
+		fmt.Println(string(data))
+	}
+}
+
 var (
 	prefsOfFlag = map[string][]string{} // "exit-node" => ExitNodeIP, ExitNodeID
 )
@@ -568,6 +725,7 @@ func init() {
 	addPrefFlagMapping("exit-node-allow-lan-access", "ExitNodeAllowLANAccess")
 	addPrefFlagMapping("unattended", "ForceDaemon")
 	addPrefFlagMapping("operator", "OperatorUser")
+	addPrefFlagMapping("ssh", "RunSSH")
 }
 
 func addPrefFlagMapping(flagName string, prefNames ...string) {
@@ -585,7 +743,7 @@ func addPrefFlagMapping(flagName string, prefNames ...string) {
 // correspond to an ipn.Pref.
 func preflessFlag(flagName string) bool {
 	switch flagName {
-	case "authkey", "force-reauth", "reset", "qr":
+	case "authkey", "force-reauth", "reset", "qr", "json":
 		return true
 	}
 	return false
@@ -619,6 +777,7 @@ type upCheckEnv struct {
 	upArgs        upArgsT
 	backendState  string
 	curExitNodeIP netaddr.IP
+	distro        distro.Distro
 }
 
 // checkForAccidentalSettingReverts (the "up checker") checks for
@@ -669,6 +828,10 @@ func checkForAccidentalSettingReverts(newPrefs, curPrefs *ipn.Prefs, env upCheck
 		if flagName == "login-server" && ipn.IsLoginServerSynonym(valCur) && ipn.IsLoginServerSynonym(valNew) {
 			continue
 		}
+		if flagName == "accept-routes" && valNew == false && env.goos == "linux" && env.distro == distro.Synology {
+			// Issue 3176. Old prefs had 'RouteAll: true' on disk, so ignore that.
+			continue
+		}
 		missing = append(missing, fmtFlagValueArg(flagName, valCur))
 	}
 	if len(missing) == 0 {
@@ -753,6 +916,8 @@ func prefsToFlags(env upCheckEnv, prefs *ipn.Prefs) (flagVal map[string]interfac
 		switch f.Name {
 		default:
 			panic(fmt.Sprintf("unhandled flag %q", f.Name))
+		case "ssh":
+			set(prefs.RunSSH)
 		case "login-server":
 			set(prefs.ControlURL)
 		case "accept-routes":

cmd/tailscale/cli/version.go

@@ -8,7 +8,6 @@ import (
 	"context"
 	"flag"
 	"fmt"
-	"log"
 
 	"github.com/peterbourgon/ff/v3/ffcli"
 	"tailscale.com/client/tailscale"
@@ -20,7 +19,7 @@ var versionCmd = &ffcli.Command{
 	ShortUsage: "version [flags]",
 	ShortHelp:  "Print Tailscale version",
 	FlagSet: (func() *flag.FlagSet {
-		fs := flag.NewFlagSet("version", flag.ExitOnError)
+		fs := newFlagSet("version")
 		fs.BoolVar(&versionArgs.daemon, "daemon", false, "also print local node's daemon version")
 		return fs
 	})(),
@@ -33,19 +32,19 @@ var versionArgs struct {
 
 func runVersion(ctx context.Context, args []string) error {
 	if len(args) > 0 {
-		log.Fatalf("too many non-flag arguments: %q", args)
+		return fmt.Errorf("too many non-flag arguments: %q", args)
 	}
 	if !versionArgs.daemon {
-		fmt.Println(version.String())
+		outln(version.String())
 		return nil
 	}
 
-	fmt.Printf("Client: %s\n", version.String())
+	printf("Client: %s\n", version.String())
 
 	st, err := tailscale.StatusWithoutPeers(ctx)
 	if err != nil {
 		return err
 	}
-	fmt.Printf("Daemon: %s\n", st.Version)
+	printf("Daemon: %s\n", st.Version)
 	return nil
 }

cmd/tailscale/cli/web.go

@@ -76,7 +76,7 @@ Tailscale, as opposed to a CLI or a native app.
 `),
 
 	FlagSet: (func() *flag.FlagSet {
-		webf := flag.NewFlagSet("web", flag.ExitOnError)
+		webf := newFlagSet("web")
 		webf.StringVar(&webArgs.listen, "listen", "localhost:8088", "listen address; use port 0 for automatic")
 		webf.BoolVar(&webArgs.cgi, "cgi", false, "run as CGI script")
 		return webf
@@ -114,7 +114,7 @@ func tlsConfigFromEnvironment() *tls.Config {
 
 func runWeb(ctx context.Context, args []string) error {
 	if len(args) > 0 {
-		log.Fatalf("too many non-flag arguments: %q", args)
+		return fmt.Errorf("too many non-flag arguments: %q", args)
 	}
 
 	if webArgs.cgi {
@@ -270,14 +270,14 @@ func synoTokenRedirect(w http.ResponseWriter, r *http.Request) bool {
 	// We need a SynoToken for authenticate.cgi.
 	// So we tell the client to get one.
 	serverURL := r.URL.Scheme + "://" + r.URL.Host
-	fmt.Fprintf(w, synoTokenRedirectHTML, serverURL)
+	synoTokenRedirectHTML.Execute(w, serverURL)
 	return true
 }
 
-const synoTokenRedirectHTML = `<html><body>
+var synoTokenRedirectHTML = template.Must(template.New("redirect").Parse(`<html><body>
 Redirecting with session token...
 <script>
-var serverURL = %q;
+var serverURL = {{ . }};
 var req = new XMLHttpRequest();
 req.overrideMimeType("application/json");
 req.open("GET", serverURL + "/webman/login.cgi", true);
@@ -289,7 +289,7 @@ req.onload = function() {
 req.send(null);
 </script>
 </body></html>
-`
+`))
 
 func webHandler(w http.ResponseWriter, r *http.Request) {
 	if authRedirect(w, r) {
@@ -375,7 +375,7 @@ func webHandler(w http.ResponseWriter, r *http.Request) {
 			data.AdvertiseExitNode = true
 		} else {
 			if data.AdvertiseRoutes != "" {
-				data.AdvertiseRoutes = ","
+				data.AdvertiseRoutes += ","
 			}
 			data.AdvertiseRoutes += r.String()
 		}

cmd/tailscale/depaware.txt

@@ -3,7 +3,9 @@ tailscale.com/cmd/tailscale dependencies: (generated by github.com/tailscale/dep
    W 💣 github.com/alexbrainman/sspi                                 from github.com/alexbrainman/sspi/negotiate+
    W    github.com/alexbrainman/sspi/internal/common                 from github.com/alexbrainman/sspi/negotiate
    W 💣 github.com/alexbrainman/sspi/negotiate                       from tailscale.com/net/tshttpproxy
+        github.com/golang/groupcache/lru                             from tailscale.com/net/dnscache
         github.com/kballard/go-shellquote                            from tailscale.com/cmd/tailscale/cli
+   L    github.com/klauspost/compress/flate                          from nhooyr.io/websocket
      💣 github.com/mitchellh/go-ps                                   from tailscale.com/cmd/tailscale/cli+
         github.com/peterbourgon/ff/v3                                from github.com/peterbourgon/ff/v3/ffcli
         github.com/peterbourgon/ff/v3/ffcli                          from tailscale.com/cmd/tailscale/cli
@@ -23,6 +25,10 @@ tailscale.com/cmd/tailscale dependencies: (generated by github.com/tailscale/dep
         go4.org/unsafe/assume-no-moving-gc                           from go4.org/intern
    W 💣 golang.zx2c4.com/wireguard/windows/tunnel/winipcfg           from tailscale.com/net/interfaces+
         inet.af/netaddr                                              from tailscale.com/cmd/tailscale/cli+
+   L    nhooyr.io/websocket                                          from tailscale.com/derp/derphttp+
+   L    nhooyr.io/websocket/internal/errd                            from nhooyr.io/websocket
+   L    nhooyr.io/websocket/internal/xsync                           from nhooyr.io/websocket
+        tailscale.com                                                from tailscale.com/version
         tailscale.com/atomicfile                                     from tailscale.com/ipn+
         tailscale.com/client/tailscale                               from tailscale.com/cmd/tailscale/cli+
         tailscale.com/client/tailscale/apitype                       from tailscale.com/client/tailscale+
@@ -30,8 +36,10 @@ tailscale.com/cmd/tailscale dependencies: (generated by github.com/tailscale/dep
         tailscale.com/control/controlknobs                           from tailscale.com/net/portmapper
         tailscale.com/derp                                           from tailscale.com/derp/derphttp
         tailscale.com/derp/derphttp                                  from tailscale.com/net/netcheck
+   L    tailscale.com/derp/wsconn                                    from tailscale.com/derp/derphttp
         tailscale.com/disco                                          from tailscale.com/derp
-        tailscale.com/hostinfo                                       from tailscale.com/net/interfaces
+        tailscale.com/envknob                                        from tailscale.com/cmd/tailscale/cli+
+        tailscale.com/hostinfo                                       from tailscale.com/net/interfaces+
         tailscale.com/ipn                                            from tailscale.com/cmd/tailscale/cli+
         tailscale.com/ipn/ipnstate                                   from tailscale.com/cmd/tailscale/cli+
         tailscale.com/kube                                           from tailscale.com/ipn
@@ -40,6 +48,7 @@ tailscale.com/cmd/tailscale dependencies: (generated by github.com/tailscale/dep
         tailscale.com/net/flowtrack                                  from tailscale.com/wgengine/filter+
      💣 tailscale.com/net/interfaces                                 from tailscale.com/cmd/tailscale/cli+
         tailscale.com/net/netcheck                                   from tailscale.com/cmd/tailscale/cli
+        tailscale.com/net/neterror                                   from tailscale.com/net/netcheck+
         tailscale.com/net/netknob                                    from tailscale.com/net/netns
         tailscale.com/net/netns                                      from tailscale.com/derp/derphttp+
         tailscale.com/net/packet                                     from tailscale.com/wgengine/filter
@@ -66,7 +75,7 @@ tailscale.com/cmd/tailscale dependencies: (generated by github.com/tailscale/dep
         tailscale.com/types/persist                                  from tailscale.com/ipn
         tailscale.com/types/preftype                                 from tailscale.com/cmd/tailscale/cli+
         tailscale.com/types/structs                                  from tailscale.com/ipn+
-        tailscale.com/types/wgkey                                    from tailscale.com/types/netmap+
+        tailscale.com/util/clientmetric                              from tailscale.com/net/netcheck+
         tailscale.com/util/dnsname                                   from tailscale.com/cmd/tailscale/cli+
    W    tailscale.com/util/endian                                    from tailscale.com/net/netns
         tailscale.com/util/groupmember                               from tailscale.com/cmd/tailscale/cli
@@ -76,16 +85,16 @@ tailscale.com/cmd/tailscale dependencies: (generated by github.com/tailscale/dep
         tailscale.com/wgengine/filter                                from tailscale.com/types/netmap
         golang.org/x/crypto/blake2b                                  from golang.org/x/crypto/nacl/box
         golang.org/x/crypto/chacha20                                 from golang.org/x/crypto/chacha20poly1305
-        golang.org/x/crypto/chacha20poly1305                         from crypto/tls+
+        golang.org/x/crypto/chacha20poly1305                         from crypto/tls
         golang.org/x/crypto/cryptobyte                               from crypto/ecdsa+
         golang.org/x/crypto/cryptobyte/asn1                          from crypto/ecdsa+
         golang.org/x/crypto/curve25519                               from crypto/tls+
         golang.org/x/crypto/hkdf                                     from crypto/tls
-        golang.org/x/crypto/nacl/box                                 from tailscale.com/derp+
+        golang.org/x/crypto/nacl/box                                 from tailscale.com/types/key
         golang.org/x/crypto/nacl/secretbox                           from golang.org/x/crypto/nacl/box
         golang.org/x/crypto/poly1305                                 from golang.org/x/crypto/chacha20poly1305
         golang.org/x/crypto/salsa20/salsa                            from golang.org/x/crypto/nacl/box+
-        golang.org/x/net/dns/dnsmessage                              from net
+        golang.org/x/net/dns/dnsmessage                              from net+
         golang.org/x/net/http/httpguts                               from net/http+
         golang.org/x/net/http/httpproxy                              from net/http
         golang.org/x/net/http2/hpack                                 from net/http
@@ -97,7 +106,7 @@ tailscale.com/cmd/tailscale dependencies: (generated by github.com/tailscale/dep
         golang.org/x/sys/cpu                                         from golang.org/x/crypto/blake2b+
   LD    golang.org/x/sys/unix                                        from tailscale.com/net/netns+
    W    golang.org/x/sys/windows                                     from golang.org/x/sys/windows/registry+
-   W    golang.org/x/sys/windows/registry                            from golang.zx2c4.com/wireguard/windows/tunnel/winipcfg
+   W    golang.org/x/sys/windows/registry                            from golang.zx2c4.com/wireguard/windows/tunnel/winipcfg+
         golang.org/x/text/secure/bidirule                            from golang.org/x/net/idna
         golang.org/x/text/transform                                  from golang.org/x/text/secure/bidirule+
         golang.org/x/text/unicode/bidi                               from golang.org/x/net/idna+
@@ -130,7 +139,7 @@ tailscale.com/cmd/tailscale dependencies: (generated by github.com/tailscale/dep
         crypto/tls                                                   from github.com/tcnksm/go-httpstat+
         crypto/x509                                                  from crypto/tls+
         crypto/x509/pkix                                             from crypto/x509+
-        embed                                                        from tailscale.com/cmd/tailscale/cli
+        embed                                                        from tailscale.com/cmd/tailscale/cli+
         encoding                                                     from encoding/json+
         encoding/asn1                                                from crypto/x509+
         encoding/base64                                              from encoding/json+

cmd/tailscaled/debug.go

@@ -24,6 +24,7 @@ import (
 
 	"inet.af/netaddr"
 	"tailscale.com/derp/derphttp"
+	"tailscale.com/envknob"
 	"tailscale.com/ipn"
 	"tailscale.com/net/interfaces"
 	"tailscale.com/net/portmapper"
@@ -193,8 +194,8 @@ func checkDerp(ctx context.Context, derpRegion string) error {
 		panic("unreachable")
 	}
 
-	priv1 := key.NewPrivate()
-	priv2 := key.NewPrivate()
+	priv1 := key.NewNode()
+	priv2 := key.NewNode()
 
 	c1 := derphttp.NewRegionClient(priv1, log.Printf, getRegion)
 	c2 := derphttp.NewRegionClient(priv2, log.Printf, getRegion)
@@ -224,7 +225,7 @@ func debugPortmap(ctx context.Context) error {
 	defer cancel()
 
 	portmapper.VerboseLogs = true
-	switch os.Getenv("TS_DEBUG_PORTMAP_TYPE") {
+	switch envknob.String("TS_DEBUG_PORTMAP_TYPE") {
 	case "":
 	case "pmp":
 		portmapper.DisablePCP = true

cmd/tailscaled/depaware.txt

@@ -3,6 +3,7 @@ tailscale.com/cmd/tailscaled dependencies: (generated by github.com/tailscale/de
    W 💣 github.com/alexbrainman/sspi                                 from github.com/alexbrainman/sspi/internal/common+
    W    github.com/alexbrainman/sspi/internal/common                 from github.com/alexbrainman/sspi/negotiate
    W 💣 github.com/alexbrainman/sspi/negotiate                       from tailscale.com/net/tshttpproxy
+   L    github.com/anmitsu/go-shlex                                  from github.com/gliderlabs/ssh
    L    github.com/aws/aws-sdk-go-v2                                 from github.com/aws/aws-sdk-go-v2/internal/ini
    L    github.com/aws/aws-sdk-go-v2/aws                             from github.com/aws/aws-sdk-go-v2/aws/middleware+
    L    github.com/aws/aws-sdk-go-v2/aws/arn                         from tailscale.com/ipn/store/aws
@@ -25,7 +26,8 @@ tailscale.com/cmd/tailscaled dependencies: (generated by github.com/tailscale/de
    L    github.com/aws/aws-sdk-go-v2/credentials/stscreds            from github.com/aws/aws-sdk-go-v2/config
    L    github.com/aws/aws-sdk-go-v2/feature/ec2/imds                from github.com/aws/aws-sdk-go-v2/config+
    L    github.com/aws/aws-sdk-go-v2/feature/ec2/imds/internal/config from github.com/aws/aws-sdk-go-v2/feature/ec2/imds
-   L    github.com/aws/aws-sdk-go-v2/internal/endpoints              from github.com/aws/aws-sdk-go-v2/service/ssm/internal/endpoints+
+   L    github.com/aws/aws-sdk-go-v2/internal/configsources          from github.com/aws/aws-sdk-go-v2/service/ssm+
+   L    github.com/aws/aws-sdk-go-v2/internal/endpoints/v2           from github.com/aws/aws-sdk-go-v2/service/ssm/internal/endpoints+
    L    github.com/aws/aws-sdk-go-v2/internal/ini                    from github.com/aws/aws-sdk-go-v2/config
    L    github.com/aws/aws-sdk-go-v2/internal/rand                   from github.com/aws/aws-sdk-go-v2/aws+
    L    github.com/aws/aws-sdk-go-v2/internal/sdk                    from github.com/aws/aws-sdk-go-v2/aws+
@@ -59,11 +61,13 @@ tailscale.com/cmd/tailscaled dependencies: (generated by github.com/tailscale/de
    L    github.com/aws/smithy-go/transport/http/internal/io          from github.com/aws/smithy-go/transport/http
    L    github.com/aws/smithy-go/waiter                              from github.com/aws/aws-sdk-go-v2/service/ssm
    L    github.com/coreos/go-iptables/iptables                       from tailscale.com/wgengine/router
-        github.com/go-multierror/multierror                          from tailscale.com/cmd/tailscaled+
+   L 💣 github.com/creack/pty                                        from tailscale.com/wgengine/netstack
+   L    github.com/gliderlabs/ssh                                    from tailscale.com/wgengine/netstack
    W 💣 github.com/go-ole/go-ole                                     from github.com/go-ole/go-ole/oleutil+
    W 💣 github.com/go-ole/go-ole/oleutil                             from tailscale.com/wgengine/winnet
    L 💣 github.com/godbus/dbus/v5                                    from tailscale.com/net/dns
-        github.com/google/btree                                      from inet.af/netstack/tcpip/header+
+        github.com/golang/groupcache/lru                             from tailscale.com/net/dnscache
+        github.com/google/btree                                      from gvisor.dev/gvisor/pkg/tcpip/header+
    L    github.com/insomniacslk/dhcp/dhcpv4                          from tailscale.com/net/tstun
    L    github.com/insomniacslk/dhcp/iana                            from github.com/insomniacslk/dhcp/dhcpv4
    L    github.com/insomniacslk/dhcp/interfaces                      from github.com/insomniacslk/dhcp/dhcpv4
@@ -73,6 +77,7 @@ tailscale.com/cmd/tailscaled dependencies: (generated by github.com/tailscale/de
    L 💣 github.com/jsimonetti/rtnetlink                              from tailscale.com/wgengine/monitor
    L    github.com/jsimonetti/rtnetlink/internal/unix                from github.com/jsimonetti/rtnetlink
         github.com/klauspost/compress                                from github.com/klauspost/compress/zstd
+   L    github.com/klauspost/compress/flate                          from nhooyr.io/websocket
         github.com/klauspost/compress/fse                            from github.com/klauspost/compress/huff0
         github.com/klauspost/compress/huff0                          from github.com/klauspost/compress/zstd
         github.com/klauspost/compress/internal/snapref               from github.com/klauspost/compress/zstd
@@ -91,60 +96,74 @@ tailscale.com/cmd/tailscaled dependencies: (generated by github.com/tailscale/de
         github.com/tailscale/goupnp/scpd                             from github.com/tailscale/goupnp
         github.com/tailscale/goupnp/soap                             from github.com/tailscale/goupnp+
         github.com/tailscale/goupnp/ssdp                             from github.com/tailscale/goupnp
+   L 💣 github.com/tailscale/netlink                                 from tailscale.com/wgengine/router
         github.com/tcnksm/go-httpstat                                from tailscale.com/net/netcheck
    L    github.com/u-root/uio/rand                                   from github.com/insomniacslk/dhcp/dhcpv4
    L    github.com/u-root/uio/ubinary                                from github.com/u-root/uio/uio
    L    github.com/u-root/uio/uio                                    from github.com/insomniacslk/dhcp/dhcpv4+
+   L 💣 github.com/vishvananda/netlink/nl                            from github.com/tailscale/netlink
+   L    github.com/vishvananda/netns                                 from github.com/tailscale/netlink+
      💣 go4.org/intern                                               from inet.af/netaddr
      💣 go4.org/mem                                                  from tailscale.com/client/tailscale+
         go4.org/unsafe/assume-no-moving-gc                           from go4.org/intern
+   W 💣 golang.zx2c4.com/wintun                                      from golang.zx2c4.com/wireguard/tun
      💣 golang.zx2c4.com/wireguard/conn                              from golang.zx2c4.com/wireguard/device+
    W 💣 golang.zx2c4.com/wireguard/conn/winrio                       from golang.zx2c4.com/wireguard/conn
      💣 golang.zx2c4.com/wireguard/device                            from tailscale.com/net/tstun+
      💣 golang.zx2c4.com/wireguard/ipc                               from golang.zx2c4.com/wireguard/device
-   W 💣 golang.zx2c4.com/wireguard/ipc/winpipe                       from golang.zx2c4.com/wireguard/ipc
+   W 💣 golang.zx2c4.com/wireguard/ipc/namedpipe                     from golang.zx2c4.com/wireguard/ipc
         golang.zx2c4.com/wireguard/ratelimiter                       from golang.zx2c4.com/wireguard/device
         golang.zx2c4.com/wireguard/replay                            from golang.zx2c4.com/wireguard/device
         golang.zx2c4.com/wireguard/rwcancel                          from golang.zx2c4.com/wireguard/device+
         golang.zx2c4.com/wireguard/tai64n                            from golang.zx2c4.com/wireguard/device
      💣 golang.zx2c4.com/wireguard/tun                               from golang.zx2c4.com/wireguard/device+
-   W 💣 golang.zx2c4.com/wireguard/tun/wintun                        from golang.zx2c4.com/wireguard/tun
    W 💣 golang.zx2c4.com/wireguard/windows/tunnel/winipcfg           from tailscale.com/cmd/tailscaled+
+        gvisor.dev/gvisor/pkg/atomicbitops                           from gvisor.dev/gvisor/pkg/tcpip+
+     💣 gvisor.dev/gvisor/pkg/buffer                                 from gvisor.dev/gvisor/pkg/tcpip/stack
+        gvisor.dev/gvisor/pkg/context                                from gvisor.dev/gvisor/pkg/refs+
+     💣 gvisor.dev/gvisor/pkg/gohacks                                from gvisor.dev/gvisor/pkg/state/wire+
+        gvisor.dev/gvisor/pkg/linewriter                             from gvisor.dev/gvisor/pkg/log
+        gvisor.dev/gvisor/pkg/log                                    from gvisor.dev/gvisor/pkg/context+
+        gvisor.dev/gvisor/pkg/rand                                   from gvisor.dev/gvisor/pkg/tcpip/network/hash+
+        gvisor.dev/gvisor/pkg/refs                                   from gvisor.dev/gvisor/pkg/refsvfs2
+        gvisor.dev/gvisor/pkg/refsvfs2                               from gvisor.dev/gvisor/pkg/tcpip/stack
+     💣 gvisor.dev/gvisor/pkg/sleep                                  from gvisor.dev/gvisor/pkg/tcpip/transport/tcp
+     💣 gvisor.dev/gvisor/pkg/state                                  from gvisor.dev/gvisor/pkg/atomicbitops+
+        gvisor.dev/gvisor/pkg/state/wire                             from gvisor.dev/gvisor/pkg/state
+     💣 gvisor.dev/gvisor/pkg/sync                                   from gvisor.dev/gvisor/pkg/linewriter+
+        gvisor.dev/gvisor/pkg/tcpip                                  from gvisor.dev/gvisor/pkg/tcpip/adapters/gonet+
+        gvisor.dev/gvisor/pkg/tcpip/adapters/gonet                   from tailscale.com/wgengine/netstack
+     💣 gvisor.dev/gvisor/pkg/tcpip/buffer                           from gvisor.dev/gvisor/pkg/tcpip/adapters/gonet+
+        gvisor.dev/gvisor/pkg/tcpip/hash/jenkins                     from gvisor.dev/gvisor/pkg/tcpip/stack+
+        gvisor.dev/gvisor/pkg/tcpip/header                           from gvisor.dev/gvisor/pkg/tcpip/header/parse+
+        gvisor.dev/gvisor/pkg/tcpip/header/parse                     from gvisor.dev/gvisor/pkg/tcpip/network/ipv4+
+        gvisor.dev/gvisor/pkg/tcpip/internal/tcp                     from gvisor.dev/gvisor/pkg/tcpip/stack+
+        gvisor.dev/gvisor/pkg/tcpip/link/channel                     from tailscale.com/wgengine/netstack
+        gvisor.dev/gvisor/pkg/tcpip/network/hash                     from gvisor.dev/gvisor/pkg/tcpip/network/ipv4+
+        gvisor.dev/gvisor/pkg/tcpip/network/internal/fragmentation   from gvisor.dev/gvisor/pkg/tcpip/network/ipv4+
+        gvisor.dev/gvisor/pkg/tcpip/network/internal/ip              from gvisor.dev/gvisor/pkg/tcpip/network/ipv4+
+        gvisor.dev/gvisor/pkg/tcpip/network/ipv4                     from tailscale.com/net/tstun+
+        gvisor.dev/gvisor/pkg/tcpip/network/ipv6                     from tailscale.com/wgengine/netstack
+        gvisor.dev/gvisor/pkg/tcpip/ports                            from gvisor.dev/gvisor/pkg/tcpip/stack+
+        gvisor.dev/gvisor/pkg/tcpip/seqnum                           from gvisor.dev/gvisor/pkg/tcpip/header+
+     💣 gvisor.dev/gvisor/pkg/tcpip/stack                            from gvisor.dev/gvisor/pkg/tcpip/adapters/gonet+
+        gvisor.dev/gvisor/pkg/tcpip/transport                        from gvisor.dev/gvisor/pkg/tcpip/transport/icmp+
+        gvisor.dev/gvisor/pkg/tcpip/transport/icmp                   from tailscale.com/wgengine/netstack
+        gvisor.dev/gvisor/pkg/tcpip/transport/internal/network       from gvisor.dev/gvisor/pkg/tcpip/transport/icmp+
+        gvisor.dev/gvisor/pkg/tcpip/transport/internal/noop          from gvisor.dev/gvisor/pkg/tcpip/transport/raw
+        gvisor.dev/gvisor/pkg/tcpip/transport/packet                 from gvisor.dev/gvisor/pkg/tcpip/transport/raw
+        gvisor.dev/gvisor/pkg/tcpip/transport/raw                    from gvisor.dev/gvisor/pkg/tcpip/transport/icmp+
+     💣 gvisor.dev/gvisor/pkg/tcpip/transport/tcp                    from gvisor.dev/gvisor/pkg/tcpip/adapters/gonet+
+        gvisor.dev/gvisor/pkg/tcpip/transport/tcpconntrack           from gvisor.dev/gvisor/pkg/tcpip/stack
+        gvisor.dev/gvisor/pkg/tcpip/transport/udp                    from gvisor.dev/gvisor/pkg/tcpip/adapters/gonet+
+        gvisor.dev/gvisor/pkg/waiter                                 from gvisor.dev/gvisor/pkg/context+
         inet.af/netaddr                                              from inet.af/wf+
-        inet.af/netstack/atomicbitops                                from inet.af/netstack/tcpip+
-     💣 inet.af/netstack/buffer                                      from inet.af/netstack/tcpip/stack
-     💣 inet.af/netstack/gohacks                                     from inet.af/netstack/state/wire+
-        inet.af/netstack/linewriter                                  from inet.af/netstack/log
-        inet.af/netstack/log                                         from inet.af/netstack/state+
-        inet.af/netstack/rand                                        from inet.af/netstack/tcpip/network/hash+
-     💣 inet.af/netstack/sleep                                       from inet.af/netstack/tcpip/transport/tcp
-     💣 inet.af/netstack/state                                       from inet.af/netstack/atomicbitops+
-        inet.af/netstack/state/wire                                  from inet.af/netstack/state
-     💣 inet.af/netstack/sync                                        from inet.af/netstack/linewriter+
-        inet.af/netstack/tcpip                                       from inet.af/netstack/tcpip/adapters/gonet+
-        inet.af/netstack/tcpip/adapters/gonet                        from tailscale.com/wgengine/netstack
-     💣 inet.af/netstack/tcpip/buffer                                from inet.af/netstack/tcpip/adapters/gonet+
-        inet.af/netstack/tcpip/hash/jenkins                          from inet.af/netstack/tcpip/stack+
-        inet.af/netstack/tcpip/header                                from inet.af/netstack/tcpip/header/parse+
-        inet.af/netstack/tcpip/header/parse                          from inet.af/netstack/tcpip/network/ipv4+
-        inet.af/netstack/tcpip/link/channel                          from tailscale.com/wgengine/netstack
-        inet.af/netstack/tcpip/network/hash                          from inet.af/netstack/tcpip/network/ipv4+
-        inet.af/netstack/tcpip/network/internal/fragmentation        from inet.af/netstack/tcpip/network/ipv4+
-        inet.af/netstack/tcpip/network/internal/ip                   from inet.af/netstack/tcpip/network/ipv4+
-        inet.af/netstack/tcpip/network/ipv4                          from tailscale.com/net/tstun+
-        inet.af/netstack/tcpip/network/ipv6                          from tailscale.com/wgengine/netstack
-        inet.af/netstack/tcpip/ports                                 from inet.af/netstack/tcpip/stack+
-        inet.af/netstack/tcpip/seqnum                                from inet.af/netstack/tcpip/header+
-     💣 inet.af/netstack/tcpip/stack                                 from inet.af/netstack/tcpip/adapters/gonet+
-        inet.af/netstack/tcpip/transport/icmp                        from tailscale.com/wgengine/netstack
-        inet.af/netstack/tcpip/transport/packet                      from inet.af/netstack/tcpip/transport/raw
-        inet.af/netstack/tcpip/transport/raw                         from inet.af/netstack/tcpip/transport/icmp+
-     💣 inet.af/netstack/tcpip/transport/tcp                         from inet.af/netstack/tcpip/adapters/gonet+
-        inet.af/netstack/tcpip/transport/tcpconntrack                from inet.af/netstack/tcpip/stack
-        inet.af/netstack/tcpip/transport/udp                         from inet.af/netstack/tcpip/adapters/gonet+
-        inet.af/netstack/waiter                                      from inet.af/netstack/tcpip+
         inet.af/peercred                                             from tailscale.com/ipn/ipnserver
    W 💣 inet.af/wf                                                   from tailscale.com/wf
+   L    nhooyr.io/websocket                                          from tailscale.com/derp/derphttp+
+   L    nhooyr.io/websocket/internal/errd                            from nhooyr.io/websocket
+   L    nhooyr.io/websocket/internal/xsync                           from nhooyr.io/websocket
+        tailscale.com                                                from tailscale.com/version
         tailscale.com/atomicfile                                     from tailscale.com/ipn+
   LD    tailscale.com/chirp                                          from tailscale.com/cmd/tailscaled
         tailscale.com/client/tailscale                               from tailscale.com/derp
@@ -153,7 +172,9 @@ tailscale.com/cmd/tailscaled dependencies: (generated by github.com/tailscale/de
         tailscale.com/control/controlknobs                           from tailscale.com/control/controlclient+
         tailscale.com/derp                                           from tailscale.com/derp/derphttp+
         tailscale.com/derp/derphttp                                  from tailscale.com/cmd/tailscaled+
+   L    tailscale.com/derp/wsconn                                    from tailscale.com/derp/derphttp
         tailscale.com/disco                                          from tailscale.com/derp+
+        tailscale.com/envknob                                        from tailscale.com/cmd/tailscaled+
         tailscale.com/health                                         from tailscale.com/control/controlclient+
         tailscale.com/hostinfo                                       from tailscale.com/control/controlclient+
         tailscale.com/ipn                                            from tailscale.com/client/tailscale+
@@ -164,30 +185,33 @@ tailscale.com/cmd/tailscaled dependencies: (generated by github.com/tailscale/de
         tailscale.com/ipn/policy                                     from tailscale.com/ipn/ipnlocal
         tailscale.com/ipn/store/aws                                  from tailscale.com/ipn/ipnserver
         tailscale.com/kube                                           from tailscale.com/ipn
-        tailscale.com/log/filelogger                                 from tailscale.com/ipn/ipnserver
+        tailscale.com/log/filelogger                                 from tailscale.com/logpolicy
         tailscale.com/log/logheap                                    from tailscale.com/control/controlclient
-        tailscale.com/logpolicy                                      from tailscale.com/cmd/tailscaled
-        tailscale.com/logtail                                        from tailscale.com/logpolicy
+        tailscale.com/logpolicy                                      from tailscale.com/cmd/tailscaled+
+        tailscale.com/logtail                                        from tailscale.com/logpolicy+
         tailscale.com/logtail/backoff                                from tailscale.com/cmd/tailscaled+
         tailscale.com/logtail/filch                                  from tailscale.com/logpolicy
-     💣 tailscale.com/metrics                                        from tailscale.com/derp
+     💣 tailscale.com/metrics                                        from tailscale.com/derp+
         tailscale.com/net/dns                                        from tailscale.com/cmd/tailscaled+
         tailscale.com/net/dns/resolver                               from tailscale.com/net/dns+
         tailscale.com/net/dnscache                                   from tailscale.com/control/controlclient+
-        tailscale.com/net/dnsfallback                                from tailscale.com/control/controlclient
+        tailscale.com/net/dnsfallback                                from tailscale.com/control/controlclient+
         tailscale.com/net/flowtrack                                  from tailscale.com/net/packet+
      💣 tailscale.com/net/interfaces                                 from tailscale.com/cmd/tailscaled+
         tailscale.com/net/netcheck                                   from tailscale.com/wgengine/magicsock
-        tailscale.com/net/netknob                                    from tailscale.com/ipn/localapi+
+        tailscale.com/net/neterror                                   from tailscale.com/net/netcheck+
+        tailscale.com/net/netknob                                    from tailscale.com/logpolicy+
         tailscale.com/net/netns                                      from tailscale.com/cmd/tailscaled+
      💣 tailscale.com/net/netstat                                    from tailscale.com/ipn/ipnserver
+        tailscale.com/net/netutil                                    from tailscale.com/ipn/ipnlocal+
         tailscale.com/net/packet                                     from tailscale.com/net/tstun+
         tailscale.com/net/portmapper                                 from tailscale.com/cmd/tailscaled+
-        tailscale.com/net/socks5                                     from tailscale.com/net/socks5/tssocks
-        tailscale.com/net/socks5/tssocks                             from tailscale.com/cmd/tailscaled
+        tailscale.com/net/proxymux                                   from tailscale.com/cmd/tailscaled
+        tailscale.com/net/socks5                                     from tailscale.com/cmd/tailscaled
         tailscale.com/net/stun                                       from tailscale.com/net/netcheck+
         tailscale.com/net/tlsdial                                    from tailscale.com/control/controlclient+
         tailscale.com/net/tsaddr                                     from tailscale.com/ipn/ipnlocal+
+        tailscale.com/net/tsdial                                     from tailscale.com/cmd/tailscaled+
      💣 tailscale.com/net/tshttpproxy                                from tailscale.com/cmd/tailscaled+
         tailscale.com/net/tstun                                      from tailscale.com/cmd/tailscaled+
      💣 tailscale.com/paths                                          from tailscale.com/client/tailscale+
@@ -200,6 +224,7 @@ tailscale.com/cmd/tailscaled dependencies: (generated by github.com/tailscale/de
         tailscale.com/tstime                                         from tailscale.com/wgengine/magicsock
      💣 tailscale.com/tstime/mono                                    from tailscale.com/net/tstun+
         tailscale.com/tstime/rate                                    from tailscale.com/wgengine/filter
+        tailscale.com/tsweb                                          from tailscale.com/cmd/tailscaled
         tailscale.com/types/dnstype                                  from tailscale.com/ipn/ipnlocal+
         tailscale.com/types/empty                                    from tailscale.com/control/controlclient+
         tailscale.com/types/flagtype                                 from tailscale.com/cmd/tailscaled
@@ -209,23 +234,25 @@ tailscale.com/cmd/tailscaled dependencies: (generated by github.com/tailscale/de
         tailscale.com/types/netmap                                   from tailscale.com/control/controlclient+
         tailscale.com/types/nettype                                  from tailscale.com/wgengine/magicsock
         tailscale.com/types/opt                                      from tailscale.com/control/controlclient+
-        tailscale.com/types/pad32                                    from tailscale.com/derp+
+        tailscale.com/types/pad32                                    from tailscale.com/derp
         tailscale.com/types/persist                                  from tailscale.com/control/controlclient+
         tailscale.com/types/preftype                                 from tailscale.com/ipn+
         tailscale.com/types/structs                                  from tailscale.com/control/controlclient+
-        tailscale.com/types/wgkey                                    from tailscale.com/control/controlclient+
+        tailscale.com/util/clientmetric                              from tailscale.com/ipn/localapi+
    L    tailscale.com/util/cmpver                                    from tailscale.com/net/dns
      💣 tailscale.com/util/deephash                                  from tailscale.com/ipn/ipnlocal+
         tailscale.com/util/dnsname                                   from tailscale.com/hostinfo+
   LW    tailscale.com/util/endian                                    from tailscale.com/net/dns+
         tailscale.com/util/groupmember                               from tailscale.com/ipn/ipnserver
         tailscale.com/util/lineread                                  from tailscale.com/hostinfo+
+        tailscale.com/util/multierr                                  from tailscale.com/cmd/tailscaled+
         tailscale.com/util/osshare                                   from tailscale.com/cmd/tailscaled+
         tailscale.com/util/pidowner                                  from tailscale.com/ipn/ipnserver
         tailscale.com/util/racebuild                                 from tailscale.com/logpolicy
         tailscale.com/util/systemd                                   from tailscale.com/control/controlclient+
         tailscale.com/util/uniq                                      from tailscale.com/wgengine/magicsock
         tailscale.com/util/winutil                                   from tailscale.com/cmd/tailscaled+
+   W 💣 tailscale.com/util/winutil/vss                               from tailscale.com/util/winutil
         tailscale.com/version                                        from tailscale.com/client/tailscale+
         tailscale.com/version/distro                                 from tailscale.com/cmd/tailscaled+
    W    tailscale.com/wf                                             from tailscale.com/cmd/tailscaled
@@ -233,7 +260,7 @@ tailscale.com/cmd/tailscaled dependencies: (generated by github.com/tailscale/de
         tailscale.com/wgengine/filter                                from tailscale.com/control/controlclient+
         tailscale.com/wgengine/magicsock                             from tailscale.com/wgengine+
         tailscale.com/wgengine/monitor                               from tailscale.com/cmd/tailscaled+
-        tailscale.com/wgengine/netstack                              from tailscale.com/cmd/tailscaled+
+     💣 tailscale.com/wgengine/netstack                              from tailscale.com/cmd/tailscaled
         tailscale.com/wgengine/router                                from tailscale.com/cmd/tailscaled+
         tailscale.com/wgengine/wgcfg                                 from tailscale.com/ipn/ipnlocal+
         tailscale.com/wgengine/wgcfg/nmcfg                           from tailscale.com/ipn/ipnlocal
@@ -242,21 +269,26 @@ tailscale.com/cmd/tailscaled dependencies: (generated by github.com/tailscale/de
         golang.org/x/crypto/acme                                     from tailscale.com/ipn/localapi
         golang.org/x/crypto/blake2b                                  from golang.org/x/crypto/nacl/box
         golang.org/x/crypto/blake2s                                  from golang.zx2c4.com/wireguard/device
-        golang.org/x/crypto/chacha20                                 from golang.org/x/crypto/chacha20poly1305
+   L    golang.org/x/crypto/blowfish                                 from golang.org/x/crypto/ssh/internal/bcrypt_pbkdf
+        golang.org/x/crypto/chacha20                                 from golang.org/x/crypto/chacha20poly1305+
         golang.org/x/crypto/chacha20poly1305                         from crypto/tls+
         golang.org/x/crypto/cryptobyte                               from crypto/ecdsa+
         golang.org/x/crypto/cryptobyte/asn1                          from crypto/ecdsa+
         golang.org/x/crypto/curve25519                               from crypto/tls+
+   L    golang.org/x/crypto/ed25519                                  from golang.org/x/crypto/ssh
         golang.org/x/crypto/hkdf                                     from crypto/tls
-        golang.org/x/crypto/nacl/box                                 from tailscale.com/derp+
+        golang.org/x/crypto/nacl/box                                 from tailscale.com/types/key
         golang.org/x/crypto/nacl/secretbox                           from golang.org/x/crypto/nacl/box
         golang.org/x/crypto/poly1305                                 from golang.org/x/crypto/chacha20poly1305+
         golang.org/x/crypto/salsa20/salsa                            from golang.org/x/crypto/nacl/box+
+   L    golang.org/x/crypto/ssh                                      from github.com/gliderlabs/ssh+
         golang.org/x/net/bpf                                         from github.com/mdlayher/netlink+
         golang.org/x/net/dns/dnsmessage                              from net+
         golang.org/x/net/http/httpguts                               from net/http+
         golang.org/x/net/http/httpproxy                              from net/http
-        golang.org/x/net/http2/hpack                                 from net/http
+        golang.org/x/net/http2                                       from golang.org/x/net/http2/h2c+
+        golang.org/x/net/http2/h2c                                   from tailscale.com/ipn/ipnlocal
+        golang.org/x/net/http2/hpack                                 from net/http+
         golang.org/x/net/idna                                        from golang.org/x/net/http/httpguts+
         golang.org/x/net/ipv4                                        from golang.zx2c4.com/wireguard/device
         golang.org/x/net/ipv6                                        from golang.zx2c4.com/wireguard/device+
@@ -275,26 +307,26 @@ tailscale.com/cmd/tailscaled dependencies: (generated by github.com/tailscale/de
         golang.org/x/text/transform                                  from golang.org/x/text/secure/bidirule+
         golang.org/x/text/unicode/bidi                               from golang.org/x/net/idna+
         golang.org/x/text/unicode/norm                               from golang.org/x/net/idna
-        golang.org/x/time/rate                                       from inet.af/netstack/tcpip/stack+
+        golang.org/x/time/rate                                       from gvisor.dev/gvisor/pkg/tcpip/stack+
         bufio                                                        from compress/flate+
         bytes                                                        from bufio+
-        compress/flate                                               from compress/gzip
+        compress/flate                                               from compress/gzip+
         compress/gzip                                                from internal/profile+
-        container/heap                                               from inet.af/netstack/tcpip/transport/tcp
+        container/heap                                               from gvisor.dev/gvisor/pkg/tcpip/transport/tcp
         container/list                                               from crypto/tls+
         context                                                      from crypto/tls+
         crypto                                                       from crypto/ecdsa+
         crypto/aes                                                   from crypto/ecdsa+
         crypto/cipher                                                from crypto/aes+
         crypto/des                                                   from crypto/tls+
-        crypto/dsa                                                   from crypto/x509
+        crypto/dsa                                                   from crypto/x509+
         crypto/ecdsa                                                 from crypto/tls+
         crypto/ed25519                                               from crypto/tls+
         crypto/elliptic                                              from crypto/ecdsa+
         crypto/hmac                                                  from crypto/tls+
         crypto/md5                                                   from crypto/tls+
         crypto/rand                                                  from crypto/ed25519+
-        crypto/rc4                                                   from crypto/tls
+        crypto/rc4                                                   from crypto/tls+
         crypto/rsa                                                   from crypto/tls+
         crypto/sha1                                                  from crypto/tls+
         crypto/sha256                                                from crypto/tls+
@@ -318,9 +350,10 @@ tailscale.com/cmd/tailscaled dependencies: (generated by github.com/tailscale/de
         fmt                                                          from compress/flate+
         hash                                                         from crypto+
         hash/crc32                                                   from compress/gzip+
-        hash/fnv                                                     from inet.af/netstack/tcpip/network/ipv6+
+        hash/fnv                                                     from gvisor.dev/gvisor/pkg/tcpip/network/ipv6+
         hash/maphash                                                 from go4.org/mem
         html                                                         from net/http/pprof+
+        html/template                                                from tailscale.com/tsweb
         io                                                           from bufio+
         io/fs                                                        from crypto/rand+
         io/ioutil                                                    from github.com/aws/aws-sdk-go-v2/aws/protocol/query+
@@ -347,7 +380,7 @@ tailscale.com/cmd/tailscaled dependencies: (generated by github.com/tailscale/de
         path                                                         from github.com/aws/aws-sdk-go-v2/credentials/ec2rolecreds+
         path/filepath                                                from crypto/x509+
         reflect                                                      from crypto/x509+
-        regexp                                                       from github.com/aws/aws-sdk-go-v2/internal/endpoints+
+        regexp                                                       from github.com/aws/aws-sdk-go-v2/internal/endpoints/v2+
         regexp/syntax                                                from regexp
         runtime/debug                                                from github.com/klauspost/compress/zstd+
         runtime/pprof                                                from net/http/pprof+
@@ -359,6 +392,8 @@ tailscale.com/cmd/tailscaled dependencies: (generated by github.com/tailscale/de
         sync/atomic                                                  from context+
         syscall                                                      from crypto/rand+
         text/tabwriter                                               from runtime/pprof
+        text/template                                                from html/template
+        text/template/parse                                          from html/template+
         time                                                         from compress/gzip+
         unicode                                                      from bytes+
         unicode/utf16                                                from crypto/x509+

cmd/tailscaled/tailscaled.go

@@ -20,24 +20,32 @@ import (
 	"net/http/pprof"
 	"os"
 	"os/signal"
+	"path/filepath"
 	"runtime"
 	"runtime/debug"
-	"strconv"
 	"strings"
 	"syscall"
 	"time"
 
-	"github.com/go-multierror/multierror"
+	"inet.af/netaddr"
+	"tailscale.com/envknob"
 	"tailscale.com/ipn"
 	"tailscale.com/ipn/ipnserver"
 	"tailscale.com/logpolicy"
+	"tailscale.com/logtail"
 	"tailscale.com/net/dns"
 	"tailscale.com/net/netns"
-	"tailscale.com/net/socks5/tssocks"
+	"tailscale.com/net/proxymux"
+	"tailscale.com/net/socks5"
+	"tailscale.com/net/tsdial"
 	"tailscale.com/net/tstun"
 	"tailscale.com/paths"
+	"tailscale.com/safesocket"
+	"tailscale.com/tsweb"
 	"tailscale.com/types/flagtype"
 	"tailscale.com/types/logger"
+	"tailscale.com/util/clientmetric"
+	"tailscale.com/util/multierr"
 	"tailscale.com/util/osshare"
 	"tailscale.com/version"
 	"tailscale.com/version/distro"
@@ -78,6 +86,7 @@ var args struct {
 	debug          string
 	port           uint16
 	statepath      string
+	statedir       string
 	socketpath     string
 	birdSocketPath string
 	verbose        int
@@ -114,7 +123,8 @@ func main() {
 	flag.StringVar(&args.httpProxyAddr, "outbound-http-proxy-listen", "", `optional [ip]:port to run an outbound HTTP proxy (e.g. "localhost:8080")`)
 	flag.StringVar(&args.tunname, "tun", defaultTunName(), `tunnel interface name; use "userspace-networking" (beta) to not use TUN`)
 	flag.Var(flagtype.PortValue(&args.port, 0), "port", "UDP port to listen on for WireGuard and peer-to-peer traffic; 0 means automatically select")
-	flag.StringVar(&args.statepath, "state", paths.DefaultTailscaledStateFile(), "path of state file; use 'kube:<secret-name>' to use Kubernetes secrets or 'arn:aws:ssm:...' to store in AWS SSM")
+	flag.StringVar(&args.statepath, "state", paths.DefaultTailscaledStateFile(), "absolute path of state file; use 'kube:<secret-name>' to use Kubernetes secrets or 'arn:aws:ssm:...' to store in AWS SSM. If empty and --statedir is provided, the default is <statedir>/tailscaled.state")
+	flag.StringVar(&args.statedir, "statedir", "", "path to directory for storage of config state, TLS certs, temporary incoming Taildrop files, etc. If empty, it's derived from --state when possible.")
 	flag.StringVar(&args.socketpath, "socket", paths.DefaultTailscaledSocket(), "path of the service unix socket")
 	flag.StringVar(&args.birdSocketPath, "bird-socket", "", "path of the bird unix socket")
 	flag.BoolVar(&printVersion, "version", false, "print version information and exit")
@@ -169,8 +179,7 @@ func main() {
 	osshare.SetFileSharingEnabled(false, logger.Discard)
 
 	if err != nil {
-		// No need to log; the func already did
-		os.Exit(1)
+		log.Fatal(err)
 	}
 }
 
@@ -202,17 +211,33 @@ func trySynologyMigration(p string) error {
 	return nil
 }
 
+func statePathOrDefault() string {
+	if args.statepath != "" {
+		return args.statepath
+	}
+	if args.statedir != "" {
+		return filepath.Join(args.statedir, "tailscaled.state")
+	}
+	return ""
+}
+
 func ipnServerOpts() (o ipnserver.Options) {
 	// Allow changing the OS-specific IPN behavior for tests
 	// so we can e.g. test Windows-specific behaviors on Linux.
-	goos := os.Getenv("TS_DEBUG_TAILSCALED_IPN_GOOS")
+	goos := envknob.String("TS_DEBUG_TAILSCALED_IPN_GOOS")
 	if goos == "" {
 		goos = runtime.GOOS
 	}
 
-	o.Port = 41112
-	o.StatePath = args.statepath
-	o.SocketPath = args.socketpath // even for goos=="windows", for tests
+	o.VarRoot = args.statedir
+
+	// If an absolute --state is provided but not --statedir, try to derive
+	// a state directory.
+	if o.VarRoot == "" && filepath.IsAbs(args.statepath) {
+		if dir := filepath.Dir(args.statepath); strings.EqualFold(filepath.Base(dir), "tailscale") {
+			o.VarRoot = dir
+		}
+	}
 
 	switch goos {
 	default:
@@ -227,7 +252,7 @@ func ipnServerOpts() (o ipnserver.Options) {
 func run() error {
 	var err error
 
-	pol := logpolicy.New("tailnode.log.tailscale.io")
+	pol := logpolicy.New(logtail.CollectionNode)
 	pol.SetVerbosityLevel(args.verbose)
 	defer func() {
 		// Finish uploading logs after closing everything else.
@@ -247,13 +272,13 @@ func run() error {
 	}
 
 	var logf logger.Logf = log.Printf
-	if v, _ := strconv.ParseBool(os.Getenv("TS_DEBUG_MEMORY")); v {
+	if envknob.Bool("TS_DEBUG_MEMORY") {
 		logf = logger.RusagePrefixLog(logf)
 	}
 	logf = logger.RateLimitedFn(logf, 5*time.Second, 5, 100)
 
 	if args.cleanup {
-		if os.Getenv("TS_PLEASE_PANIC") != "" {
+		if envknob.Bool("TS_PLEASE_PANIC") {
 			panic("TS_PLEASE_PANIC asked us to panic")
 		}
 		dns.Cleanup(logf, args.tunname)
@@ -261,51 +286,73 @@ func run() error {
 		return nil
 	}
 
-	if args.statepath == "" {
-		log.Fatalf("--state is required")
+	if args.statepath == "" && args.statedir == "" {
+		log.Fatalf("--statedir (or at least --state) is required")
 	}
-	if err := trySynologyMigration(args.statepath); err != nil {
+	if err := trySynologyMigration(statePathOrDefault()); err != nil {
 		log.Printf("error in synology migration: %v", err)
 	}
 
 	var debugMux *http.ServeMux
 	if args.debug != "" {
 		debugMux = newDebugMux()
-		go runDebugServer(debugMux, args.debug)
 	}
 
 	linkMon, err := monitor.New(logf)
 	if err != nil {
-		log.Fatalf("creating link monitor: %v", err)
+		return fmt.Errorf("monitor.New: %w", err)
 	}
 	pol.Logtail.SetLinkMonitor(linkMon)
 
-	socksListener := mustStartTCPListener("SOCKS5", args.socksAddr)
-	httpProxyListener := mustStartTCPListener("HTTP proxy", args.httpProxyAddr)
+	socksListener, httpProxyListener := mustStartProxyListeners(args.socksAddr, args.httpProxyAddr)
 
-	e, useNetstack, err := createEngine(logf, linkMon)
+	dialer := new(tsdial.Dialer) // mutated below (before used)
+	e, useNetstack, err := createEngine(logf, linkMon, dialer)
 	if err != nil {
-		logf("wgengine.New: %v", err)
-		return err
+		return fmt.Errorf("createEngine: %w", err)
+	}
+	if _, ok := e.(wgengine.ResolvingEngine).GetResolver(); !ok {
+		panic("internal error: exit node resolver not wired up")
+	}
+	if debugMux != nil {
+		if ig, ok := e.(wgengine.InternalsGetter); ok {
+			if _, mc, ok := ig.GetInternals(); ok {
+				debugMux.HandleFunc("/debug/magicsock", mc.ServeHTTPDebug)
+			}
+		}
+		go runDebugServer(debugMux, args.debug)
 	}
 
-	var ns *netstack.Impl
-	if useNetstack || wrapNetstack {
-		onlySubnets := wrapNetstack && !useNetstack
-		ns = mustStartNetstack(logf, e, onlySubnets)
+	ns, err := newNetstack(logf, dialer, e)
+	if err != nil {
+		return fmt.Errorf("newNetstack: %w", err)
 	}
+	ns.ProcessLocalIPs = useNetstack
+	ns.ProcessSubnets = useNetstack || wrapNetstack
 
+	if useNetstack {
+		dialer.UseNetstackForIP = func(ip netaddr.IP) bool {
+			_, ok := e.PeerForIP(ip)
+			return ok
+		}
+		dialer.NetstackDialTCP = func(ctx context.Context, dst netaddr.IPPort) (net.Conn, error) {
+			return ns.DialContextTCP(ctx, dst)
+		}
+	}
 	if socksListener != nil || httpProxyListener != nil {
-		srv := tssocks.NewServer(logger.WithPrefix(logf, "socks5: "), e, ns)
 		if httpProxyListener != nil {
-			hs := &http.Server{Handler: httpProxyHandler(srv.Dialer)}
+			hs := &http.Server{Handler: httpProxyHandler(dialer.UserDial)}
 			go func() {
 				log.Fatalf("HTTP proxy exited: %v", hs.Serve(httpProxyListener))
 			}()
 		}
 		if socksListener != nil {
+			ss := &socks5.Server{
+				Logf:   logger.WithPrefix(logf, "socks5: "),
+				Dialer: dialer.UserDial,
+			}
 			go func() {
-				log.Fatalf("SOCKS5 server exited: %v", srv.Serve(socksListener))
+				log.Fatalf("SOCKS5 server exited: %v", ss.Serve(socksListener))
 			}()
 		}
 	}
@@ -332,42 +379,59 @@ func run() error {
 	}()
 
 	opts := ipnServerOpts()
-	opts.DebugMux = debugMux
-	err = ipnserver.Run(ctx, logf, pol.PublicID.String(), ipnserver.FixedEngine(e), opts)
+
+	store, err := ipnserver.StateStore(statePathOrDefault(), logf)
+	if err != nil {
+		return fmt.Errorf("ipnserver.StateStore: %w", err)
+	}
+	srv, err := ipnserver.New(logf, pol.PublicID.String(), store, e, dialer, nil, opts)
+	if err != nil {
+		return fmt.Errorf("ipnserver.New: %w", err)
+	}
+	ns.SetLocalBackend(srv.LocalBackend())
+	if err := ns.Start(); err != nil {
+		log.Fatalf("failed to start netstack: %v", err)
+	}
+
+	if debugMux != nil {
+		debugMux.HandleFunc("/debug/ipn", srv.ServeHTMLStatus)
+	}
+
+	ln, _, err := safesocket.Listen(args.socketpath, safesocket.WindowsLocalPort)
+	if err != nil {
+		return fmt.Errorf("safesocket.Listen: %v", err)
+	}
+
+	err = srv.Run(ctx, ln)
 	// Cancelation is not an error: it is the only way to stop ipnserver.
 	if err != nil && err != context.Canceled {
-		logf("ipnserver.Run: %v", err)
-		return err
+		return fmt.Errorf("ipnserver.Run: %w", err)
 	}
 
 	return nil
 }
 
-func createEngine(logf logger.Logf, linkMon *monitor.Mon) (e wgengine.Engine, useNetstack bool, err error) {
+func createEngine(logf logger.Logf, linkMon *monitor.Mon, dialer *tsdial.Dialer) (e wgengine.Engine, useNetstack bool, err error) {
 	if args.tunname == "" {
 		return nil, false, errors.New("no --tun value specified")
 	}
 	var errs []error
 	for _, name := range strings.Split(args.tunname, ",") {
 		logf("wgengine.NewUserspaceEngine(tun %q) ...", name)
-		e, useNetstack, err = tryEngine(logf, linkMon, name)
+		e, useNetstack, err = tryEngine(logf, linkMon, dialer, name)
 		if err == nil {
 			return e, useNetstack, nil
 		}
 		logf("wgengine.NewUserspaceEngine(tun %q) error: %v", name, err)
 		errs = append(errs, err)
 	}
-	return nil, false, multierror.New(errs)
+	return nil, false, multierr.New(errs...)
 }
 
 var wrapNetstack = shouldWrapNetstack()
 
 func shouldWrapNetstack() bool {
-	if e := os.Getenv("TS_DEBUG_WRAP_NETSTACK"); e != "" {
-		v, err := strconv.ParseBool(e)
-		if err != nil {
-			log.Fatalf("invalid TS_DEBUG_WRAP_NETSTACK value: %v", err)
-		}
+	if v, ok := envknob.LookupBool("TS_DEBUG_WRAP_NETSTACK"); ok {
 		return v
 	}
 	if distro.Get() == distro.Synology {
@@ -382,10 +446,11 @@ func shouldWrapNetstack() bool {
 	return false
 }
 
-func tryEngine(logf logger.Logf, linkMon *monitor.Mon, name string) (e wgengine.Engine, useNetstack bool, err error) {
+func tryEngine(logf logger.Logf, linkMon *monitor.Mon, dialer *tsdial.Dialer, name string) (e wgengine.Engine, useNetstack bool, err error) {
 	conf := wgengine.Config{
 		ListenPort:  args.port,
 		LinkMonitor: linkMon,
+		Dialer:      dialer,
 	}
 
 	useNetstack = name == "userspace-networking"
@@ -395,14 +460,14 @@ func tryEngine(logf logger.Logf, linkMon *monitor.Mon, name string) (e wgengine.
 		log.Printf("Connecting to BIRD at %s ...", args.birdSocketPath)
 		conf.BIRDClient, err = createBIRDClient(args.birdSocketPath)
 		if err != nil {
-			return nil, false, err
+			return nil, false, fmt.Errorf("createBIRDClient: %w", err)
 		}
 	}
 	if !useNetstack {
 		dev, devName, err := tstun.New(logf, name)
 		if err != nil {
 			tstun.Diagnose(logf, name)
-			return nil, false, err
+			return nil, false, fmt.Errorf("tstun.New(%q): %w", name, err)
 		}
 		conf.Tun = dev
 		if strings.HasPrefix(name, "tap:") {
@@ -414,11 +479,11 @@ func tryEngine(logf logger.Logf, linkMon *monitor.Mon, name string) (e wgengine.
 		r, err := router.New(logf, dev, linkMon)
 		if err != nil {
 			dev.Close()
-			return nil, false, err
+			return nil, false, fmt.Errorf("creating router: %w", err)
 		}
 		d, err := dns.NewOSConfigurator(logf, devName)
 		if err != nil {
-			return nil, false, err
+			return nil, false, fmt.Errorf("dns.NewOSConfigurator: %w", err)
 		}
 		conf.DNS = d
 		conf.Router = r
@@ -435,6 +500,7 @@ func tryEngine(logf logger.Logf, linkMon *monitor.Mon, name string) (e wgengine.
 
 func newDebugMux() *http.ServeMux {
 	mux := http.NewServeMux()
+	mux.HandleFunc("/debug/metrics", servePrometheusMetrics)
 	mux.HandleFunc("/debug/pprof/", pprof.Index)
 	mux.HandleFunc("/debug/pprof/cmdline", pprof.Cmdline)
 	mux.HandleFunc("/debug/pprof/profile", pprof.Profile)
@@ -443,6 +509,12 @@ func newDebugMux() *http.ServeMux {
 	return mux
 }
 
+func servePrometheusMetrics(w http.ResponseWriter, r *http.Request) {
+	w.Header().Set("Content-Type", "text/plain")
+	tsweb.VarzHandler(w, r)
+	clientmetric.WritePrometheusExpositionFormat(w)
+}
+
 func runDebugServer(mux *http.ServeMux, addr string) {
 	srv := &http.Server{
 		Addr:    addr,
@@ -453,33 +525,54 @@ func runDebugServer(mux *http.ServeMux, addr string) {
 	}
 }
 
-func mustStartNetstack(logf logger.Logf, e wgengine.Engine, onlySubnets bool) *netstack.Impl {
+func newNetstack(logf logger.Logf, dialer *tsdial.Dialer, e wgengine.Engine) (*netstack.Impl, error) {
 	tunDev, magicConn, ok := e.(wgengine.InternalsGetter).GetInternals()
 	if !ok {
-		log.Fatalf("%T is not a wgengine.InternalsGetter", e)
+		return nil, fmt.Errorf("%T is not a wgengine.InternalsGetter", e)
 	}
-	ns, err := netstack.Create(logf, tunDev, e, magicConn, onlySubnets)
-	if err != nil {
-		log.Fatalf("netstack.Create: %v", err)
-	}
-	if err := ns.Start(); err != nil {
-		log.Fatalf("failed to start netstack: %v", err)
-	}
-	return ns
+	return netstack.Create(logf, tunDev, e, magicConn, dialer)
 }
 
-func mustStartTCPListener(name, addr string) net.Listener {
-	if addr == "" {
-		return nil
+// mustStartProxyListeners creates listeners for local SOCKS and HTTP
+// proxies, if the respective addresses are not empty. socksAddr and
+// httpAddr can be the same, in which case socksListener will receive
+// connections that look like they're speaking SOCKS and httpListener
+// will receive everything else.
+//
+// socksListener and httpListener can be nil, if their respective
+// addrs are empty.
+func mustStartProxyListeners(socksAddr, httpAddr string) (socksListener, httpListener net.Listener) {
+	if socksAddr == httpAddr && socksAddr != "" && !strings.HasSuffix(socksAddr, ":0") {
+		ln, err := net.Listen("tcp", socksAddr)
+		if err != nil {
+			log.Fatalf("proxy listener: %v", err)
+		}
+		return proxymux.SplitSOCKSAndHTTP(ln)
 	}
-	ln, err := net.Listen("tcp", addr)
-	if err != nil {
-		log.Fatalf("%v listener: %v", name, err)
+
+	var err error
+	if socksAddr != "" {
+		socksListener, err = net.Listen("tcp", socksAddr)
+		if err != nil {
+			log.Fatalf("SOCKS5 listener: %v", err)
+		}
+		if strings.HasSuffix(socksAddr, ":0") {
+			// Log kernel-selected port number so integration tests
+			// can find it portably.
+			log.Printf("SOCKS5 listening on %v", socksListener.Addr())
+		}
 	}
-	if strings.HasSuffix(addr, ":0") {
-		// Log kernel-selected port number so integration tests
-		// can find it portably.
-		log.Printf("%v listening on %v", name, ln.Addr())
+	if httpAddr != "" {
+		httpListener, err = net.Listen("tcp", httpAddr)
+		if err != nil {
+			log.Fatalf("HTTP proxy listener: %v", err)
+		}
+		if strings.HasSuffix(httpAddr, ":0") {
+			// Log kernel-selected port number so integration tests
+			// can find it portably.
+			log.Printf("HTTP proxy listening on %v", httpListener.Addr())
+		}
 	}
-	return ln
+
+	return socksListener, httpListener
 }

cmd/tailscaled/tailscaled_windows.go

@@ -29,15 +29,19 @@ import (
 	"golang.org/x/sys/windows/svc"
 	"golang.zx2c4.com/wireguard/windows/tunnel/winipcfg"
 	"inet.af/netaddr"
+	"tailscale.com/envknob"
 	"tailscale.com/ipn/ipnserver"
 	"tailscale.com/logpolicy"
 	"tailscale.com/net/dns"
+	"tailscale.com/net/tsdial"
 	"tailscale.com/net/tstun"
+	"tailscale.com/safesocket"
 	"tailscale.com/types/logger"
 	"tailscale.com/util/winutil"
 	"tailscale.com/version"
 	"tailscale.com/wf"
 	"tailscale.com/wgengine"
+	"tailscale.com/wgengine/monitor"
 	"tailscale.com/wgengine/netstack"
 	"tailscale.com/wgengine/router"
 )
@@ -52,6 +56,11 @@ func isWindowsService() bool {
 	return v
 }
 
+// runWindowsService starts running Tailscale under the Windows
+// Service environment.
+//
+// At this point we're still the parent process that
+// Windows started.
 func runWindowsService(pol *logpolicy.Policy) error {
 	return svc.Run(serviceName, &ipnService{Policy: pol})
 }
@@ -65,7 +74,7 @@ func (service *ipnService) Execute(args []string, r <-chan svc.ChangeRequest, ch
 	changes <- svc.Status{State: svc.StartPending}
 
 	svcAccepts := svc.AcceptStop
-	if winutil.GetRegInteger("FlushDNSOnSessionUnlock", 0) != 0 {
+	if winutil.GetPolicyInteger("FlushDNSOnSessionUnlock", 0) != 0 {
 		svcAccepts |= svc.AcceptSessionChange
 	}
 
@@ -74,7 +83,14 @@ func (service *ipnService) Execute(args []string, r <-chan svc.ChangeRequest, ch
 	go func() {
 		defer close(doneCh)
 		args := []string{"/subproc", service.Policy.PublicID.String()}
-		ipnserver.BabysitProc(ctx, args, log.Printf)
+		// Make a logger without a date prefix, as filelogger
+		// and logtail both already add their own. All we really want
+		// from the log package is the automatic newline.
+		// We start with log.Default().Writer(), which is the logtail
+		// writer that logpolicy already installed as the global
+		// output.
+		logger := log.New(log.Default().Writer(), "", 0)
+		ipnserver.BabysitProc(ctx, args, logger.Printf)
 	}()
 
 	changes <- svc.Status{State: svc.Running, Accepts: svcAccepts}
@@ -83,6 +99,7 @@ func (service *ipnService) Execute(args []string, r <-chan svc.ChangeRequest, ch
 		select {
 		case <-doneCh:
 		case cmd := <-r:
+			log.Printf("Got Windows Service event: %v", cmdName(cmd.Cmd))
 			switch cmd.Cmd {
 			case svc.Stop:
 				cancel()
@@ -99,6 +116,42 @@ func (service *ipnService) Execute(args []string, r <-chan svc.ChangeRequest, ch
 	return false, windows.NO_ERROR
 }
 
+func cmdName(c svc.Cmd) string {
+	switch c {
+	case svc.Stop:
+		return "Stop"
+	case svc.Pause:
+		return "Pause"
+	case svc.Continue:
+		return "Continue"
+	case svc.Interrogate:
+		return "Interrogate"
+	case svc.Shutdown:
+		return "Shutdown"
+	case svc.ParamChange:
+		return "ParamChange"
+	case svc.NetBindAdd:
+		return "NetBindAdd"
+	case svc.NetBindRemove:
+		return "NetBindRemove"
+	case svc.NetBindEnable:
+		return "NetBindEnable"
+	case svc.NetBindDisable:
+		return "NetBindDisable"
+	case svc.DeviceEvent:
+		return "DeviceEvent"
+	case svc.HardwareProfileChange:
+		return "HardwareProfileChange"
+	case svc.PowerEvent:
+		return "PowerEvent"
+	case svc.SessionChange:
+		return "SessionChange"
+	case svc.PreShutdown:
+		return "PreShutdown"
+	}
+	return fmt.Sprintf("Unknown-Service-Cmd-%d", c)
+}
+
 func beWindowsSubprocess() bool {
 	if beFirewallKillswitch() {
 		return true
@@ -109,6 +162,9 @@ func beWindowsSubprocess() bool {
 	}
 	logid := os.Args[2]
 
+	// Remove the date/time prefix; the logtail + file logggers add it.
+	log.SetFlags(0)
+
 	log.Printf("Program starting: v%v: %#v", version.Long, os.Args)
 	log.Printf("subproc mode: logid=%v", logid)
 
@@ -172,6 +228,12 @@ func beFirewallKillswitch() bool {
 func startIPNServer(ctx context.Context, logid string) error {
 	var logf logger.Logf = log.Printf
 
+	linkMon, err := monitor.New(logf)
+	if err != nil {
+		return err
+	}
+	dialer := new(tsdial.Dialer)
+
 	getEngineRaw := func() (wgengine.Engine, error) {
 		dev, devName, err := tstun.New(logf, "Tailscale")
 		if err != nil {
@@ -192,19 +254,26 @@ func startIPNServer(ctx context.Context, logid string) error {
 			return nil, fmt.Errorf("DNS: %w", err)
 		}
 		eng, err := wgengine.NewUserspaceEngine(logf, wgengine.Config{
-			Tun:        dev,
-			Router:     r,
-			DNS:        d,
-			ListenPort: 41641,
+			Tun:         dev,
+			Router:      r,
+			DNS:         d,
+			ListenPort:  41641,
+			LinkMonitor: linkMon,
+			Dialer:      dialer,
 		})
 		if err != nil {
 			r.Close()
 			dev.Close()
 			return nil, fmt.Errorf("engine: %w", err)
 		}
-		onlySubnets := true
-		if wrapNetstack {
-			mustStartNetstack(logf, eng, onlySubnets)
+		ns, err := newNetstack(logf, dialer, eng)
+		if err != nil {
+			return nil, fmt.Errorf("newNetstack: %w", err)
+		}
+		ns.ProcessLocalIPs = false
+		ns.ProcessSubnets = wrapNetstack
+		if err := ns.Start(); err != nil {
+			return nil, fmt.Errorf("failed to start netstack: %w", err)
 		}
 		return wgengine.NewWatchdog(eng), nil
 	}
@@ -246,7 +315,7 @@ func startIPNServer(ctx context.Context, logid string) error {
 	// not called concurrently and is not called again once it
 	// successfully returns an engine.
 	getEngine := func() (wgengine.Engine, error) {
-		if msg := os.Getenv("TS_DEBUG_WIN_FAIL"); msg != "" {
+		if msg := envknob.String("TS_DEBUG_WIN_FAIL"); msg != "" {
 			return nil, fmt.Errorf("pretending to be a service failure: %v", msg)
 		}
 		for {
@@ -266,7 +335,18 @@ func startIPNServer(ctx context.Context, logid string) error {
 			return nil, fmt.Errorf("%w\n\nlogid: %v", res.Err, logid)
 		}
 	}
-	err := ipnserver.Run(ctx, logf, logid, getEngine, ipnServerOpts())
+
+	store, err := ipnserver.StateStore(statePathOrDefault(), logf)
+	if err != nil {
+		return err
+	}
+
+	ln, _, err := safesocket.Listen(args.socketpath, safesocket.WindowsLocalPort)
+	if err != nil {
+		return fmt.Errorf("safesocket.Listen: %v", err)
+	}
+
+	err = ipnserver.Run(ctx, logf, ln, store, linkMon, dialer, logid, getEngine, ipnServerOpts())
 	if err != nil {
 		logf("ipnserver.Run: %v", err)
 	}

cmd/tsshd/tsshd.go

@@ -157,8 +157,9 @@ func handleSSH(s ssh.Session) {
 		cmd.Process.Kill()
 		if err := cmd.Wait(); err != nil {
 			s.Exit(1)
+		} else {
+			s.Exit(0)
 		}
-		s.Exit(0)
 		return
 	}
 

control/controlbase/conn.go

@@ -0,0 +1,359 @@
+// Copyright (c) 2021 Tailscale Inc & AUTHORS All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+// Package controlbase implements the base transport of the Tailscale
+// 2021 control protocol.
+//
+// The base transport implements Noise IK, instantiated with
+// Curve25519, ChaCha20Poly1305 and BLAKE2s.
+package controlbase
+
+import (
+	"crypto/cipher"
+	"encoding/binary"
+	"fmt"
+	"net"
+	"sync"
+	"time"
+
+	"golang.org/x/crypto/blake2s"
+	chp "golang.org/x/crypto/chacha20poly1305"
+	"tailscale.com/types/key"
+)
+
+const (
+	// maxMessageSize is the maximum size of a protocol frame on the
+	// wire, including header and payload.
+	maxMessageSize = 4096
+	// maxCiphertextSize is the maximum amount of ciphertext bytes
+	// that one protocol frame can carry, after framing.
+	maxCiphertextSize = maxMessageSize - 3
+	// maxPlaintextSize is the maximum amount of plaintext bytes that
+	// one protocol frame can carry, after encryption and framing.
+	maxPlaintextSize = maxCiphertextSize - chp.Overhead
+)
+
+// A Conn is a secured Noise connection. It implements the net.Conn
+// interface, with the unusual trait that any write error (including a
+// SetWriteDeadline induced i/o timeout) causes all future writes to
+// fail.
+type Conn struct {
+	conn          net.Conn
+	version       uint16
+	peer          key.MachinePublic
+	handshakeHash [blake2s.Size]byte
+	rx            rxState
+	tx            txState
+}
+
+// rxState is all the Conn state that Read uses.
+type rxState struct {
+	sync.Mutex
+	cipher    cipher.AEAD
+	nonce     nonce
+	buf       [maxMessageSize]byte
+	n         int    // number of valid bytes in buf
+	next      int    // offset of next undecrypted packet
+	plaintext []byte // slice into buf of decrypted bytes
+}
+
+// txState is all the Conn state that Write uses.
+type txState struct {
+	sync.Mutex
+	cipher cipher.AEAD
+	nonce  nonce
+	buf    [maxMessageSize]byte
+	err    error // records the first partial write error for all future calls
+}
+
+// ProtocolVersion returns the protocol version that was used to
+// establish this Conn.
+func (c *Conn) ProtocolVersion() int {
+	return int(c.version)
+}
+
+// HandshakeHash returns the Noise handshake hash for the connection,
+// which can be used to bind other messages to this connection
+// (i.e. to ensure that the message wasn't replayed from a different
+// connection).
+func (c *Conn) HandshakeHash() [blake2s.Size]byte {
+	return c.handshakeHash
+}
+
+// Peer returns the peer's long-term public key.
+func (c *Conn) Peer() key.MachinePublic {
+	return c.peer
+}
+
+// readNLocked reads into c.rx.buf until buf contains at least total
+// bytes. Returns a slice of the total bytes in rxBuf, or an
+// error if fewer than total bytes are available.
+func (c *Conn) readNLocked(total int) ([]byte, error) {
+	if total > maxMessageSize {
+		return nil, errReadTooBig{total}
+	}
+	for {
+		if total <= c.rx.n {
+			return c.rx.buf[:total], nil
+		}
+
+		n, err := c.conn.Read(c.rx.buf[c.rx.n:])
+		c.rx.n += n
+		if err != nil {
+			return nil, err
+		}
+	}
+}
+
+// decryptLocked decrypts msg (which is header+ciphertext) in-place
+// and sets c.rx.plaintext to the decrypted bytes.
+func (c *Conn) decryptLocked(msg []byte) (err error) {
+	if msgType := msg[0]; msgType != msgTypeRecord {
+		return fmt.Errorf("received message with unexpected type %d, want %d", msgType, msgTypeRecord)
+	}
+	// We don't check the length field here, because the caller
+	// already did in order to figure out how big the msg slice should
+	// be.
+	ciphertext := msg[headerLen:]
+
+	if !c.rx.nonce.Valid() {
+		return errCipherExhausted{}
+	}
+
+	c.rx.plaintext, err = c.rx.cipher.Open(ciphertext[:0], c.rx.nonce[:], ciphertext, nil)
+	c.rx.nonce.Increment()
+
+	if err != nil {
+		// Once a decryption has failed, our Conn is no longer
+		// synchronized with our peer. Nuke the cipher state to be
+		// safe, so that no further decryptions are attempted. Future
+		// read attempts will return net.ErrClosed.
+		c.rx.cipher = nil
+	}
+	return err
+}
+
+// encryptLocked encrypts plaintext into c.tx.buf (including the
+// packet header) and returns a slice of the ciphertext, or an error
+// if the cipher is exhausted (i.e. can no longer be used safely).
+func (c *Conn) encryptLocked(plaintext []byte) ([]byte, error) {
+	if !c.tx.nonce.Valid() {
+		// Received 2^64-1 messages on this cipher state. Connection
+		// is no longer usable.
+		return nil, errCipherExhausted{}
+	}
+
+	c.tx.buf[0] = msgTypeRecord
+	binary.BigEndian.PutUint16(c.tx.buf[1:headerLen], uint16(len(plaintext)+chp.Overhead))
+	ret := c.tx.cipher.Seal(c.tx.buf[:headerLen], c.tx.nonce[:], plaintext, nil)
+	c.tx.nonce.Increment()
+
+	return ret, nil
+}
+
+// wholeMessageLocked returns a slice of one whole Noise transport
+// message from c.rx.buf, if one whole message is available, and
+// advances the read state to the next Noise message in the
+// buffer. Returns nil without advancing read state if there isn't one
+// whole message in c.rx.buf.
+func (c *Conn) wholeMessageLocked() []byte {
+	available := c.rx.n - c.rx.next
+	if available < headerLen {
+		return nil
+	}
+	bs := c.rx.buf[c.rx.next:c.rx.n]
+	totalSize := headerLen + int(binary.BigEndian.Uint16(bs[1:3]))
+	if len(bs) < totalSize {
+		return nil
+	}
+	c.rx.next += totalSize
+	return bs[:totalSize]
+}
+
+// decryptOneLocked decrypts one Noise transport message, reading from
+// c.conn as needed, and sets c.rx.plaintext to point to the decrypted
+// bytes. c.rx.plaintext is only valid if err == nil.
+func (c *Conn) decryptOneLocked() error {
+	c.rx.plaintext = nil
+
+	// Fast path: do we have one whole ciphertext frame buffered
+	// already?
+	if bs := c.wholeMessageLocked(); bs != nil {
+		return c.decryptLocked(bs)
+	}
+
+	if c.rx.next != 0 {
+		// To simplify the read logic, move the remainder of the
+		// buffered bytes back to the head of the buffer, so we can
+		// grow it without worrying about wraparound.
+		c.rx.n = copy(c.rx.buf[:], c.rx.buf[c.rx.next:c.rx.n])
+		c.rx.next = 0
+	}
+
+	bs, err := c.readNLocked(headerLen)
+	if err != nil {
+		return err
+	}
+	// The rest of the header (besides the length field) gets verified
+	// in decryptLocked, not here.
+	messageLen := headerLen + int(binary.BigEndian.Uint16(bs[1:3]))
+	bs, err = c.readNLocked(messageLen)
+	if err != nil {
+		return err
+	}
+
+	c.rx.next = len(bs)
+
+	return c.decryptLocked(bs)
+}
+
+// Read implements io.Reader.
+func (c *Conn) Read(bs []byte) (int, error) {
+	c.rx.Lock()
+	defer c.rx.Unlock()
+
+	if c.rx.cipher == nil {
+		return 0, net.ErrClosed
+	}
+	// If no plaintext is buffered, decrypt incoming frames until we
+	// have some plaintext. Zero-byte Noise frames are allowed in this
+	// protocol, which is why we have to loop here rather than decrypt
+	// a single additional frame.
+	for len(c.rx.plaintext) == 0 {
+		if err := c.decryptOneLocked(); err != nil {
+			return 0, err
+		}
+	}
+	n := copy(bs, c.rx.plaintext)
+	c.rx.plaintext = c.rx.plaintext[n:]
+	return n, nil
+}
+
+// Write implements io.Writer.
+func (c *Conn) Write(bs []byte) (n int, err error) {
+	c.tx.Lock()
+	defer c.tx.Unlock()
+
+	if c.tx.err != nil {
+		return 0, c.tx.err
+	}
+	defer func() {
+		if err != nil {
+			// All write errors are fatal for this conn, so clear the
+			// cipher state whenever an error happens.
+			c.tx.cipher = nil
+		}
+		if c.tx.err == nil {
+			// Only set c.tx.err if not nil so that we can return one
+			// error on the first failure, and a different one for
+			// subsequent calls. See the error handling around Write
+			// below for why.
+			c.tx.err = err
+		}
+	}()
+
+	if c.tx.cipher == nil {
+		return 0, net.ErrClosed
+	}
+
+	var sent int
+	for len(bs) > 0 {
+		toSend := bs
+		if len(toSend) > maxPlaintextSize {
+			toSend = bs[:maxPlaintextSize]
+		}
+		bs = bs[len(toSend):]
+
+		ciphertext, err := c.encryptLocked(toSend)
+		if err != nil {
+			return 0, err
+		}
+
+		n, err := c.conn.Write(ciphertext)
+		sent += n
+		if err != nil {
+			// Return the raw error on the Write that actually
+			// failed. For future writes, return that error wrapped in
+			// a desync error.
+			c.tx.err = errPartialWrite{err}
+			return sent, err
+		}
+	}
+	return sent, nil
+}
+
+// Close implements io.Closer.
+func (c *Conn) Close() error {
+	closeErr := c.conn.Close() // unblocks any waiting reads or writes
+
+	// Remove references to live cipher state. Strictly speaking this
+	// is unnecessary, but we want to try and hand the active cipher
+	// state to the garbage collector promptly, to preserve perfect
+	// forward secrecy as much as we can.
+	c.rx.Lock()
+	c.rx.cipher = nil
+	c.rx.Unlock()
+	c.tx.Lock()
+	c.tx.cipher = nil
+	c.tx.Unlock()
+	return closeErr
+}
+
+func (c *Conn) LocalAddr() net.Addr                { return c.conn.LocalAddr() }
+func (c *Conn) RemoteAddr() net.Addr               { return c.conn.RemoteAddr() }
+func (c *Conn) SetDeadline(t time.Time) error      { return c.conn.SetDeadline(t) }
+func (c *Conn) SetReadDeadline(t time.Time) error  { return c.conn.SetReadDeadline(t) }
+func (c *Conn) SetWriteDeadline(t time.Time) error { return c.conn.SetWriteDeadline(t) }
+
+// errCipherExhausted is the error returned when we run out of nonces
+// on a cipher.
+type errCipherExhausted struct{}
+
+func (errCipherExhausted) Error() string {
+	return "cipher exhausted, no more nonces available for current key"
+}
+func (errCipherExhausted) Timeout() bool   { return false }
+func (errCipherExhausted) Temporary() bool { return false }
+
+// errPartialWrite is the error returned when the cipher state has
+// become unusable due to a past partial write.
+type errPartialWrite struct {
+	err error
+}
+
+func (e errPartialWrite) Error() string {
+	return fmt.Sprintf("cipher state desynchronized due to partial write (%v)", e.err)
+}
+func (e errPartialWrite) Unwrap() error   { return e.err }
+func (e errPartialWrite) Temporary() bool { return false }
+func (e errPartialWrite) Timeout() bool   { return false }
+
+// errReadTooBig is the error returned when the peer sent an
+// unacceptably large Noise frame.
+type errReadTooBig struct {
+	requested int
+}
+
+func (e errReadTooBig) Error() string {
+	return fmt.Sprintf("requested read of %d bytes exceeds max allowed Noise frame size", e.requested)
+}
+func (e errReadTooBig) Temporary() bool {
+	// permanent error because this error only occurs when our peer
+	// sends us a frame so large we're unwilling to ever decode it.
+	return false
+}
+func (e errReadTooBig) Timeout() bool { return false }
+
+type nonce [chp.NonceSize]byte
+
+func (n *nonce) Valid() bool {
+	return binary.BigEndian.Uint32(n[:4]) == 0 && binary.BigEndian.Uint64(n[4:]) != invalidNonce
+}
+
+func (n *nonce) Increment() {
+	if !n.Valid() {
+		panic("increment of invalid nonce")
+	}
+	binary.BigEndian.PutUint64(n[4:], 1+binary.BigEndian.Uint64(n[4:]))
+}

control/controlbase/conn_test.go

@@ -0,0 +1,339 @@
+// Copyright (c) 2021 Tailscale Inc & AUTHORS All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package controlbase
+
+import (
+	"bufio"
+	"bytes"
+	"context"
+	"crypto/rand"
+	"encoding/binary"
+	"fmt"
+	"io"
+	"net"
+	"strings"
+	"sync"
+	"testing"
+	"testing/iotest"
+
+	chp "golang.org/x/crypto/chacha20poly1305"
+	"golang.org/x/net/nettest"
+	tsnettest "tailscale.com/net/nettest"
+	"tailscale.com/types/key"
+)
+
+func TestMessageSize(t *testing.T) {
+	// This test is a regression guard against someone looking at
+	// maxCiphertextSize, going "huh, we could be more efficient if it
+	// were larger, and accidentally violating the Noise spec. Do not
+	// change this max value, it's a deliberate limitation of the
+	// cryptographic protocol we use (see Section 3 "Message Format"
+	// of the Noise spec).
+	const max = 65535
+	if maxCiphertextSize > max {
+		t.Fatalf("max ciphertext size is %d, which is larger than the maximum noise message size %d", maxCiphertextSize, max)
+	}
+}
+
+func TestConnBasic(t *testing.T) {
+	client, server := pair(t)
+
+	sb := sinkReads(server)
+
+	want := "test"
+	if _, err := io.WriteString(client, want); err != nil {
+		t.Fatalf("client write failed: %v", err)
+	}
+	client.Close()
+
+	if got := sb.String(4); got != want {
+		t.Fatalf("wrong content received: got %q, want %q", got, want)
+	}
+	if err := sb.Error(); err != io.EOF {
+		t.Fatal("client close wasn't seen by server")
+	}
+	if sb.Total() != 4 {
+		t.Fatalf("wrong amount of bytes received: got %d, want 4", sb.Total())
+	}
+}
+
+// bufferedWriteConn wraps a net.Conn and gives control over how
+// Writes get batched out.
+type bufferedWriteConn struct {
+	net.Conn
+	w           *bufio.Writer
+	manualFlush bool
+}
+
+func (c *bufferedWriteConn) Write(bs []byte) (int, error) {
+	n, err := c.w.Write(bs)
+	if err == nil && !c.manualFlush {
+		err = c.w.Flush()
+	}
+	return n, err
+}
+
+// TestFastPath exercises the Read codepath that can receive multiple
+// Noise frames at once and decode each in turn without making another
+// syscall.
+func TestFastPath(t *testing.T) {
+	s1, s2 := tsnettest.NewConn("noise", 128000)
+	b := &bufferedWriteConn{s1, bufio.NewWriterSize(s1, 10000), false}
+	client, server := pairWithConns(t, b, s2)
+
+	b.manualFlush = true
+
+	sb := sinkReads(server)
+
+	const packets = 10
+	s := "test"
+	for i := 0; i < packets; i++ {
+		// Many separate writes, to force separate Noise frames that
+		// all get buffered up and then all sent as a single slice to
+		// the server.
+		if _, err := io.WriteString(client, s); err != nil {
+			t.Fatalf("client write1 failed: %v", err)
+		}
+	}
+	if err := b.w.Flush(); err != nil {
+		t.Fatalf("client flush failed: %v", err)
+	}
+	client.Close()
+
+	want := strings.Repeat(s, packets)
+	if got := sb.String(len(want)); got != want {
+		t.Fatalf("wrong content received: got %q, want %q", got, want)
+	}
+	if err := sb.Error(); err != io.EOF {
+		t.Fatalf("client close wasn't seen by server")
+	}
+}
+
+// Writes things larger than a single Noise frame, to check the
+// chunking on the encoder and decoder.
+func TestBigData(t *testing.T) {
+	client, server := pair(t)
+
+	serverReads := sinkReads(server)
+	clientReads := sinkReads(client)
+
+	const sz = 15 * 1024 // 15KiB
+	clientStr := strings.Repeat("abcde", sz/5)
+	serverStr := strings.Repeat("fghij", sz/5*2)
+
+	if _, err := io.WriteString(client, clientStr); err != nil {
+		t.Fatalf("writing client>server: %v", err)
+	}
+	if _, err := io.WriteString(server, serverStr); err != nil {
+		t.Fatalf("writing server>client: %v", err)
+	}
+
+	if serverGot := serverReads.String(sz); serverGot != clientStr {
+		t.Error("server didn't receive what client sent")
+	}
+	if clientGot := clientReads.String(2 * sz); clientGot != serverStr {
+		t.Error("client didn't receive what server sent")
+	}
+
+	getNonce := func(n [chp.NonceSize]byte) uint64 {
+		if binary.BigEndian.Uint32(n[:4]) != 0 {
+			panic("unexpected nonce")
+		}
+		return binary.BigEndian.Uint64(n[4:])
+	}
+
+	// Reach into the Conns and verify the cipher nonces advanced as
+	// expected.
+	if getNonce(client.tx.nonce) != getNonce(server.rx.nonce) {
+		t.Error("desynchronized client tx nonce")
+	}
+	if getNonce(server.tx.nonce) != getNonce(client.rx.nonce) {
+		t.Error("desynchronized server tx nonce")
+	}
+	if n := getNonce(client.tx.nonce); n != 4 {
+		t.Errorf("wrong client tx nonce, got %d want 4", n)
+	}
+	if n := getNonce(server.tx.nonce); n != 8 {
+		t.Errorf("wrong client tx nonce, got %d want 8", n)
+	}
+}
+
+// readerConn wraps a net.Conn and routes its Reads through a separate
+// io.Reader.
+type readerConn struct {
+	net.Conn
+	r io.Reader
+}
+
+func (c readerConn) Read(bs []byte) (int, error) { return c.r.Read(bs) }
+
+// Check that the receiver can handle not being able to read an entire
+// frame in a single syscall.
+func TestDataTrickle(t *testing.T) {
+	s1, s2 := tsnettest.NewConn("noise", 128000)
+	client, server := pairWithConns(t, s1, readerConn{s2, iotest.OneByteReader(s2)})
+	serverReads := sinkReads(server)
+
+	const sz = 10000
+	clientStr := strings.Repeat("abcde", sz/5)
+	if _, err := io.WriteString(client, clientStr); err != nil {
+		t.Fatalf("writing client>server: %v", err)
+	}
+
+	serverGot := serverReads.String(sz)
+	if serverGot != clientStr {
+		t.Error("server didn't receive what client sent")
+	}
+}
+
+func TestConnStd(t *testing.T) {
+	// You can run this test manually, and noise.Conn should pass all
+	// of them except for TestConn/PastTimeout,
+	// TestConn/FutureTimeout, TestConn/ConcurrentMethods, because
+	// those tests assume that write errors are recoverable, and
+	// they're not on our Conn due to cipher security.
+	t.Skip("not all tests can pass on this Conn, see https://github.com/golang/go/issues/46977")
+	nettest.TestConn(t, func() (c1 net.Conn, c2 net.Conn, stop func(), err error) {
+		s1, s2 := tsnettest.NewConn("noise", 4096)
+		controlKey := key.NewMachine()
+		machineKey := key.NewMachine()
+		serverErr := make(chan error, 1)
+		go func() {
+			var err error
+			c2, err = Server(context.Background(), s2, controlKey, nil)
+			serverErr <- err
+		}()
+		c1, err = Client(context.Background(), s1, machineKey, controlKey.Public())
+		if err != nil {
+			s1.Close()
+			s2.Close()
+			return nil, nil, nil, fmt.Errorf("connecting client: %w", err)
+		}
+		if err := <-serverErr; err != nil {
+			c1.Close()
+			s1.Close()
+			s2.Close()
+			return nil, nil, nil, fmt.Errorf("connecting server: %w", err)
+		}
+		return c1, c2, func() {
+			c1.Close()
+			c2.Close()
+		}, nil
+	})
+}
+
+// mkConns creates synthetic Noise Conns wrapping the given net.Conns.
+// This function is for testing just the Conn transport logic without
+// having to muck about with Noise handshakes.
+func mkConns(s1, s2 net.Conn) (*Conn, *Conn) {
+	var k1, k2 [chp.KeySize]byte
+	if _, err := rand.Read(k1[:]); err != nil {
+		panic(err)
+	}
+	if _, err := rand.Read(k2[:]); err != nil {
+		panic(err)
+	}
+
+	ret1 := &Conn{
+		conn: s1,
+		tx:   txState{cipher: newCHP(k1)},
+		rx:   rxState{cipher: newCHP(k2)},
+	}
+	ret2 := &Conn{
+		conn: s2,
+		tx:   txState{cipher: newCHP(k2)},
+		rx:   rxState{cipher: newCHP(k1)},
+	}
+
+	return ret1, ret2
+}
+
+type readSink struct {
+	r io.Reader
+
+	cond *sync.Cond
+	sync.Mutex
+	bs  bytes.Buffer
+	err error
+}
+
+func sinkReads(r io.Reader) *readSink {
+	ret := &readSink{
+		r: r,
+	}
+	ret.cond = sync.NewCond(&ret.Mutex)
+	go func() {
+		var buf [4096]byte
+		for {
+			n, err := r.Read(buf[:])
+			ret.Lock()
+			ret.bs.Write(buf[:n])
+			if err != nil {
+				ret.err = err
+			}
+			ret.cond.Broadcast()
+			ret.Unlock()
+			if err != nil {
+				return
+			}
+		}
+	}()
+	return ret
+}
+
+func (s *readSink) String(total int) string {
+	s.Lock()
+	defer s.Unlock()
+	for s.bs.Len() < total && s.err == nil {
+		s.cond.Wait()
+	}
+	if s.err != nil {
+		total = s.bs.Len()
+	}
+	return string(s.bs.Bytes()[:total])
+}
+
+func (s *readSink) Error() error {
+	s.Lock()
+	defer s.Unlock()
+	for s.err == nil {
+		s.cond.Wait()
+	}
+	return s.err
+}
+
+func (s *readSink) Total() int {
+	s.Lock()
+	defer s.Unlock()
+	return s.bs.Len()
+}
+
+func pairWithConns(t *testing.T, clientConn, serverConn net.Conn) (*Conn, *Conn) {
+	var (
+		controlKey = key.NewMachine()
+		machineKey = key.NewMachine()
+		server     *Conn
+		serverErr  = make(chan error, 1)
+	)
+	go func() {
+		var err error
+		server, err = Server(context.Background(), serverConn, controlKey, nil)
+		serverErr <- err
+	}()
+
+	client, err := Client(context.Background(), clientConn, machineKey, controlKey.Public())
+	if err != nil {
+		t.Fatalf("client connection failed: %v", err)
+	}
+	if err := <-serverErr; err != nil {
+		t.Fatalf("server connection failed: %v", err)
+	}
+	return client, server
+}
+
+func pair(t *testing.T) (*Conn, *Conn) {
+	s1, s2 := tsnettest.NewConn("noise", 128000)
+	return pairWithConns(t, s1, s2)
+}

control/controlbase/handshake.go

@@ -0,0 +1,492 @@
+// Copyright (c) 2021 Tailscale Inc & AUTHORS All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package controlbase
+
+import (
+	"context"
+	"crypto/cipher"
+	"encoding/binary"
+	"errors"
+	"fmt"
+	"hash"
+	"io"
+	"net"
+	"strconv"
+	"time"
+
+	"go4.org/mem"
+	"golang.org/x/crypto/blake2s"
+	chp "golang.org/x/crypto/chacha20poly1305"
+	"golang.org/x/crypto/curve25519"
+	"golang.org/x/crypto/hkdf"
+	"tailscale.com/types/key"
+)
+
+const (
+	// protocolName is the name of the specific instantiation of Noise
+	// that the control protocol uses. This string's value is fixed by
+	// the Noise spec, and shouldn't be changed unless we're updating
+	// the control protocol to use a different Noise instance.
+	protocolName = "Noise_IK_25519_ChaChaPoly_BLAKE2s"
+	// protocolVersion is the version of the control protocol that
+	// Client will use when initiating a handshake.
+	protocolVersion uint16 = 1
+	// protocolVersionPrefix is the name portion of the protocol
+	// name+version string that gets mixed into the handshake as a
+	// prologue.
+	//
+	// This mixing verifies that both clients agree that they're
+	// executing the control protocol at a specific version that
+	// matches the advertised version in the cleartext packet header.
+	protocolVersionPrefix = "Tailscale Control Protocol v"
+	invalidNonce          = ^uint64(0)
+)
+
+func protocolVersionPrologue(version uint16) []byte {
+	ret := make([]byte, 0, len(protocolVersionPrefix)+5) // 5 bytes is enough to encode all possible version numbers.
+	ret = append(ret, protocolVersionPrefix...)
+	return strconv.AppendUint(ret, uint64(version), 10)
+}
+
+// HandshakeContinuation upgrades a net.Conn to a Conn. The net.Conn
+// is assumed to have already sent the client>server handshake
+// initiation message.
+type HandshakeContinuation func(context.Context, net.Conn) (*Conn, error)
+
+// ClientDeferred initiates a control client handshake, returning the
+// initial message to send to the server and a continuation to
+// finalize the handshake.
+//
+// ClientDeferred is split in this way for RTT reduction: we run this
+// protocol after negotiating a protocol switch from HTTP/HTTPS. If we
+// completely serialized the negotiation followed by the handshake,
+// we'd pay an extra RTT to transmit the handshake initiation after
+// protocol switching. By splitting the handshake into an initial
+// message and a continuation, we can embed the handshake initiation
+// into the HTTP protocol switching request and avoid a bit of delay.
+func ClientDeferred(machineKey key.MachinePrivate, controlKey key.MachinePublic) (initialHandshake []byte, continueHandshake HandshakeContinuation, err error) {
+	var s symmetricState
+	s.Initialize()
+
+	// prologue
+	s.MixHash(protocolVersionPrologue(protocolVersion))
+
+	// <- s
+	// ...
+	s.MixHash(controlKey.UntypedBytes())
+
+	// -> e, es, s, ss
+	init := mkInitiationMessage()
+	machineEphemeral := key.NewMachine()
+	machineEphemeralPub := machineEphemeral.Public()
+	copy(init.EphemeralPub(), machineEphemeralPub.UntypedBytes())
+	s.MixHash(machineEphemeralPub.UntypedBytes())
+	cipher, err := s.MixDH(machineEphemeral, controlKey)
+	if err != nil {
+		return nil, nil, fmt.Errorf("computing es: %w", err)
+	}
+	machineKeyPub := machineKey.Public()
+	s.EncryptAndHash(cipher, init.MachinePub(), machineKeyPub.UntypedBytes())
+	cipher, err = s.MixDH(machineKey, controlKey)
+	if err != nil {
+		return nil, nil, fmt.Errorf("computing ss: %w", err)
+	}
+	s.EncryptAndHash(cipher, init.Tag(), nil) // empty message payload
+
+	cont := func(ctx context.Context, conn net.Conn) (*Conn, error) {
+		return continueClientHandshake(ctx, conn, &s, machineKey, machineEphemeral, controlKey)
+	}
+	return init[:], cont, nil
+}
+
+// Client wraps ClientDeferred and immediately invokes the returned
+// continuation with conn.
+//
+// This is a helper for when you don't need the fancy
+// continuation-style handshake, and just want to synchronously
+// upgrade a net.Conn to a secure transport.
+func Client(ctx context.Context, conn net.Conn, machineKey key.MachinePrivate, controlKey key.MachinePublic) (*Conn, error) {
+	init, cont, err := ClientDeferred(machineKey, controlKey)
+	if err != nil {
+		return nil, err
+	}
+	if _, err := conn.Write(init); err != nil {
+		return nil, err
+	}
+	return cont(ctx, conn)
+}
+
+func continueClientHandshake(ctx context.Context, conn net.Conn, s *symmetricState, machineKey, machineEphemeral key.MachinePrivate, controlKey key.MachinePublic) (*Conn, error) {
+	// No matter what, this function can only run once per s. Ensure
+	// attempted reuse causes a panic.
+	defer func() {
+		s.finished = true
+	}()
+
+	if deadline, ok := ctx.Deadline(); ok {
+		if err := conn.SetDeadline(deadline); err != nil {
+			return nil, fmt.Errorf("setting conn deadline: %w", err)
+		}
+		defer func() {
+			conn.SetDeadline(time.Time{})
+		}()
+	}
+
+	// Read in the payload and look for errors/protocol violations from the server.
+	var resp responseMessage
+	if _, err := io.ReadFull(conn, resp.Header()); err != nil {
+		return nil, fmt.Errorf("reading response header: %w", err)
+	}
+	if resp.Type() != msgTypeResponse {
+		if resp.Type() != msgTypeError {
+			return nil, fmt.Errorf("unexpected response message type %d", resp.Type())
+		}
+		msg := make([]byte, resp.Length())
+		if _, err := io.ReadFull(conn, msg); err != nil {
+			return nil, err
+		}
+		return nil, fmt.Errorf("server error: %q", msg)
+	}
+	if resp.Length() != len(resp.Payload()) {
+		return nil, fmt.Errorf("wrong length %d received for handshake response", resp.Length())
+	}
+	if _, err := io.ReadFull(conn, resp.Payload()); err != nil {
+		return nil, err
+	}
+
+	// <- e, ee, se
+	controlEphemeralPub := key.MachinePublicFromRaw32(mem.B(resp.EphemeralPub()))
+	s.MixHash(controlEphemeralPub.UntypedBytes())
+	if _, err := s.MixDH(machineEphemeral, controlEphemeralPub); err != nil {
+		return nil, fmt.Errorf("computing ee: %w", err)
+	}
+	cipher, err := s.MixDH(machineKey, controlEphemeralPub)
+	if err != nil {
+		return nil, fmt.Errorf("computing se: %w", err)
+	}
+	if err := s.DecryptAndHash(cipher, nil, resp.Tag()); err != nil {
+		return nil, fmt.Errorf("decrypting payload: %w", err)
+	}
+
+	c1, c2, err := s.Split()
+	if err != nil {
+		return nil, fmt.Errorf("finalizing handshake: %w", err)
+	}
+
+	c := &Conn{
+		conn:          conn,
+		version:       protocolVersion,
+		peer:          controlKey,
+		handshakeHash: s.h,
+		tx: txState{
+			cipher: c1,
+		},
+		rx: rxState{
+			cipher: c2,
+		},
+	}
+	return c, nil
+}
+
+// Server initiates a control server handshake, returning the resulting
+// control connection.
+//
+// optionalInit can be the client's initial handshake message as
+// returned by ClientDeferred, or nil in which case the initial
+// message is read from conn.
+//
+// The context deadline, if any, covers the entire handshaking
+// process.
+func Server(ctx context.Context, conn net.Conn, controlKey key.MachinePrivate, optionalInit []byte) (*Conn, error) {
+	if deadline, ok := ctx.Deadline(); ok {
+		if err := conn.SetDeadline(deadline); err != nil {
+			return nil, fmt.Errorf("setting conn deadline: %w", err)
+		}
+		defer func() {
+			conn.SetDeadline(time.Time{})
+		}()
+	}
+
+	// Deliberately does not support formatting, so that we don't echo
+	// attacker-controlled input back to them.
+	sendErr := func(msg string) error {
+		if len(msg) >= 1<<16 {
+			msg = msg[:1<<16]
+		}
+		var hdr [headerLen]byte
+		hdr[0] = msgTypeError
+		binary.BigEndian.PutUint16(hdr[1:3], uint16(len(msg)))
+		if _, err := conn.Write(hdr[:]); err != nil {
+			return fmt.Errorf("sending %q error to client: %w", msg, err)
+		}
+		if _, err := io.WriteString(conn, msg); err != nil {
+			return fmt.Errorf("sending %q error to client: %w", msg, err)
+		}
+		return fmt.Errorf("refused client handshake: %q", msg)
+	}
+
+	var s symmetricState
+	s.Initialize()
+
+	var init initiationMessage
+	if optionalInit != nil {
+		if len(optionalInit) != len(init) {
+			return nil, sendErr("wrong handshake initiation size")
+		}
+		copy(init[:], optionalInit)
+	} else if _, err := io.ReadFull(conn, init.Header()); err != nil {
+		return nil, err
+	}
+	if init.Version() != protocolVersion {
+		return nil, sendErr("unsupported protocol version")
+	}
+	if init.Type() != msgTypeInitiation {
+		return nil, sendErr("unexpected handshake message type")
+	}
+	if init.Length() != len(init.Payload()) {
+		return nil, sendErr("wrong handshake initiation length")
+	}
+	// if optionalInit was provided, we have the payload already.
+	if optionalInit == nil {
+		if _, err := io.ReadFull(conn, init.Payload()); err != nil {
+			return nil, err
+		}
+	}
+
+	// prologue. Can only do this once we at least think the client is
+	// handshaking using a supported version.
+	s.MixHash(protocolVersionPrologue(protocolVersion))
+
+	// <- s
+	// ...
+	controlKeyPub := controlKey.Public()
+	s.MixHash(controlKeyPub.UntypedBytes())
+
+	// -> e, es, s, ss
+	machineEphemeralPub := key.MachinePublicFromRaw32(mem.B(init.EphemeralPub()))
+	s.MixHash(machineEphemeralPub.UntypedBytes())
+	cipher, err := s.MixDH(controlKey, machineEphemeralPub)
+	if err != nil {
+		return nil, fmt.Errorf("computing es: %w", err)
+	}
+	var machineKeyBytes [32]byte
+	if err := s.DecryptAndHash(cipher, machineKeyBytes[:], init.MachinePub()); err != nil {
+		return nil, fmt.Errorf("decrypting machine key: %w", err)
+	}
+	machineKey := key.MachinePublicFromRaw32(mem.B(machineKeyBytes[:]))
+	cipher, err = s.MixDH(controlKey, machineKey)
+	if err != nil {
+		return nil, fmt.Errorf("computing ss: %w", err)
+	}
+	if err := s.DecryptAndHash(cipher, nil, init.Tag()); err != nil {
+		return nil, fmt.Errorf("decrypting initiation tag: %w", err)
+	}
+
+	// <- e, ee, se
+	resp := mkResponseMessage()
+	controlEphemeral := key.NewMachine()
+	controlEphemeralPub := controlEphemeral.Public()
+	copy(resp.EphemeralPub(), controlEphemeralPub.UntypedBytes())
+	s.MixHash(controlEphemeralPub.UntypedBytes())
+	if _, err := s.MixDH(controlEphemeral, machineEphemeralPub); err != nil {
+		return nil, fmt.Errorf("computing ee: %w", err)
+	}
+	cipher, err = s.MixDH(controlEphemeral, machineKey)
+	if err != nil {
+		return nil, fmt.Errorf("computing se: %w", err)
+	}
+	s.EncryptAndHash(cipher, resp.Tag(), nil) // empty message payload
+
+	c1, c2, err := s.Split()
+	if err != nil {
+		return nil, fmt.Errorf("finalizing handshake: %w", err)
+	}
+
+	if _, err := conn.Write(resp[:]); err != nil {
+		return nil, err
+	}
+
+	c := &Conn{
+		conn:          conn,
+		version:       protocolVersion,
+		peer:          machineKey,
+		handshakeHash: s.h,
+		tx: txState{
+			cipher: c2,
+		},
+		rx: rxState{
+			cipher: c1,
+		},
+	}
+	return c, nil
+}
+
+// symmetricState contains the state of an in-flight handshake.
+type symmetricState struct {
+	finished bool
+
+	h  [blake2s.Size]byte // hash of currently-processed handshake state
+	ck [blake2s.Size]byte // chaining key used to construct session keys at the end of the handshake
+}
+
+func (s *symmetricState) checkFinished() {
+	if s.finished {
+		panic("attempted to use symmetricState after Split was called")
+	}
+}
+
+// Initialize sets s to the initial handshake state, prior to
+// processing any handshake messages.
+func (s *symmetricState) Initialize() {
+	s.checkFinished()
+	s.h = blake2s.Sum256([]byte(protocolName))
+	s.ck = s.h
+}
+
+// MixHash updates s.h to be BLAKE2s(s.h || data), where || is
+// concatenation.
+func (s *symmetricState) MixHash(data []byte) {
+	s.checkFinished()
+	h := newBLAKE2s()
+	h.Write(s.h[:])
+	h.Write(data)
+	h.Sum(s.h[:0])
+}
+
+// MixDH updates s.ck with the result of X25519(priv, pub) and returns
+// a singleUseCHP that can be used to encrypt or decrypt handshake
+// data.
+//
+// MixDH corresponds to MixKey(X25519(...))) in the spec. Implementing
+// it as a single function allows for strongly-typed arguments that
+// reduce the risk of error in the caller (e.g. invoking X25519 with
+// two private keys, or two public keys), and thus producing the wrong
+// calculation.
+func (s *symmetricState) MixDH(priv key.MachinePrivate, pub key.MachinePublic) (*singleUseCHP, error) {
+	s.checkFinished()
+	keyData, err := curve25519.X25519(priv.UntypedBytes(), pub.UntypedBytes())
+	if err != nil {
+		return nil, fmt.Errorf("computing X25519: %w", err)
+	}
+
+	r := hkdf.New(newBLAKE2s, keyData, s.ck[:], nil)
+	if _, err := io.ReadFull(r, s.ck[:]); err != nil {
+		return nil, fmt.Errorf("extracting ck: %w", err)
+	}
+	var k [chp.KeySize]byte
+	if _, err := io.ReadFull(r, k[:]); err != nil {
+		return nil, fmt.Errorf("extracting k: %w", err)
+	}
+	return newSingleUseCHP(k), nil
+}
+
+// EncryptAndHash encrypts plaintext into ciphertext (which must be
+// the correct size to hold the encrypted plaintext) using cipher,
+// mixes the ciphertext into s.h, and returns the ciphertext.
+func (s *symmetricState) EncryptAndHash(cipher *singleUseCHP, ciphertext, plaintext []byte) {
+	s.checkFinished()
+	if len(ciphertext) != len(plaintext)+chp.Overhead {
+		panic("ciphertext is wrong size for given plaintext")
+	}
+	ret := cipher.Seal(ciphertext[:0], plaintext, s.h[:])
+	s.MixHash(ret)
+}
+
+// DecryptAndHash decrypts the given ciphertext into plaintext (which
+// must be the correct size to hold the decrypted ciphertext) using
+// cipher. If decryption is successful, it mixes the ciphertext into
+// s.h.
+func (s *symmetricState) DecryptAndHash(cipher *singleUseCHP, plaintext, ciphertext []byte) error {
+	s.checkFinished()
+	if len(ciphertext) != len(plaintext)+chp.Overhead {
+		return errors.New("plaintext is wrong size for given ciphertext")
+	}
+	if _, err := cipher.Open(plaintext[:0], ciphertext, s.h[:]); err != nil {
+		return err
+	}
+	s.MixHash(ciphertext)
+	return nil
+}
+
+// Split returns two ChaCha20Poly1305 ciphers with keys derived from
+// the current handshake state. Methods on s cannot be used again
+// after calling Split.
+func (s *symmetricState) Split() (c1, c2 cipher.AEAD, err error) {
+	s.finished = true
+
+	var k1, k2 [chp.KeySize]byte
+	r := hkdf.New(newBLAKE2s, nil, s.ck[:], nil)
+	if _, err := io.ReadFull(r, k1[:]); err != nil {
+		return nil, nil, fmt.Errorf("extracting k1: %w", err)
+	}
+	if _, err := io.ReadFull(r, k2[:]); err != nil {
+		return nil, nil, fmt.Errorf("extracting k2: %w", err)
+	}
+	c1, err = chp.New(k1[:])
+	if err != nil {
+		return nil, nil, fmt.Errorf("constructing AEAD c1: %w", err)
+	}
+	c2, err = chp.New(k2[:])
+	if err != nil {
+		return nil, nil, fmt.Errorf("constructing AEAD c2: %w", err)
+	}
+	return c1, c2, nil
+}
+
+// newBLAKE2s returns a hash.Hash implementing BLAKE2s, or panics on
+// error.
+func newBLAKE2s() hash.Hash {
+	h, err := blake2s.New256(nil)
+	if err != nil {
+		// Should never happen, errors only happen when using BLAKE2s
+		// in MAC mode with a key.
+		panic(err)
+	}
+	return h
+}
+
+// newCHP returns a cipher.AEAD implementing ChaCha20Poly1305, or
+// panics on error.
+func newCHP(key [chp.KeySize]byte) cipher.AEAD {
+	aead, err := chp.New(key[:])
+	if err != nil {
+		// Can only happen if we passed a key of the wrong length. The
+		// function signature prevents that.
+		panic(err)
+	}
+	return aead
+}
+
+// singleUseCHP is an instance of ChaCha20Poly1305 that can be used
+// only once, either for encrypting or decrypting, but not both. The
+// chosen operation is always executed with an all-zeros
+// nonce. Subsequent calls to either Seal or Open panic.
+type singleUseCHP struct {
+	c cipher.AEAD
+}
+
+func newSingleUseCHP(key [chp.KeySize]byte) *singleUseCHP {
+	return &singleUseCHP{newCHP(key)}
+}
+
+func (c *singleUseCHP) Seal(dst, plaintext, additionalData []byte) []byte {
+	if c.c == nil {
+		panic("Attempted reuse of singleUseAEAD")
+	}
+	cipher := c.c
+	c.c = nil
+	var nonce [chp.NonceSize]byte
+	return cipher.Seal(dst, nonce[:], plaintext, additionalData)
+}
+
+func (c *singleUseCHP) Open(dst, ciphertext, additionalData []byte) ([]byte, error) {
+	if c.c == nil {
+		panic("Attempted reuse of singleUseAEAD")
+	}
+	cipher := c.c
+	c.c = nil
+	var nonce [chp.NonceSize]byte
+	return cipher.Open(dst, nonce[:], ciphertext, additionalData)
+}

control/controlbase/handshake_test.go

@@ -0,0 +1,299 @@
+// Copyright (c) 2021 Tailscale Inc & AUTHORS All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package controlbase
+
+import (
+	"bytes"
+	"context"
+	"io"
+	"strings"
+	"testing"
+	"time"
+
+	tsnettest "tailscale.com/net/nettest"
+	"tailscale.com/types/key"
+)
+
+func TestHandshake(t *testing.T) {
+	var (
+		clientConn, serverConn = tsnettest.NewConn("noise", 128000)
+		serverKey              = key.NewMachine()
+		clientKey              = key.NewMachine()
+		server                 *Conn
+		serverErr              = make(chan error, 1)
+	)
+	go func() {
+		var err error
+		server, err = Server(context.Background(), serverConn, serverKey, nil)
+		serverErr <- err
+	}()
+
+	client, err := Client(context.Background(), clientConn, clientKey, serverKey.Public())
+	if err != nil {
+		t.Fatalf("client connection failed: %v", err)
+	}
+	if err := <-serverErr; err != nil {
+		t.Fatalf("server connection failed: %v", err)
+	}
+
+	if client.HandshakeHash() != server.HandshakeHash() {
+		t.Fatal("client and server disagree on handshake hash")
+	}
+
+	if client.ProtocolVersion() != int(protocolVersion) {
+		t.Fatalf("client reporting wrong protocol version %d, want %d", client.ProtocolVersion(), protocolVersion)
+	}
+	if client.ProtocolVersion() != server.ProtocolVersion() {
+		t.Fatalf("peers disagree on protocol version, client=%d server=%d", client.ProtocolVersion(), server.ProtocolVersion())
+	}
+	if client.Peer() != serverKey.Public() {
+		t.Fatal("client peer key isn't serverKey")
+	}
+	if server.Peer() != clientKey.Public() {
+		t.Fatal("client peer key isn't serverKey")
+	}
+}
+
+// Check that handshaking repeatedly with the same long-term keys
+// result in different handshake hashes and wire traffic.
+func TestNoReuse(t *testing.T) {
+	var (
+		hashes           = map[[32]byte]bool{}
+		clientHandshakes = map[[96]byte]bool{}
+		serverHandshakes = map[[48]byte]bool{}
+		packets          = map[[32]byte]bool{}
+	)
+	for i := 0; i < 10; i++ {
+		var (
+			clientRaw, serverRaw = tsnettest.NewConn("noise", 128000)
+			clientBuf, serverBuf bytes.Buffer
+			clientConn           = &readerConn{clientRaw, io.TeeReader(clientRaw, &clientBuf)}
+			serverConn           = &readerConn{serverRaw, io.TeeReader(serverRaw, &serverBuf)}
+			serverKey            = key.NewMachine()
+			clientKey            = key.NewMachine()
+			server               *Conn
+			serverErr            = make(chan error, 1)
+		)
+		go func() {
+			var err error
+			server, err = Server(context.Background(), serverConn, serverKey, nil)
+			serverErr <- err
+		}()
+
+		client, err := Client(context.Background(), clientConn, clientKey, serverKey.Public())
+		if err != nil {
+			t.Fatalf("client connection failed: %v", err)
+		}
+		if err := <-serverErr; err != nil {
+			t.Fatalf("server connection failed: %v", err)
+		}
+
+		var clientHS [96]byte
+		copy(clientHS[:], serverBuf.Bytes())
+		if clientHandshakes[clientHS] {
+			t.Fatal("client handshake seen twice")
+		}
+		clientHandshakes[clientHS] = true
+
+		var serverHS [48]byte
+		copy(serverHS[:], clientBuf.Bytes())
+		if serverHandshakes[serverHS] {
+			t.Fatal("server handshake seen twice")
+		}
+		serverHandshakes[serverHS] = true
+
+		clientBuf.Reset()
+		serverBuf.Reset()
+		cb := sinkReads(client)
+		sb := sinkReads(server)
+
+		if hashes[client.HandshakeHash()] {
+			t.Fatalf("handshake hash %v seen twice", client.HandshakeHash())
+		}
+		hashes[client.HandshakeHash()] = true
+
+		// Sending 14 bytes turns into 32 bytes on the wire (+16 for
+		// the chacha20poly1305 overhead, +2 length header)
+		if _, err := io.WriteString(client, strings.Repeat("a", 14)); err != nil {
+			t.Fatalf("client>server write failed: %v", err)
+		}
+		if _, err := io.WriteString(server, strings.Repeat("b", 14)); err != nil {
+			t.Fatalf("server>client write failed: %v", err)
+		}
+
+		// Wait for the bytes to be read, so we know they've traveled end to end
+		cb.String(14)
+		sb.String(14)
+
+		var clientWire, serverWire [32]byte
+		copy(clientWire[:], clientBuf.Bytes())
+		copy(serverWire[:], serverBuf.Bytes())
+
+		if packets[clientWire] {
+			t.Fatalf("client wire traffic seen twice")
+		}
+		packets[clientWire] = true
+		if packets[serverWire] {
+			t.Fatalf("server wire traffic seen twice")
+		}
+		packets[serverWire] = true
+
+		server.Close()
+		client.Close()
+	}
+}
+
+// tamperReader wraps a reader and mutates the Nth byte.
+type tamperReader struct {
+	r     io.Reader
+	n     int
+	total int
+}
+
+func (r *tamperReader) Read(bs []byte) (int, error) {
+	n, err := r.r.Read(bs)
+	if off := r.n - r.total; off >= 0 && off < n {
+		bs[off] += 1
+	}
+	r.total += n
+	return n, err
+}
+
+func TestTampering(t *testing.T) {
+	// Tamper with every byte of the client initiation message.
+	for i := 0; i < 101; i++ {
+		var (
+			clientConn, serverRaw = tsnettest.NewConn("noise", 128000)
+			serverConn            = &readerConn{serverRaw, &tamperReader{serverRaw, i, 0}}
+			serverKey             = key.NewMachine()
+			clientKey             = key.NewMachine()
+			serverErr             = make(chan error, 1)
+		)
+		go func() {
+			_, err := Server(context.Background(), serverConn, serverKey, nil)
+			// If the server failed, we have to close the Conn to
+			// unblock the client.
+			if err != nil {
+				serverConn.Close()
+			}
+			serverErr <- err
+		}()
+
+		_, err := Client(context.Background(), clientConn, clientKey, serverKey.Public())
+		if err == nil {
+			t.Fatal("client connection succeeded despite tampering")
+		}
+		if err := <-serverErr; err == nil {
+			t.Fatalf("server connection succeeded despite tampering")
+		}
+	}
+
+	// Tamper with every byte of the server response message.
+	for i := 0; i < 51; i++ {
+		var (
+			clientRaw, serverConn = tsnettest.NewConn("noise", 128000)
+			clientConn            = &readerConn{clientRaw, &tamperReader{clientRaw, i, 0}}
+			serverKey             = key.NewMachine()
+			clientKey             = key.NewMachine()
+			serverErr             = make(chan error, 1)
+		)
+		go func() {
+			_, err := Server(context.Background(), serverConn, serverKey, nil)
+			serverErr <- err
+		}()
+
+		_, err := Client(context.Background(), clientConn, clientKey, serverKey.Public())
+		if err == nil {
+			t.Fatal("client connection succeeded despite tampering")
+		}
+		// The server shouldn't fail, because the tampering took place
+		// in its response.
+		if err := <-serverErr; err != nil {
+			t.Fatalf("server connection failed despite no tampering: %v", err)
+		}
+	}
+
+	// Tamper with every byte of the first server>client transport message.
+	for i := 0; i < 30; i++ {
+		var (
+			clientRaw, serverConn = tsnettest.NewConn("noise", 128000)
+			clientConn            = &readerConn{clientRaw, &tamperReader{clientRaw, 51 + i, 0}}
+			serverKey             = key.NewMachine()
+			clientKey             = key.NewMachine()
+			serverErr             = make(chan error, 1)
+		)
+		go func() {
+			server, err := Server(context.Background(), serverConn, serverKey, nil)
+			serverErr <- err
+			_, err = io.WriteString(server, strings.Repeat("a", 14))
+			serverErr <- err
+		}()
+
+		client, err := Client(context.Background(), clientConn, clientKey, serverKey.Public())
+		if err != nil {
+			t.Fatalf("client handshake failed: %v", err)
+		}
+		// The server shouldn't fail, because the tampering took place
+		// in its response.
+		if err := <-serverErr; err != nil {
+			t.Fatalf("server handshake failed: %v", err)
+		}
+
+		// The client needs a timeout if the tampering is hitting the length header.
+		if i == 1 || i == 2 {
+			client.SetReadDeadline(time.Now().Add(10 * time.Millisecond))
+		}
+
+		var bs [100]byte
+		n, err := client.Read(bs[:])
+		if err == nil {
+			t.Fatal("read succeeded despite tampering")
+		}
+		if n != 0 {
+			t.Fatal("conn yielded some bytes despite tampering")
+		}
+	}
+
+	// Tamper with every byte of the first client>server transport message.
+	for i := 0; i < 30; i++ {
+		var (
+			clientConn, serverRaw = tsnettest.NewConn("noise", 128000)
+			serverConn            = &readerConn{serverRaw, &tamperReader{serverRaw, 101 + i, 0}}
+			serverKey             = key.NewMachine()
+			clientKey             = key.NewMachine()
+			serverErr             = make(chan error, 1)
+		)
+		go func() {
+			server, err := Server(context.Background(), serverConn, serverKey, nil)
+			serverErr <- err
+			var bs [100]byte
+			// The server needs a timeout if the tampering is hitting the length header.
+			if i == 1 || i == 2 {
+				server.SetReadDeadline(time.Now().Add(10 * time.Millisecond))
+			}
+			n, err := server.Read(bs[:])
+			if n != 0 {
+				panic("server got bytes despite tampering")
+			} else {
+				serverErr <- err
+			}
+		}()
+
+		client, err := Client(context.Background(), clientConn, clientKey, serverKey.Public())
+		if err != nil {
+			t.Fatalf("client handshake failed: %v", err)
+		}
+		if err := <-serverErr; err != nil {
+			t.Fatalf("server handshake failed: %v", err)
+		}
+
+		if _, err := io.WriteString(client, strings.Repeat("a", 14)); err != nil {
+			t.Fatalf("client>server write failed: %v", err)
+		}
+		if err := <-serverErr; err == nil {
+			t.Fatal("server successfully received bytes despite tampering")
+		}
+	}
+}

control/controlbase/interop_test.go

@@ -0,0 +1,257 @@
+// Copyright (c) 2021 Tailscale Inc & AUTHORS All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package controlbase
+
+import (
+	"context"
+	"encoding/binary"
+	"errors"
+	"io"
+	"net"
+	"testing"
+
+	tsnettest "tailscale.com/net/nettest"
+	"tailscale.com/types/key"
+)
+
+// Can a reference Noise IK client talk to our server?
+func TestInteropClient(t *testing.T) {
+	var (
+		s1, s2      = tsnettest.NewConn("noise", 128000)
+		controlKey  = key.NewMachine()
+		machineKey  = key.NewMachine()
+		serverErr   = make(chan error, 2)
+		serverBytes = make(chan []byte, 1)
+		c2s         = "client>server"
+		s2c         = "server>client"
+	)
+
+	go func() {
+		server, err := Server(context.Background(), s2, controlKey, nil)
+		serverErr <- err
+		if err != nil {
+			return
+		}
+		var buf [1024]byte
+		_, err = io.ReadFull(server, buf[:len(c2s)])
+		serverBytes <- buf[:len(c2s)]
+		if err != nil {
+			serverErr <- err
+			return
+		}
+		_, err = server.Write([]byte(s2c))
+		serverErr <- err
+	}()
+
+	gotS2C, err := noiseExplorerClient(s1, controlKey.Public(), machineKey, []byte(c2s))
+	if err != nil {
+		t.Fatalf("failed client interop: %v", err)
+	}
+	if string(gotS2C) != s2c {
+		t.Fatalf("server sent unexpected data %q, want %q", string(gotS2C), s2c)
+	}
+
+	if err := <-serverErr; err != nil {
+		t.Fatalf("server handshake failed: %v", err)
+	}
+	if err := <-serverErr; err != nil {
+		t.Fatalf("server read/write failed: %v", err)
+	}
+	if got := string(<-serverBytes); got != c2s {
+		t.Fatalf("server received %q, want %q", got, c2s)
+	}
+}
+
+// Can our client talk to a reference Noise IK server?
+func TestInteropServer(t *testing.T) {
+	var (
+		s1, s2      = tsnettest.NewConn("noise", 128000)
+		controlKey  = key.NewMachine()
+		machineKey  = key.NewMachine()
+		clientErr   = make(chan error, 2)
+		clientBytes = make(chan []byte, 1)
+		c2s         = "client>server"
+		s2c         = "server>client"
+	)
+
+	go func() {
+		client, err := Client(context.Background(), s1, machineKey, controlKey.Public())
+		clientErr <- err
+		if err != nil {
+			return
+		}
+		_, err = client.Write([]byte(c2s))
+		if err != nil {
+			clientErr <- err
+			return
+		}
+		var buf [1024]byte
+		_, err = io.ReadFull(client, buf[:len(s2c)])
+		clientBytes <- buf[:len(s2c)]
+		clientErr <- err
+	}()
+
+	gotC2S, err := noiseExplorerServer(s2, controlKey, machineKey.Public(), []byte(s2c))
+	if err != nil {
+		t.Fatalf("failed server interop: %v", err)
+	}
+	if string(gotC2S) != c2s {
+		t.Fatalf("server sent unexpected data %q, want %q", string(gotC2S), c2s)
+	}
+
+	if err := <-clientErr; err != nil {
+		t.Fatalf("client handshake failed: %v", err)
+	}
+	if err := <-clientErr; err != nil {
+		t.Fatalf("client read/write failed: %v", err)
+	}
+	if got := string(<-clientBytes); got != s2c {
+		t.Fatalf("client received %q, want %q", got, s2c)
+	}
+}
+
+// noiseExplorerClient uses the Noise Explorer implementation of Noise
+// IK to handshake as a Noise client on conn, transmit payload, and
+// read+return a payload from the peer.
+func noiseExplorerClient(conn net.Conn, controlKey key.MachinePublic, machineKey key.MachinePrivate, payload []byte) ([]byte, error) {
+	var mk keypair
+	copy(mk.private_key[:], machineKey.UntypedBytes())
+	copy(mk.public_key[:], machineKey.Public().UntypedBytes())
+	var peerKey [32]byte
+	copy(peerKey[:], controlKey.UntypedBytes())
+	session := InitSession(true, protocolVersionPrologue(protocolVersion), mk, peerKey)
+
+	_, msg1 := SendMessage(&session, nil)
+	var hdr [initiationHeaderLen]byte
+	binary.BigEndian.PutUint16(hdr[:2], protocolVersion)
+	hdr[2] = msgTypeInitiation
+	binary.BigEndian.PutUint16(hdr[3:5], 96)
+	if _, err := conn.Write(hdr[:]); err != nil {
+		return nil, err
+	}
+	if _, err := conn.Write(msg1.ne[:]); err != nil {
+		return nil, err
+	}
+	if _, err := conn.Write(msg1.ns); err != nil {
+		return nil, err
+	}
+	if _, err := conn.Write(msg1.ciphertext); err != nil {
+		return nil, err
+	}
+
+	var buf [1024]byte
+	if _, err := io.ReadFull(conn, buf[:51]); err != nil {
+		return nil, err
+	}
+	// ignore the header for this test, we're only checking the noise
+	// implementation.
+	msg2 := messagebuffer{
+		ciphertext: buf[35:51],
+	}
+	copy(msg2.ne[:], buf[3:35])
+	_, p, valid := RecvMessage(&session, &msg2)
+	if !valid {
+		return nil, errors.New("handshake failed")
+	}
+	if len(p) != 0 {
+		return nil, errors.New("non-empty payload")
+	}
+
+	_, msg3 := SendMessage(&session, payload)
+	hdr[0] = msgTypeRecord
+	binary.BigEndian.PutUint16(hdr[1:3], uint16(len(msg3.ciphertext)))
+	if _, err := conn.Write(hdr[:3]); err != nil {
+		return nil, err
+	}
+	if _, err := conn.Write(msg3.ciphertext); err != nil {
+		return nil, err
+	}
+
+	if _, err := io.ReadFull(conn, buf[:3]); err != nil {
+		return nil, err
+	}
+	// Ignore all of the header except the payload length
+	plen := int(binary.BigEndian.Uint16(buf[1:3]))
+	if _, err := io.ReadFull(conn, buf[:plen]); err != nil {
+		return nil, err
+	}
+
+	msg4 := messagebuffer{
+		ciphertext: buf[:plen],
+	}
+	_, p, valid = RecvMessage(&session, &msg4)
+	if !valid {
+		return nil, errors.New("transport message decryption failed")
+	}
+
+	return p, nil
+}
+
+func noiseExplorerServer(conn net.Conn, controlKey key.MachinePrivate, wantMachineKey key.MachinePublic, payload []byte) ([]byte, error) {
+	var mk keypair
+	copy(mk.private_key[:], controlKey.UntypedBytes())
+	copy(mk.public_key[:], controlKey.Public().UntypedBytes())
+	session := InitSession(false, protocolVersionPrologue(protocolVersion), mk, [32]byte{})
+
+	var buf [1024]byte
+	if _, err := io.ReadFull(conn, buf[:101]); err != nil {
+		return nil, err
+	}
+	// Ignore the header, we're just checking the noise implementation.
+	msg1 := messagebuffer{
+		ns:         buf[37:85],
+		ciphertext: buf[85:101],
+	}
+	copy(msg1.ne[:], buf[5:37])
+	_, p, valid := RecvMessage(&session, &msg1)
+	if !valid {
+		return nil, errors.New("handshake failed")
+	}
+	if len(p) != 0 {
+		return nil, errors.New("non-empty payload")
+	}
+
+	_, msg2 := SendMessage(&session, nil)
+	var hdr [headerLen]byte
+	hdr[0] = msgTypeResponse
+	binary.BigEndian.PutUint16(hdr[1:3], 48)
+	if _, err := conn.Write(hdr[:]); err != nil {
+		return nil, err
+	}
+	if _, err := conn.Write(msg2.ne[:]); err != nil {
+		return nil, err
+	}
+	if _, err := conn.Write(msg2.ciphertext[:]); err != nil {
+		return nil, err
+	}
+
+	if _, err := io.ReadFull(conn, buf[:3]); err != nil {
+		return nil, err
+	}
+	plen := int(binary.BigEndian.Uint16(buf[1:3]))
+	if _, err := io.ReadFull(conn, buf[:plen]); err != nil {
+		return nil, err
+	}
+
+	msg3 := messagebuffer{
+		ciphertext: buf[:plen],
+	}
+	_, p, valid = RecvMessage(&session, &msg3)
+	if !valid {
+		return nil, errors.New("transport message decryption failed")
+	}
+
+	_, msg4 := SendMessage(&session, payload)
+	hdr[0] = msgTypeRecord
+	binary.BigEndian.PutUint16(hdr[1:3], uint16(len(msg4.ciphertext)))
+	if _, err := conn.Write(hdr[:]); err != nil {
+		return nil, err
+	}
+	if _, err := conn.Write(msg4.ciphertext); err != nil {
+		return nil, err
+	}
+
+	return p, nil
+}

control/controlbase/messages.go

@@ -0,0 +1,88 @@
+// Copyright (c) 2021 Tailscale Inc & AUTHORS All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package controlbase
+
+import "encoding/binary"
+
+const (
+	// msgTypeInitiation frames carry a Noise IK handshake initiation message.
+	msgTypeInitiation = 1
+	// msgTypeResponse frames carry a Noise IK handshake response message.
+	msgTypeResponse = 2
+	// msgTypeError frames carry an unauthenticated human-readable
+	// error message.
+	//
+	// Errors reported in this message type must be treated as public
+	// hints only. They are not encrypted or authenticated, and so can
+	// be seen and tampered with on the wire.
+	msgTypeError = 3
+	// msgTypeRecord frames carry session data bytes.
+	msgTypeRecord = 4
+
+	// headerLen is the size of the header on all messages except msgTypeInitiation.
+	headerLen = 3
+	// initiationHeaderLen is the size of the header on all msgTypeInitiation messages.
+	initiationHeaderLen = 5
+)
+
+// initiationMessage is the protocol message sent from a client
+// machine to a control server.
+//
+// 2b: protocol version
+// 1b: message type (0x01)
+// 2b: payload length (96)
+// 5b: header (see headerLen for fields)
+// 32b: client ephemeral public key (cleartext)
+// 48b: client machine public key (encrypted)
+// 16b: message tag (authenticates the whole message)
+type initiationMessage [101]byte
+
+func mkInitiationMessage() initiationMessage {
+	var ret initiationMessage
+	binary.BigEndian.PutUint16(ret[:2], uint16(protocolVersion))
+	ret[2] = msgTypeInitiation
+	binary.BigEndian.PutUint16(ret[3:5], uint16(len(ret.Payload())))
+	return ret
+}
+
+func (m *initiationMessage) Header() []byte  { return m[:initiationHeaderLen] }
+func (m *initiationMessage) Payload() []byte { return m[initiationHeaderLen:] }
+
+func (m *initiationMessage) Version() uint16 { return binary.BigEndian.Uint16(m[:2]) }
+func (m *initiationMessage) Type() byte      { return m[2] }
+func (m *initiationMessage) Length() int     { return int(binary.BigEndian.Uint16(m[3:5])) }
+
+func (m *initiationMessage) EphemeralPub() []byte {
+	return m[initiationHeaderLen : initiationHeaderLen+32]
+}
+func (m *initiationMessage) MachinePub() []byte {
+	return m[initiationHeaderLen+32 : initiationHeaderLen+32+48]
+}
+func (m *initiationMessage) Tag() []byte { return m[initiationHeaderLen+32+48:] }
+
+// responseMessage is the protocol message sent from a control server
+// to a client machine.
+//
+// 1b: message type (0x02)
+// 2b: payload length (48)
+// 32b: control ephemeral public key (cleartext)
+// 16b: message tag (authenticates the whole message)
+type responseMessage [51]byte
+
+func mkResponseMessage() responseMessage {
+	var ret responseMessage
+	ret[0] = msgTypeResponse
+	binary.BigEndian.PutUint16(ret[1:], uint16(len(ret.Payload())))
+	return ret
+}
+
+func (m *responseMessage) Header() []byte  { return m[:headerLen] }
+func (m *responseMessage) Payload() []byte { return m[headerLen:] }
+
+func (m *responseMessage) Type() byte  { return m[0] }
+func (m *responseMessage) Length() int { return int(binary.BigEndian.Uint16(m[1:3])) }
+
+func (m *responseMessage) EphemeralPub() []byte { return m[headerLen : headerLen+32] }
+func (m *responseMessage) Tag() []byte          { return m[headerLen+32:] }

control/controlbase/noiseexplorer_test.go

@@ -0,0 +1,475 @@
+// This file contains the implementation of Noise IK from
+// https://noiseexplorer.com/ . Unlike the rest of this repository,
+// this file is licensed under the terms of the GNU GPL v3. See
+// https://source.symbolic.software/noiseexplorer/noiseexplorer for
+// more information.
+//
+// This file is used here to verify that Tailscale's implementation of
+// Noise IK is interoperable with another implementation.
+//lint:file-ignore SA4006 not our code.
+
+/*
+IK:
+  <- s
+  ...
+  -> e, es, s, ss
+  <- e, ee, se
+  ->
+  <-
+*/
+
+// Implementation Version: 1.0.2
+
+/* ---------------------------------------------------------------- *
+ * PARAMETERS                                                       *
+ * ---------------------------------------------------------------- */
+
+package controlbase
+
+import (
+	"crypto/rand"
+	"crypto/subtle"
+	"encoding/binary"
+	"hash"
+	"io"
+	"math"
+
+	"golang.org/x/crypto/blake2s"
+	"golang.org/x/crypto/chacha20poly1305"
+	"golang.org/x/crypto/curve25519"
+	"golang.org/x/crypto/hkdf"
+)
+
+/* ---------------------------------------------------------------- *
+ * TYPES                                                            *
+ * ---------------------------------------------------------------- */
+
+type keypair struct {
+	public_key  [32]byte
+	private_key [32]byte
+}
+
+type messagebuffer struct {
+	ne         [32]byte
+	ns         []byte
+	ciphertext []byte
+}
+
+type cipherstate struct {
+	k [32]byte
+	n uint32
+}
+
+type symmetricstate struct {
+	cs cipherstate
+	ck [32]byte
+	h  [32]byte
+}
+
+type handshakestate struct {
+	ss  symmetricstate
+	s   keypair
+	e   keypair
+	rs  [32]byte
+	re  [32]byte
+	psk [32]byte
+}
+
+type noisesession struct {
+	hs  handshakestate
+	h   [32]byte
+	cs1 cipherstate
+	cs2 cipherstate
+	mc  uint64
+	i   bool
+}
+
+/* ---------------------------------------------------------------- *
+ * CONSTANTS                                                        *
+ * ---------------------------------------------------------------- */
+
+var emptyKey = [32]byte{
+	0x00, 0x00, 0x00, 0x00,
+	0x00, 0x00, 0x00, 0x00,
+	0x00, 0x00, 0x00, 0x00,
+	0x00, 0x00, 0x00, 0x00,
+	0x00, 0x00, 0x00, 0x00,
+	0x00, 0x00, 0x00, 0x00,
+	0x00, 0x00, 0x00, 0x00,
+	0x00, 0x00, 0x00, 0x00,
+}
+
+var minNonce = uint32(0)
+
+/* ---------------------------------------------------------------- *
+ * UTILITY FUNCTIONS                                                *
+ * ---------------------------------------------------------------- */
+
+func getPublicKey(kp *keypair) [32]byte {
+	return kp.public_key
+}
+
+func isEmptyKey(k [32]byte) bool {
+	return subtle.ConstantTimeCompare(k[:], emptyKey[:]) == 1
+}
+
+func validatePublicKey(k []byte) bool {
+	forbiddenCurveValues := [12][]byte{
+		{0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0},
+		{1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0},
+		{224, 235, 122, 124, 59, 65, 184, 174, 22, 86, 227, 250, 241, 159, 196, 106, 218, 9, 141, 235, 156, 50, 177, 253, 134, 98, 5, 22, 95, 73, 184, 0},
+		{95, 156, 149, 188, 163, 80, 140, 36, 177, 208, 177, 85, 156, 131, 239, 91, 4, 68, 92, 196, 88, 28, 142, 134, 216, 34, 78, 221, 208, 159, 17, 87},
+		{236, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 127},
+		{237, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 127},
+		{238, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 127},
+		{205, 235, 122, 124, 59, 65, 184, 174, 22, 86, 227, 250, 241, 159, 196, 106, 218, 9, 141, 235, 156, 50, 177, 253, 134, 98, 5, 22, 95, 73, 184, 128},
+		{76, 156, 149, 188, 163, 80, 140, 36, 177, 208, 177, 85, 156, 131, 239, 91, 4, 68, 92, 196, 88, 28, 142, 134, 216, 34, 78, 221, 208, 159, 17, 215},
+		{217, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255},
+		{218, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255},
+		{219, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 25},
+	}
+
+	for _, testValue := range forbiddenCurveValues {
+		if subtle.ConstantTimeCompare(k[:], testValue[:]) == 1 {
+			panic("Invalid public key")
+		}
+	}
+	return true
+}
+
+/* ---------------------------------------------------------------- *
+ * PRIMITIVES                                                       *
+ * ---------------------------------------------------------------- */
+
+func incrementNonce(n uint32) uint32 {
+	return n + 1
+}
+
+func dh(private_key [32]byte, public_key [32]byte) [32]byte {
+	var ss [32]byte
+	curve25519.ScalarMult(&ss, &private_key, &public_key)
+	return ss
+}
+
+func generateKeypair() keypair {
+	var public_key [32]byte
+	var private_key [32]byte
+	_, _ = rand.Read(private_key[:])
+	curve25519.ScalarBaseMult(&public_key, &private_key)
+	if validatePublicKey(public_key[:]) {
+		return keypair{public_key, private_key}
+	}
+	return generateKeypair()
+}
+
+func generatePublicKey(private_key [32]byte) [32]byte {
+	var public_key [32]byte
+	curve25519.ScalarBaseMult(&public_key, &private_key)
+	return public_key
+}
+
+func encrypt(k [32]byte, n uint32, ad []byte, plaintext []byte) []byte {
+	var nonce [12]byte
+	var ciphertext []byte
+	enc, _ := chacha20poly1305.New(k[:])
+	binary.LittleEndian.PutUint32(nonce[4:], n)
+	ciphertext = enc.Seal(nil, nonce[:], plaintext, ad)
+	return ciphertext
+}
+
+func decrypt(k [32]byte, n uint32, ad []byte, ciphertext []byte) (bool, []byte, []byte) {
+	var nonce [12]byte
+	var plaintext []byte
+	enc, err := chacha20poly1305.New(k[:])
+	binary.LittleEndian.PutUint32(nonce[4:], n)
+	plaintext, err = enc.Open(nil, nonce[:], ciphertext, ad)
+	return (err == nil), ad, plaintext
+}
+
+func getHash(a []byte, b []byte) [32]byte {
+	return blake2s.Sum256(append(a, b...))
+}
+
+func hashProtocolName(protocolName []byte) [32]byte {
+	var h [32]byte
+	if len(protocolName) <= 32 {
+		copy(h[:], protocolName)
+	} else {
+		h = getHash(protocolName, []byte{})
+	}
+	return h
+}
+
+func blake2HkdfInterface() hash.Hash {
+	h, _ := blake2s.New256([]byte{})
+	return h
+}
+
+func getHkdf(ck [32]byte, ikm []byte) ([32]byte, [32]byte, [32]byte) {
+	var k1 [32]byte
+	var k2 [32]byte
+	var k3 [32]byte
+	output := hkdf.New(blake2HkdfInterface, ikm[:], ck[:], []byte{})
+	io.ReadFull(output, k1[:])
+	io.ReadFull(output, k2[:])
+	io.ReadFull(output, k3[:])
+	return k1, k2, k3
+}
+
+/* ---------------------------------------------------------------- *
+ * STATE MANAGEMENT                                                 *
+ * ---------------------------------------------------------------- */
+
+/* CipherState */
+func initializeKey(k [32]byte) cipherstate {
+	return cipherstate{k, minNonce}
+}
+
+func hasKey(cs *cipherstate) bool {
+	return !isEmptyKey(cs.k)
+}
+
+func setNonce(cs *cipherstate, newNonce uint32) *cipherstate {
+	cs.n = newNonce
+	return cs
+}
+
+func encryptWithAd(cs *cipherstate, ad []byte, plaintext []byte) (*cipherstate, []byte) {
+	e := encrypt(cs.k, cs.n, ad, plaintext)
+	cs = setNonce(cs, incrementNonce(cs.n))
+	return cs, e
+}
+
+func decryptWithAd(cs *cipherstate, ad []byte, ciphertext []byte) (*cipherstate, []byte, bool) {
+	valid, ad, plaintext := decrypt(cs.k, cs.n, ad, ciphertext)
+	cs = setNonce(cs, incrementNonce(cs.n))
+	return cs, plaintext, valid
+}
+
+func reKey(cs *cipherstate) *cipherstate {
+	e := encrypt(cs.k, math.MaxUint32, []byte{}, emptyKey[:])
+	copy(cs.k[:], e)
+	return cs
+}
+
+/* SymmetricState */
+
+func initializeSymmetric(protocolName []byte) symmetricstate {
+	h := hashProtocolName(protocolName)
+	ck := h
+	cs := initializeKey(emptyKey)
+	return symmetricstate{cs, ck, h}
+}
+
+func mixKey(ss *symmetricstate, ikm [32]byte) *symmetricstate {
+	ck, tempK, _ := getHkdf(ss.ck, ikm[:])
+	ss.cs = initializeKey(tempK)
+	ss.ck = ck
+	return ss
+}
+
+func mixHash(ss *symmetricstate, data []byte) *symmetricstate {
+	ss.h = getHash(ss.h[:], data)
+	return ss
+}
+
+func mixKeyAndHash(ss *symmetricstate, ikm [32]byte) *symmetricstate {
+	var tempH [32]byte
+	var tempK [32]byte
+	ss.ck, tempH, tempK = getHkdf(ss.ck, ikm[:])
+	ss = mixHash(ss, tempH[:])
+	ss.cs = initializeKey(tempK)
+	return ss
+}
+
+func getHandshakeHash(ss *symmetricstate) [32]byte {
+	return ss.h
+}
+
+func encryptAndHash(ss *symmetricstate, plaintext []byte) (*symmetricstate, []byte) {
+	var ciphertext []byte
+	if hasKey(&ss.cs) {
+		_, ciphertext = encryptWithAd(&ss.cs, ss.h[:], plaintext)
+	} else {
+		ciphertext = plaintext
+	}
+	ss = mixHash(ss, ciphertext)
+	return ss, ciphertext
+}
+
+func decryptAndHash(ss *symmetricstate, ciphertext []byte) (*symmetricstate, []byte, bool) {
+	var plaintext []byte
+	var valid bool
+	if hasKey(&ss.cs) {
+		_, plaintext, valid = decryptWithAd(&ss.cs, ss.h[:], ciphertext)
+	} else {
+		plaintext, valid = ciphertext, true
+	}
+	ss = mixHash(ss, ciphertext)
+	return ss, plaintext, valid
+}
+
+func split(ss *symmetricstate) (cipherstate, cipherstate) {
+	tempK1, tempK2, _ := getHkdf(ss.ck, []byte{})
+	cs1 := initializeKey(tempK1)
+	cs2 := initializeKey(tempK2)
+	return cs1, cs2
+}
+
+/* HandshakeState */
+
+func initializeInitiator(prologue []byte, s keypair, rs [32]byte, psk [32]byte) handshakestate {
+	var ss symmetricstate
+	var e keypair
+	var re [32]byte
+	name := []byte("Noise_IK_25519_ChaChaPoly_BLAKE2s")
+	ss = initializeSymmetric(name)
+	mixHash(&ss, prologue)
+	mixHash(&ss, rs[:])
+	return handshakestate{ss, s, e, rs, re, psk}
+}
+
+func initializeResponder(prologue []byte, s keypair, rs [32]byte, psk [32]byte) handshakestate {
+	var ss symmetricstate
+	var e keypair
+	var re [32]byte
+	name := []byte("Noise_IK_25519_ChaChaPoly_BLAKE2s")
+	ss = initializeSymmetric(name)
+	mixHash(&ss, prologue)
+	mixHash(&ss, s.public_key[:])
+	return handshakestate{ss, s, e, rs, re, psk}
+}
+
+func writeMessageA(hs *handshakestate, payload []byte) (*handshakestate, messagebuffer) {
+	ne, ns, ciphertext := emptyKey, []byte{}, []byte{}
+	hs.e = generateKeypair()
+	ne = hs.e.public_key
+	mixHash(&hs.ss, ne[:])
+	/* No PSK, so skipping mixKey */
+	mixKey(&hs.ss, dh(hs.e.private_key, hs.rs))
+	spk := make([]byte, len(hs.s.public_key))
+	copy(spk[:], hs.s.public_key[:])
+	_, ns = encryptAndHash(&hs.ss, spk)
+	mixKey(&hs.ss, dh(hs.s.private_key, hs.rs))
+	_, ciphertext = encryptAndHash(&hs.ss, payload)
+	messageBuffer := messagebuffer{ne, ns, ciphertext}
+	return hs, messageBuffer
+}
+
+func writeMessageB(hs *handshakestate, payload []byte) ([32]byte, messagebuffer, cipherstate, cipherstate) {
+	ne, ns, ciphertext := emptyKey, []byte{}, []byte{}
+	hs.e = generateKeypair()
+	ne = hs.e.public_key
+	mixHash(&hs.ss, ne[:])
+	/* No PSK, so skipping mixKey */
+	mixKey(&hs.ss, dh(hs.e.private_key, hs.re))
+	mixKey(&hs.ss, dh(hs.e.private_key, hs.rs))
+	_, ciphertext = encryptAndHash(&hs.ss, payload)
+	messageBuffer := messagebuffer{ne, ns, ciphertext}
+	cs1, cs2 := split(&hs.ss)
+	return hs.ss.h, messageBuffer, cs1, cs2
+}
+
+func writeMessageRegular(cs *cipherstate, payload []byte) (*cipherstate, messagebuffer) {
+	ne, ns, ciphertext := emptyKey, []byte{}, []byte{}
+	cs, ciphertext = encryptWithAd(cs, []byte{}, payload)
+	messageBuffer := messagebuffer{ne, ns, ciphertext}
+	return cs, messageBuffer
+}
+
+func readMessageA(hs *handshakestate, message *messagebuffer) (*handshakestate, []byte, bool) {
+	valid1 := true
+	if validatePublicKey(message.ne[:]) {
+		hs.re = message.ne
+	}
+	mixHash(&hs.ss, hs.re[:])
+	/* No PSK, so skipping mixKey */
+	mixKey(&hs.ss, dh(hs.s.private_key, hs.re))
+	_, ns, valid1 := decryptAndHash(&hs.ss, message.ns)
+	if valid1 && len(ns) == 32 && validatePublicKey(message.ns[:]) {
+		copy(hs.rs[:], ns)
+	}
+	mixKey(&hs.ss, dh(hs.s.private_key, hs.rs))
+	_, plaintext, valid2 := decryptAndHash(&hs.ss, message.ciphertext)
+	return hs, plaintext, (valid1 && valid2)
+}
+
+func readMessageB(hs *handshakestate, message *messagebuffer) ([32]byte, []byte, bool, cipherstate, cipherstate) {
+	valid1 := true
+	if validatePublicKey(message.ne[:]) {
+		hs.re = message.ne
+	}
+	mixHash(&hs.ss, hs.re[:])
+	/* No PSK, so skipping mixKey */
+	mixKey(&hs.ss, dh(hs.e.private_key, hs.re))
+	mixKey(&hs.ss, dh(hs.s.private_key, hs.re))
+	_, plaintext, valid2 := decryptAndHash(&hs.ss, message.ciphertext)
+	cs1, cs2 := split(&hs.ss)
+	return hs.ss.h, plaintext, (valid1 && valid2), cs1, cs2
+}
+
+func readMessageRegular(cs *cipherstate, message *messagebuffer) (*cipherstate, []byte, bool) {
+	/* No encrypted keys */
+	_, plaintext, valid2 := decryptWithAd(cs, []byte{}, message.ciphertext)
+	return cs, plaintext, valid2
+}
+
+/* ---------------------------------------------------------------- *
+ * PROCESSES                                                        *
+ * ---------------------------------------------------------------- */
+
+func InitSession(initiator bool, prologue []byte, s keypair, rs [32]byte) noisesession {
+	var session noisesession
+	psk := emptyKey
+	if initiator {
+		session.hs = initializeInitiator(prologue, s, rs, psk)
+	} else {
+		session.hs = initializeResponder(prologue, s, rs, psk)
+	}
+	session.i = initiator
+	session.mc = 0
+	return session
+}
+
+func SendMessage(session *noisesession, message []byte) (*noisesession, messagebuffer) {
+	var messageBuffer messagebuffer
+	if session.mc == 0 {
+		_, messageBuffer = writeMessageA(&session.hs, message)
+	}
+	if session.mc == 1 {
+		session.h, messageBuffer, session.cs1, session.cs2 = writeMessageB(&session.hs, message)
+		session.hs = handshakestate{}
+	}
+	if session.mc > 1 {
+		if session.i {
+			_, messageBuffer = writeMessageRegular(&session.cs1, message)
+		} else {
+			_, messageBuffer = writeMessageRegular(&session.cs2, message)
+		}
+	}
+	session.mc = session.mc + 1
+	return session, messageBuffer
+}
+
+func RecvMessage(session *noisesession, message *messagebuffer) (*noisesession, []byte, bool) {
+	var plaintext []byte
+	var valid bool
+	if session.mc == 0 {
+		_, plaintext, valid = readMessageA(&session.hs, message)
+	}
+	if session.mc == 1 {
+		session.h, plaintext, valid, session.cs1, session.cs2 = readMessageB(&session.hs, message)
+		session.hs = handshakestate{}
+	}
+	if session.mc > 1 {
+		if session.i {
+			_, plaintext, valid = readMessageRegular(&session.cs2, message)
+		} else {
+			_, plaintext, valid = readMessageRegular(&session.cs1, message)
+		}
+	}
+	session.mc = session.mc + 1
+	return session, plaintext, valid
+}
+
+func main() {}

control/controlclient/auto.go

@@ -14,11 +14,11 @@ import (
 	"tailscale.com/logtail/backoff"
 	"tailscale.com/tailcfg"
 	"tailscale.com/types/empty"
+	"tailscale.com/types/key"
 	"tailscale.com/types/logger"
 	"tailscale.com/types/netmap"
 	"tailscale.com/types/persist"
 	"tailscale.com/types/structs"
-	"tailscale.com/types/wgkey"
 )
 
 type LoginGoal struct {
@@ -266,9 +266,9 @@ func (c *Auto) authRoutine() {
 		goal := c.loginGoal
 		ctx := c.authCtx
 		if goal != nil {
-			c.logf("authRoutine: %s; wantLoggedIn=%v", c.state, goal.wantLoggedIn)
+			c.logf("[v1] authRoutine: %s; wantLoggedIn=%v", c.state, goal.wantLoggedIn)
 		} else {
-			c.logf("authRoutine: %s; goal=nil paused=%v", c.state, c.paused)
+			c.logf("[v1] authRoutine: %s; goal=nil paused=%v", c.state, c.paused)
 		}
 		c.mu.Unlock()
 
@@ -281,7 +281,6 @@ func (c *Auto) authRoutine() {
 
 		report := func(err error, msg string) {
 			c.logf("[v1] %s: %v", msg, err)
-			err = fmt.Errorf("%s: %v", msg, err)
 			// don't send status updates for context errors,
 			// since context cancelation is always on purpose.
 			if ctx.Err() == nil {
@@ -340,11 +339,9 @@ func (c *Auto) authRoutine() {
 				continue
 			}
 			if url != "" {
-				if goal.url != "" {
-					err = fmt.Errorf("[unexpected] server required a new URL?")
-					report(err, "WaitLoginURL")
-				}
-
+				// goal.url ought to be empty here.
+				// However, not all control servers get this right,
+				// and logging about it here just generates noise.
 				c.mu.Lock()
 				c.loginGoal = &LoginGoal{
 					wantLoggedIn: true,
@@ -417,7 +414,7 @@ func (c *Auto) mapRoutine() {
 			}
 			continue
 		}
-		c.logf("mapRoutine: %s", c.state)
+		c.logf("[v1] mapRoutine: %s", c.state)
 		loggedIn := c.loggedIn
 		ctx := c.mapCtx
 		c.mu.Unlock()
@@ -431,7 +428,7 @@ func (c *Auto) mapRoutine() {
 
 		report := func(err error, msg string) {
 			c.logf("[v1] %s: %v", msg, err)
-			err = fmt.Errorf("%s: %v", msg, err)
+			err = fmt.Errorf("%s: %w", msg, err)
 			// don't send status updates for context errors,
 			// since context cancelation is always on purpose.
 			if ctx.Err() == nil {
@@ -448,9 +445,9 @@ func (c *Auto) mapRoutine() {
 
 			select {
 			case <-ctx.Done():
-				c.logf("mapRoutine: context done.")
+				c.logf("[v1] mapRoutine: context done.")
 			case <-c.newMapCh:
-				c.logf("mapRoutine: new map needed while idle.")
+				c.logf("[v1] mapRoutine: new map needed while idle.")
 			}
 		} else {
 			// Be sure this is false when we're not inside
@@ -599,9 +596,7 @@ func (c *Auto) sendStatus(who string, err error, url string, nm *netmap.NetworkM
 		NetMap:         nm,
 		Hostinfo:       hi,
 		State:          state,
-	}
-	if err != nil {
-		new.Err = err.Error()
+		Err:            err,
 	}
 	if statusFunc != nil {
 		statusFunc(new)
@@ -702,7 +697,7 @@ func (c *Auto) Shutdown() {
 
 // NodePublicKey returns the node public key currently in use. This is
 // used exclusively in tests.
-func (c *Auto) TestOnlyNodePublicKey() wgkey.Key {
+func (c *Auto) TestOnlyNodePublicKey() key.NodePublic {
 	priv := c.direct.GetPersist()
 	return priv.PrivateNodeKey.Public()
 }

control/controlclient/client.go

@@ -20,6 +20,7 @@ type LoginFlags int
 const (
 	LoginDefault     = LoginFlags(0)
 	LoginInteractive = LoginFlags(1 << iota) // force user login and key refresh
+	LoginEphemeral                           // set RegisterRequest.Ephemeral
 )
 
 // Client represents a client connection to the control server.
@@ -78,3 +79,9 @@ type Client interface {
 	// requesting a DNS record be created or updated.
 	SetDNS(context.Context, *tailcfg.SetDNSRequest) error
 }
+
+// UserVisibleError is an error that should be shown to users.
+type UserVisibleError string
+
+func (e UserVisibleError) Error() string            { return string(e) }
+func (e UserVisibleError) UserVisibleError() string { return string(e) }

control/controlclient/direct.go

@@ -21,7 +21,6 @@ import (
 	"os/exec"
 	"reflect"
 	"runtime"
-	"strconv"
 	"strings"
 	"sync"
 	"sync/atomic"
@@ -30,6 +29,7 @@ import (
 	"go4.org/mem"
 	"inet.af/netaddr"
 	"tailscale.com/control/controlknobs"
+	"tailscale.com/envknob"
 	"tailscale.com/health"
 	"tailscale.com/hostinfo"
 	"tailscale.com/ipn/ipnstate"
@@ -46,7 +46,7 @@ import (
 	"tailscale.com/types/netmap"
 	"tailscale.com/types/opt"
 	"tailscale.com/types/persist"
-	"tailscale.com/types/wgkey"
+	"tailscale.com/util/clientmetric"
 	"tailscale.com/util/systemd"
 	"tailscale.com/wgengine/monitor"
 )
@@ -61,7 +61,7 @@ type Direct struct {
 	keepAlive              bool
 	logf                   logger.Logf
 	linkMon                *monitor.Mon // or nil
-	discoPubKey            tailcfg.DiscoKey
+	discoPubKey            key.DiscoPublic
 	getMachinePrivKey      func() (key.MachinePrivate, error)
 	debugFlags             []string
 	keepSharerAndUserSplit bool
@@ -72,7 +72,7 @@ type Direct struct {
 	serverKey    key.MachinePublic
 	persist      persist.Persist
 	authKey      string
-	tryingNewKey wgkey.Private
+	tryingNewKey key.NodePrivate
 	expiry       *time.Time
 	// hostinfo is mutated in-place while mu is held.
 	hostinfo      *tailcfg.Hostinfo // always non-nil
@@ -89,7 +89,7 @@ type Options struct {
 	AuthKey              string                             // optional node auth key for auto registration
 	TimeNow              func() time.Time                   // time.Now implementation used by Client
 	Hostinfo             *tailcfg.Hostinfo                  // non-nil passes ownership, nil means to use default using os.Hostname, etc
-	DiscoPublicKey       tailcfg.DiscoKey
+	DiscoPublicKey       key.DiscoPublic
 	NewDecompressor      func() (Decompressor, error)
 	KeepAlive            bool
 	Logf                 logger.Logf
@@ -147,13 +147,20 @@ func NewDirect(opts Options) (*Direct, error) {
 	}
 
 	httpc := opts.HTTPTestClient
+	if httpc == nil && runtime.GOOS == "js" {
+		// In js/wasm, net/http.Transport (as of Go 1.18) will
+		// only use the browser's Fetch API if you're using
+		// the DefaultClient (or a client without dial hooks
+		// etc set).
+		httpc = http.DefaultClient
+	}
 	if httpc == nil {
 		dnsCache := &dnscache.Resolver{
 			Forward:          dnscache.Get().Forward, // use default cache's forwarder
 			UseLastGood:      true,
 			LookupIPFallback: dnsfallback.Lookup,
 		}
-		dialer := netns.NewDialer()
+		dialer := netns.NewDialer(opts.Logf)
 		tr := http.DefaultTransport.(*http.Transport).Clone()
 		tr.Proxy = tshttpproxy.ProxyFromEnvironment
 		tshttpproxy.SetTransportGetProxyConnectHeader(tr)
@@ -161,6 +168,10 @@ func NewDirect(opts Options) (*Direct, error) {
 		tr.DialContext = dnscache.Dialer(dialer.DialContext, dnsCache)
 		tr.DialTLSContext = dnscache.TLSDialer(dialer.DialContext, dnsCache, tr.TLSClientConfig)
 		tr.ForceAttemptHTTP2 = true
+		// Disable implicit gzip compression; the various
+		// handlers (register, map, set-dns, etc) do their own
+		// zstd compression per naclbox.
+		tr.DisableCompression = true
 		httpc = &http.Client{Transport: tr}
 	}
 
@@ -203,7 +214,7 @@ func (c *Direct) SetHostinfo(hi *tailcfg.Hostinfo) bool {
 	}
 	c.hostinfo = hi.Clone()
 	j, _ := json.Marshal(c.hostinfo)
-	c.logf("HostInfo: %s", j)
+	c.logf("[v1] HostInfo: %s", j)
 	return true
 }
 
@@ -234,10 +245,10 @@ func (c *Direct) GetPersist() persist.Persist {
 }
 
 func (c *Direct) TryLogout(ctx context.Context) error {
-	c.logf("direct.TryLogout()")
+	c.logf("[v1] direct.TryLogout()")
 
 	mustRegen, newURL, err := c.doLogin(ctx, loginOpt{Logout: true})
-	c.logf("TryLogout control response: mustRegen=%v, newURL=%v, err=%v", mustRegen, newURL, err)
+	c.logf("[v1] TryLogout control response: mustRegen=%v, newURL=%v, err=%v", mustRegen, newURL, err)
 
 	c.mu.Lock()
 	c.persist = persist.Persist{}
@@ -247,7 +258,7 @@ func (c *Direct) TryLogout(ctx context.Context) error {
 }
 
 func (c *Direct) TryLogin(ctx context.Context, t *tailcfg.Oauth2Token, flags LoginFlags) (url string, err error) {
-	c.logf("direct.TryLogin(token=%v, flags=%v)", t != nil, flags)
+	c.logf("[v1] direct.TryLogin(token=%v, flags=%v)", t != nil, flags)
 	return c.doLoginOrRegen(ctx, loginOpt{Token: t, Flags: flags})
 }
 
@@ -255,7 +266,7 @@ func (c *Direct) TryLogin(ctx context.Context, t *tailcfg.Oauth2Token, flags Log
 //
 // On success, newURL and err will both be nil.
 func (c *Direct) WaitLoginURL(ctx context.Context, url string) (newURL string, err error) {
-	c.logf("direct.WaitLoginURL")
+	c.logf("[v1] direct.WaitLoginURL")
 	return c.doLoginOrRegen(ctx, loginOpt{URL: url})
 }
 
@@ -285,8 +296,8 @@ func (c *Direct) doLogin(ctx context.Context, opt loginOpt) (mustRegen bool, new
 	tryingNewKey := c.tryingNewKey
 	serverKey := c.serverKey
 	authKey := c.authKey
-	hostinfo := c.hostinfo.Clone()
-	backendLogID := hostinfo.BackendLogID
+	hi := c.hostinfo.Clone()
+	backendLogID := hi.BackendLogID
 	expired := c.expiry != nil && !c.expiry.IsZero() && c.expiry.Before(c.timeNow())
 	c.mu.Unlock()
 
@@ -327,7 +338,7 @@ func (c *Direct) doLogin(ctx context.Context, opt loginOpt) (mustRegen bool, new
 		c.mu.Unlock()
 	}
 
-	var oldNodeKey wgkey.Key
+	var oldNodeKey key.NodePublic
 	switch {
 	case opt.Logout:
 		tryingNewKey = persist.PrivateNodeKey
@@ -336,12 +347,7 @@ func (c *Direct) doLogin(ctx context.Context, opt loginOpt) (mustRegen bool, new
 	case regen || persist.PrivateNodeKey.IsZero():
 		c.logf("Generating a new nodekey.")
 		persist.OldPrivateNodeKey = persist.PrivateNodeKey
-		key, err := wgkey.NewPrivate()
-		if err != nil {
-			c.logf("login keygen: %v", err)
-			return regen, opt.URL, err
-		}
-		tryingNewKey = key
+		tryingNewKey = key.NewNode()
 	default:
 		// Try refreshing the current key first
 		tryingNewKey = persist.PrivateNodeKey
@@ -363,11 +369,12 @@ func (c *Direct) doLogin(ctx context.Context, opt loginOpt) (mustRegen bool, new
 	now := time.Now().Round(time.Second)
 	request := tailcfg.RegisterRequest{
 		Version:    1,
-		OldNodeKey: tailcfg.NodeKey(oldNodeKey),
-		NodeKey:    tailcfg.NodeKey(tryingNewKey.Public()),
-		Hostinfo:   hostinfo,
+		OldNodeKey: oldNodeKey,
+		NodeKey:    tryingNewKey.Public(),
+		Hostinfo:   hi,
 		Followup:   opt.URL,
 		Timestamp:  &now,
+		Ephemeral:  (opt.Flags & LoginEphemeral) != 0,
 	}
 	if opt.Logout {
 		request.Expiry = time.Unix(123, 0) // far in the past
@@ -435,6 +442,9 @@ func (c *Direct) doLogin(ctx context.Context, opt loginOpt) (mustRegen bool, new
 	c.logf("RegisterReq: got response; nodeKeyExpired=%v, machineAuthorized=%v; authURL=%v",
 		resp.NodeKeyExpired, resp.MachineAuthorized, resp.AuthURL != "")
 
+	if resp.Error != "" {
+		return false, "", UserVisibleError(resp.Error)
+	}
 	if resp.NodeKeyExpired {
 		if regen {
 			return true, "", fmt.Errorf("weird: regen=true but server says NodeKeyExpired: %v", request.NodeKey)
@@ -459,7 +469,7 @@ func (c *Direct) doLogin(ctx context.Context, opt loginOpt) (mustRegen bool, new
 	if resp.AuthURL != "" {
 		c.logf("AuthURL is %v", resp.AuthURL)
 	} else {
-		c.logf("No AuthURL")
+		c.logf("[v1] No AuthURL")
 	}
 
 	c.mu.Lock()
@@ -510,7 +520,7 @@ func (c *Direct) newEndpoints(localPort uint16, endpoints []tailcfg.Endpoint) (c
 	for _, ep := range endpoints {
 		epStrs = append(epStrs, ep.Addr.String())
 	}
-	c.logf("client.newEndpoints(%v, %v)", localPort, epStrs)
+	c.logf("[v2] client.newEndpoints(%v, %v)", localPort, epStrs)
 	c.localPort = localPort
 	c.endpoints = append(c.endpoints[:0], endpoints...)
 	if len(endpoints) > 0 {
@@ -553,12 +563,21 @@ const pollTimeout = 120 * time.Second
 
 // cb nil means to omit peers.
 func (c *Direct) sendMapRequest(ctx context.Context, maxPolls int, cb func(*netmap.NetworkMap)) error {
+	metricMapRequests.Add(1)
+	metricMapRequestsActive.Add(1)
+	defer metricMapRequestsActive.Add(-1)
+	if maxPolls == -1 {
+		metricMapRequestsPoll.Add(1)
+	} else {
+		metricMapRequestsLite.Add(1)
+	}
+
 	c.mu.Lock()
 	persist := c.persist
 	serverURL := c.serverURL
 	serverKey := c.serverKey
-	hostinfo := c.hostinfo.Clone()
-	backendLogID := hostinfo.BackendLogID
+	hi := c.hostinfo.Clone()
+	backendLogID := hi.BackendLogID
 	localPort := c.localPort
 	var epStrs []string
 	var epTypes []tailcfg.EndpointType
@@ -597,18 +616,18 @@ func (c *Direct) sendMapRequest(ctx context.Context, maxPolls int, cb func(*netm
 	request := &tailcfg.MapRequest{
 		Version:       tailcfg.CurrentMapRequestVersion,
 		KeepAlive:     c.keepAlive,
-		NodeKey:       tailcfg.NodeKey(persist.PrivateNodeKey.Public()),
+		NodeKey:       persist.PrivateNodeKey.Public(),
 		DiscoKey:      c.discoPubKey,
 		Endpoints:     epStrs,
 		EndpointTypes: epTypes,
 		Stream:        allowStream,
-		Hostinfo:      hostinfo,
+		Hostinfo:      hi,
 		DebugFlags:    c.debugFlags,
 		OmitPeers:     cb == nil,
 	}
 	var extraDebugFlags []string
-	if hostinfo != nil && c.linkMon != nil && !c.skipIPForwardingCheck &&
-		ipForwardingBroken(hostinfo.RoutableIPs, c.linkMon.InterfaceState()) {
+	if hi != nil && c.linkMon != nil && !c.skipIPForwardingCheck &&
+		ipForwardingBroken(hi.RoutableIPs, c.linkMon.InterfaceState()) {
 		extraDebugFlags = append(extraDebugFlags, "warn-ip-forwarding-off")
 	}
 	if health.RouterHealth() != nil {
@@ -617,6 +636,9 @@ func (c *Direct) sendMapRequest(ctx context.Context, maxPolls int, cb func(*netm
 	if health.NetworkCategoryHealth() != nil {
 		extraDebugFlags = append(extraDebugFlags, "warn-network-category-unhealthy")
 	}
+	if hostinfo.DisabledEtcAptSource() {
+		extraDebugFlags = append(extraDebugFlags, "warn-etc-apt-source-disabled")
+	}
 	if len(extraDebugFlags) > 0 {
 		old := request.DebugFlags
 		request.DebugFlags = append(old[:len(old):len(old)], extraDebugFlags...)
@@ -739,11 +761,14 @@ func (c *Direct) sendMapRequest(ctx context.Context, maxPolls int, cb func(*netm
 			return err
 		}
 
+		metricMapResponseMessages.Add(1)
+
 		if allowStream {
 			health.GotStreamedMapResponse()
 		}
 
 		if pr := resp.PingRequest; pr != nil && c.isUniquePingRequest(pr) {
+			metricMapResponsePings.Add(1)
 			go answerPing(c.logf, c.httpc, pr)
 		}
 
@@ -760,13 +785,23 @@ func (c *Direct) sendMapRequest(ctx context.Context, maxPolls int, cb func(*netm
 			return ctx.Err()
 		}
 		if resp.KeepAlive {
+			metricMapResponseKeepAlives.Add(1)
 			continue
 		}
 
+		metricMapResponseMap.Add(1)
+		if i > 0 {
+			metricMapResponseMapDelta.Add(1)
+		}
+
 		hasDebug := resp.Debug != nil
 		// being conservative here, if Debug not present set to False
 		controlknobs.SetDisableUPnP(hasDebug && resp.Debug.DisableUPnP.EqualBool(true))
 		if hasDebug {
+			if code := resp.Debug.Exit; code != nil {
+				c.logf("exiting process with status %v per controlplane", *code)
+				os.Exit(*code)
+			}
 			if resp.Debug.LogHeapPprof {
 				go logheap.LogHeap(resp.Debug.LogHeapURL)
 			}
@@ -790,10 +825,7 @@ func (c *Direct) sendMapRequest(ctx context.Context, maxPolls int, cb func(*netm
 
 		if Debug.StripEndpoints {
 			for _, p := range resp.Peers {
-				// We need at least one endpoint here for now else
-				// other code doesn't even create the discoEndpoint.
-				// TODO(bradfitz): fix that and then just nil this out.
-				p.Endpoints = []string{"127.9.9.9:456"}
+				p.Endpoints = nil
 			}
 		}
 		if Debug.StripCaps {
@@ -843,8 +875,8 @@ func decode(res *http.Response, v interface{}, serverKey key.MachinePublic, mkey
 }
 
 var (
-	debugMap, _      = strconv.ParseBool(os.Getenv("TS_DEBUG_MAP"))
-	debugRegister, _ = strconv.ParseBool(os.Getenv("TS_DEBUG_REGISTER"))
+	debugMap      = envknob.Bool("TS_DEBUG_MAP")
+	debugRegister = envknob.Bool("TS_DEBUG_REGISTER")
 )
 
 var jsonEscapedZero = []byte(`\u0000`)
@@ -954,26 +986,14 @@ type debug struct {
 
 func initDebug() debug {
 	return debug{
-		NetMap:         envBool("TS_DEBUG_NETMAP"),
-		ProxyDNS:       envBool("TS_DEBUG_PROXY_DNS"),
-		StripEndpoints: envBool("TS_DEBUG_STRIP_ENDPOINTS"),
-		StripCaps:      envBool("TS_DEBUG_STRIP_CAPS"),
-		Disco:          os.Getenv("TS_DEBUG_USE_DISCO") == "" || envBool("TS_DEBUG_USE_DISCO"),
+		NetMap:         envknob.Bool("TS_DEBUG_NETMAP"),
+		ProxyDNS:       envknob.Bool("TS_DEBUG_PROXY_DNS"),
+		StripEndpoints: envknob.Bool("TS_DEBUG_STRIP_ENDPOINTS"),
+		StripCaps:      envknob.Bool("TS_DEBUG_STRIP_CAPS"),
+		Disco:          envknob.BoolDefaultTrue("TS_DEBUG_USE_DISCO"),
 	}
 }
 
-func envBool(k string) bool {
-	e := os.Getenv(k)
-	if e == "" {
-		return false
-	}
-	v, err := strconv.ParseBool(e)
-	if err != nil {
-		panic(fmt.Sprintf("invalid non-bool %q for env var %q", e, k))
-	}
-	return v
-}
-
 var clockNow = time.Now
 
 // opt.Bool configs from control.
@@ -1169,7 +1189,13 @@ func sleepAsRequested(ctx context.Context, logf logger.Logf, timeoutReset chan<-
 
 // SetDNS sends the SetDNSRequest request to the control plane server,
 // requesting a DNS record be created or updated.
-func (c *Direct) SetDNS(ctx context.Context, req *tailcfg.SetDNSRequest) error {
+func (c *Direct) SetDNS(ctx context.Context, req *tailcfg.SetDNSRequest) (err error) {
+	metricSetDNS.Add(1)
+	defer func() {
+		if err != nil {
+			metricSetDNSError.Add(1)
+		}
+	}()
 	c.mu.Lock()
 	serverKey := c.serverKey
 	c.mu.Unlock()
@@ -1269,3 +1295,20 @@ func postPingResult(now time.Time, logf logger.Logf, c *http.Client, pr *tailcfg
 	}
 	return nil
 }
+
+var (
+	metricMapRequestsActive = clientmetric.NewGauge("controlclient_map_requests_active")
+
+	metricMapRequests     = clientmetric.NewCounter("controlclient_map_requests")
+	metricMapRequestsLite = clientmetric.NewCounter("controlclient_map_requests_lite")
+	metricMapRequestsPoll = clientmetric.NewCounter("controlclient_map_requests_poll")
+
+	metricMapResponseMessages   = clientmetric.NewCounter("controlclient_map_response_message") // any message type
+	metricMapResponsePings      = clientmetric.NewCounter("controlclient_map_response_ping")
+	metricMapResponseKeepAlives = clientmetric.NewCounter("controlclient_map_response_keepalive")
+	metricMapResponseMap        = clientmetric.NewCounter("controlclient_map_response_map")       // any non-keepalive map response
+	metricMapResponseMapDelta   = clientmetric.NewCounter("controlclient_map_response_map_delta") // 2nd+ non-keepalive map response
+
+	metricSetDNS      = clientmetric.NewCounter("controlclient_setdns")
+	metricSetDNSError = clientmetric.NewCounter("controlclient_setdns_error")
+)

control/controlclient/map.go

@@ -6,16 +6,14 @@ package controlclient
 
 import (
 	"log"
-	"os"
 	"sort"
-	"strconv"
 
 	"inet.af/netaddr"
+	"tailscale.com/envknob"
 	"tailscale.com/tailcfg"
 	"tailscale.com/types/key"
 	"tailscale.com/types/logger"
 	"tailscale.com/types/netmap"
-	"tailscale.com/types/wgkey"
 	"tailscale.com/wgengine/filter"
 )
 
@@ -29,7 +27,7 @@ import (
 // one MapRequest).
 type mapSession struct {
 	// Immutable fields.
-	privateNodeKey         wgkey.Private
+	privateNodeKey         key.NodePrivate
 	logf                   logger.Logf
 	vlogf                  logger.Logf
 	machinePubKey          key.MachinePublic
@@ -51,7 +49,7 @@ type mapSession struct {
 	netMapBuilding *netmap.NetworkMap
 }
 
-func newMapSession(privateNodeKey wgkey.Private) *mapSession {
+func newMapSession(privateNodeKey key.NodePrivate) *mapSession {
 	ms := &mapSession{
 		privateNodeKey:  privateNodeKey,
 		logf:            logger.Discard,
@@ -111,7 +109,7 @@ func (ms *mapSession) netmapForResponse(resp *tailcfg.MapResponse) *netmap.Netwo
 	}
 
 	nm := &netmap.NetworkMap{
-		NodeKey:         tailcfg.NodeKey(ms.privateNodeKey.Public()),
+		NodeKey:         ms.privateNodeKey.Public(),
 		PrivateKey:      ms.privateNodeKey,
 		MachineKey:      ms.machinePubKey,
 		Peers:           resp.Peers,
@@ -290,7 +288,7 @@ func cloneNodes(v1 []*tailcfg.Node) []*tailcfg.Node {
 	return v2
 }
 
-var debugSelfIPv6Only, _ = strconv.ParseBool(os.Getenv("TS_DEBUG_SELF_V6_ONLY"))
+var debugSelfIPv6Only = envknob.Bool("TS_DEBUG_SELF_V6_ONLY")
 
 func filterSelfAddresses(in []netaddr.IPPrefix) (ret []netaddr.IPPrefix) {
 	switch {

control/controlclient/map_test.go

@@ -13,8 +13,8 @@ import (
 	"time"
 
 	"tailscale.com/tailcfg"
+	"tailscale.com/types/key"
 	"tailscale.com/types/netmap"
-	"tailscale.com/types/wgkey"
 )
 
 func TestUndeltaPeers(t *testing.T) {
@@ -170,11 +170,7 @@ func formatNodes(nodes []*tailcfg.Node) string {
 }
 
 func newTestMapSession(t *testing.T) *mapSession {
-	k, err := wgkey.NewPrivate()
-	if err != nil {
-		t.Fatal(err)
-	}
-	return newMapSession(k)
+	return newMapSession(key.NewNode())
 }
 
 func TestNetmapForResponse(t *testing.T) {

control/controlclient/sign_supported.go

@@ -18,6 +18,7 @@ import (
 	"errors"
 	"fmt"
 	"sync"
+	"time"
 
 	"github.com/tailscale/certstore"
 	"tailscale.com/tailcfg"
@@ -73,23 +74,46 @@ func isSubjectInChain(subject string, chain []*x509.Certificate) bool {
 	return false
 }
 
-func selectIdentityFromSlice(subject string, ids []certstore.Identity) (certstore.Identity, []*x509.Certificate) {
+func selectIdentityFromSlice(subject string, ids []certstore.Identity, now time.Time) (certstore.Identity, []*x509.Certificate) {
+	var bestCandidate struct {
+		id    certstore.Identity
+		chain []*x509.Certificate
+	}
+
 	for _, id := range ids {
 		chain, err := id.CertificateChain()
 		if err != nil {
 			continue
 		}
 
+		if len(chain) < 1 {
+			continue
+		}
+
 		if !isSupportedCertificate(chain[0]) {
 			continue
 		}
 
-		if isSubjectInChain(subject, chain) {
-			return id, chain
+		if now.Before(chain[0].NotBefore) || now.After(chain[0].NotAfter) {
+			// Certificate is not valid at this time
+			continue
 		}
+
+		if !isSubjectInChain(subject, chain) {
+			continue
+		}
+
+		// Select the most recently issued certificate. If there is a tie, pick
+		// one arbitrarily.
+		if len(bestCandidate.chain) > 0 && bestCandidate.chain[0].NotBefore.After(chain[0].NotBefore) {
+			continue
+		}
+
+		bestCandidate.id = id
+		bestCandidate.chain = chain
 	}
 
-	return nil, nil
+	return bestCandidate.id, bestCandidate.chain
 }
 
 // findIdentity locates an identity from the Windows or Darwin certificate
@@ -105,7 +129,7 @@ func findIdentity(subject string, st certstore.Store) (certstore.Identity, []*x5
 		return nil, nil, err
 	}
 
-	selected, chain := selectIdentityFromSlice(subject, ids)
+	selected, chain := selectIdentityFromSlice(subject, ids, time.Now())
 
 	for _, id := range ids {
 		if id != selected {

control/controlclient/sign_supported_test.go

@@ -0,0 +1,238 @@
+// Copyright (c) 2022 Tailscale Inc & AUTHORS All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+//go:build windows && cgo
+// +build windows,cgo
+
+package controlclient
+
+import (
+	"crypto"
+	"crypto/x509"
+	"crypto/x509/pkix"
+	"errors"
+	"reflect"
+	"testing"
+	"time"
+
+	"github.com/tailscale/certstore"
+)
+
+const (
+	testRootCommonName = "testroot"
+	testRootSubject    = "CN=testroot"
+)
+
+type testIdentity struct {
+	chain []*x509.Certificate
+}
+
+func makeChain(rootCommonName string, notBefore, notAfter time.Time) []*x509.Certificate {
+	return []*x509.Certificate{
+		{
+			NotBefore:          notBefore,
+			NotAfter:           notAfter,
+			PublicKeyAlgorithm: x509.RSA,
+		},
+		{
+			Subject: pkix.Name{
+				CommonName: rootCommonName,
+			},
+			PublicKeyAlgorithm: x509.RSA,
+		},
+	}
+}
+
+func (t *testIdentity) Certificate() (*x509.Certificate, error) {
+	return t.chain[0], nil
+}
+
+func (t *testIdentity) CertificateChain() ([]*x509.Certificate, error) {
+	return t.chain, nil
+}
+
+func (t *testIdentity) Signer() (crypto.Signer, error) {
+	return nil, errors.New("not implemented")
+}
+
+func (t *testIdentity) Delete() error {
+	return errors.New("not implemented")
+}
+
+func (t *testIdentity) Close() {}
+
+func TestSelectIdentityFromSlice(t *testing.T) {
+	var times []time.Time
+	for _, ts := range []string{
+		"2000-01-01T00:00:00Z",
+		"2001-01-01T00:00:00Z",
+		"2002-01-01T00:00:00Z",
+		"2003-01-01T00:00:00Z",
+	} {
+		tm, err := time.Parse(time.RFC3339, ts)
+		if err != nil {
+			t.Fatal(err)
+		}
+		times = append(times, tm)
+	}
+
+	tests := []struct {
+		name    string
+		subject string
+		ids     []certstore.Identity
+		now     time.Time
+		// wantIndex is an index into ids, or -1 for nil.
+		wantIndex int
+	}{
+		{
+			name:    "single unexpired identity",
+			subject: testRootSubject,
+			ids: []certstore.Identity{
+				&testIdentity{
+					chain: makeChain(testRootCommonName, times[0], times[2]),
+				},
+			},
+			now:       times[1],
+			wantIndex: 0,
+		},
+		{
+			name:    "single expired identity",
+			subject: testRootSubject,
+			ids: []certstore.Identity{
+				&testIdentity{
+					chain: makeChain(testRootCommonName, times[0], times[1]),
+				},
+			},
+			now:       times[2],
+			wantIndex: -1,
+		},
+		{
+			name:    "unrelated ids",
+			subject: testRootSubject,
+			ids: []certstore.Identity{
+				&testIdentity{
+					chain: makeChain("something", times[0], times[2]),
+				},
+				&testIdentity{
+					chain: makeChain(testRootCommonName, times[0], times[2]),
+				},
+				&testIdentity{
+					chain: makeChain("else", times[0], times[2]),
+				},
+			},
+			now:       times[1],
+			wantIndex: 1,
+		},
+		{
+			name:    "expired with unrelated ids",
+			subject: testRootSubject,
+			ids: []certstore.Identity{
+				&testIdentity{
+					chain: makeChain("something", times[0], times[3]),
+				},
+				&testIdentity{
+					chain: makeChain(testRootCommonName, times[0], times[1]),
+				},
+				&testIdentity{
+					chain: makeChain("else", times[0], times[3]),
+				},
+			},
+			now:       times[2],
+			wantIndex: -1,
+		},
+		{
+			name:    "one expired",
+			subject: testRootSubject,
+			ids: []certstore.Identity{
+				&testIdentity{
+					chain: makeChain(testRootCommonName, times[0], times[1]),
+				},
+				&testIdentity{
+					chain: makeChain(testRootCommonName, times[1], times[3]),
+				},
+			},
+			now:       times[2],
+			wantIndex: 1,
+		},
+		{
+			name:    "two certs both unexpired",
+			subject: testRootSubject,
+			ids: []certstore.Identity{
+				&testIdentity{
+					chain: makeChain(testRootCommonName, times[0], times[3]),
+				},
+				&testIdentity{
+					chain: makeChain(testRootCommonName, times[1], times[3]),
+				},
+			},
+			now:       times[2],
+			wantIndex: 1,
+		},
+		{
+			name:    "two unexpired one expired",
+			subject: testRootSubject,
+			ids: []certstore.Identity{
+				&testIdentity{
+					chain: makeChain(testRootCommonName, times[0], times[3]),
+				},
+				&testIdentity{
+					chain: makeChain(testRootCommonName, times[1], times[3]),
+				},
+				&testIdentity{
+					chain: makeChain(testRootCommonName, times[0], times[1]),
+				},
+			},
+			now:       times[2],
+			wantIndex: 1,
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			gotId, gotChain := selectIdentityFromSlice(tt.subject, tt.ids, tt.now)
+
+			if gotId == nil && gotChain != nil {
+				t.Error("id is nil: got non-nil chain, want nil chain")
+				return
+			}
+			if gotId != nil && gotChain == nil {
+				t.Error("id is not nil: got nil chain, want non-nil chain")
+				return
+			}
+			if tt.wantIndex == -1 {
+				if gotId != nil {
+					t.Error("got non-nil id, want nil id")
+				}
+				return
+			}
+			if gotId == nil {
+				t.Error("got nil id, want non-nil id")
+				return
+			}
+			if gotId != tt.ids[tt.wantIndex] {
+				found := -1
+				for i := range tt.ids {
+					if tt.ids[i] == gotId {
+						found = i
+						break
+					}
+				}
+				if found == -1 {
+					t.Errorf("got unknown id, want id at index %v", tt.wantIndex)
+				} else {
+					t.Errorf("got id at index %v, want id at index %v", found, tt.wantIndex)
+				}
+			}
+
+			tid, ok := tt.ids[tt.wantIndex].(*testIdentity)
+			if !ok {
+				t.Error("got non-testIdentity, want testIdentity")
+				return
+			}
+
+			if !reflect.DeepEqual(tid.chain, gotChain) {
+				t.Errorf("got unknown chain, want chain from id at index %v", tt.wantIndex)
+			}
+		})
+	}
+}

control/controlclient/status.go

@@ -67,7 +67,7 @@ type Status struct {
 	_              structs.Incomparable
 	LoginFinished  *empty.Message // nonempty when login finishes
 	LogoutFinished *empty.Message // nonempty when logout finishes
-	Err            string
+	Err            error
 	URL            string             // interactive URL to visit to finish logging in
 	NetMap         *netmap.NetworkMap // server-pushed configuration
 

control/controlhttp/client.go

@@ -0,0 +1,242 @@
+// Copyright (c) 2021 Tailscale Inc & AUTHORS All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+// Package controlhttp implements the Tailscale 2021 control protocol
+// base transport over HTTP.
+//
+// This tunnels the protocol in control/controlbase over HTTP with a
+// variety of compatibility fallbacks for handling picky or deep
+// inspecting proxies.
+//
+// In the happy path, a client makes a single cleartext HTTP request
+// to the server, the server responds with 101 Switching Protocols,
+// and the control base protocol takes place over plain TCP.
+//
+// In the compatibility path, the client does the above over HTTPS,
+// resulting in double encryption (once for the control transport, and
+// once for the outer TLS layer).
+package controlhttp
+
+import (
+	"context"
+	"crypto/tls"
+	"encoding/base64"
+	"errors"
+	"fmt"
+	"io"
+	"log"
+	"net"
+	"net/http"
+	"net/http/httptrace"
+	"net/url"
+
+	"tailscale.com/control/controlbase"
+	"tailscale.com/net/dnscache"
+	"tailscale.com/net/dnsfallback"
+	"tailscale.com/net/netns"
+	"tailscale.com/net/tlsdial"
+	"tailscale.com/net/tshttpproxy"
+	"tailscale.com/types/key"
+)
+
+// upgradeHeader is the value of the Upgrade HTTP header used to
+// indicate the Tailscale control protocol.
+const (
+	upgradeHeaderValue  = "tailscale-control-protocol"
+	handshakeHeaderName = "X-Tailscale-Handshake"
+)
+
+// Dial connects to the HTTP server at addr, requests to switch to the
+// Tailscale control protocol, and returns an established control
+// protocol connection.
+//
+// If Dial fails to connect using addr, it also tries to tunnel over
+// TLS to <addr's host>:443 as a compatibility fallback.
+func Dial(ctx context.Context, addr string, machineKey key.MachinePrivate, controlKey key.MachinePublic) (*controlbase.Conn, error) {
+	host, port, err := net.SplitHostPort(addr)
+	if err != nil {
+		return nil, err
+	}
+	a := &dialParams{
+		ctx:        ctx,
+		host:       host,
+		httpPort:   port,
+		httpsPort:  "443",
+		machineKey: machineKey,
+		controlKey: controlKey,
+		proxyFunc:  tshttpproxy.ProxyFromEnvironment,
+	}
+	return a.dial()
+}
+
+type dialParams struct {
+	ctx        context.Context
+	host       string
+	httpPort   string
+	httpsPort  string
+	machineKey key.MachinePrivate
+	controlKey key.MachinePublic
+	proxyFunc  func(*http.Request) (*url.URL, error) // or nil
+
+	// For tests only
+	insecureTLS bool
+}
+
+func (a *dialParams) dial() (*controlbase.Conn, error) {
+	init, cont, err := controlbase.ClientDeferred(a.machineKey, a.controlKey)
+	if err != nil {
+		return nil, err
+	}
+
+	u := &url.URL{
+		Scheme: "http",
+		Host:   net.JoinHostPort(a.host, a.httpPort),
+		Path:   "/switch",
+	}
+	conn, httpErr := a.tryURL(u, init)
+	if httpErr == nil {
+		ret, err := cont(a.ctx, conn)
+		if err != nil {
+			conn.Close()
+			return nil, err
+		}
+		return ret, nil
+	}
+
+	// Connecting over plain HTTP failed, assume it's an HTTP proxy
+	// being difficult and see if we can get through over HTTPS.
+	u.Scheme = "https"
+	u.Host = net.JoinHostPort(a.host, a.httpsPort)
+	init, cont, err = controlbase.ClientDeferred(a.machineKey, a.controlKey)
+	if err != nil {
+		return nil, err
+	}
+	conn, tlsErr := a.tryURL(u, init)
+	if tlsErr == nil {
+		ret, err := cont(a.ctx, conn)
+		if err != nil {
+			conn.Close()
+			return nil, err
+		}
+		return ret, nil
+	}
+
+	return nil, fmt.Errorf("all connection attempts failed (HTTP: %v, HTTPS: %v)", httpErr, tlsErr)
+}
+
+func (a *dialParams) tryURL(u *url.URL, init []byte) (net.Conn, error) {
+	dns := &dnscache.Resolver{
+		Forward:          dnscache.Get().Forward,
+		LookupIPFallback: dnsfallback.Lookup,
+		UseLastGood:      true,
+	}
+	dialer := netns.NewDialer(log.Printf)
+	tr := http.DefaultTransport.(*http.Transport).Clone()
+	defer tr.CloseIdleConnections()
+	tr.Proxy = a.proxyFunc
+	tshttpproxy.SetTransportGetProxyConnectHeader(tr)
+	tr.DialContext = dnscache.Dialer(dialer.DialContext, dns)
+	// Disable HTTP2, since h2 can't do protocol switching.
+	tr.TLSClientConfig.NextProtos = []string{}
+	tr.TLSNextProto = map[string]func(string, *tls.Conn) http.RoundTripper{}
+	tr.TLSClientConfig = tlsdial.Config(a.host, tr.TLSClientConfig)
+	if a.insecureTLS {
+		tr.TLSClientConfig.InsecureSkipVerify = true
+		tr.TLSClientConfig.VerifyConnection = nil
+	}
+	tr.DialTLSContext = dnscache.TLSDialer(dialer.DialContext, dns, tr.TLSClientConfig)
+	tr.DisableCompression = true
+
+	// (mis)use httptrace to extract the underlying net.Conn from the
+	// transport. We make exactly 1 request using this transport, so
+	// there will be exactly 1 GotConn call. Additionally, the
+	// transport handles 101 Switching Protocols correctly, such that
+	// the Conn will not be reused or kept alive by the transport once
+	// the response has been handed back from RoundTrip.
+	//
+	// In theory, the machinery of net/http should make it such that
+	// the trace callback happens-before we get the response, but
+	// there's no promise of that. So, to make sure, we use a buffered
+	// channel as a synchronization step to avoid data races.
+	//
+	// Note that even though we're able to extract a net.Conn via this
+	// mechanism, we must still keep using the eventual resp.Body to
+	// read from, because it includes a buffer we can't get rid of. If
+	// the server never sends any data after sending the HTTP
+	// response, we could get away with it, but violating this
+	// assumption leads to very mysterious transport errors (lockups,
+	// unexpected EOFs...), and we're bound to forget someday and
+	// introduce a protocol optimization at a higher level that starts
+	// eagerly transmitting from the server.
+	connCh := make(chan net.Conn, 1)
+	trace := httptrace.ClientTrace{
+		GotConn: func(info httptrace.GotConnInfo) {
+			connCh <- info.Conn
+		},
+	}
+	ctx := httptrace.WithClientTrace(a.ctx, &trace)
+	req := &http.Request{
+		Method: "POST",
+		URL:    u,
+		Header: http.Header{
+			"Upgrade":           []string{upgradeHeaderValue},
+			"Connection":        []string{"upgrade"},
+			handshakeHeaderName: []string{base64.StdEncoding.EncodeToString(init)},
+		},
+	}
+	req = req.WithContext(ctx)
+
+	resp, err := tr.RoundTrip(req)
+	if err != nil {
+		return nil, err
+	}
+
+	if resp.StatusCode != http.StatusSwitchingProtocols {
+		return nil, fmt.Errorf("unexpected HTTP response: %s", resp.Status)
+	}
+
+	// From here on, the underlying net.Conn is ours to use, but there
+	// is still a read buffer attached to it within resp.Body. So, we
+	// must direct I/O through resp.Body, but we can still use the
+	// underlying net.Conn for stuff like deadlines.
+	var switchedConn net.Conn
+	select {
+	case switchedConn = <-connCh:
+	default:
+	}
+	if switchedConn == nil {
+		resp.Body.Close()
+		return nil, fmt.Errorf("httptrace didn't provide a connection")
+	}
+
+	if next := resp.Header.Get("Upgrade"); next != upgradeHeaderValue {
+		resp.Body.Close()
+		return nil, fmt.Errorf("server switched to unexpected protocol %q", next)
+	}
+
+	rwc, ok := resp.Body.(io.ReadWriteCloser)
+	if !ok {
+		resp.Body.Close()
+		return nil, errors.New("http Transport did not provide a writable body")
+	}
+
+	return &wrappedConn{switchedConn, rwc}, nil
+}
+
+type wrappedConn struct {
+	net.Conn
+	rwc io.ReadWriteCloser
+}
+
+func (w *wrappedConn) Read(bs []byte) (int, error) {
+	return w.rwc.Read(bs)
+}
+
+func (w *wrappedConn) Write(bs []byte) (int, error) {
+	return w.rwc.Write(bs)
+}
+
+func (w *wrappedConn) Close() error {
+	return w.rwc.Close()
+}

control/controlhttp/http_test.go

@@ -0,0 +1,398 @@
+// Copyright (c) 2021 Tailscale Inc & AUTHORS All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package controlhttp
+
+import (
+	"context"
+	"crypto/tls"
+	"fmt"
+	"io"
+	"log"
+	"net"
+	"net/http"
+	"net/http/httputil"
+	"net/url"
+	"strconv"
+	"sync"
+	"testing"
+
+	"tailscale.com/control/controlbase"
+	"tailscale.com/net/socks5"
+	"tailscale.com/types/key"
+)
+
+func TestControlHTTP(t *testing.T) {
+	tests := []struct {
+		name  string
+		proxy proxy
+	}{
+		// direct connection
+		{
+			name:  "no_proxy",
+			proxy: nil,
+		},
+		// SOCKS5
+		{
+			name:  "socks5",
+			proxy: &socksProxy{},
+		},
+		// HTTP->HTTP
+		{
+			name: "http_to_http",
+			proxy: &httpProxy{
+				useTLS:       false,
+				allowConnect: false,
+				allowHTTP:    true,
+			},
+		},
+		// HTTP->HTTPS
+		{
+			name: "http_to_https",
+			proxy: &httpProxy{
+				useTLS:       false,
+				allowConnect: true,
+				allowHTTP:    false,
+			},
+		},
+		// HTTP->any (will pick HTTP)
+		{
+			name: "http_to_any",
+			proxy: &httpProxy{
+				useTLS:       false,
+				allowConnect: true,
+				allowHTTP:    true,
+			},
+		},
+		// HTTPS->HTTP
+		{
+			name: "https_to_http",
+			proxy: &httpProxy{
+				useTLS:       true,
+				allowConnect: false,
+				allowHTTP:    true,
+			},
+		},
+		// HTTPS->HTTPS
+		{
+			name: "https_to_https",
+			proxy: &httpProxy{
+				useTLS:       true,
+				allowConnect: true,
+				allowHTTP:    false,
+			},
+		},
+		// HTTPS->any (will pick HTTP)
+		{
+			name: "https_to_any",
+			proxy: &httpProxy{
+				useTLS:       true,
+				allowConnect: true,
+				allowHTTP:    true,
+			},
+		},
+	}
+
+	for _, test := range tests {
+		t.Run(test.name, func(t *testing.T) {
+			testControlHTTP(t, test.proxy)
+		})
+	}
+}
+
+func testControlHTTP(t *testing.T, proxy proxy) {
+	client, server := key.NewMachine(), key.NewMachine()
+
+	sch := make(chan serverResult, 1)
+	handler := http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		conn, err := AcceptHTTP(context.Background(), w, r, server)
+		if err != nil {
+			log.Print(err)
+		}
+		res := serverResult{
+			err: err,
+		}
+		if conn != nil {
+			res.clientAddr = conn.RemoteAddr().String()
+			res.version = conn.ProtocolVersion()
+			res.peer = conn.Peer()
+			res.conn = conn
+		}
+		sch <- res
+	})
+
+	httpLn, err := net.Listen("tcp", "127.0.0.1:0")
+	if err != nil {
+		t.Fatalf("HTTP listen: %v", err)
+	}
+	httpsLn, err := net.Listen("tcp", "127.0.0.1:0")
+	if err != nil {
+		t.Fatalf("HTTPS listen: %v", err)
+	}
+
+	httpServer := &http.Server{Handler: handler}
+	go httpServer.Serve(httpLn)
+	defer httpServer.Close()
+
+	httpsServer := &http.Server{
+		Handler:   handler,
+		TLSConfig: tlsConfig(t),
+	}
+	go httpsServer.ServeTLS(httpsLn, "", "")
+	defer httpsServer.Close()
+
+	//ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
+	//defer cancel()
+
+	a := dialParams{
+		ctx:         context.Background(), //ctx,
+		host:        "localhost",
+		httpPort:    strconv.Itoa(httpLn.Addr().(*net.TCPAddr).Port),
+		httpsPort:   strconv.Itoa(httpsLn.Addr().(*net.TCPAddr).Port),
+		machineKey:  client,
+		controlKey:  server.Public(),
+		insecureTLS: true,
+	}
+
+	if proxy != nil {
+		proxyEnv := proxy.Start(t)
+		defer proxy.Close()
+		proxyURL, err := url.Parse(proxyEnv)
+		if err != nil {
+			t.Fatal(err)
+		}
+		a.proxyFunc = func(*http.Request) (*url.URL, error) {
+			return proxyURL, nil
+		}
+	} else {
+		a.proxyFunc = func(*http.Request) (*url.URL, error) {
+			return nil, nil
+		}
+	}
+
+	conn, err := a.dial()
+	if err != nil {
+		t.Fatalf("dialing controlhttp: %v", err)
+	}
+	defer conn.Close()
+	si := <-sch
+	if si.conn != nil {
+		defer si.conn.Close()
+	}
+	if si.err != nil {
+		t.Fatalf("controlhttp server got error: %v", err)
+	}
+	if clientVersion := conn.ProtocolVersion(); si.version != clientVersion {
+		t.Fatalf("client and server don't agree on protocol version: %d vs %d", clientVersion, si.version)
+	}
+	if si.peer != client.Public() {
+		t.Fatalf("server got peer pubkey %s, want %s", si.peer, client.Public())
+	}
+	if spub := conn.Peer(); spub != server.Public() {
+		t.Fatalf("client got peer pubkey %s, want %s", spub, server.Public())
+	}
+	if proxy != nil && !proxy.ConnIsFromProxy(si.clientAddr) {
+		t.Fatalf("client connected from %s, which isn't the proxy", si.clientAddr)
+	}
+}
+
+type serverResult struct {
+	err        error
+	clientAddr string
+	version    int
+	peer       key.MachinePublic
+	conn       *controlbase.Conn
+}
+
+type proxy interface {
+	Start(*testing.T) string
+	Close()
+	ConnIsFromProxy(string) bool
+}
+
+type socksProxy struct {
+	sync.Mutex
+	proxy           socks5.Server
+	ln              net.Listener
+	clientConnAddrs map[string]bool // addrs of the local end of outgoing conns from proxy
+}
+
+func (s *socksProxy) Start(t *testing.T) (url string) {
+	t.Helper()
+	s.Lock()
+	defer s.Unlock()
+	ln, err := net.Listen("tcp", "127.0.0.1:0")
+	if err != nil {
+		t.Fatalf("listening for SOCKS server: %v", err)
+	}
+	s.ln = ln
+	s.clientConnAddrs = map[string]bool{}
+	s.proxy.Logf = t.Logf
+	s.proxy.Dialer = s.dialAndRecord
+	go s.proxy.Serve(ln)
+	return fmt.Sprintf("socks5://%s", ln.Addr().String())
+}
+
+func (s *socksProxy) Close() {
+	s.Lock()
+	defer s.Unlock()
+	s.ln.Close()
+}
+
+func (s *socksProxy) dialAndRecord(ctx context.Context, network, addr string) (net.Conn, error) {
+	var d net.Dialer
+	conn, err := d.DialContext(ctx, network, addr)
+	if err != nil {
+		return nil, err
+	}
+	s.Lock()
+	defer s.Unlock()
+	s.clientConnAddrs[conn.LocalAddr().String()] = true
+	return conn, nil
+}
+
+func (s *socksProxy) ConnIsFromProxy(addr string) bool {
+	s.Lock()
+	defer s.Unlock()
+	return s.clientConnAddrs[addr]
+}
+
+type httpProxy struct {
+	useTLS       bool // take incoming connections over TLS
+	allowConnect bool // allow CONNECT for TLS
+	allowHTTP    bool // allow plain HTTP proxying
+
+	sync.Mutex
+	ln              net.Listener
+	rp              httputil.ReverseProxy
+	s               http.Server
+	clientConnAddrs map[string]bool // addrs of the local end of outgoing conns from proxy
+}
+
+func (h *httpProxy) Start(t *testing.T) (url string) {
+	t.Helper()
+	h.Lock()
+	defer h.Unlock()
+	ln, err := net.Listen("tcp", "127.0.0.1:0")
+	if err != nil {
+		t.Fatalf("listening for HTTP proxy: %v", err)
+	}
+	h.ln = ln
+	h.rp = httputil.ReverseProxy{
+		Director: func(*http.Request) {},
+		Transport: &http.Transport{
+			DialContext: h.dialAndRecord,
+			TLSClientConfig: &tls.Config{
+				InsecureSkipVerify: true,
+			},
+			TLSNextProto: map[string]func(string, *tls.Conn) http.RoundTripper{},
+		},
+	}
+	h.clientConnAddrs = map[string]bool{}
+	h.s.Handler = h
+	if h.useTLS {
+		h.s.TLSConfig = tlsConfig(t)
+		go h.s.ServeTLS(h.ln, "", "")
+		return fmt.Sprintf("https://%s", ln.Addr().String())
+	} else {
+		go h.s.Serve(h.ln)
+		return fmt.Sprintf("http://%s", ln.Addr().String())
+	}
+}
+
+func (h *httpProxy) Close() {
+	h.Lock()
+	defer h.Unlock()
+	h.s.Close()
+}
+
+func (h *httpProxy) ServeHTTP(w http.ResponseWriter, r *http.Request) {
+	if r.Method != "CONNECT" {
+		if !h.allowHTTP {
+			http.Error(w, "http proxy not allowed", 500)
+			return
+		}
+		h.rp.ServeHTTP(w, r)
+		return
+	}
+
+	if !h.allowConnect {
+		http.Error(w, "connect not allowed", 500)
+		return
+	}
+
+	dst := r.RequestURI
+	c, err := h.dialAndRecord(context.Background(), "tcp", dst)
+	if err != nil {
+		http.Error(w, err.Error(), 500)
+		return
+	}
+	defer c.Close()
+
+	cc, ccbuf, err := w.(http.Hijacker).Hijack()
+	if err != nil {
+		http.Error(w, err.Error(), 500)
+		return
+	}
+	defer cc.Close()
+
+	io.WriteString(cc, "HTTP/1.1 200 OK\r\n\r\n")
+
+	errc := make(chan error, 1)
+	go func() {
+		_, err := io.Copy(cc, c)
+		errc <- err
+	}()
+	go func() {
+		_, err := io.Copy(c, ccbuf)
+		errc <- err
+	}()
+	<-errc
+}
+
+func (h *httpProxy) dialAndRecord(ctx context.Context, network, addr string) (net.Conn, error) {
+	var d net.Dialer
+	conn, err := d.DialContext(ctx, network, addr)
+	if err != nil {
+		return nil, err
+	}
+	h.Lock()
+	defer h.Unlock()
+	h.clientConnAddrs[conn.LocalAddr().String()] = true
+	return conn, nil
+}
+
+func (h *httpProxy) ConnIsFromProxy(addr string) bool {
+	h.Lock()
+	defer h.Unlock()
+	return h.clientConnAddrs[addr]
+}
+
+func tlsConfig(t *testing.T) *tls.Config {
+	// Cert and key taken from the example code in the crypto/tls
+	// package.
+	certPem := []byte(`-----BEGIN CERTIFICATE-----
+MIIBhTCCASugAwIBAgIQIRi6zePL6mKjOipn+dNuaTAKBggqhkjOPQQDAjASMRAw
+DgYDVQQKEwdBY21lIENvMB4XDTE3MTAyMDE5NDMwNloXDTE4MTAyMDE5NDMwNlow
+EjEQMA4GA1UEChMHQWNtZSBDbzBZMBMGByqGSM49AgEGCCqGSM49AwEHA0IABD0d
+7VNhbWvZLWPuj/RtHFjvtJBEwOkhbN/BnnE8rnZR8+sbwnc/KhCk3FhnpHZnQz7B
+5aETbbIgmuvewdjvSBSjYzBhMA4GA1UdDwEB/wQEAwICpDATBgNVHSUEDDAKBggr
+BgEFBQcDATAPBgNVHRMBAf8EBTADAQH/MCkGA1UdEQQiMCCCDmxvY2FsaG9zdDo1
+NDUzgg4xMjcuMC4wLjE6NTQ1MzAKBggqhkjOPQQDAgNIADBFAiEA2zpJEPQyz6/l
+Wf86aX6PepsntZv2GYlA5UpabfT2EZICICpJ5h/iI+i341gBmLiAFQOyTDT+/wQc
+6MF9+Yw1Yy0t
+-----END CERTIFICATE-----`)
+	keyPem := []byte(`-----BEGIN EC PRIVATE KEY-----
+MHcCAQEEIIrYSSNQFaA2Hwf1duRSxKtLYX5CB04fSeQ6tF1aY/PuoAoGCCqGSM49
+AwEHoUQDQgAEPR3tU2Fta9ktY+6P9G0cWO+0kETA6SFs38GecTyudlHz6xvCdz8q
+EKTcWGekdmdDPsHloRNtsiCa697B2O9IFA==
+-----END EC PRIVATE KEY-----`)
+	cert, err := tls.X509KeyPair(certPem, keyPem)
+	if err != nil {
+		t.Fatal(err)
+	}
+	return &tls.Config{
+		Certificates: []tls.Certificate{cert},
+	}
+}

control/controlhttp/server.go

@@ -0,0 +1,95 @@
+// Copyright (c) 2021 Tailscale Inc & AUTHORS All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package controlhttp
+
+import (
+	"bufio"
+	"context"
+	"encoding/base64"
+	"errors"
+	"fmt"
+	"net"
+	"net/http"
+
+	"tailscale.com/control/controlbase"
+	"tailscale.com/types/key"
+)
+
+// AcceptHTTP upgrades the HTTP request given by w and r into a
+// Tailscale control protocol base transport connection.
+//
+// AcceptHTTP always writes an HTTP response to w. The caller must not
+// attempt their own response after calling AcceptHTTP.
+func AcceptHTTP(ctx context.Context, w http.ResponseWriter, r *http.Request, private key.MachinePrivate) (*controlbase.Conn, error) {
+	next := r.Header.Get("Upgrade")
+	if next == "" {
+		http.Error(w, "missing next protocol", http.StatusBadRequest)
+		return nil, errors.New("no next protocol in HTTP request")
+	}
+	if next != upgradeHeaderValue {
+		http.Error(w, "unknown next protocol", http.StatusBadRequest)
+		return nil, fmt.Errorf("client requested unhandled next protocol %q", next)
+	}
+
+	initB64 := r.Header.Get(handshakeHeaderName)
+	if initB64 == "" {
+		http.Error(w, "missing Tailscale handshake header", http.StatusBadRequest)
+		return nil, errors.New("no tailscale handshake header in HTTP request")
+	}
+	init, err := base64.StdEncoding.DecodeString(initB64)
+	if err != nil {
+		http.Error(w, "invalid tailscale handshake header", http.StatusBadRequest)
+		return nil, fmt.Errorf("decoding base64 handshake header: %v", err)
+	}
+
+	hijacker, ok := w.(http.Hijacker)
+	if !ok {
+		http.Error(w, "make request over HTTP/1", http.StatusBadRequest)
+		return nil, errors.New("can't hijack client connection")
+	}
+
+	w.Header().Set("Upgrade", upgradeHeaderValue)
+	w.Header().Set("Connection", "upgrade")
+	w.WriteHeader(http.StatusSwitchingProtocols)
+
+	conn, brw, err := hijacker.Hijack()
+	if err != nil {
+		return nil, fmt.Errorf("hijacking client connection: %w", err)
+	}
+	if err := brw.Flush(); err != nil {
+		conn.Close()
+		return nil, fmt.Errorf("flushing hijacked HTTP buffer: %w", err)
+	}
+	if brw.Reader.Buffered() > 0 {
+		conn = &drainBufConn{conn, brw.Reader}
+	}
+
+	nc, err := controlbase.Server(ctx, conn, private, init)
+	if err != nil {
+		conn.Close()
+		return nil, fmt.Errorf("noise handshake failed: %w", err)
+	}
+
+	return nc, nil
+}
+
+// drainBufConn is a net.Conn with an initial bunch of bytes in a
+// bufio.Reader. Read drains the bufio.Reader until empty, then passes
+// through subsequent reads to the Conn directly.
+type drainBufConn struct {
+	net.Conn
+	r *bufio.Reader
+}
+
+func (b *drainBufConn) Read(bs []byte) (int, error) {
+	if b.r == nil {
+		return b.Conn.Read(bs)
+	}
+	n, err := b.r.Read(bs)
+	if b.r.Buffered() == 0 {
+		b.r = nil
+	}
+	return n, err
+}

control/controlknobs/controlknobs.go

@@ -7,9 +7,7 @@
 package controlknobs
 
 import (
-	"os"
-	"strconv"
-
+	"tailscale.com/envknob"
 	"tailscale.com/syncs"
 )
 
@@ -17,8 +15,7 @@ import (
 var disableUPnP syncs.AtomicBool
 
 func init() {
-	v, _ := strconv.ParseBool(os.Getenv("TS_DISABLE_UPNP"))
-	SetDisableUPnP(v)
+	SetDisableUPnP(envknob.Bool("TS_DISABLE_UPNP"))
 }
 
 // DisableUPnP reports the last reported value from control

derp/derp_client.go

@@ -6,26 +6,27 @@ package derp
 
 import (
 	"bufio"
-	crand "crypto/rand"
 	"encoding/binary"
 	"encoding/json"
 	"errors"
 	"fmt"
 	"io"
 	"sync"
+	"sync/atomic"
 	"time"
 
-	"golang.org/x/crypto/nacl/box"
+	"go4.org/mem"
 	"golang.org/x/time/rate"
+	"inet.af/netaddr"
 	"tailscale.com/types/key"
 	"tailscale.com/types/logger"
 )
 
 // Client is a DERP client.
 type Client struct {
-	serverKey   key.Public // of the DERP server; not a machine or node key
-	privateKey  key.Private
-	publicKey   key.Public // of privateKey
+	serverKey   key.NodePublic // of the DERP server; not a machine or node key
+	privateKey  key.NodePrivate
+	publicKey   key.NodePublic // of privateKey
 	logf        logger.Logf
 	nc          Conn
 	br          *bufio.Reader
@@ -38,8 +39,8 @@ type Client struct {
 	rate *rate.Limiter // if non-nil, rate limiter to use
 
 	// Owned by Recv:
-	peeked  int   // bytes to discard on next Recv
-	readErr error // sticky read error
+	peeked  int          // bytes to discard on next Recv
+	readErr atomic.Value // of error; sticky (set by Recv)
 }
 
 // ClientOpt is an option passed to NewClient.
@@ -54,7 +55,7 @@ func (f clientOptFunc) update(o *clientOpt) { f(o) }
 // clientOpt are the options passed to newClient.
 type clientOpt struct {
 	MeshKey     string
-	ServerPub   key.Public
+	ServerPub   key.NodePublic
 	CanAckPings bool
 	IsProber    bool
 }
@@ -71,7 +72,7 @@ func IsProber(v bool) ClientOpt { return clientOptFunc(func(o *clientOpt) { o.Is
 
 // ServerPublicKey returns a ClientOpt to declare that the server's DERP public key is known.
 // If key is the zero value, the returned ClientOpt is a no-op.
-func ServerPublicKey(key key.Public) ClientOpt {
+func ServerPublicKey(key key.NodePublic) ClientOpt {
 	return clientOptFunc(func(o *clientOpt) { o.ServerPub = key })
 }
 
@@ -81,7 +82,7 @@ func CanAckPings(v bool) ClientOpt {
 	return clientOptFunc(func(o *clientOpt) { o.CanAckPings = v })
 }
 
-func NewClient(privateKey key.Private, nc Conn, brw *bufio.ReadWriter, logf logger.Logf, opts ...ClientOpt) (*Client, error) {
+func NewClient(privateKey key.NodePrivate, nc Conn, brw *bufio.ReadWriter, logf logger.Logf, opts ...ClientOpt) (*Client, error) {
 	var opt clientOpt
 	for _, o := range opts {
 		if o == nil {
@@ -92,7 +93,7 @@ func NewClient(privateKey key.Private, nc Conn, brw *bufio.ReadWriter, logf logg
 	return newClient(privateKey, nc, brw, logf, opt)
 }
 
-func newClient(privateKey key.Private, nc Conn, brw *bufio.ReadWriter, logf logger.Logf, opt clientOpt) (*Client, error) {
+func newClient(privateKey key.NodePrivate, nc Conn, brw *bufio.ReadWriter, logf logger.Logf, opt clientOpt) (*Client, error) {
 	c := &Client{
 		privateKey:  privateKey,
 		publicKey:   privateKey.Public(),
@@ -130,7 +131,7 @@ func (c *Client) recvServerKey() error {
 	if flen < uint32(len(buf)) || t != frameServerKey || string(buf[:len(magic)]) != magic {
 		return errors.New("invalid server greeting")
 	}
-	copy(c.serverKey[:], buf[len(magic):])
+	c.serverKey = key.NodePublicFromRaw32(mem.B(buf[len(magic):]))
 	return nil
 }
 
@@ -143,13 +144,9 @@ func (c *Client) parseServerInfo(b []byte) (*serverInfo, error) {
 	if fl > maxLength {
 		return nil, fmt.Errorf("long serverInfo frame")
 	}
-	// TODO: add a read-nonce-and-box helper
-	var nonce [nonceLen]byte
-	copy(nonce[:], b)
-	msgbox := b[nonceLen:]
-	msg, ok := box.Open(nil, msgbox, &nonce, c.serverKey.B32(), c.privateKey.B32())
+	msg, ok := c.privateKey.OpenFrom(c.serverKey, b)
 	if !ok {
-		return nil, fmt.Errorf("failed to open naclbox from server key %x", c.serverKey[:])
+		return nil, fmt.Errorf("failed to open naclbox from server key %s", c.serverKey)
 	}
 	info := new(serverInfo)
 	if err := json.Unmarshal(msg, info); err != nil {
@@ -176,10 +173,6 @@ type clientInfo struct {
 }
 
 func (c *Client) sendClientKey() error {
-	var nonce [nonceLen]byte
-	if _, err := crand.Read(nonce[:]); err != nil {
-		return err
-	}
 	msg, err := json.Marshal(clientInfo{
 		Version:     ProtocolVersion,
 		MeshKey:     c.meshKey,
@@ -189,24 +182,23 @@ func (c *Client) sendClientKey() error {
 	if err != nil {
 		return err
 	}
-	msgbox := box.Seal(nil, msg, &nonce, c.serverKey.B32(), c.privateKey.B32())
+	msgbox := c.privateKey.SealTo(c.serverKey, msg)
 
-	buf := make([]byte, 0, nonceLen+keyLen+len(msgbox))
-	buf = append(buf, c.publicKey[:]...)
-	buf = append(buf, nonce[:]...)
+	buf := make([]byte, 0, keyLen+len(msgbox))
+	buf = c.publicKey.AppendTo(buf)
 	buf = append(buf, msgbox...)
 	return writeFrame(c.bw, frameClientInfo, buf)
 }
 
 // ServerPublicKey returns the server's public key.
-func (c *Client) ServerPublicKey() key.Public { return c.serverKey }
+func (c *Client) ServerPublicKey() key.NodePublic { return c.serverKey }
 
 // Send sends a packet to the Tailscale node identified by dstKey.
 //
 // It is an error if the packet is larger than 64KB.
-func (c *Client) Send(dstKey key.Public, pkt []byte) error { return c.send(dstKey, pkt) }
+func (c *Client) Send(dstKey key.NodePublic, pkt []byte) error { return c.send(dstKey, pkt) }
 
-func (c *Client) send(dstKey key.Public, pkt []byte) (ret error) {
+func (c *Client) send(dstKey key.NodePublic, pkt []byte) (ret error) {
 	defer func() {
 		if ret != nil {
 			ret = fmt.Errorf("derp.Send: %w", ret)
@@ -220,15 +212,15 @@ func (c *Client) send(dstKey key.Public, pkt []byte) (ret error) {
 	c.wmu.Lock()
 	defer c.wmu.Unlock()
 	if c.rate != nil {
-		pktLen := frameHeaderLen + len(dstKey) + len(pkt)
+		pktLen := frameHeaderLen + key.NodePublicRawLen + len(pkt)
 		if !c.rate.AllowN(time.Now(), pktLen) {
 			return nil // drop
 		}
 	}
-	if err := writeFrameHeader(c.bw, frameSendPacket, uint32(len(dstKey)+len(pkt))); err != nil {
+	if err := writeFrameHeader(c.bw, frameSendPacket, uint32(key.NodePublicRawLen+len(pkt))); err != nil {
 		return err
 	}
-	if _, err := c.bw.Write(dstKey[:]); err != nil {
+	if _, err := c.bw.Write(dstKey.AppendTo(nil)); err != nil {
 		return err
 	}
 	if _, err := c.bw.Write(pkt); err != nil {
@@ -237,7 +229,7 @@ func (c *Client) send(dstKey key.Public, pkt []byte) (ret error) {
 	return c.bw.Flush()
 }
 
-func (c *Client) ForwardPacket(srcKey, dstKey key.Public, pkt []byte) (err error) {
+func (c *Client) ForwardPacket(srcKey, dstKey key.NodePublic, pkt []byte) (err error) {
 	defer func() {
 		if err != nil {
 			err = fmt.Errorf("derp.ForwardPacket: %w", err)
@@ -257,10 +249,10 @@ func (c *Client) ForwardPacket(srcKey, dstKey key.Public, pkt []byte) (err error
 	if err := writeFrameHeader(c.bw, frameForwardPacket, uint32(keyLen*2+len(pkt))); err != nil {
 		return err
 	}
-	if _, err := c.bw.Write(srcKey[:]); err != nil {
+	if _, err := c.bw.Write(srcKey.AppendTo(nil)); err != nil {
 		return err
 	}
-	if _, err := c.bw.Write(dstKey[:]); err != nil {
+	if _, err := c.bw.Write(dstKey.AppendTo(nil)); err != nil {
 		return err
 	}
 	if _, err := c.bw.Write(pkt); err != nil {
@@ -271,10 +263,18 @@ func (c *Client) ForwardPacket(srcKey, dstKey key.Public, pkt []byte) (err error
 
 func (c *Client) writeTimeoutFired() { c.nc.Close() }
 
+func (c *Client) SendPing(data [8]byte) error {
+	return c.sendPingOrPong(framePing, data)
+}
+
 func (c *Client) SendPong(data [8]byte) error {
+	return c.sendPingOrPong(framePong, data)
+}
+
+func (c *Client) sendPingOrPong(typ frameType, data [8]byte) error {
 	c.wmu.Lock()
 	defer c.wmu.Unlock()
-	if err := writeFrameHeader(c.bw, framePong, 8); err != nil {
+	if err := writeFrameHeader(c.bw, typ, 8); err != nil {
 		return err
 	}
 	if _, err := c.bw.Write(data[:]); err != nil {
@@ -322,10 +322,10 @@ func (c *Client) WatchConnectionChanges() error {
 
 // ClosePeer asks the server to close target's TCP connection.
 // It's a fatal error if the client wasn't created using MeshKey.
-func (c *Client) ClosePeer(target key.Public) error {
+func (c *Client) ClosePeer(target key.NodePublic) error {
 	c.wmu.Lock()
 	defer c.wmu.Unlock()
-	return writeFrame(c.bw, frameClosePeer, target[:])
+	return writeFrame(c.bw, frameClosePeer, target.AppendTo(nil))
 }
 
 // ReceivedMessage represents a type returned by Client.Recv. Unless
@@ -338,7 +338,7 @@ type ReceivedMessage interface {
 
 // ReceivedPacket is a ReceivedMessage representing an incoming packet.
 type ReceivedPacket struct {
-	Source key.Public
+	Source key.NodePublic
 	// Data is the received packet bytes. It aliases the memory
 	// passed to Client.Recv.
 	Data []byte
@@ -349,13 +349,13 @@ func (ReceivedPacket) msg() {}
 // PeerGoneMessage is a ReceivedMessage that indicates that the client
 // identified by the underlying public key had previously sent you a
 // packet but has now disconnected from the server.
-type PeerGoneMessage key.Public
+type PeerGoneMessage key.NodePublic
 
 func (PeerGoneMessage) msg() {}
 
 // PeerPresentMessage is a ReceivedMessage that indicates that the client
 // is connected to the server. (Only used by trusted mesh clients)
-type PeerPresentMessage key.Public
+type PeerPresentMessage key.NodePublic
 
 func (PeerPresentMessage) msg() {}
 
@@ -385,6 +385,12 @@ type PingMessage [8]byte
 
 func (PingMessage) msg() {}
 
+// PongMessage is a reply to a PingMessage from a client or server
+// with the payload sent previously in a PingMessage.
+type PongMessage [8]byte
+
+func (PongMessage) msg() {}
+
 // KeepAliveMessage is a one-way empty message from server to client, just to
 // keep the connection alive. It's like a PingMessage, but doesn't solicit
 // a reply from the client.
@@ -437,13 +443,14 @@ func (c *Client) Recv() (m ReceivedMessage, err error) {
 }
 
 func (c *Client) recvTimeout(timeout time.Duration) (m ReceivedMessage, err error) {
-	if c.readErr != nil {
-		return nil, c.readErr
+	readErr, _ := c.readErr.Load().(error)
+	if readErr != nil {
+		return nil, readErr
 	}
 	defer func() {
 		if err != nil {
 			err = fmt.Errorf("derp.Recv: %w", err)
-			c.readErr = err
+			c.readErr.Store(err)
 		}
 	}()
 
@@ -516,8 +523,7 @@ func (c *Client) recvTimeout(timeout time.Duration) (m ReceivedMessage, err erro
 				c.logf("[unexpected] dropping short peerGone frame from DERP server")
 				continue
 			}
-			var pg PeerGoneMessage
-			copy(pg[:], b[:keyLen])
+			pg := PeerGoneMessage(key.NodePublicFromRaw32(mem.B(b[:keyLen])))
 			return pg, nil
 
 		case framePeerPresent:
@@ -525,8 +531,7 @@ func (c *Client) recvTimeout(timeout time.Duration) (m ReceivedMessage, err erro
 				c.logf("[unexpected] dropping short peerPresent frame from DERP server")
 				continue
 			}
-			var pg PeerPresentMessage
-			copy(pg[:], b[:keyLen])
+			pg := PeerPresentMessage(key.NodePublicFromRaw32(mem.B(b[:keyLen])))
 			return pg, nil
 
 		case frameRecvPacket:
@@ -535,7 +540,7 @@ func (c *Client) recvTimeout(timeout time.Duration) (m ReceivedMessage, err erro
 				c.logf("[unexpected] dropping short packet from DERP server")
 				continue
 			}
-			copy(rp.Source[:], b[:keyLen])
+			rp.Source = key.NodePublicFromRaw32(mem.B(b[:keyLen]))
 			rp.Data = b[keyLen:n]
 			return rp, nil
 
@@ -548,6 +553,15 @@ func (c *Client) recvTimeout(timeout time.Duration) (m ReceivedMessage, err erro
 			copy(pm[:], b[:])
 			return pm, nil
 
+		case framePong:
+			var pm PongMessage
+			if n < 8 {
+				c.logf("[unexpected] dropping short ping frame")
+				continue
+			}
+			copy(pm[:], b[:])
+			return pm, nil
+
 		case frameHealth:
 			return HealthMessage{Problem: string(b[:])}, nil
 
@@ -576,3 +590,22 @@ func (c *Client) setSendRateLimiter(sm ServerInfoMessage) {
 			sm.TokenBucketBytesBurst)
 	}
 }
+
+// LocalAddr returns the TCP connection's local address.
+//
+// If the client is broken in some previously detectable way, it
+// returns an error.
+func (c *Client) LocalAddr() (netaddr.IPPort, error) {
+	readErr, _ := c.readErr.Load().(error)
+	if readErr != nil {
+		return netaddr.IPPort{}, readErr
+	}
+	if c.nc == nil {
+		return netaddr.IPPort{}, errors.New("nil conn")
+	}
+	a := c.nc.LocalAddr()
+	if a == nil {
+		return netaddr.IPPort{}, errors.New("nil addr")
+	}
+	return netaddr.ParseIPPort(a.String())
+}

derp/derp_server.go

@@ -23,8 +23,8 @@ import (
 	"math"
 	"math/big"
 	"math/rand"
+	"net"
 	"net/http"
-	"os"
 	"os/exec"
 	"runtime"
 	"strconv"
@@ -34,12 +34,12 @@ import (
 	"time"
 
 	"go4.org/mem"
-	"golang.org/x/crypto/nacl/box"
 	"golang.org/x/sync/errgroup"
 	"golang.org/x/time/rate"
 	"inet.af/netaddr"
 	"tailscale.com/client/tailscale"
 	"tailscale.com/disco"
+	"tailscale.com/envknob"
 	"tailscale.com/metrics"
 	"tailscale.com/syncs"
 	"tailscale.com/types/key"
@@ -48,19 +48,19 @@ import (
 	"tailscale.com/version"
 )
 
-var debug, _ = strconv.ParseBool(os.Getenv("DERP_DEBUG_LOGS"))
+var debug = envknob.Bool("DERP_DEBUG_LOGS")
 
 // verboseDropKeys is the set of destination public keys that should
 // verbosely log whenever DERP drops a packet.
-var verboseDropKeys = map[key.Public]bool{}
+var verboseDropKeys = map[key.NodePublic]bool{}
 
 func init() {
-	keys := os.Getenv("TS_DEBUG_VERBOSE_DROPS")
+	keys := envknob.String("TS_DEBUG_VERBOSE_DROPS")
 	if keys == "" {
 		return
 	}
 	for _, keyStr := range strings.Split(keys, ",") {
-		k, err := key.NewPublicFromHexMem(mem.S(keyStr))
+		k, err := key.ParseNodePublicUntyped(mem.S(keyStr))
 		if err != nil {
 			log.Printf("ignoring invalid debug key %q: %v", keyStr, err)
 		} else {
@@ -99,8 +99,8 @@ type Server struct {
 	// before failing when writing to a client.
 	WriteTimeout time.Duration
 
-	privateKey  key.Private
-	publicKey   key.Public
+	privateKey  key.NodePrivate
+	publicKey   key.NodePublic
 	logf        logger.Logf
 	memSys0     uint64 // runtime.MemStats.Sys at start (or early-ish)
 	meshKey     string
@@ -125,6 +125,8 @@ type Server struct {
 	packetsForwardedOut          expvar.Int
 	packetsForwardedIn           expvar.Int
 	peerGoneFrames               expvar.Int // number of peer gone frames sent
+	gotPing                      expvar.Int // number of ping frames from client
+	sentPong                     expvar.Int // number of pong frames enqueued to client
 	accepts                      expvar.Int
 	curClients                   expvar.Int
 	curHomeClients               expvar.Int // ones with preferred
@@ -146,22 +148,22 @@ type Server struct {
 	mu       sync.Mutex
 	closed   bool
 	netConns map[Conn]chan struct{} // chan is closed when conn closes
-	clients  map[key.Public]clientSet
+	clients  map[key.NodePublic]clientSet
 	watchers map[*sclient]bool // mesh peer -> true
 	// clientsMesh tracks all clients in the cluster, both locally
 	// and to mesh peers.  If the value is nil, that means the
 	// peer is only local (and thus in the clients Map, but not
 	// remote). If the value is non-nil, it's remote (+ maybe also
 	// local).
-	clientsMesh map[key.Public]PacketForwarder
+	clientsMesh map[key.NodePublic]PacketForwarder
 	// sentTo tracks which peers have sent to which other peers,
 	// and at which connection number. This isn't on sclient
 	// because it includes intra-region forwarded packets as the
 	// src.
-	sentTo map[key.Public]map[key.Public]int64 // src => dst => dst's latest sclient.connNum
+	sentTo map[key.NodePublic]map[key.NodePublic]int64 // src => dst => dst's latest sclient.connNum
 
 	// maps from netaddr.IPPort to a client's public key
-	keyOfAddr map[netaddr.IPPort]key.Public
+	keyOfAddr map[netaddr.IPPort]key.NodePublic
 }
 
 // clientSet represents 1 or more *sclients.
@@ -277,16 +279,15 @@ func (s *dupClientSet) removeClient(c *sclient) bool {
 // is a multiForwarder, which this package creates as needed if a
 // public key gets more than one PacketForwarder registered for it.
 type PacketForwarder interface {
-	ForwardPacket(src, dst key.Public, payload []byte) error
+	ForwardPacket(src, dst key.NodePublic, payload []byte) error
 }
 
 // Conn is the subset of the underlying net.Conn the DERP Server needs.
 // It is a defined type so that non-net connections can be used.
 type Conn interface {
 	io.WriteCloser
-
+	LocalAddr() net.Addr
 	// The *Deadline methods follow the semantics of net.Conn.
-
 	SetDeadline(time.Time) error
 	SetReadDeadline(time.Time) error
 	SetWriteDeadline(time.Time) error
@@ -294,7 +295,7 @@ type Conn interface {
 
 // NewServer returns a new DERP server. It doesn't listen on its own.
 // Connections are given to it via Server.Accept.
-func NewServer(privateKey key.Private, logf logger.Logf) *Server {
+func NewServer(privateKey key.NodePrivate, logf logger.Logf) *Server {
 	var ms runtime.MemStats
 	runtime.ReadMemStats(&ms)
 
@@ -306,14 +307,14 @@ func NewServer(privateKey key.Private, logf logger.Logf) *Server {
 		packetsRecvByKind:    metrics.LabelMap{Label: "kind"},
 		packetsDroppedReason: metrics.LabelMap{Label: "reason"},
 		packetsDroppedType:   metrics.LabelMap{Label: "type"},
-		clients:              map[key.Public]clientSet{},
-		clientsMesh:          map[key.Public]PacketForwarder{},
+		clients:              map[key.NodePublic]clientSet{},
+		clientsMesh:          map[key.NodePublic]PacketForwarder{},
 		netConns:             map[Conn]chan struct{}{},
 		memSys0:              ms.Sys,
 		watchers:             map[*sclient]bool{},
-		sentTo:               map[key.Public]map[key.Public]int64{},
+		sentTo:               map[key.NodePublic]map[key.NodePublic]int64{},
 		avgQueueDuration:     new(uint64),
-		keyOfAddr:            map[netaddr.IPPort]key.Public{},
+		keyOfAddr:            map[netaddr.IPPort]key.NodePublic{},
 	}
 	s.initMetacert()
 	s.packetsRecvDisco = s.packetsRecvByKind.Get("disco")
@@ -353,10 +354,10 @@ func (s *Server) HasMeshKey() bool { return s.meshKey != "" }
 func (s *Server) MeshKey() string { return s.meshKey }
 
 // PrivateKey returns the server's private key.
-func (s *Server) PrivateKey() key.Private { return s.privateKey }
+func (s *Server) PrivateKey() key.NodePrivate { return s.privateKey }
 
 // PublicKey returns the server's public key.
-func (s *Server) PublicKey() key.Public { return s.publicKey }
+func (s *Server) PublicKey() key.NodePublic { return s.publicKey }
 
 // Close closes the server and waits for the connections to disconnect.
 func (s *Server) Close() error {
@@ -447,7 +448,7 @@ func (s *Server) initMetacert() {
 	tmpl := &x509.Certificate{
 		SerialNumber: big.NewInt(ProtocolVersion),
 		Subject: pkix.Name{
-			CommonName: fmt.Sprintf("derpkey%x", s.publicKey[:]),
+			CommonName: fmt.Sprintf("derpkey%s", s.publicKey.UntypedHexString()),
 		},
 		// Windows requires NotAfter and NotBefore set:
 		NotAfter:  time.Now().Add(30 * 24 * time.Hour),
@@ -515,7 +516,7 @@ func (s *Server) registerClient(c *sclient) {
 // presence changed.
 //
 // s.mu must be held.
-func (s *Server) broadcastPeerStateChangeLocked(peer key.Public, present bool) {
+func (s *Server) broadcastPeerStateChangeLocked(peer key.NodePublic, present bool) {
 	for w := range s.watchers {
 		w.peerStateChange = append(w.peerStateChange, peerConnState{peer: peer, present: present})
 		go w.requestMeshUpdate()
@@ -577,7 +578,7 @@ func (s *Server) unregisterClient(c *sclient) {
 // key has sent to previously (whether those sends were from a local
 // client or forwarded).  It must only be called after the key has
 // been removed from clientsMesh.
-func (s *Server) notePeerGoneFromRegionLocked(key key.Public) {
+func (s *Server) notePeerGoneFromRegionLocked(key key.NodePublic) {
 	if _, ok := s.clientsMesh[key]; ok {
 		panic("usage")
 	}
@@ -663,7 +664,8 @@ func (s *Server) accept(nc Conn, brw *bufio.ReadWriter, remoteAddr string, connN
 		connectedAt:    time.Now(),
 		sendQueue:      make(chan pkt, perClientSendQueueDepth),
 		discoSendQueue: make(chan pkt, perClientSendQueueDepth),
-		peerGone:       make(chan key.Public),
+		sendPongCh:     make(chan [8]byte, 1),
+		peerGone:       make(chan key.NodePublic),
 		canMesh:        clientInfo.MeshKey != "" && clientInfo.MeshKey == s.meshKey,
 	}
 
@@ -730,6 +732,8 @@ func (c *sclient) run(ctx context.Context) error {
 			err = c.handleFrameWatchConns(ft, fl)
 		case frameClosePeer:
 			err = c.handleFrameClosePeer(ft, fl)
+		case framePing:
+			err = c.handleFramePing(ft, fl)
 		default:
 			err = c.handleUnknownFrame(ft, fl)
 		}
@@ -767,6 +771,33 @@ func (c *sclient) handleFrameWatchConns(ft frameType, fl uint32) error {
 	return nil
 }
 
+func (c *sclient) handleFramePing(ft frameType, fl uint32) error {
+	c.s.gotPing.Add(1)
+	var m PingMessage
+	if fl < uint32(len(m)) {
+		return fmt.Errorf("short ping: %v", fl)
+	}
+	if fl > 1000 {
+		// unreasonably extra large. We leave some extra
+		// space for future extensibility, but not too much.
+		return fmt.Errorf("ping body too large: %v", fl)
+	}
+	_, err := io.ReadFull(c.br, m[:])
+	if err != nil {
+		return err
+	}
+	if extra := int64(fl) - int64(len(m)); extra > 0 {
+		_, err = io.CopyN(ioutil.Discard, c.br, extra)
+	}
+	select {
+	case c.sendPongCh <- [8]byte(m):
+	default:
+		// They're pinging too fast. Ignore.
+		// TODO(bradfitz): add a rate limiter too.
+	}
+	return err
+}
+
 func (c *sclient) handleFrameClosePeer(ft frameType, fl uint32) error {
 	if fl != keyLen {
 		return fmt.Errorf("handleFrameClosePeer wrong size")
@@ -774,8 +805,8 @@ func (c *sclient) handleFrameClosePeer(ft frameType, fl uint32) error {
 	if !c.canMesh {
 		return fmt.Errorf("insufficient permissions")
 	}
-	var targetKey key.Public
-	if _, err := io.ReadFull(c.br, targetKey[:]); err != nil {
+	var targetKey key.NodePublic
+	if err := targetKey.ReadRawWithoutAllocating(c.br); err != nil {
 		return err
 	}
 	s := c.s
@@ -845,10 +876,10 @@ func (c *sclient) handleFrameForwardPacket(ft frameType, fl uint32) error {
 // notePeerSendLocked records that src sent to dst.  We keep track of
 // that so when src disconnects, we can tell dst (if it's still
 // around) that src is gone (a peerGone frame).
-func (s *Server) notePeerSendLocked(src key.Public, dst *sclient) {
+func (s *Server) notePeerSendLocked(src key.NodePublic, dst *sclient) {
 	m, ok := s.sentTo[src]
 	if !ok {
-		m = map[key.Public]int64{}
+		m = map[key.NodePublic]int64{}
 		s.sentTo[src] = m
 	}
 	m[dst.key] = dst.connNum
@@ -919,7 +950,7 @@ const (
 	dropReasonDupClient                          // the public key is connected 2+ times (active/active, fighting)
 )
 
-func (s *Server) recordDrop(packetBytes []byte, srcKey, dstKey key.Public, reason dropReason) {
+func (s *Server) recordDrop(packetBytes []byte, srcKey, dstKey key.NodePublic, reason dropReason) {
 	s.packetsDropped.Add(1)
 	s.packetsDroppedReasonCounters[reason].Add(1)
 	if disco.LooksLikeDiscoWrapper(packetBytes) {
@@ -982,7 +1013,7 @@ func (c *sclient) sendPkt(dst *sclient, p pkt) error {
 // requestPeerGoneWrite sends a request to write a "peer gone" frame
 // that the provided peer has disconnected. It blocks until either the
 // write request is scheduled, or the client has closed.
-func (c *sclient) requestPeerGoneWrite(peer key.Public) {
+func (c *sclient) requestPeerGoneWrite(peer key.NodePublic) {
 	select {
 	case c.peerGone <- peer:
 	case <-c.done:
@@ -999,7 +1030,7 @@ func (c *sclient) requestMeshUpdate() {
 	}
 }
 
-func (s *Server) verifyClient(clientKey key.Public, info *clientInfo) error {
+func (s *Server) verifyClient(clientKey key.NodePublic, info *clientInfo) error {
 	if !s.verifyClients {
 		return nil
 	}
@@ -1018,9 +1049,9 @@ func (s *Server) verifyClient(clientKey key.Public, info *clientInfo) error {
 }
 
 func (s *Server) sendServerKey(lw *lazyBufioWriter) error {
-	buf := make([]byte, 0, len(magic)+len(s.publicKey))
+	buf := make([]byte, 0, len(magic)+key.NodePublicRawLen)
 	buf = append(buf, magic...)
-	buf = append(buf, s.publicKey[:]...)
+	buf = s.publicKey.AppendTo(buf)
 	err := writeFrame(lw.bw(), frameServerKey, buf)
 	lw.Flush() // redundant (no-op) flush to release bufio.Writer
 	return err
@@ -1084,21 +1115,14 @@ type serverInfo struct {
 	TokenBucketBytesBurst     int `json:",omitempty"`
 }
 
-func (s *Server) sendServerInfo(bw *lazyBufioWriter, clientKey key.Public) error {
-	var nonce [24]byte
-	if _, err := crand.Read(nonce[:]); err != nil {
-		return err
-	}
+func (s *Server) sendServerInfo(bw *lazyBufioWriter, clientKey key.NodePublic) error {
 	msg, err := json.Marshal(serverInfo{Version: ProtocolVersion})
 	if err != nil {
 		return err
 	}
 
-	msgbox := box.Seal(nil, msg, &nonce, clientKey.B32(), s.privateKey.B32())
-	if err := writeFrameHeader(bw.bw(), frameServerInfo, nonceLen+uint32(len(msgbox))); err != nil {
-		return err
-	}
-	if _, err := bw.Write(nonce[:]); err != nil {
+	msgbox := s.privateKey.SealTo(clientKey, msg)
+	if err := writeFrameHeader(bw.bw(), frameServerInfo, uint32(len(msgbox))); err != nil {
 		return err
 	}
 	if _, err := bw.Write(msgbox); err != nil {
@@ -1110,7 +1134,7 @@ func (s *Server) sendServerInfo(bw *lazyBufioWriter, clientKey key.Public) error
 // recvClientKey reads the frameClientInfo frame from the client (its
 // proof of identity) upon its initial connection. It should be
 // considered especially untrusted at this point.
-func (s *Server) recvClientKey(br *bufio.Reader) (clientKey key.Public, info *clientInfo, err error) {
+func (s *Server) recvClientKey(br *bufio.Reader) (clientKey key.NodePublic, info *clientInfo, err error) {
 	fl, err := readFrameTypeHeader(br, frameClientInfo)
 	if err != nil {
 		return zpub, nil, err
@@ -1124,21 +1148,17 @@ func (s *Server) recvClientKey(br *bufio.Reader) (clientKey key.Public, info *cl
 	if fl > 256<<10 {
 		return zpub, nil, errors.New("long client info")
 	}
-	if _, err := io.ReadFull(br, clientKey[:]); err != nil {
+	if err := clientKey.ReadRawWithoutAllocating(br); err != nil {
 		return zpub, nil, err
 	}
-	var nonce [24]byte
-	if _, err := io.ReadFull(br, nonce[:]); err != nil {
-		return zpub, nil, fmt.Errorf("nonce: %v", err)
-	}
-	msgLen := int(fl - minLen)
+	msgLen := int(fl - keyLen)
 	msgbox := make([]byte, msgLen)
 	if _, err := io.ReadFull(br, msgbox); err != nil {
 		return zpub, nil, fmt.Errorf("msgbox: %v", err)
 	}
-	msg, ok := box.Open(nil, msgbox, &nonce, (*[32]byte)(&clientKey), s.privateKey.B32())
+	msg, ok := s.privateKey.OpenFrom(clientKey, msgbox)
 	if !ok {
-		return zpub, nil, fmt.Errorf("msgbox: cannot open len=%d with client key %x", msgLen, clientKey[:])
+		return zpub, nil, fmt.Errorf("msgbox: cannot open len=%d with client key %s", msgLen, clientKey)
 	}
 	info = new(clientInfo)
 	if err := json.Unmarshal(msg, info); err != nil {
@@ -1147,11 +1167,11 @@ func (s *Server) recvClientKey(br *bufio.Reader) (clientKey key.Public, info *cl
 	return clientKey, info, nil
 }
 
-func (s *Server) recvPacket(br *bufio.Reader, frameLen uint32) (dstKey key.Public, contents []byte, err error) {
+func (s *Server) recvPacket(br *bufio.Reader, frameLen uint32) (dstKey key.NodePublic, contents []byte, err error) {
 	if frameLen < keyLen {
 		return zpub, nil, errors.New("short send packet frame")
 	}
-	if err := readPublicKey(br, &dstKey); err != nil {
+	if err := dstKey.ReadRawWithoutAllocating(br); err != nil {
 		return zpub, nil, err
 	}
 	packetLen := frameLen - keyLen
@@ -1172,17 +1192,17 @@ func (s *Server) recvPacket(br *bufio.Reader, frameLen uint32) (dstKey key.Publi
 	return dstKey, contents, nil
 }
 
-// zpub is the key.Public zero value.
-var zpub key.Public
+// zpub is the key.NodePublic zero value.
+var zpub key.NodePublic
 
-func (s *Server) recvForwardPacket(br *bufio.Reader, frameLen uint32) (srcKey, dstKey key.Public, contents []byte, err error) {
+func (s *Server) recvForwardPacket(br *bufio.Reader, frameLen uint32) (srcKey, dstKey key.NodePublic, contents []byte, err error) {
 	if frameLen < keyLen*2 {
 		return zpub, zpub, nil, errors.New("short send packet frame")
 	}
-	if _, err := io.ReadFull(br, srcKey[:]); err != nil {
+	if err := srcKey.ReadRawWithoutAllocating(br); err != nil {
 		return zpub, zpub, nil, err
 	}
-	if _, err := io.ReadFull(br, dstKey[:]); err != nil {
+	if err := dstKey.ReadRawWithoutAllocating(br); err != nil {
 		return zpub, zpub, nil, err
 	}
 	packetLen := frameLen - keyLen*2
@@ -1206,19 +1226,20 @@ type sclient struct {
 	connNum        int64 // process-wide unique counter, incremented each Accept
 	s              *Server
 	nc             Conn
-	key            key.Public
+	key            key.NodePublic
 	info           clientInfo
 	logf           logger.Logf
-	done           <-chan struct{}  // closed when connection closes
-	remoteAddr     string           // usually ip:port from net.Conn.RemoteAddr().String()
-	remoteIPPort   netaddr.IPPort   // zero if remoteAddr is not ip:port.
-	sendQueue      chan pkt         // packets queued to this client; never closed
-	discoSendQueue chan pkt         // important packets queued to this client; never closed
-	peerGone       chan key.Public  // write request that a previous sender has disconnected (not used by mesh peers)
-	meshUpdate     chan struct{}    // write request to write peerStateChange
-	canMesh        bool             // clientInfo had correct mesh token for inter-region routing
-	isDup          syncs.AtomicBool // whether more than 1 sclient for key is connected
-	isDisabled     syncs.AtomicBool // whether sends to this peer are disabled due to active/active dups
+	done           <-chan struct{}     // closed when connection closes
+	remoteAddr     string              // usually ip:port from net.Conn.RemoteAddr().String()
+	remoteIPPort   netaddr.IPPort      // zero if remoteAddr is not ip:port.
+	sendQueue      chan pkt            // packets queued to this client; never closed
+	discoSendQueue chan pkt            // important packets queued to this client; never closed
+	sendPongCh     chan [8]byte        // pong replies to send to the client; never closed
+	peerGone       chan key.NodePublic // write request that a previous sender has disconnected (not used by mesh peers)
+	meshUpdate     chan struct{}       // write request to write peerStateChange
+	canMesh        bool                // clientInfo had correct mesh token for inter-region routing
+	isDup          syncs.AtomicBool    // whether more than 1 sclient for key is connected
+	isDisabled     syncs.AtomicBool    // whether sends to this peer are disabled due to active/active dups
 
 	// replaceLimiter controls how quickly two connections with
 	// the same client key can kick each other off the server by
@@ -1245,14 +1266,14 @@ type sclient struct {
 // peerConnState represents whether a peer is connected to the server
 // or not.
 type peerConnState struct {
-	peer    key.Public
+	peer    key.NodePublic
 	present bool
 }
 
 // pkt is a request to write a data frame to an sclient.
 type pkt struct {
 	// src is the who's the sender of the packet.
-	src key.Public
+	src key.NodePublic
 
 	// enqueuedAt is when a packet was put onto a queue before it was sent,
 	// and is used for reporting metrics on the duration of packets in the queue.
@@ -1354,6 +1375,9 @@ func (c *sclient) sendLoop(ctx context.Context) error {
 			werr = c.sendPacket(msg.src, msg.bs)
 			c.recordQueueTime(msg.enqueuedAt)
 			continue
+		case msg := <-c.sendPongCh:
+			werr = c.sendPong(msg)
+			continue
 		case <-keepAliveTick.C:
 			werr = c.sendKeepAlive()
 			continue
@@ -1380,6 +1404,9 @@ func (c *sclient) sendLoop(ctx context.Context) error {
 		case msg := <-c.discoSendQueue:
 			werr = c.sendPacket(msg.src, msg.bs)
 			c.recordQueueTime(msg.enqueuedAt)
+		case msg := <-c.sendPongCh:
+			werr = c.sendPong(msg)
+			continue
 		case <-keepAliveTick.C:
 			werr = c.sendKeepAlive()
 		}
@@ -1396,24 +1423,35 @@ func (c *sclient) sendKeepAlive() error {
 	return writeFrameHeader(c.bw.bw(), frameKeepAlive, 0)
 }
 
+// sendPong sends a pong reply, without flushing.
+func (c *sclient) sendPong(data [8]byte) error {
+	c.s.sentPong.Add(1)
+	c.setWriteDeadline()
+	if err := writeFrameHeader(c.bw.bw(), framePong, uint32(len(data))); err != nil {
+		return err
+	}
+	_, err := c.bw.Write(data[:])
+	return err
+}
+
 // sendPeerGone sends a peerGone frame, without flushing.
-func (c *sclient) sendPeerGone(peer key.Public) error {
+func (c *sclient) sendPeerGone(peer key.NodePublic) error {
 	c.s.peerGoneFrames.Add(1)
 	c.setWriteDeadline()
 	if err := writeFrameHeader(c.bw.bw(), framePeerGone, keyLen); err != nil {
 		return err
 	}
-	_, err := c.bw.Write(peer[:])
+	_, err := c.bw.Write(peer.AppendTo(nil))
 	return err
 }
 
 // sendPeerPresent sends a peerPresent frame, without flushing.
-func (c *sclient) sendPeerPresent(peer key.Public) error {
+func (c *sclient) sendPeerPresent(peer key.NodePublic) error {
 	c.setWriteDeadline()
 	if err := writeFrameHeader(c.bw.bw(), framePeerPresent, keyLen); err != nil {
 		return err
 	}
-	_, err := c.bw.Write(peer[:])
+	_, err := c.bw.Write(peer.AppendTo(nil))
 	return err
 }
 
@@ -1465,7 +1503,7 @@ func (c *sclient) sendMeshUpdates() error {
 // DERPv2. The bytes of contents are only valid until this function
 // returns, do not retain slices.
 // It does not flush its bufio.Writer.
-func (c *sclient) sendPacket(srcKey key.Public, contents []byte) (err error) {
+func (c *sclient) sendPacket(srcKey key.NodePublic, contents []byte) (err error) {
 	defer func() {
 		// Stats update.
 		if err != nil {
@@ -1481,14 +1519,13 @@ func (c *sclient) sendPacket(srcKey key.Public, contents []byte) (err error) {
 	withKey := !srcKey.IsZero()
 	pktLen := len(contents)
 	if withKey {
-		pktLen += len(srcKey)
+		pktLen += key.NodePublicRawLen
 	}
 	if err = writeFrameHeader(c.bw.bw(), frameRecvPacket, uint32(pktLen)); err != nil {
 		return err
 	}
 	if withKey {
-		err := writePublicKey(c.bw.bw(), &srcKey)
-		if err != nil {
+		if err := srcKey.WriteRawWithoutAllocating(c.bw.bw()); err != nil {
 			return err
 		}
 	}
@@ -1498,7 +1535,7 @@ func (c *sclient) sendPacket(srcKey key.Public, contents []byte) (err error) {
 
 // AddPacketForwarder registers fwd as a packet forwarder for dst.
 // fwd must be comparable.
-func (s *Server) AddPacketForwarder(dst key.Public, fwd PacketForwarder) {
+func (s *Server) AddPacketForwarder(dst key.NodePublic, fwd PacketForwarder) {
 	s.mu.Lock()
 	defer s.mu.Unlock()
 	if prev, ok := s.clientsMesh[dst]; ok {
@@ -1530,7 +1567,7 @@ func (s *Server) AddPacketForwarder(dst key.Public, fwd PacketForwarder) {
 
 // RemovePacketForwarder removes fwd as a packet forwarder for dst.
 // fwd must be comparable.
-func (s *Server) RemovePacketForwarder(dst key.Public, fwd PacketForwarder) {
+func (s *Server) RemovePacketForwarder(dst key.NodePublic, fwd PacketForwarder) {
 	s.mu.Lock()
 	defer s.mu.Unlock()
 	v, ok := s.clientsMesh[dst]
@@ -1592,7 +1629,7 @@ func (m multiForwarder) maxVal() (max uint8) {
 	return
 }
 
-func (m multiForwarder) ForwardPacket(src, dst key.Public, payload []byte) error {
+func (m multiForwarder) ForwardPacket(src, dst key.NodePublic, payload []byte) error {
 	var fwd PacketForwarder
 	var lowest uint8
 	for k, v := range m {
@@ -1638,6 +1675,8 @@ func (s *Server) ExpVar() expvar.Var {
 	m.Set("unknown_frames", &s.unknownFrames)
 	m.Set("home_moves_in", &s.homeMovesIn)
 	m.Set("home_moves_out", &s.homeMovesOut)
+	m.Set("got_ping", &s.gotPing)
+	m.Set("sent_pong", &s.sentPong)
 	m.Set("peer_gone_frames", &s.peerGoneFrames)
 	m.Set("packets_forwarded_out", &s.packetsForwardedOut)
 	m.Set("packets_forwarded_in", &s.packetsForwardedIn)
@@ -1692,37 +1731,6 @@ func (s *Server) ConsistencyCheck() error {
 	return errors.New(strings.Join(errs, ", "))
 }
 
-// readPublicKey reads key from br.
-// It is ~4x slower than io.ReadFull(br, key),
-// but it prevents key from escaping and thus being allocated.
-// If io.ReadFull(br, key) does not cause key to escape, use that instead.
-func readPublicKey(br *bufio.Reader, key *key.Public) error {
-	// Do io.ReadFull(br, key), but one byte at a time, to avoid allocation.
-	for i := range key {
-		b, err := br.ReadByte()
-		if err != nil {
-			return err
-		}
-		key[i] = b
-	}
-	return nil
-}
-
-// writePublicKey writes key to bw.
-// It is ~3x slower than bw.Write(key[:]),
-// but it prevents key from escaping and thus being allocated.
-// If bw.Write(key[:]) does not cause key to escape, use that instead.
-func writePublicKey(bw *bufio.Writer, key *key.Public) error {
-	// Do bw.Write(key[:]), but one byte at a time to avoid allocation.
-	for _, b := range key {
-		err := bw.WriteByte(b)
-		if err != nil {
-			return err
-		}
-	}
-	return nil
-}
-
 const minTimeBetweenLogs = 2 * time.Second
 
 // BytesSentRecv records the number of bytes that have been sent since the last traffic check
@@ -1731,7 +1739,7 @@ type BytesSentRecv struct {
 	Sent uint64
 	Recv uint64
 	// Key is the public key of the client which sent/received these bytes.
-	Key key.Public
+	Key key.NodePublic
 }
 
 // parseSSOutput parses the output from the specific call to ss in ServeDebugTraffic.

derp/derp_test.go

@@ -8,7 +8,6 @@ import (
 	"bufio"
 	"bytes"
 	"context"
-	crand "crypto/rand"
 	"crypto/x509"
 	"encoding/json"
 	"errors"
@@ -23,20 +22,13 @@ import (
 	"testing"
 	"time"
 
+	"go4.org/mem"
 	"golang.org/x/time/rate"
 	"tailscale.com/net/nettest"
 	"tailscale.com/types/key"
 	"tailscale.com/types/logger"
 )
 
-func newPrivateKey(tb testing.TB) (k key.Private) {
-	tb.Helper()
-	if _, err := crand.Read(k[:]); err != nil {
-		tb.Fatal(err)
-	}
-	return
-}
-
 func TestClientInfoUnmarshal(t *testing.T) {
 	for i, in := range []string{
 		`{"Version":5,"MeshKey":"abc"}`,
@@ -54,15 +46,15 @@ func TestClientInfoUnmarshal(t *testing.T) {
 }
 
 func TestSendRecv(t *testing.T) {
-	serverPrivateKey := newPrivateKey(t)
+	serverPrivateKey := key.NewNode()
 	s := NewServer(serverPrivateKey, t.Logf)
 	defer s.Close()
 
 	const numClients = 3
-	var clientPrivateKeys []key.Private
-	var clientKeys []key.Public
+	var clientPrivateKeys []key.NodePrivate
+	var clientKeys []key.NodePublic
 	for i := 0; i < numClients; i++ {
-		priv := newPrivateKey(t)
+		priv := key.NewNode()
 		clientPrivateKeys = append(clientPrivateKeys, priv)
 		clientKeys = append(clientKeys, priv.Public())
 	}
@@ -225,7 +217,7 @@ func TestSendRecv(t *testing.T) {
 }
 
 func TestSendFreeze(t *testing.T) {
-	serverPrivateKey := newPrivateKey(t)
+	serverPrivateKey := key.NewNode()
 	s := NewServer(serverPrivateKey, t.Logf)
 	defer s.Close()
 	s.WriteTimeout = 100 * time.Millisecond
@@ -238,7 +230,7 @@ func TestSendFreeze(t *testing.T) {
 	// Then cathy stops processing messsages.
 	// That should not interfere with alice talking to bob.
 
-	newClient := func(name string, k key.Private) (c *Client, clientConn nettest.Conn) {
+	newClient := func(name string, k key.NodePrivate) (c *Client, clientConn nettest.Conn) {
 		t.Helper()
 		c1, c2 := nettest.NewConn(name, 1024)
 		go s.Accept(c1, bufio.NewReadWriter(bufio.NewReader(c1), bufio.NewWriter(c1)), name)
@@ -252,13 +244,13 @@ func TestSendFreeze(t *testing.T) {
 		return c, c2
 	}
 
-	aliceKey := newPrivateKey(t)
+	aliceKey := key.NewNode()
 	aliceClient, aliceConn := newClient("alice", aliceKey)
 
-	bobKey := newPrivateKey(t)
+	bobKey := key.NewNode()
 	bobClient, bobConn := newClient("bob", bobKey)
 
-	cathyKey := newPrivateKey(t)
+	cathyKey := key.NewNode()
 	cathyClient, cathyConn := newClient("cathy", cathyKey)
 
 	var (
@@ -427,7 +419,7 @@ type testServer struct {
 	logf logger.Logf
 
 	mu      sync.Mutex
-	pubName map[key.Public]string
+	pubName map[key.NodePublic]string
 	clients map[*testClient]bool
 }
 
@@ -437,14 +429,14 @@ func (ts *testServer) addTestClient(c *testClient) {
 	ts.clients[c] = true
 }
 
-func (ts *testServer) addKeyName(k key.Public, name string) {
+func (ts *testServer) addKeyName(k key.NodePublic, name string) {
 	ts.mu.Lock()
 	defer ts.mu.Unlock()
 	ts.pubName[k] = name
 	ts.logf("test adding named key %q for %x", name, k)
 }
 
-func (ts *testServer) keyName(k key.Public) string {
+func (ts *testServer) keyName(k key.NodePublic) string {
 	ts.mu.Lock()
 	defer ts.mu.Unlock()
 	if name, ok := ts.pubName[k]; ok {
@@ -465,7 +457,7 @@ func (ts *testServer) close(t *testing.T) error {
 func newTestServer(t *testing.T) *testServer {
 	t.Helper()
 	logf := logger.WithPrefix(t.Logf, "derp-server: ")
-	s := NewServer(newPrivateKey(t), logf)
+	s := NewServer(key.NewNode(), logf)
 	s.SetMeshKey("mesh-key")
 	ln, err := net.Listen("tcp", "127.0.0.1:0")
 	if err != nil {
@@ -491,7 +483,7 @@ func newTestServer(t *testing.T) *testServer {
 		ln:      ln,
 		logf:    logf,
 		clients: map[*testClient]bool{},
-		pubName: map[key.Public]string{},
+		pubName: map[key.NodePublic]string{},
 	}
 }
 
@@ -499,20 +491,20 @@ type testClient struct {
 	name   string
 	c      *Client
 	nc     net.Conn
-	pub    key.Public
+	pub    key.NodePublic
 	ts     *testServer
 	closed bool
 }
 
-func newTestClient(t *testing.T, ts *testServer, name string, newClient func(net.Conn, key.Private, logger.Logf) (*Client, error)) *testClient {
+func newTestClient(t *testing.T, ts *testServer, name string, newClient func(net.Conn, key.NodePrivate, logger.Logf) (*Client, error)) *testClient {
 	t.Helper()
 	nc, err := net.Dial("tcp", ts.ln.Addr().String())
 	if err != nil {
 		t.Fatal(err)
 	}
-	key := newPrivateKey(t)
-	ts.addKeyName(key.Public(), name)
-	c, err := newClient(nc, key, logger.WithPrefix(t.Logf, "client-"+name+": "))
+	k := key.NewNode()
+	ts.addKeyName(k.Public(), name)
+	c, err := newClient(nc, k, logger.WithPrefix(t.Logf, "client-"+name+": "))
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -521,14 +513,14 @@ func newTestClient(t *testing.T, ts *testServer, name string, newClient func(net
 		nc:   nc,
 		c:    c,
 		ts:   ts,
-		pub:  key.Public(),
+		pub:  k.Public(),
 	}
 	ts.addTestClient(tc)
 	return tc
 }
 
 func newRegularClient(t *testing.T, ts *testServer, name string) *testClient {
-	return newTestClient(t, ts, name, func(nc net.Conn, priv key.Private, logf logger.Logf) (*Client, error) {
+	return newTestClient(t, ts, name, func(nc net.Conn, priv key.NodePrivate, logf logger.Logf) (*Client, error) {
 		brw := bufio.NewReadWriter(bufio.NewReader(nc), bufio.NewWriter(nc))
 		c, err := NewClient(priv, nc, brw, logf)
 		if err != nil {
@@ -541,7 +533,7 @@ func newRegularClient(t *testing.T, ts *testServer, name string) *testClient {
 }
 
 func newTestWatcher(t *testing.T, ts *testServer, name string) *testClient {
-	return newTestClient(t, ts, name, func(nc net.Conn, priv key.Private, logf logger.Logf) (*Client, error) {
+	return newTestClient(t, ts, name, func(nc net.Conn, priv key.NodePrivate, logf logger.Logf) (*Client, error) {
 		brw := bufio.NewReadWriter(bufio.NewReader(nc), bufio.NewWriter(nc))
 		c, err := NewClient(priv, nc, brw, logf, MeshKey("mesh-key"))
 		if err != nil {
@@ -555,9 +547,9 @@ func newTestWatcher(t *testing.T, ts *testServer, name string) *testClient {
 	})
 }
 
-func (tc *testClient) wantPresent(t *testing.T, peers ...key.Public) {
+func (tc *testClient) wantPresent(t *testing.T, peers ...key.NodePublic) {
 	t.Helper()
-	want := map[key.Public]bool{}
+	want := map[key.NodePublic]bool{}
 	for _, k := range peers {
 		want[k] = true
 	}
@@ -569,7 +561,7 @@ func (tc *testClient) wantPresent(t *testing.T, peers ...key.Public) {
 		}
 		switch m := m.(type) {
 		case PeerPresentMessage:
-			got := key.Public(m)
+			got := key.NodePublic(m)
 			if !want[got] {
 				t.Fatalf("got peer present for %v; want present for %v", tc.ts.keyName(got), logger.ArgWriter(func(bw *bufio.Writer) {
 					for _, pub := range peers {
@@ -587,7 +579,7 @@ func (tc *testClient) wantPresent(t *testing.T, peers ...key.Public) {
 	}
 }
 
-func (tc *testClient) wantGone(t *testing.T, peer key.Public) {
+func (tc *testClient) wantGone(t *testing.T, peer key.NodePublic) {
 	t.Helper()
 	m, err := tc.c.recvTimeout(time.Second)
 	if err != nil {
@@ -595,7 +587,7 @@ func (tc *testClient) wantGone(t *testing.T, peer key.Public) {
 	}
 	switch m := m.(type) {
 	case PeerGoneMessage:
-		got := key.Public(m)
+		got := key.NodePublic(m)
 		if peer != got {
 			t.Errorf("got gone message for %v; want gone for %v", tc.ts.keyName(got), tc.ts.keyName(peer))
 		}
@@ -654,21 +646,24 @@ func TestWatch(t *testing.T) {
 
 type testFwd int
 
-func (testFwd) ForwardPacket(key.Public, key.Public, []byte) error { panic("not called in tests") }
+func (testFwd) ForwardPacket(key.NodePublic, key.NodePublic, []byte) error {
+	panic("not called in tests")
+}
 
-func pubAll(b byte) (ret key.Public) {
-	for i := range ret {
-		ret[i] = b
+func pubAll(b byte) (ret key.NodePublic) {
+	var bs [32]byte
+	for i := range bs {
+		bs[i] = b
 	}
-	return
+	return key.NodePublicFromRaw32(mem.B(bs[:]))
 }
 
 func TestForwarderRegistration(t *testing.T) {
 	s := &Server{
-		clients:     make(map[key.Public]clientSet),
-		clientsMesh: map[key.Public]PacketForwarder{},
+		clients:     make(map[key.NodePublic]clientSet),
+		clientsMesh: map[key.NodePublic]PacketForwarder{},
 	}
-	want := func(want map[key.Public]PacketForwarder) {
+	want := func(want map[key.NodePublic]PacketForwarder) {
 		t.Helper()
 		if got := s.clientsMesh; !reflect.DeepEqual(got, want) {
 			t.Fatalf("mismatch\n got: %v\nwant: %v\n", got, want)
@@ -687,28 +682,28 @@ func TestForwarderRegistration(t *testing.T) {
 
 	s.AddPacketForwarder(u1, testFwd(1))
 	s.AddPacketForwarder(u2, testFwd(2))
-	want(map[key.Public]PacketForwarder{
+	want(map[key.NodePublic]PacketForwarder{
 		u1: testFwd(1),
 		u2: testFwd(2),
 	})
 
 	// Verify a remove of non-registered forwarder is no-op.
 	s.RemovePacketForwarder(u2, testFwd(999))
-	want(map[key.Public]PacketForwarder{
+	want(map[key.NodePublic]PacketForwarder{
 		u1: testFwd(1),
 		u2: testFwd(2),
 	})
 
 	// Verify a remove of non-registered user is no-op.
 	s.RemovePacketForwarder(u3, testFwd(1))
-	want(map[key.Public]PacketForwarder{
+	want(map[key.NodePublic]PacketForwarder{
 		u1: testFwd(1),
 		u2: testFwd(2),
 	})
 
 	// Actual removal.
 	s.RemovePacketForwarder(u2, testFwd(2))
-	want(map[key.Public]PacketForwarder{
+	want(map[key.NodePublic]PacketForwarder{
 		u1: testFwd(1),
 	})
 
@@ -716,7 +711,7 @@ func TestForwarderRegistration(t *testing.T) {
 	wantCounter(&s.multiForwarderCreated, 0)
 	s.AddPacketForwarder(u1, testFwd(100))
 	s.AddPacketForwarder(u1, testFwd(100)) // dup to trigger dup path
-	want(map[key.Public]PacketForwarder{
+	want(map[key.NodePublic]PacketForwarder{
 		u1: multiForwarder{
 			testFwd(1):   1,
 			testFwd(100): 2,
@@ -726,7 +721,7 @@ func TestForwarderRegistration(t *testing.T) {
 
 	// Removing a forwarder in a multi set that doesn't exist; does nothing.
 	s.RemovePacketForwarder(u1, testFwd(55))
-	want(map[key.Public]PacketForwarder{
+	want(map[key.NodePublic]PacketForwarder{
 		u1: multiForwarder{
 			testFwd(1):   1,
 			testFwd(100): 2,
@@ -737,7 +732,7 @@ func TestForwarderRegistration(t *testing.T) {
 	// from being a multiForwarder.
 	wantCounter(&s.multiForwarderDeleted, 0)
 	s.RemovePacketForwarder(u1, testFwd(1))
-	want(map[key.Public]PacketForwarder{
+	want(map[key.NodePublic]PacketForwarder{
 		u1: testFwd(100),
 	})
 	wantCounter(&s.multiForwarderDeleted, 1)
@@ -750,18 +745,18 @@ func TestForwarderRegistration(t *testing.T) {
 	}
 	s.clients[u1] = singleClient{u1c}
 	s.RemovePacketForwarder(u1, testFwd(100))
-	want(map[key.Public]PacketForwarder{
+	want(map[key.NodePublic]PacketForwarder{
 		u1: nil,
 	})
 
 	// But once that client disconnects, it should go away.
 	s.unregisterClient(u1c)
-	want(map[key.Public]PacketForwarder{})
+	want(map[key.NodePublic]PacketForwarder{})
 
 	// But if it already has a forwarder, it's not removed.
 	s.AddPacketForwarder(u1, testFwd(2))
 	s.unregisterClient(u1c)
-	want(map[key.Public]PacketForwarder{
+	want(map[key.NodePublic]PacketForwarder{
 		u1: testFwd(2),
 	})
 
@@ -770,17 +765,17 @@ func TestForwarderRegistration(t *testing.T) {
 	// from nil to the new one, not a multiForwarder.
 	s.clients[u1] = singleClient{u1c}
 	s.clientsMesh[u1] = nil
-	want(map[key.Public]PacketForwarder{
+	want(map[key.NodePublic]PacketForwarder{
 		u1: nil,
 	})
 	s.AddPacketForwarder(u1, testFwd(3))
-	want(map[key.Public]PacketForwarder{
+	want(map[key.NodePublic]PacketForwarder{
 		u1: testFwd(3),
 	})
 }
 
 func TestMetaCert(t *testing.T) {
-	priv := newPrivateKey(t)
+	priv := key.NewNode()
 	pub := priv.Public()
 	s := NewServer(priv, t.Logf)
 
@@ -792,7 +787,7 @@ func TestMetaCert(t *testing.T) {
 	if fmt.Sprint(cert.SerialNumber) != fmt.Sprint(ProtocolVersion) {
 		t.Errorf("serial = %v; want %v", cert.SerialNumber, ProtocolVersion)
 	}
-	if g, w := cert.Subject.CommonName, fmt.Sprintf("derpkey%x", pub[:]); g != w {
+	if g, w := cert.Subject.CommonName, fmt.Sprintf("derpkey%s", pub.UntypedHexString()); g != w {
 		t.Errorf("CommonName = %q; want %q", g, w)
 	}
 }
@@ -817,6 +812,14 @@ func TestClientRecv(t *testing.T) {
 			},
 			want: PingMessage{1, 2, 3, 4, 5, 6, 7, 8},
 		},
+		{
+			name: "pong",
+			input: []byte{
+				byte(framePong), 0, 0, 0, 8,
+				1, 2, 3, 4, 5, 6, 7, 8,
+			},
+			want: PongMessage{1, 2, 3, 4, 5, 6, 7, 8},
+		},
 		{
 			name: "health_bad",
 			input: []byte{
@@ -863,6 +866,23 @@ func TestClientRecv(t *testing.T) {
 	}
 }
 
+func TestClientSendPing(t *testing.T) {
+	var buf bytes.Buffer
+	c := &Client{
+		bw: bufio.NewWriter(&buf),
+	}
+	if err := c.SendPing([8]byte{1, 2, 3, 4, 5, 6, 7, 8}); err != nil {
+		t.Fatal(err)
+	}
+	want := []byte{
+		byte(framePing), 0, 0, 0, 8,
+		1, 2, 3, 4, 5, 6, 7, 8,
+	}
+	if !bytes.Equal(buf.Bytes(), want) {
+		t.Errorf("unexpected output\nwrote: % 02x\n want: % 02x", buf.Bytes(), want)
+	}
+}
+
 func TestClientSendPong(t *testing.T) {
 	var buf bytes.Buffer
 	c := &Client{
@@ -878,14 +898,13 @@ func TestClientSendPong(t *testing.T) {
 	if !bytes.Equal(buf.Bytes(), want) {
 		t.Errorf("unexpected output\nwrote: % 02x\n want: % 02x", buf.Bytes(), want)
 	}
-
 }
 
 func TestServerDupClients(t *testing.T) {
-	serverPriv := newPrivateKey(t)
+	serverPriv := key.NewNode()
 	var s *Server
 
-	clientPriv := newPrivateKey(t)
+	clientPriv := key.NewNode()
 	clientPub := clientPriv.Public()
 
 	var c1, c2, c3 *sclient
@@ -1141,12 +1160,12 @@ func BenchmarkSendRecv(b *testing.B) {
 }
 
 func benchmarkSendRecvSize(b *testing.B, packetSize int) {
-	serverPrivateKey := newPrivateKey(b)
+	serverPrivateKey := key.NewNode()
 	s := NewServer(serverPrivateKey, logger.Discard)
 	defer s.Close()
 
-	key := newPrivateKey(b)
-	clientKey := key.Public()
+	k := key.NewNode()
+	clientKey := k.Public()
 
 	ln, err := net.Listen("tcp", "127.0.0.1:0")
 	if err != nil {
@@ -1170,7 +1189,7 @@ func benchmarkSendRecvSize(b *testing.B, packetSize int) {
 	go s.Accept(connIn, brwServer, "test-client")
 
 	brw := bufio.NewReadWriter(bufio.NewReader(connOut), bufio.NewWriter(connOut))
-	client, err := NewClient(key, connOut, brw, logger.Discard)
+	client, err := NewClient(k, connOut, brw, logger.Discard)
 	if err != nil {
 		b.Fatalf("client: %v", err)
 	}
@@ -1279,7 +1298,7 @@ func TestClientSendRateLimiting(t *testing.T) {
 	c.setSendRateLimiter(ServerInfoMessage{})
 
 	pkt := make([]byte, 1000)
-	if err := c.send(key.Public{}, pkt); err != nil {
+	if err := c.send(key.NodePublic{}, pkt); err != nil {
 		t.Fatal(err)
 	}
 	writes1, bytes1 := cw.Stats()
@@ -1290,7 +1309,7 @@ func TestClientSendRateLimiting(t *testing.T) {
 	// Flood should all succeed.
 	cw.ResetStats()
 	for i := 0; i < 1000; i++ {
-		if err := c.send(key.Public{}, pkt); err != nil {
+		if err := c.send(key.NodePublic{}, pkt); err != nil {
 			t.Fatal(err)
 		}
 	}
@@ -1309,7 +1328,7 @@ func TestClientSendRateLimiting(t *testing.T) {
 		TokenBucketBytesBurst:     int(bytes1 * 2),
 	})
 	for i := 0; i < 1000; i++ {
-		if err := c.send(key.Public{}, pkt); err != nil {
+		if err := c.send(key.NodePublic{}, pkt); err != nil {
 			t.Fatal(err)
 		}
 	}
@@ -1321,3 +1340,30 @@ func TestClientSendRateLimiting(t *testing.T) {
 		t.Errorf("limited conn's bytes count = %v; want >=%v, <%v", bytesLimited, bytes1K*2, bytes1K)
 	}
 }
+
+func TestServerRepliesToPing(t *testing.T) {
+	ts := newTestServer(t)
+	defer ts.close(t)
+
+	tc := newRegularClient(t, ts, "alice")
+
+	data := [8]byte{1, 2, 3, 4, 5, 6, 7, 42}
+
+	if err := tc.c.SendPing(data); err != nil {
+		t.Fatal(err)
+	}
+
+	for {
+		m, err := tc.c.recvTimeout(time.Second)
+		if err != nil {
+			t.Fatal(err)
+		}
+		switch m := m.(type) {
+		case PongMessage:
+			if ([8]byte(m)) != data {
+				t.Fatalf("got pong %2x; want %2x", [8]byte(m), data)
+			}
+			return
+		}
+	}
+}

derp/derphttp/derphttp_client.go

@@ -13,6 +13,7 @@ package derphttp
 import (
 	"bufio"
 	"context"
+	"crypto/rand"
 	"crypto/tls"
 	"crypto/x509"
 	"errors"
@@ -22,13 +23,16 @@ import (
 	"net"
 	"net/http"
 	"net/url"
+	"runtime"
 	"strings"
 	"sync"
+	"sync/atomic"
 	"time"
 
 	"go4.org/mem"
 	"inet.af/netaddr"
 	"tailscale.com/derp"
+	"tailscale.com/envknob"
 	"tailscale.com/net/dnscache"
 	"tailscale.com/net/netns"
 	"tailscale.com/net/tlsdial"
@@ -50,7 +54,7 @@ type Client struct {
 	MeshKey   string             // optional; for trusted clients
 	IsProber  bool               // optional; for probers to optional declare themselves as such
 
-	privateKey key.Private
+	privateKey key.NodePrivate
 	logf       logger.Logf
 	dialer     func(ctx context.Context, network, addr string) (net.Conn, error)
 
@@ -61,6 +65,12 @@ type Client struct {
 	ctx       context.Context // closed via cancelCtx in Client.Close
 	cancelCtx context.CancelFunc
 
+	// addrFamSelAtomic is the last AddressFamilySelector set
+	// by SetAddressFamilySelector. It's an atomic because it needs
+	// to be accessed by multiple racing routines started while
+	// Client.conn holds mu.
+	addrFamSelAtomic atomic.Value // of AddressFamilySelector
+
 	mu           sync.Mutex
 	preferred    bool
 	canAckPings  bool
@@ -68,12 +78,14 @@ type Client struct {
 	netConn      io.Closer
 	client       *derp.Client
 	connGen      int // incremented once per new connection; valid values are >0
-	serverPubKey key.Public
+	serverPubKey key.NodePublic
+	tlsState     *tls.ConnectionState
+	pingOut      map[derp.PingMessage]chan<- bool // chan to send to on pong
 }
 
 // NewRegionClient returns a new DERP-over-HTTP client. It connects lazily.
 // To trigger a connection, use Connect.
-func NewRegionClient(privateKey key.Private, logf logger.Logf, getRegion func() *tailcfg.DERPRegion) *Client {
+func NewRegionClient(privateKey key.NodePrivate, logf logger.Logf, getRegion func() *tailcfg.DERPRegion) *Client {
 	ctx, cancel := context.WithCancel(context.Background())
 	c := &Client{
 		privateKey: privateKey,
@@ -93,7 +105,7 @@ func NewNetcheckClient(logf logger.Logf) *Client {
 
 // NewClient returns a new DERP-over-HTTP client. It connects lazily.
 // To trigger a connection, use Connect.
-func NewClient(privateKey key.Private, serverURL string, logf logger.Logf) (*Client, error) {
+func NewClient(privateKey key.NodePrivate, serverURL string, logf logger.Logf) (*Client, error) {
 	u, err := url.Parse(serverURL)
 	if err != nil {
 		return nil, fmt.Errorf("derphttp.NewClient: %v", err)
@@ -120,18 +132,29 @@ func (c *Client) Connect(ctx context.Context) error {
 	return err
 }
 
+// TLSConnectionState returns the last TLS connection state, if any.
+// The client must already be connected.
+func (c *Client) TLSConnectionState() (_ *tls.ConnectionState, ok bool) {
+	c.mu.Lock()
+	defer c.mu.Unlock()
+	if c.closed || c.client == nil {
+		return nil, false
+	}
+	return c.tlsState, c.tlsState != nil
+}
+
 // ServerPublicKey returns the server's public key.
 //
 // It only returns a non-zero value once a connection has succeeded
 // from an earlier call.
-func (c *Client) ServerPublicKey() key.Public {
+func (c *Client) ServerPublicKey() key.NodePublic {
 	c.mu.Lock()
 	defer c.mu.Unlock()
 	return c.serverPubKey
 }
 
 // SelfPublicKey returns our own public key.
-func (c *Client) SelfPublicKey() key.Public {
+func (c *Client) SelfPublicKey() key.NodePublic {
 	return c.privateKey.Public()
 }
 
@@ -177,6 +200,45 @@ func (c *Client) urlString(node *tailcfg.DERPNode) string {
 	return fmt.Sprintf("https://%s/derp", node.HostName)
 }
 
+// AddressFamilySelector decides whethers IPv6 is preferred for
+// outbound dials.
+type AddressFamilySelector interface {
+	// PreferIPv6 reports whether IPv4 dials should be slightly
+	// delayed to give IPv6 a better chance of winning dial races.
+	// Implementations should only return true if IPv6 is expected
+	// to succeed. (otherwise delaying IPv4 will delay the
+	// connection overall)
+	PreferIPv6() bool
+}
+
+// SetAddressFamilySelector sets the AddressFamilySelector that this
+// connection will use. It should be called before any dials.
+// The value must not be nil. If called more than once, s must
+// be the same concrete type as any prior calls.
+func (c *Client) SetAddressFamilySelector(s AddressFamilySelector) {
+	c.addrFamSelAtomic.Store(s)
+}
+
+func (c *Client) preferIPv6() bool {
+	if s, ok := c.addrFamSelAtomic.Load().(AddressFamilySelector); ok {
+		return s.PreferIPv6()
+	}
+	return false
+}
+
+// dialWebsocketFunc is non-nil (set by websocket.go's init) when compiled in.
+var dialWebsocketFunc func(ctx context.Context, urlStr string) (net.Conn, error)
+
+func useWebsockets() bool {
+	if runtime.GOOS == "js" {
+		return true
+	}
+	if dialWebsocketFunc != nil {
+		return envknob.Bool("TS_DEBUG_DERP_WS_CLIENT")
+	}
+	return false
+}
+
 func (c *Client) connect(ctx context.Context, caller string) (client *derp.Client, connGen int, err error) {
 	c.mu.Lock()
 	defer c.mu.Unlock()
@@ -229,10 +291,44 @@ func (c *Client) connect(ctx context.Context, caller string) (client *derp.Clien
 	}()
 
 	var node *tailcfg.DERPNode // nil when using c.url to dial
-	if c.url != nil {
+	switch {
+	case useWebsockets():
+		var urlStr string
+		if c.url != nil {
+			urlStr = c.url.String()
+		} else {
+			urlStr = c.urlString(reg.Nodes[0])
+		}
+		c.logf("%s: connecting websocket to %v", caller, urlStr)
+		conn, err := dialWebsocketFunc(ctx, urlStr)
+		if err != nil {
+			c.logf("%s: websocket to %v error: %v", caller, urlStr, err)
+			return nil, 0, err
+		}
+		brw := bufio.NewReadWriter(bufio.NewReader(conn), bufio.NewWriter(conn))
+		derpClient, err := derp.NewClient(c.privateKey, conn, brw, c.logf,
+			derp.MeshKey(c.MeshKey),
+			derp.CanAckPings(c.canAckPings),
+			derp.IsProber(c.IsProber),
+		)
+		if err != nil {
+			return nil, 0, err
+		}
+		if c.preferred {
+			if err := derpClient.NotePreferred(true); err != nil {
+				go conn.Close()
+				return nil, 0, err
+			}
+		}
+		c.serverPubKey = derpClient.ServerPublicKey()
+		c.client = derpClient
+		c.netConn = tcpConn
+		c.connGen++
+		return c.client, c.connGen, nil
+	case c.url != nil:
 		c.logf("%s: connecting to %v", caller, c.url)
 		tcpConn, err = c.dialURL(ctx)
-	} else {
+	default:
 		c.logf("%s: connecting to derp-%d (%v)", caller, reg.RegionID, reg.RegionCode)
 		tcpConn, node, err = c.dialRegion(ctx, reg)
 	}
@@ -264,9 +360,10 @@ func (c *Client) connect(ctx context.Context, caller string) (client *derp.Clien
 		}
 	}()
 
-	var httpConn net.Conn    // a TCP conn or a TLS conn; what we speak HTTP to
-	var serverPub key.Public // or zero if unknown (if not using TLS or TLS middlebox eats it)
+	var httpConn net.Conn        // a TCP conn or a TLS conn; what we speak HTTP to
+	var serverPub key.NodePublic // or zero if unknown (if not using TLS or TLS middlebox eats it)
 	var serverProtoVersion int
+	var tlsState *tls.ConnectionState
 	if c.useHTTPS() {
 		tlsConn := c.tlsClient(tcpConn, node)
 		httpConn = tlsConn
@@ -289,9 +386,10 @@ func (c *Client) connect(ctx context.Context, caller string) (client *derp.Clien
 		// Note that we're not specifically concerned about TLS downgrade
 		// attacks. TLS handles that fine:
 		// https://blog.gypsyengineer.com/en/security/how-does-tls-1-3-protect-against-downgrade-attacks.html
-		connState := tlsConn.ConnectionState()
-		if connState.Version >= tls.VersionTLS13 {
-			serverPub, serverProtoVersion = parseMetaCert(connState.PeerCertificates)
+		cs := tlsConn.ConnectionState()
+		tlsState = &cs
+		if cs.Version >= tls.VersionTLS13 {
+			serverPub, serverProtoVersion = parseMetaCert(cs.PeerCertificates)
 		}
 	} else {
 		httpConn = tcpConn
@@ -358,6 +456,7 @@ func (c *Client) connect(ctx context.Context, caller string) (client *derp.Clien
 	c.serverPubKey = derpClient.ServerPublicKey()
 	c.client = derpClient
 	c.netConn = tcpConn
+	c.tlsState = tlsState
 	c.connGen++
 	return c.client, c.connGen, nil
 }
@@ -378,7 +477,7 @@ func (c *Client) dialURL(ctx context.Context) (net.Conn, error) {
 		return c.dialer(ctx, "tcp", net.JoinHostPort(host, urlPort(c.url)))
 	}
 	hostOrIP := host
-	dialer := netns.NewDialer()
+	dialer := netns.NewDialer(c.logf)
 
 	if c.DNSCache != nil {
 		ip, _, _, err := c.DNSCache.LookupIP(ctx, host)
@@ -468,7 +567,7 @@ func (c *Client) DialRegionTLS(ctx context.Context, reg *tailcfg.DERPRegion) (tl
 }
 
 func (c *Client) dialContext(ctx context.Context, proto, addr string) (net.Conn, error) {
-	return netns.NewDialer().DialContext(ctx, proto, addr)
+	return netns.NewDialer(c.logf).DialContext(ctx, proto, addr)
 }
 
 // shouldDialProto reports whether an explicitly provided IPv4 or IPv6
@@ -517,6 +616,18 @@ func (c *Client) dialNode(ctx context.Context, n *tailcfg.DERPNode) (net.Conn, e
 	startDial := func(dstPrimary, proto string) {
 		nwait++
 		go func() {
+			if proto == "tcp4" && c.preferIPv6() {
+				t := time.NewTimer(200 * time.Millisecond)
+				select {
+				case <-ctx.Done():
+					// Either user canceled original context,
+					// it timed out, or the v6 dial succeeded.
+					t.Stop()
+					return
+				case <-t.C:
+					// Start v4 dial
+				}
+			}
 			dst := dstPrimary
 			if dst == "" {
 				dst = n.HostName
@@ -636,7 +747,7 @@ func (c *Client) dialNodeUsingProxy(ctx context.Context, n *tailcfg.DERPNode, pr
 	return proxyConn, nil
 }
 
-func (c *Client) Send(dstKey key.Public, b []byte) error {
+func (c *Client) Send(dstKey key.NodePublic, b []byte) error {
 	client, _, err := c.connect(context.TODO(), "derphttp.Client.Send")
 	if err != nil {
 		return err
@@ -647,7 +758,96 @@ func (c *Client) Send(dstKey key.Public, b []byte) error {
 	return err
 }
 
-func (c *Client) ForwardPacket(from, to key.Public, b []byte) error {
+func (c *Client) registerPing(m derp.PingMessage, ch chan<- bool) {
+	c.mu.Lock()
+	defer c.mu.Unlock()
+	if c.pingOut == nil {
+		c.pingOut = map[derp.PingMessage]chan<- bool{}
+	}
+	c.pingOut[m] = ch
+}
+
+func (c *Client) unregisterPing(m derp.PingMessage) {
+	c.mu.Lock()
+	defer c.mu.Unlock()
+	delete(c.pingOut, m)
+}
+
+func (c *Client) handledPong(m derp.PongMessage) bool {
+	c.mu.Lock()
+	defer c.mu.Unlock()
+	k := derp.PingMessage(m)
+	if ch, ok := c.pingOut[k]; ok {
+		ch <- true
+		delete(c.pingOut, k)
+		return true
+	}
+	return false
+}
+
+// Ping sends a ping to the peer and waits for it either to be
+// acknowledged (in which case Ping returns nil) or waits for ctx to
+// be over and returns an error. It will wait at most 5 seconds
+// before returning an error.
+//
+// Another goroutine must be in a loop calling Recv or
+// RecvDetail or ping responses won't be handled.
+func (c *Client) Ping(ctx context.Context) error {
+	maxDL := time.Now().Add(5 * time.Second)
+	if dl, ok := ctx.Deadline(); !ok || dl.After(maxDL) {
+		var cancel context.CancelFunc
+		ctx, cancel = context.WithDeadline(ctx, maxDL)
+		defer cancel()
+	}
+	var data derp.PingMessage
+	rand.Read(data[:])
+	gotPing := make(chan bool, 1)
+	c.registerPing(data, gotPing)
+	defer c.unregisterPing(data)
+	if err := c.SendPing(data); err != nil {
+		return err
+	}
+	select {
+	case <-gotPing:
+		return nil
+	case <-ctx.Done():
+		return ctx.Err()
+	}
+}
+
+// SendPing writes a ping message, without any implicit connect or
+// reconnect. This is a lower-level interface that writes a frame
+// without any implicit handling of the response pong, if any. For a
+// higher-level interface, use Ping.
+func (c *Client) SendPing(data [8]byte) error {
+	c.mu.Lock()
+	closed, client := c.closed, c.client
+	c.mu.Unlock()
+	if closed {
+		return ErrClientClosed
+	}
+	if client == nil {
+		return errors.New("client not connected")
+	}
+	return client.SendPing(data)
+}
+
+// LocalAddr reports c's local TCP address, without any implicit
+// connect or reconnect.
+func (c *Client) LocalAddr() (netaddr.IPPort, error) {
+	c.mu.Lock()
+	closed, client := c.closed, c.client
+	c.mu.Unlock()
+	if closed {
+		return netaddr.IPPort{}, ErrClientClosed
+	}
+	if client == nil {
+		return netaddr.IPPort{}, errors.New("client not connected")
+	}
+	return client.LocalAddr()
+}
+
+func (c *Client) ForwardPacket(from, to key.NodePublic, b []byte) error {
 	client, _, err := c.connect(context.TODO(), "derphttp.Client.ForwardPacket")
 	if err != nil {
 		return err
@@ -728,7 +928,7 @@ func (c *Client) WatchConnectionChanges() error {
 // ClosePeer asks the server to close target's TCP connection.
 //
 // Only trusted connections (using MeshKey) are allowed to use this.
-func (c *Client) ClosePeer(target key.Public) error {
+func (c *Client) ClosePeer(target key.NodePublic) error {
 	client, _, err := c.connect(context.TODO(), "derphttp.Client.ClosePeer")
 	if err != nil {
 		return err
@@ -754,14 +954,22 @@ func (c *Client) RecvDetail() (m derp.ReceivedMessage, connGen int, err error) {
 	if err != nil {
 		return nil, 0, err
 	}
-	m, err = client.Recv()
-	if err != nil {
-		c.closeForReconnect(client)
-		if c.isClosed() {
-			err = ErrClientClosed
+	for {
+		m, err = client.Recv()
+		switch m := m.(type) {
+		case derp.PongMessage:
+			if c.handledPong(m) {
+				continue
+			}
 		}
+		if err != nil {
+			c.closeForReconnect(client)
+			if c.isClosed() {
+				err = ErrClientClosed
+			}
+		}
+		return m, connGen, err
 	}
-	return m, connGen, err
 }
 
 func (c *Client) isClosed() bool {
@@ -812,15 +1020,15 @@ func (c *Client) closeForReconnect(brokenClient *derp.Client) {
 
 var ErrClientClosed = errors.New("derphttp.Client closed")
 
-func parseMetaCert(certs []*x509.Certificate) (serverPub key.Public, serverProtoVersion int) {
+func parseMetaCert(certs []*x509.Certificate) (serverPub key.NodePublic, serverProtoVersion int) {
 	for _, cert := range certs {
 		if cn := cert.Subject.CommonName; strings.HasPrefix(cn, "derpkey") {
 			var err error
-			serverPub, err = key.NewPublicFromHexMem(mem.S(strings.TrimPrefix(cn, "derpkey")))
+			serverPub, err = key.ParseNodePublicUntyped(mem.S(strings.TrimPrefix(cn, "derpkey")))
 			if err == nil && cert.SerialNumber.BitLen() <= 8 { // supports up to version 255
 				return serverPub, int(cert.SerialNumber.Int64())
 			}
 		}
 	}
-	return key.Public{}, 0
+	return key.NodePublic{}, 0
 }

derp/derphttp/derphttp_server.go

@@ -8,6 +8,7 @@ import (
 	"fmt"
 	"log"
 	"net/http"
+	"strings"
 
 	"tailscale.com/derp"
 )
@@ -20,10 +21,15 @@ const fastStartHeader = "Derp-Fast-Start"
 
 func Handler(s *derp.Server) http.Handler {
 	return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
-		if p := r.Header.Get("Upgrade"); p != "WebSocket" && p != "DERP" {
+		up := strings.ToLower(r.Header.Get("Upgrade"))
+		if up != "websocket" && up != "derp" {
+			if up != "" {
+				log.Printf("Weird upgrade: %q", up)
+			}
 			http.Error(w, "DERP requires connection upgrade", http.StatusUpgradeRequired)
 			return
 		}
+
 		fastStart := r.Header.Get(fastStartHeader) == "1"
 
 		h, ok := w.(http.Hijacker)
@@ -45,9 +51,9 @@ func Handler(s *derp.Server) http.Handler {
 				"Upgrade: DERP\r\n"+
 				"Connection: Upgrade\r\n"+
 				"Derp-Version: %v\r\n"+
-				"Derp-Public-Key: %x\r\n\r\n",
+				"Derp-Public-Key: %s\r\n\r\n",
 				derp.ProtocolVersion,
-				pubKey[:])
+				pubKey.UntypedHexString())
 		}
 
 		s.Accept(netConn, conn, netConn.RemoteAddr().String())

derp/derphttp/derphttp_test.go

@@ -18,13 +18,13 @@ import (
 )
 
 func TestSendRecv(t *testing.T) {
-	serverPrivateKey := key.NewPrivate()
+	serverPrivateKey := key.NewNode()
 
 	const numClients = 3
-	var clientPrivateKeys []key.Private
-	var clientKeys []key.Public
+	var clientPrivateKeys []key.NodePrivate
+	var clientKeys []key.NodePublic
 	for i := 0; i < numClients; i++ {
-		priv := key.NewPrivate()
+		priv := key.NewNode()
 		clientPrivateKeys = append(clientPrivateKeys, priv)
 		clientKeys = append(clientKeys, priv.Public())
 	}
@@ -154,3 +154,55 @@ func waitConnect(t testing.TB, c *Client) {
 		t.Fatalf("client first Recv was unexpected type %T", v)
 	}
 }
+
+func TestPing(t *testing.T) {
+	serverPrivateKey := key.NewNode()
+	s := derp.NewServer(serverPrivateKey, t.Logf)
+	defer s.Close()
+
+	httpsrv := &http.Server{
+		TLSNextProto: make(map[string]func(*http.Server, *tls.Conn, http.Handler)),
+		Handler:      Handler(s),
+	}
+
+	ln, err := net.Listen("tcp4", "localhost:0")
+	if err != nil {
+		t.Fatal(err)
+	}
+	serverURL := "http://" + ln.Addr().String()
+	t.Logf("server URL: %s", serverURL)
+
+	go func() {
+		if err := httpsrv.Serve(ln); err != nil {
+			if err == http.ErrServerClosed {
+				return
+			}
+			panic(err)
+		}
+	}()
+
+	c, err := NewClient(key.NewNode(), serverURL, t.Logf)
+	if err != nil {
+		t.Fatalf("NewClient: %v", err)
+	}
+	defer c.Close()
+	if err := c.Connect(context.Background()); err != nil {
+		t.Fatalf("client Connect: %v", err)
+	}
+
+	errc := make(chan error, 1)
+	go func() {
+		for {
+			m, err := c.Recv()
+			if err != nil {
+				errc <- err
+				return
+			}
+			t.Logf("Recv: %T", m)
+		}
+	}()
+	err = c.Ping(context.Background())
+	if err != nil {
+		t.Fatalf("Ping: %v", err)
+	}
+}

derp/derphttp/mesh_client.go

@@ -27,7 +27,7 @@ import (
 //
 // To force RunWatchConnectionLoop to return quickly, its ctx needs to
 // be closed, and c itself needs to be closed.
-func (c *Client) RunWatchConnectionLoop(ctx context.Context, ignoreServerKey key.Public, infoLogf logger.Logf, add, remove func(key.Public)) {
+func (c *Client) RunWatchConnectionLoop(ctx context.Context, ignoreServerKey key.NodePublic, infoLogf logger.Logf, add, remove func(key.NodePublic)) {
 	if infoLogf == nil {
 		infoLogf = logger.Discard
 	}
@@ -36,7 +36,7 @@ func (c *Client) RunWatchConnectionLoop(ctx context.Context, ignoreServerKey key
 	const statusInterval = 10 * time.Second
 	var (
 		mu              sync.Mutex
-		present         = map[key.Public]bool{}
+		present         = map[key.NodePublic]bool{}
 		loggedConnected = false
 	)
 	clear := func() {
@@ -49,7 +49,7 @@ func (c *Client) RunWatchConnectionLoop(ctx context.Context, ignoreServerKey key
 		for k := range present {
 			remove(k)
 		}
-		present = map[key.Public]bool{}
+		present = map[key.NodePublic]bool{}
 	}
 	lastConnGen := 0
 	lastStatus := time.Now()
@@ -69,7 +69,7 @@ func (c *Client) RunWatchConnectionLoop(ctx context.Context, ignoreServerKey key
 	})
 	defer timer.Stop()
 
-	updatePeer := func(k key.Public, isPresent bool) {
+	updatePeer := func(k key.NodePublic, isPresent bool) {
 		if isPresent {
 			add(k)
 		} else {
@@ -127,9 +127,9 @@ func (c *Client) RunWatchConnectionLoop(ctx context.Context, ignoreServerKey key
 			}
 			switch m := m.(type) {
 			case derp.PeerPresentMessage:
-				updatePeer(key.Public(m), true)
+				updatePeer(key.NodePublic(m), true)
 			case derp.PeerGoneMessage:
-				updatePeer(key.Public(m), false)
+				updatePeer(key.NodePublic(m), false)
 			default:
 				continue
 			}

derp/derphttp/websocket.go

@@ -0,0 +1,33 @@
+// Copyright (c) 2021 Tailscale Inc & AUTHORS All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+//go:build linux || js
+// +build linux js
+
+package derphttp
+
+import (
+	"context"
+	"log"
+	"net"
+
+	"nhooyr.io/websocket"
+	"tailscale.com/derp/wsconn"
+)
+
+func init() {
+	dialWebsocketFunc = dialWebsocket
+}
+
+func dialWebsocket(ctx context.Context, urlStr string) (net.Conn, error) {
+	c, res, err := websocket.Dial(ctx, urlStr, &websocket.DialOptions{
+		Subprotocols: []string{"derp"},
+	})
+	if err != nil {
+		log.Printf("websocket Dial: %v, %+v", err, res)
+		return nil, err
+	}
+	log.Printf("websocket: connected to %v", urlStr)
+	return wsconn.New(c), nil
+}

derp/wsconn/wsconn.go

@@ -0,0 +1,104 @@
+// Copyright (c) 2021 Tailscale Inc & AUTHORS All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+// Package wsconn contains an adapter type that turns
+// a websocket connection into a net.Conn.
+package wsconn
+
+import (
+	"context"
+	"net"
+	"sync"
+	"time"
+
+	"nhooyr.io/websocket"
+)
+
+// New returns a net.Conn wrapper around c,
+// using c to send and receive binary messages with
+// chunks of bytes with no defined framing, effectively
+// discarding all WebSocket-level message framing.
+func New(c *websocket.Conn) net.Conn {
+	return &websocketConn{c: c}
+}
+
+// websocketConn implements derp.Conn around a *websocket.Conn,
+// treating a websocket.Conn as a byte stream, ignoring the WebSocket
+// frame/message boundaries.
+type websocketConn struct {
+	c *websocket.Conn
+
+	// rextra are extra bytes owned by the reader.
+	rextra []byte
+
+	mu         sync.Mutex
+	rdeadline  time.Time
+	cancelRead context.CancelFunc
+}
+
+func (wc *websocketConn) LocalAddr() net.Addr  { return addr{} }
+func (wc *websocketConn) RemoteAddr() net.Addr { return addr{} }
+
+type addr struct{}
+
+func (addr) Network() string { return "websocket" }
+func (addr) String() string  { return "websocket" }
+
+func (wc *websocketConn) Read(p []byte) (n int, err error) {
+	// Drain any leftover from previously.
+	n = copy(p, wc.rextra)
+	if n > 0 {
+		wc.rextra = wc.rextra[n:]
+		return n, nil
+	}
+
+	var ctx context.Context
+	var cancel context.CancelFunc
+
+	wc.mu.Lock()
+	if dl := wc.rdeadline; !dl.IsZero() {
+		ctx, cancel = context.WithDeadline(context.Background(), wc.rdeadline)
+	} else {
+		ctx, cancel = context.WithDeadline(context.Background(), time.Now().Add(30*24*time.Hour))
+		wc.rdeadline = time.Time{}
+	}
+	wc.cancelRead = cancel
+	wc.mu.Unlock()
+	defer cancel()
+
+	_, buf, err := wc.c.Read(ctx)
+	n = copy(p, buf)
+	wc.rextra = buf[n:]
+	return n, err
+}
+
+func (wc *websocketConn) Write(p []byte) (n int, err error) {
+	err = wc.c.Write(context.Background(), websocket.MessageBinary, p)
+	if err != nil {
+		return 0, err
+	}
+	return len(p), nil
+}
+
+func (wc *websocketConn) Close() error { return wc.c.Close(websocket.StatusNormalClosure, "close") }
+
+func (wc *websocketConn) SetDeadline(t time.Time) error {
+	wc.SetReadDeadline(t)
+	wc.SetWriteDeadline(t)
+	return nil
+}
+
+func (wc *websocketConn) SetReadDeadline(t time.Time) error {
+	wc.mu.Lock()
+	defer wc.mu.Unlock()
+	if !t.IsZero() && (wc.rdeadline.IsZero() || t.Before(wc.rdeadline)) && wc.cancelRead != nil {
+		wc.cancelRead()
+	}
+	wc.rdeadline = t
+	return nil
+}
+
+func (wc *websocketConn) SetWriteDeadline(t time.Time) error {
+	return nil
+}

disco/disco.go

@@ -25,7 +25,9 @@ import (
 	"fmt"
 	"net"
 
+	"go4.org/mem"
 	"inet.af/netaddr"
+	"tailscale.com/types/key"
 )
 
 // Magic is the 6 byte header of all discovery messages.
@@ -106,12 +108,28 @@ func appendMsgHeader(b []byte, t MessageType, ver uint8, dataLen int) (all, data
 }
 
 type Ping struct {
+	// TxID is a random client-generated per-ping transaction ID.
 	TxID [12]byte
+
+	// NodeKey is allegedly the ping sender's wireguard public key.
+	// Old clients (~1.16.0 and earlier) don't send this field.
+	// It shouldn't be trusted by itself, but can be combined with
+	// netmap data to reduce the discokey:nodekey relation from 1:N to
+	// 1:1.
+	NodeKey key.NodePublic
 }
 
 func (m *Ping) AppendMarshal(b []byte) []byte {
-	ret, d := appendMsgHeader(b, TypePing, v0, 12)
-	copy(d, m.TxID[:])
+	dataLen := 12
+	hasKey := !m.NodeKey.IsZero()
+	if hasKey {
+		dataLen += key.NodePublicRawLen
+	}
+	ret, d := appendMsgHeader(b, TypePing, v0, dataLen)
+	n := copy(d, m.TxID[:])
+	if hasKey {
+		m.NodeKey.AppendTo(d[:n])
+	}
 	return ret
 }
 
@@ -120,7 +138,12 @@ func parsePing(ver uint8, p []byte) (m *Ping, err error) {
 		return nil, errShort
 	}
 	m = new(Ping)
-	copy(m.TxID[:], p)
+	p = p[copy(m.TxID[:], p):]
+	// Deliberately lax on longer-than-expected messages, for future
+	// compatibility.
+	if len(p) >= key.NodePublicRawLen {
+		m.NodeKey = key.NodePublicFromRaw32(mem.B(p[:key.NodePublicRawLen]))
+	}
 	return m, nil
 }
 

disco/disco_test.go

@@ -10,7 +10,9 @@ import (
 	"strings"
 	"testing"
 
+	"go4.org/mem"
 	"inet.af/netaddr"
+	"tailscale.com/types/key"
 )
 
 func TestMarshalAndParse(t *testing.T) {
@@ -26,6 +28,14 @@ func TestMarshalAndParse(t *testing.T) {
 			},
 			want: "01 00 01 02 03 04 05 06 07 08 09 0a 0b 0c",
 		},
+		{
+			name: "ping_with_nodekey_src",
+			m: &Ping{
+				TxID:    [12]byte{1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12},
+				NodeKey: key.NodePublicFromRaw32(mem.B([]byte{1: 1, 2: 2, 30: 30, 31: 31})),
+			},
+			want: "01 00 01 02 03 04 05 06 07 08 09 0a 0b 0c 00 01 02 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 1e 1f",
+		},
 		{
 			name: "pong",
 			m: &Pong{

docs/k8s/Makefile

@@ -32,3 +32,7 @@ userspace-sidecar:
 proxy:
 	@kubectl delete -f proxy.yaml --ignore-not-found --grace-period=0
 	@sed -e "s;{{KUBE_SECRET}};$(KUBE_SECRET);g" proxy.yaml | sed -e "s;{{SA_NAME}};$(SA_NAME);g" | sed -e "s;{{IMAGE_TAG}};$(IMAGE_TAG);g" | sed -e "s;{{DEST_IP}};$(DEST_IP);g" | kubectl create -f-
+
+subnet-router:
+	@kubectl delete -f subnet.yaml --ignore-not-found --grace-period=0
+	@sed -e "s;{{KUBE_SECRET}};$(KUBE_SECRET);g" subnet.yaml | sed -e "s;{{SA_NAME}};$(SA_NAME);g" | sed -e "s;{{IMAGE_TAG}};$(IMAGE_TAG);g" | sed -e "s;{{ROUTES}};$(ROUTES);g" | kubectl create -f-

docs/k8s/README.md

@@ -27,12 +27,12 @@ There are quite a few ways of running Tailscale inside a Kubernetes Cluster, som
    Configure RBAC to allow the Tailscale pod to read/write the `tailscale` secret.
    ```bash
    export SA_NAME=tailscale
-   export KUBE_SECRET=tailscale
+   export KUBE_SECRET=tailscale-auth
    make rbac
    ```
 
 ### Sample Sidecar
-Running as a sidecar allows you to directly expose a Kubernetes pod over Tailscale. This is particularly useful if you do not wish to expose a service on the public internet. This method allows bi-directional connectivty between the pod and other devices on the Tailnet. You can use [ACLs](https://tailscale.com/kb/1018/acls/) to control traffic flow.
+Running as a sidecar allows you to directly expose a Kubernetes pod over Tailscale. This is particularly useful if you do not wish to expose a service on the public internet. This method allows bi-directional connectivity between the pod and other devices on the Tailnet. You can use [ACLs](https://tailscale.com/kb/1018/acls/) to control traffic flow.
 
 1. Create and login to the sample nginx pod with a Tailscale sidecar
 
@@ -107,4 +107,41 @@ Running a Tailscale proxy allows you to provide inbound connectivity to a Kubern
 
    ```bash
    curl "http://$(tailscale ip -4 proxy)"
-   ```
+   ```
+
+### Subnet Router
+
+Running a Tailscale [subnet router](https://tailscale.com/kb/1019/subnets/) allows you to access
+the entire Kubernetes cluster network (assuming NetworkPolicies allow) over Tailscale.
+
+1. Identify the Pod/Service CIDRs that cover your Kubernetes cluster.  These will vary depending on [which CNI](https://kubernetes.io/docs/concepts/cluster-administration/networking/) you are using and on the Cloud Provider you are using. Add these to the `ROUTES` variable as comma-separated values.
+
+   ```bash
+   SERVICE_CIDR=10.20.0.0/16
+   POD_CIDR=10.42.0.0/15
+   export ROUTES=$SERVICE_CIDR,$POD_CIDR
+   ```
+
+1. Deploy the subnet-router pod.
+
+   ```bash
+   make subnet-router
+   # If not using an auth key, authenticate by grabbing the Login URL here:
+   kubectl logs subnet-router
+   ```
+
+1. In the [Tailscale admin console](https://login.tailscale.com/admin/machines), ensure that the
+routes for the subnet-router are enabled.
+
+1. Make sure that any client you want to connect from has `--accept-routes` enabled.
+
+1. Check if you can connect to a `ClusterIP` or a `PodIP` over Tailscale:
+
+   ```bash
+   # Get the Service IP
+   INTERNAL_IP="$(kubectl get svc <SVC_NAME> -o=jsonpath='{.spec.clusterIP}')"
+   # or, the Pod IP 
+   # INTERNAL_IP="$(kubectl get po <POD_NAME> -o=jsonpath='{.status.podIP}')" 
+   INTERNAL_PORT=8080 
+   curl http://$INTERNAL_IP:$INTERNAL_PORT
+   ```

docs/k8s/run.sh

@@ -53,7 +53,7 @@ tailscale --socket=/tmp/tailscaled.sock up ${UP_ARGS}
 
 if [[ ! -z "${DEST_IP}" ]]; then
   echo "Adding iptables rule for DNAT"
-  iptables -t nat -I PREROUTING -d "$(tailscale ip -4)" -j DNAT --to-destination "${DEST_IP}"
+  iptables -t nat -I PREROUTING -d "$(tailscale --socket=/tmp/tailscaled.sock ip -4)" -j DNAT --to-destination "${DEST_IP}"
 fi
 
-wait ${PID}
+wait ${PID}

docs/k8s/subnet.yaml

@@ -0,0 +1,32 @@
+# Copyright (c) 2021 Tailscale Inc & AUTHORS All rights reserved.
+# Use of this source code is governed by a BSD-style
+# license that can be found in the LICENSE file.
+apiVersion: v1
+kind: Pod
+metadata:
+  name: subnet-router
+  labels:
+    app: tailscale
+spec:
+  serviceAccountName: "{{SA_NAME}}"
+  containers:
+  - name: tailscale
+    imagePullPolicy: Always
+    image: "{{IMAGE_TAG}}"
+    env:
+    # Store the state in a k8s secret
+    - name: KUBE_SECRET
+      value: "{{KUBE_SECRET}}"
+    - name: USERSPACE
+      value: "true"
+    - name: AUTH_KEY
+      valueFrom:
+        secretKeyRef:
+          name: tailscale-auth
+          key: AUTH_KEY
+          optional: true
+    - name: ROUTES
+      value: "{{ROUTES}}"
+    securityContext:
+      runAsUser: 1000
+      runAsGroup: 1000

envknob/envknob.go

@@ -0,0 +1,144 @@
+// Copyright (c) 2022 Tailscale Inc & AUTHORS All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+// Package envknob provides access to environment-variable tweakable
+// debug settings.
+//
+// These are primarily knobs used by Tailscale developers during
+// development or by users when instructed to by Tailscale developers
+// when debugging something. They are not a stable interface and may
+// be removed or any time.
+//
+// A related package, control/controlknobs, are knobs that can be
+// changed at runtime by the control plane. Sometimes both are used:
+// an envknob for the default/explicit value, else falling back
+// to the controlknob value.
+package envknob
+
+import (
+	"log"
+	"os"
+	"strconv"
+	"sync"
+
+	"tailscale.com/types/opt"
+)
+
+var (
+	mu   sync.Mutex
+	set  = map[string]string{}
+	list []string
+)
+
+func noteEnv(k, v string) {
+	if v == "" {
+		return
+	}
+	mu.Lock()
+	defer mu.Unlock()
+	if _, ok := set[v]; !ok {
+		list = append(list, k)
+	}
+	set[k] = v
+}
+
+// logf is logger.Logf, but logger depends on envknob, so for circular
+// dependency reasons, make a type alias (so it's still assignable,
+// but has nice docs here).
+type logf = func(format string, args ...interface{})
+
+// LogCurrent logs the currently set environment knobs.
+func LogCurrent(logf logf) {
+	mu.Lock()
+	defer mu.Unlock()
+	for _, k := range list {
+		logf("envknob: %s=%q", k, set[k])
+	}
+}
+
+// String returns the named environment variable, using os.Getenv.
+//
+// If the variable is non-empty, it's also tracked & logged as being
+// an in-use knob.
+func String(envVar string) string {
+	v := os.Getenv(envVar)
+	noteEnv(envVar, v)
+	return v
+}
+
+// Bool returns the boolean value of the named environment variable.
+// If the variable is not set, it returns false.
+// An invalid value exits the binary with a failure.
+func Bool(envVar string) bool {
+	return boolOr(envVar, false)
+}
+
+// BoolDefaultTrue is like Bool, but returns true by default if the
+// environment variable isn't present.
+func BoolDefaultTrue(envVar string) bool {
+	return boolOr(envVar, true)
+}
+
+func boolOr(envVar string, implicitValue bool) bool {
+	val := os.Getenv(envVar)
+	if val == "" {
+		return implicitValue
+	}
+	b, err := strconv.ParseBool(val)
+	if err == nil {
+		noteEnv(envVar, strconv.FormatBool(b)) // canonicalize
+		return b
+	}
+	log.Fatalf("invalid boolean environment variable %s value %q", envVar, val)
+	panic("unreachable")
+}
+
+// LookupBool returns the boolean value of the named environment value.
+// The ok result is whether a value was set.
+// If the value isn't a valid int, it exits the program with a failure.
+func LookupBool(envVar string) (v bool, ok bool) {
+	val := os.Getenv(envVar)
+	if val == "" {
+		return false, false
+	}
+	b, err := strconv.ParseBool(val)
+	if err == nil {
+		return b, true
+	}
+	log.Fatalf("invalid boolean environment variable %s value %q", envVar, val)
+	panic("unreachable")
+}
+
+// OptBool is like Bool, but returns an opt.Bool, so the caller can
+// distinguish between implicitly and explicitly false.
+func OptBool(envVar string) opt.Bool {
+	b, ok := LookupBool(envVar)
+	if !ok {
+		return ""
+	}
+	var ret opt.Bool
+	ret.Set(b)
+	return ret
+}
+
+// LookupInt returns the integer value of the named environment value.
+// The ok result is whether a value was set.
+// If the value isn't a valid int, it exits the program with a failure.
+func LookupInt(envVar string) (v int, ok bool) {
+	val := os.Getenv(envVar)
+	if val == "" {
+		return 0, false
+	}
+	v, err := strconv.Atoi(val)
+	if err == nil {
+		noteEnv(envVar, val)
+		return v, true
+	}
+	log.Fatalf("invalid integer environment variable %s: %v", envVar, val)
+	panic("unreachable")
+}
+
+// UseWIPCode is whether TAILSCALE_USE_WIP_CODE is set to permit use
+// of Work-In-Progress code.
+func UseWIPCode() bool { return Bool("TAILSCALE_USE_WIP_CODE") }