debug

Signed-off-by: David Crawshaw <crawshaw@tailscale.com>

.github/workflows/cross-darwin.yml

@@ -1,53 +0,0 @@
-name: Darwin-Cross
-
-on:
-  push:
-    branches:
-      - main
-  pull_request:
-    branches:
-      - '*'
-
-jobs:
-  build:
-    runs-on: ubuntu-latest
-
-    if: "!contains(github.event.head_commit.message, '[ci skip]')"
-
-    steps:
-
-    - name: Set up Go
-      uses: actions/setup-go@v1
-      with:
-        go-version: 1.16
-      id: go
-
-    - name: Check out code into the Go module directory
-      uses: actions/checkout@v1
-
-    - name: macOS build cmd
-      env:
-        GOOS: darwin
-        GOARCH: amd64
-      run: go build ./cmd/...
-
-    - name: macOS build tests
-      env:
-        GOOS: darwin
-        GOARCH: amd64
-      run: for d in $(go list -f '{{if .TestGoFiles}}{{.Dir}}{{end}}' ./... ); do (echo $d; cd $d && go test -c ); done
-
-    - uses: k0kubun/action-slack@v2.0.0
-      with:
-        payload: |
-          {
-            "attachments": [{
-              "text": "${{ job.status }}: ${{ github.workflow }} <https://github.com/${{ github.repository }}/commit/${{ github.sha }}/checks|${{ env.COMMIT_DATE }} #${{ env.COMMIT_NUMBER_OF_DAY }}> " +
-                      "(<https://github.com/${{ github.repository }}/commit/${{ github.sha }}|" + "${{ github.sha }}".substring(0, 10) + ">) " +
-                      "of ${{ github.repository }}@" + "${{ github.ref }}".split('/').reverse()[0] + " by ${{ github.event.head_commit.committer.name }}",
-              "color": "danger"
-            }]
-          }
-      env:
-        SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK_URL }}
-      if: failure() && github.event_name == 'push'

.github/workflows/cross-freebsd.yml

@@ -1,53 +0,0 @@
-name: FreeBSD-Cross
-
-on:
-  push:
-    branches:
-      - main
-  pull_request:
-    branches:
-      - '*'
-
-jobs:
-  build:
-    runs-on: ubuntu-latest
-
-    if: "!contains(github.event.head_commit.message, '[ci skip]')"
-
-    steps:
-
-    - name: Set up Go
-      uses: actions/setup-go@v1
-      with:
-        go-version: 1.16
-      id: go
-
-    - name: Check out code into the Go module directory
-      uses: actions/checkout@v1
-
-    - name: FreeBSD build cmd
-      env:
-        GOOS: freebsd
-        GOARCH: amd64
-      run: go build ./cmd/...
-
-    - name: FreeBSD build tests
-      env:
-        GOOS: freebsd
-        GOARCH: amd64
-      run: for d in $(go list -f '{{if .TestGoFiles}}{{.Dir}}{{end}}' ./... ); do (echo $d; cd $d && go test -c ); done
-
-    - uses: k0kubun/action-slack@v2.0.0
-      with:
-        payload: |
-          {
-            "attachments": [{
-              "text": "${{ job.status }}: ${{ github.workflow }} <https://github.com/${{ github.repository }}/commit/${{ github.sha }}/checks|${{ env.COMMIT_DATE }} #${{ env.COMMIT_NUMBER_OF_DAY }}> " +
-                      "(<https://github.com/${{ github.repository }}/commit/${{ github.sha }}|" + "${{ github.sha }}".substring(0, 10) + ">) " +
-                      "of ${{ github.repository }}@" + "${{ github.ref }}".split('/').reverse()[0] + " by ${{ github.event.head_commit.committer.name }}",
-              "color": "danger"
-            }]
-          }
-      env:
-        SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK_URL }}
-      if: failure() && github.event_name == 'push'

.github/workflows/cross-openbsd.yml

@@ -1,53 +0,0 @@
-name: OpenBSD-Cross
-
-on:
-  push:
-    branches:
-      - main
-  pull_request:
-    branches:
-      - '*'
-
-jobs:
-  build:
-    runs-on: ubuntu-latest
-
-    if: "!contains(github.event.head_commit.message, '[ci skip]')"
-
-    steps:
-
-    - name: Set up Go
-      uses: actions/setup-go@v1
-      with:
-        go-version: 1.16
-      id: go
-
-    - name: Check out code into the Go module directory
-      uses: actions/checkout@v1
-
-    - name: OpenBSD build cmd
-      env:
-        GOOS: openbsd
-        GOARCH: amd64
-      run: go build ./cmd/...
-
-    - name: OpenBSD build tests
-      env:
-        GOOS: openbsd
-        GOARCH: amd64
-      run: for d in $(go list -f '{{if .TestGoFiles}}{{.Dir}}{{end}}' ./... ); do (echo $d; cd $d && go test -c ); done
-
-    - uses: k0kubun/action-slack@v2.0.0
-      with:
-        payload: |
-          {
-            "attachments": [{
-              "text": "${{ job.status }}: ${{ github.workflow }} <https://github.com/${{ github.repository }}/commit/${{ github.sha }}/checks|${{ env.COMMIT_DATE }} #${{ env.COMMIT_NUMBER_OF_DAY }}> " +
-                      "(<https://github.com/${{ github.repository }}/commit/${{ github.sha }}|" + "${{ github.sha }}".substring(0, 10) + ">) " +
-                      "of ${{ github.repository }}@" + "${{ github.ref }}".split('/').reverse()[0] + " by ${{ github.event.head_commit.committer.name }}",
-              "color": "danger"
-            }]
-          }
-      env:
-        SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK_URL }}
-      if: failure() && github.event_name == 'push'

.github/workflows/cross-windows.yml

@@ -1,53 +0,0 @@
-name: Windows-Cross
-
-on:
-  push:
-    branches:
-      - main
-  pull_request:
-    branches:
-      - '*'
-
-jobs:
-  build:
-    runs-on: ubuntu-latest
-
-    if: "!contains(github.event.head_commit.message, '[ci skip]')"
-
-    steps:
-
-    - name: Set up Go
-      uses: actions/setup-go@v1
-      with:
-        go-version: 1.16
-      id: go
-
-    - name: Check out code into the Go module directory
-      uses: actions/checkout@v1
-
-    - name: Windows build cmd
-      env:
-        GOOS: windows
-        GOARCH: amd64
-      run: go build ./cmd/...
-
-    - name: Windows build tests
-      env:
-        GOOS: windows
-        GOARCH: amd64
-      run: for d in $(go list -f '{{if .TestGoFiles}}{{.Dir}}{{end}}' ./... ); do (echo $d; cd $d && go test -c ); done
-
-    - uses: k0kubun/action-slack@v2.0.0
-      with:
-        payload: |
-          {
-            "attachments": [{
-              "text": "${{ job.status }}: ${{ github.workflow }} <https://github.com/${{ github.repository }}/commit/${{ github.sha }}/checks|${{ env.COMMIT_DATE }} #${{ env.COMMIT_NUMBER_OF_DAY }}> " +
-                      "(<https://github.com/${{ github.repository }}/commit/${{ github.sha }}|" + "${{ github.sha }}".substring(0, 10) + ">) " +
-                      "of ${{ github.repository }}@" + "${{ github.ref }}".split('/').reverse()[0] + " by ${{ github.event.head_commit.committer.name }}",
-              "color": "danger"
-            }]
-          }
-      env:
-        SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK_URL }}
-      if: failure() && github.event_name == 'push'

.github/workflows/depaware.yml

@@ -1,28 +0,0 @@
-name: depaware
-
-on:
-  push:
-    branches:
-      - main
-  pull_request:
-    branches:
-      - '*'
-
-jobs:
-  build:
-    runs-on: ubuntu-latest
-
-    steps:
-    - name: Set up Go
-      uses: actions/setup-go@v1
-      with:
-        go-version: 1.16
-
-    - name: Check out code
-      uses: actions/checkout@v1
-
-    - name: depaware tailscaled
-      run: go run github.com/tailscale/depaware --check tailscale.com/cmd/tailscaled
-
-    - name: depaware tailscale
-      run: go run github.com/tailscale/depaware --check tailscale.com/cmd/tailscale

.github/workflows/license.yml

@@ -1,40 +0,0 @@
-name: license
-
-on:
-  push:
-    branches:
-      - main
-  pull_request:
-    branches:
-      - '*'
-
-jobs:
-  build:
-    runs-on: ubuntu-latest
-
-    steps:
-    - name: Set up Go
-      uses: actions/setup-go@v1
-      with:
-        go-version: 1.16
-
-    - name: Check out code
-      uses: actions/checkout@v1
-
-    - name: Run license checker
-      run: ./scripts/check_license_headers.sh .
-
-    - uses: k0kubun/action-slack@v2.0.0
-      with:
-        payload: |
-          {
-            "attachments": [{
-              "text": "${{ job.status }}: ${{ github.workflow }} <https://github.com/${{ github.repository }}/commit/${{ github.sha }}/checks|${{ env.COMMIT_DATE }} #${{ env.COMMIT_NUMBER_OF_DAY }}> " +
-                      "(<https://github.com/${{ github.repository }}/commit/${{ github.sha }}|" + "${{ github.sha }}".substring(0, 10) + ">) " +
-                      "of ${{ github.repository }}@" + "${{ github.ref }}".split('/').reverse()[0] + " by ${{ github.event.head_commit.committer.name }}",
-              "color": "danger"
-            }]
-          }
-      env:
-        SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK_URL }}
-      if: failure() && github.event_name == 'push'

.github/workflows/linux-race.yml

@@ -1,48 +0,0 @@
-name: Linux race
-
-on:
-  push:
-    branches:
-      - main
-  pull_request:
-    branches:
-      - '*'
-
-jobs:
-  build:
-    runs-on: ubuntu-latest
-
-    if: "!contains(github.event.head_commit.message, '[ci skip]')"
-
-    steps:
-
-    - name: Set up Go
-      uses: actions/setup-go@v1
-      with:
-        go-version: 1.16
-      id: go
-
-    - name: Check out code into the Go module directory
-      uses: actions/checkout@v1
-
-    - name: Basic build
-      run: go build ./cmd/...
-
-    - name: Run tests and benchmarks with -race flag on linux
-      run: go test -race -bench=. -benchtime=1x ./...
-
-    - uses: k0kubun/action-slack@v2.0.0
-      with:
-        payload: |
-          {
-            "attachments": [{
-              "text": "${{ job.status }}: ${{ github.workflow }} <https://github.com/${{ github.repository }}/commit/${{ github.sha }}/checks|${{ env.COMMIT_DATE }} #${{ env.COMMIT_NUMBER_OF_DAY }}> " +
-                      "(<https://github.com/${{ github.repository }}/commit/${{ github.sha }}|" + "${{ github.sha }}".substring(0, 10) + ">) " +
-                      "of ${{ github.repository }}@" + "${{ github.ref }}".split('/').reverse()[0] + " by ${{ github.event.head_commit.committer.name }}",
-              "color": "danger"
-            }]
-          }
-      env:
-        SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK_URL }}
-      if: failure() && github.event_name == 'push'
-

.github/workflows/linux.yml

@@ -1,48 +0,0 @@
-name: Linux
-
-on:
-  push:
-    branches:
-      - main
-  pull_request:
-    branches:
-      - '*'
-
-jobs:
-  build:
-    runs-on: ubuntu-latest
-
-    if: "!contains(github.event.head_commit.message, '[ci skip]')"
-
-    steps:
-
-    - name: Set up Go
-      uses: actions/setup-go@v1
-      with:
-        go-version: 1.16
-      id: go
-
-    - name: Check out code into the Go module directory
-      uses: actions/checkout@v1
-
-    - name: Basic build
-      run: go build ./cmd/...
-
-    - name: Run tests on linux
-      run: go test -bench=. -benchtime=1x ./...
-
-    - uses: k0kubun/action-slack@v2.0.0
-      with:
-        payload: |
-          {
-            "attachments": [{
-              "text": "${{ job.status }}: ${{ github.workflow }} <https://github.com/${{ github.repository }}/commit/${{ github.sha }}/checks|${{ env.COMMIT_DATE }} #${{ env.COMMIT_NUMBER_OF_DAY }}> " +
-                      "(<https://github.com/${{ github.repository }}/commit/${{ github.sha }}|" + "${{ github.sha }}".substring(0, 10) + ">) " +
-                      "of ${{ github.repository }}@" + "${{ github.ref }}".split('/').reverse()[0] + " by ${{ github.event.head_commit.committer.name }}",
-              "color": "danger"
-            }]
-          }
-      env:
-        SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK_URL }}
-      if: failure() && github.event_name == 'push'
-

.github/workflows/linux32.yml

@@ -1,48 +0,0 @@
-name: Linux 32-bit
-
-on:
-  push:
-    branches:
-      - main
-  pull_request:
-    branches:
-      - '*'
-
-jobs:
-  build:
-    runs-on: ubuntu-latest
-
-    if: "!contains(github.event.head_commit.message, '[ci skip]')"
-
-    steps:
-
-    - name: Set up Go
-      uses: actions/setup-go@v1
-      with:
-        go-version: 1.16
-      id: go
-
-    - name: Check out code into the Go module directory
-      uses: actions/checkout@v1
-
-    - name: Basic build
-      run: GOARCH=386 go build ./cmd/...
-
-    - name: Run tests on linux
-      run: GOARCH=386 go test -bench=. -benchtime=1x ./...
-
-    - uses: k0kubun/action-slack@v2.0.0
-      with:
-        payload: |
-          {
-            "attachments": [{
-              "text": "${{ job.status }}: ${{ github.workflow }} <https://github.com/${{ github.repository }}/commit/${{ github.sha }}/checks|${{ env.COMMIT_DATE }} #${{ env.COMMIT_NUMBER_OF_DAY }}> " +
-                      "(<https://github.com/${{ github.repository }}/commit/${{ github.sha }}|" + "${{ github.sha }}".substring(0, 10) + ">) " +
-                      "of ${{ github.repository }}@" + "${{ github.ref }}".split('/').reverse()[0] + " by ${{ github.event.head_commit.committer.name }}",
-              "color": "danger"
-            }]
-          }
-      env:
-        SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK_URL }}
-      if: failure() && github.event_name == 'push'
-

.github/workflows/staticcheck.yml

@@ -1,58 +0,0 @@
-name: staticcheck
-
-on:
-  push:
-    branches:
-      - main
-  pull_request:
-    branches:
-      - '*'
-
-jobs:
-  build:
-    runs-on: ubuntu-latest
-
-    steps:
-    - name: Set up Go
-      uses: actions/setup-go@v1
-      with:
-        go-version: 1.16
-
-    - name: Check out code
-      uses: actions/checkout@v1
-
-    - name: Run go vet
-      run: go vet ./...
-
-    - name: Install staticcheck
-      run: "GOBIN=~/.local/bin go install honnef.co/go/tools/cmd/staticcheck"
-
-    - name: Print staticcheck version
-      run: "staticcheck -version"
-
-    - name: Run staticcheck (linux/amd64)
-      run: "GOOS=linux GOARCH=amd64 staticcheck -- $(go list ./... | grep -v tempfork)"
-
-    - name: Run staticcheck (darwin/amd64)
-      run: "GOOS=darwin GOARCH=amd64 staticcheck -- $(go list ./... | grep -v tempfork)"
-
-    - name: Run staticcheck (windows/amd64)
-      run: "GOOS=windows GOARCH=amd64 staticcheck -- $(go list ./... | grep -v tempfork)"
-
-    - name: Run staticcheck (windows/386)
-      run: "GOOS=windows GOARCH=386 staticcheck -- $(go list ./... | grep -v tempfork)"
-
-    - uses: k0kubun/action-slack@v2.0.0
-      with:
-        payload: |
-          {
-            "attachments": [{
-              "text": "${{ job.status }}: ${{ github.workflow }} <https://github.com/${{ github.repository }}/commit/${{ github.sha }}/checks|${{ env.COMMIT_DATE }} #${{ env.COMMIT_NUMBER_OF_DAY }}> " +
-                      "(<https://github.com/${{ github.repository }}/commit/${{ github.sha }}|" + "${{ github.sha }}".substring(0, 10) + ">) " +
-                      "of ${{ github.repository }}@" + "${{ github.ref }}".split('/').reverse()[0] + " by ${{ github.event.head_commit.committer.name }}",
-              "color": "danger"
-            }]
-          }
-      env:
-        SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK_URL }}
-      if: failure() && github.event_name == 'push'

.github/workflows/windows-race.yml

@@ -36,7 +36,7 @@ jobs:
       # Don't use -bench=. -benchtime=1x.
       # Somewhere in the layers (powershell?)
       # the equals signs cause great confusion.
-      run: go test -race -bench . -benchtime 1x ./...
+      run: go test -race -bench . -benchtime 1x -timeout 5m -v ./...
 
     - uses: k0kubun/action-slack@v2.0.0
       with:

.github/workflows/xe-experimental-vm-test.yml

@@ -1,47 +0,0 @@
-name: "integration-vms"
-
-on:
-  pull_request:
-    paths:
-      - "tstest/integration/vms/**"
-  push:
-    branches: [ main ]
-  release:
-    types: [ created ]
-
-jobs:
-  experimental-linux-vm-test:
-    # To set up a new runner, see tstest/integration/vms/runner.nix
-    runs-on: [ self-hosted, linux, vm_integration_test ]
-
-    if: "!contains(github.event.head_commit.message, '[ci skip]')"
-
-    steps:
-      - name: Checkout Code
-        uses: actions/checkout@v1
-
-      - name: Download VM Images
-        run: go test ./tstest/integration/vms -run-vm-tests -run=Download -timeout=60m -no-s3
-        env:
-          XDG_CACHE_HOME: "/var/lib/ghrunner/cache"
-
-      - name: Run VM tests
-        run: go test ./tstest/integration/vms -v -run-vm-tests
-        env:
-          TMPDIR: "/tmp"
-          XDG_CACHE_HOME: "/var/lib/ghrunner/cache"
-
-      - uses: k0kubun/action-slack@v2.0.0
-        with:
-          payload: |
-            {
-              "attachments": [{
-                "text": "${{ job.status }}: ${{ github.workflow }} <https://github.com/${{ github.repository }}/commit/${{ github.sha }}/checks|${{ env.COMMIT_DATE }} #${{ env.COMMIT_NUMBER_OF_DAY }}> " +
-                        "(<https://github.com/${{ github.repository }}/commit/${{ github.sha }}|" + "${{ github.sha }}".substring(0, 10) + ">) " +
-                        "of ${{ github.repository }}@" + "${{ github.ref }}".split('/').reverse()[0] + " by ${{ github.event.head_commit.committer.name }}",
-                "color": "danger"
-              }]
-            }
-        env:
-          SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK_URL }}
-        if: failure() && github.event_name == 'push'

Dockerfile

@@ -42,8 +42,7 @@ FROM golang:1.16-alpine AS build-env
 
 WORKDIR /go/src/tailscale
 
-COPY go.mod .
-COPY go.sum .
+COPY go.mod go.sum ./
 RUN go mod download
 
 COPY . .

VERSION.txt

@@ -1 +1 @@
-1.11.0
+1.13.0

cmd/addlicense/main.go

@@ -0,0 +1,77 @@
+// Copyright (c) 2021 Tailscale Inc & AUTHORS All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+// Program addlicense adds a license header to a file.
+// It is intended for use with 'go generate',
+// so it has a slightly weird usage.
+package main
+
+import (
+	"flag"
+	"fmt"
+	"os"
+	"os/exec"
+)
+
+var (
+	year = flag.Int("year", 0, "copyright year")
+	file = flag.String("file", "", "file to modify")
+)
+
+func usage() {
+	fmt.Fprintf(os.Stderr, `
+usage: addlicense -year YEAR -file FILE <subcommand args...>
+`[1:])
+
+	flag.PrintDefaults()
+	fmt.Fprintf(os.Stderr, `
+addlicense adds a Tailscale license to the beginning of file,
+using year as the copyright year.
+
+It is intended for use with 'go generate', so it also runs a subcommand,
+which presumably creates the file.
+
+Sample usage:
+
+addlicense -year 2021 -file pull_strings.go stringer -type=pull
+`[1:])
+	os.Exit(2)
+}
+
+func main() {
+	flag.Usage = usage
+	flag.Parse()
+	if len(flag.Args()) == 0 {
+		flag.Usage()
+	}
+	cmd := exec.Command(flag.Arg(0), flag.Args()[1:]...)
+	cmd.Stdout = os.Stdout
+	cmd.Stderr = os.Stderr
+	err := cmd.Run()
+	check(err)
+	b, err := os.ReadFile(*file)
+	check(err)
+	f, err := os.OpenFile(*file, os.O_TRUNC|os.O_WRONLY, 0644)
+	check(err)
+	_, err = fmt.Fprintf(f, license, *year)
+	check(err)
+	_, err = f.Write(b)
+	check(err)
+	err = f.Close()
+	check(err)
+}
+
+func check(err error) {
+	if err != nil {
+		fmt.Fprintln(os.Stderr, err)
+		os.Exit(1)
+	}
+}
+
+var license = `
+// Copyright (c) %d Tailscale Inc & AUTHORS All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+`[1:]

cmd/derper/derper.go

@@ -58,7 +58,12 @@ func loadConfig() config {
 		return config{PrivateKey: mustNewKey()}
 	}
 	if *configPath == "" {
-		log.Fatalf("derper: -c <config path> not specified")
+		if os.Getuid() == 0 {
+			*configPath = "/var/lib/derper/derper.key"
+		} else {
+			log.Fatalf("derper: -c <config path> not specified")
+		}
+		log.Printf("no config path specified; using %s", *configPath)
 	}
 	b, err := ioutil.ReadFile(*configPath)
 	switch {

cmd/derpprobe/derpprobe.go

@@ -0,0 +1,354 @@
+// Copyright (c) 2021 Tailscale Inc & AUTHORS All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+// The derpprobe binary probes derpers.
+package main // import "tailscale.com/cmd/derper/derpprobe"
+
+import (
+	"bytes"
+	"context"
+	crand "crypto/rand"
+	"encoding/json"
+	"flag"
+	"fmt"
+	"html"
+	"io"
+	"log"
+	"net/http"
+	"sort"
+	"sync"
+	"time"
+
+	"tailscale.com/derp"
+	"tailscale.com/derp/derphttp"
+	"tailscale.com/tailcfg"
+	"tailscale.com/types/key"
+)
+
+var (
+	derpMapURL = flag.String("derp-map", "https://login.tailscale.com/derpmap/default", "URL to DERP map (https:// or file://)")
+	listen     = flag.String("listen", ":8030", "HTTP listen address")
+)
+
+var (
+	mu            sync.Mutex
+	state         = map[nodePair]pairStatus{}
+	lastDERPMap   *tailcfg.DERPMap
+	lastDERPMapAt time.Time
+)
+
+func main() {
+	flag.Parse()
+	go probeLoop()
+	log.Fatal(http.ListenAndServe(*listen, http.HandlerFunc(serve)))
+}
+
+type overallStatus struct {
+	good, bad []string
+}
+
+func (st *overallStatus) addBadf(format string, a ...interface{}) {
+	st.bad = append(st.bad, fmt.Sprintf(format, a...))
+}
+
+func (st *overallStatus) addGoodf(format string, a ...interface{}) {
+	st.good = append(st.good, fmt.Sprintf(format, a...))
+}
+
+func getOverallStatus() (o overallStatus) {
+	mu.Lock()
+	defer mu.Unlock()
+	if lastDERPMap == nil {
+		o.addBadf("no DERP map")
+		return
+	}
+	now := time.Now()
+	if age := now.Sub(lastDERPMapAt); age > time.Minute {
+		o.addBadf("DERPMap hasn't been successfully refreshed in %v", age.Round(time.Second))
+	}
+	for _, reg := range sortedRegions(lastDERPMap) {
+		for _, from := range reg.Nodes {
+			for _, to := range reg.Nodes {
+				pair := nodePair{from.Name, to.Name}
+				st, ok := state[pair]
+				age := now.Sub(st.at).Round(time.Second)
+				switch {
+				case !ok:
+					o.addBadf("no state for %v", pair)
+				case st.err != nil:
+					o.addBadf("%v: %v", pair, st.err)
+				case age > 90*time.Second:
+					o.addBadf("%v: update is %v old", pair, age)
+				default:
+					o.addGoodf("%v: %v, %v ago", pair, st.latency.Round(time.Millisecond), age)
+				}
+			}
+		}
+	}
+	return
+}
+
+func serve(w http.ResponseWriter, r *http.Request) {
+	st := getOverallStatus()
+	summary := "All good"
+	if len(st.bad) > 0 {
+		w.WriteHeader(500)
+		summary = fmt.Sprintf("%d problems", len(st.bad))
+	}
+	io.WriteString(w, "<html><head><style>.bad { font-weight: bold; color: #700; }</style></head>\n")
+	fmt.Fprintf(w, "<body><h1>derp probe</h1>\n%s:<ul>", summary)
+	for _, s := range st.bad {
+		fmt.Fprintf(w, "<li class=bad>%s</li>\n", html.EscapeString(s))
+	}
+	for _, s := range st.good {
+		fmt.Fprintf(w, "<li>%s</li>\n", html.EscapeString(s))
+	}
+	io.WriteString(w, "</ul></body></html>\n")
+}
+
+func sortedRegions(dm *tailcfg.DERPMap) []*tailcfg.DERPRegion {
+	ret := make([]*tailcfg.DERPRegion, 0, len(dm.Regions))
+	for _, r := range dm.Regions {
+		ret = append(ret, r)
+	}
+	sort.Slice(ret, func(i, j int) bool { return ret[i].RegionID < ret[j].RegionID })
+	return ret
+}
+
+type nodePair struct {
+	from, to string // DERPNode.Name
+}
+
+func (p nodePair) String() string { return fmt.Sprintf("(%s→%s)", p.from, p.to) }
+
+type pairStatus struct {
+	err     error
+	latency time.Duration
+	at      time.Time
+}
+
+func setDERPMap(dm *tailcfg.DERPMap) {
+	mu.Lock()
+	defer mu.Unlock()
+	lastDERPMap = dm
+	lastDERPMapAt = time.Now()
+}
+
+func setState(p nodePair, latency time.Duration, err error) {
+	mu.Lock()
+	defer mu.Unlock()
+	st := pairStatus{
+		err:     err,
+		latency: latency,
+		at:      time.Now(),
+	}
+	state[p] = st
+	if err != nil {
+		log.Printf("%+v error: %v", p, err)
+	} else {
+		log.Printf("%+v: %v", p, latency.Round(time.Millisecond))
+	}
+}
+
+func probeLoop() {
+	ticker := time.NewTicker(15 * time.Second)
+	for {
+		err := probe()
+		if err != nil {
+			log.Printf("probe: %v", err)
+		}
+		<-ticker.C
+	}
+}
+
+func probe() error {
+	ctx, cancel := context.WithTimeout(context.Background(), 60*time.Second)
+	defer cancel()
+	dm, err := getDERPMap(ctx)
+	if err != nil {
+		return err
+	}
+
+	var wg sync.WaitGroup
+	wg.Add(len(dm.Regions))
+	for _, reg := range dm.Regions {
+		reg := reg
+		go func() {
+			defer wg.Done()
+			for _, from := range reg.Nodes {
+				for _, to := range reg.Nodes {
+					latency, err := probeNodePair(ctx, dm, from, to)
+					setState(nodePair{from.Name, to.Name}, latency, err)
+				}
+			}
+		}()
+	}
+
+	wg.Wait()
+	return ctx.Err()
+}
+
+func probeNodePair(ctx context.Context, dm *tailcfg.DERPMap, from, to *tailcfg.DERPNode) (latency time.Duration, err error) {
+	// The passed in context is a minute for the whole region. The
+	// idea is that each node pair in the region will be done
+	// serially and regularly in the future, reusing connections
+	// (at least in the happy path). For now they don't reuse
+	// connections and probe at most once every 15 seconds. We
+	// bound the duration of a single node pair within a region
+	// so one bad one can't starve others.
+	ctx, cancel := context.WithTimeout(ctx, 10*time.Second)
+	defer cancel()
+
+	fromc, err := newConn(ctx, dm, from)
+	if err != nil {
+		return 0, err
+	}
+	defer fromc.Close()
+	toc, err := newConn(ctx, dm, to)
+	if err != nil {
+		return 0, err
+	}
+	defer toc.Close()
+
+	// Wait a bit for from's node to hear about to existing on the
+	// other node in the region, in the case where the two nodes
+	// are different.
+	if from.Name != to.Name {
+		time.Sleep(100 * time.Millisecond) // pretty arbitrary
+	}
+
+	// Make a random packet
+	pkt := make([]byte, 8)
+	crand.Read(pkt)
+
+	t0 := time.Now()
+
+	// Send the random packet.
+	sendc := make(chan error, 1)
+	go func() {
+		sendc <- fromc.Send(toc.SelfPublicKey(), pkt)
+	}()
+	select {
+	case <-ctx.Done():
+		return 0, fmt.Errorf("timeout sending via %q: %w", from.Name, ctx.Err())
+	case err := <-sendc:
+		if err != nil {
+			return 0, fmt.Errorf("error sending via %q: %w", from.Name, err)
+		}
+	}
+
+	// Receive the random packet.
+	recvc := make(chan interface{}, 1) // either derp.ReceivedPacket or error
+	go func() {
+		for {
+			m, err := toc.Recv()
+			if err != nil {
+				recvc <- err
+				return
+			}
+			switch v := m.(type) {
+			case derp.ReceivedPacket:
+				recvc <- v
+			default:
+				log.Printf("%v: ignoring Recv frame type %T", to.Name, v)
+				// Loop.
+			}
+		}
+	}()
+	select {
+	case <-ctx.Done():
+		return 0, fmt.Errorf("timeout receiving from %q: %w", to.Name, ctx.Err())
+	case v := <-recvc:
+		if err, ok := v.(error); ok {
+			return 0, fmt.Errorf("error receiving from %q: %w", to.Name, err)
+		}
+		p := v.(derp.ReceivedPacket)
+		if p.Source != fromc.SelfPublicKey() {
+			return 0, fmt.Errorf("got data packet from unexpected source, %v", p.Source)
+		}
+		if !bytes.Equal(p.Data, pkt) {
+			return 0, fmt.Errorf("unexpected data packet %q", p.Data)
+		}
+	}
+	return time.Since(t0), nil
+}
+
+func newConn(ctx context.Context, dm *tailcfg.DERPMap, n *tailcfg.DERPNode) (*derphttp.Client, error) {
+	priv := key.NewPrivate()
+	dc := derphttp.NewRegionClient(priv, log.Printf, func() *tailcfg.DERPRegion {
+		rid := n.RegionID
+		return &tailcfg.DERPRegion{
+			RegionID:   rid,
+			RegionCode: fmt.Sprintf("%s-%s", dm.Regions[rid].RegionCode, n.Name),
+			RegionName: dm.Regions[rid].RegionName,
+			Nodes:      []*tailcfg.DERPNode{n},
+		}
+	})
+	dc.IsProber = true
+	err := dc.Connect(ctx)
+	if err != nil {
+		return nil, err
+	}
+	errc := make(chan error, 1)
+	go func() {
+		m, err := dc.Recv()
+		if err != nil {
+			errc <- err
+			return
+		}
+		switch m.(type) {
+		case derp.ServerInfoMessage:
+			errc <- nil
+		default:
+			errc <- fmt.Errorf("unexpected first message type %T", errc)
+		}
+	}()
+	select {
+	case err := <-errc:
+		if err != nil {
+			go dc.Close()
+			return nil, err
+		}
+	case <-ctx.Done():
+		go dc.Close()
+		return nil, fmt.Errorf("timeout waiting for ServerInfoMessage: %w", ctx.Err())
+	}
+	return dc, nil
+}
+
+var httpOrFileClient = &http.Client{Transport: httpOrFileTransport()}
+
+func httpOrFileTransport() http.RoundTripper {
+	tr := http.DefaultTransport.(*http.Transport).Clone()
+	tr.RegisterProtocol("file", http.NewFileTransport(http.Dir("/")))
+	return tr
+}
+
+func getDERPMap(ctx context.Context) (*tailcfg.DERPMap, error) {
+	req, err := http.NewRequestWithContext(ctx, "GET", *derpMapURL, nil)
+	if err != nil {
+		return nil, err
+	}
+	res, err := httpOrFileClient.Do(req)
+	if err != nil {
+		mu.Lock()
+		defer mu.Unlock()
+		if lastDERPMap != nil && time.Since(lastDERPMapAt) < 10*time.Minute {
+			// Assume that control is restarting and use
+			// the same one for a bit.
+			return lastDERPMap, nil
+		}
+		return nil, err
+	}
+	defer res.Body.Close()
+	if res.StatusCode != 200 {
+		return nil, fmt.Errorf("fetching %s: %s", *derpMapURL, res.Status)
+	}
+	dm := new(tailcfg.DERPMap)
+	if err := json.NewDecoder(res.Body).Decode(dm); err != nil {
+		return nil, fmt.Errorf("decoding %s JSON: %v", *derpMapURL, err)
+	}
+	setDERPMap(dm)
+	return dm, nil
+}

cmd/speedtest/speedtest.go

@@ -0,0 +1,121 @@
+// Copyright (c) 2021 Tailscale Inc & AUTHORS All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+// Program speedtest provides the speedtest command. The reason to keep it separate from
+// the normal tailscale cli is because it is not yet ready to go in the tailscale binary.
+// It will be included in the tailscale cli after it has been added to tailscaled.
+
+// Example usage for client command: go run cmd/speedtest -host 127.0.0.1:20333 -t 5s
+// This will connect to the server on 127.0.0.1:20333 and start a 5 second download speedtest.
+// Example usage for server command: go run cmd/speedtest -s -host :20333
+// This will start a speedtest server on port 20333.
+package main
+
+import (
+	"context"
+	"errors"
+	"flag"
+	"fmt"
+	"net"
+	"os"
+	"strconv"
+	"text/tabwriter"
+	"time"
+
+	"github.com/peterbourgon/ff/v2/ffcli"
+	"tailscale.com/net/speedtest"
+)
+
+// Runs the speedtest command as a commandline program
+func main() {
+	args := os.Args[1:]
+	if err := speedtestCmd.Parse(args); err != nil {
+		fmt.Fprintln(os.Stderr, err.Error())
+		os.Exit(1)
+	}
+
+	err := speedtestCmd.Run(context.Background())
+	if errors.Is(err, flag.ErrHelp) {
+		fmt.Fprintln(os.Stderr, speedtestCmd.ShortUsage)
+		os.Exit(2)
+	}
+	if err != nil {
+		fmt.Fprintln(os.Stderr, err.Error())
+		os.Exit(1)
+	}
+}
+
+// speedtestCmd is the root command. It runs either the server or client depending on the
+// flags passed to it.
+var speedtestCmd = &ffcli.Command{
+	Name:       "speedtest",
+	ShortUsage: "speedtest [-host <host:port>] [-s] [-r] [-t <test duration>]",
+	ShortHelp:  "Run a speed test",
+	FlagSet: (func() *flag.FlagSet {
+		fs := flag.NewFlagSet("speedtest", flag.ExitOnError)
+		fs.StringVar(&speedtestArgs.host, "host", ":20333", "host:port pair to connect to or listen on")
+		fs.DurationVar(&speedtestArgs.testDuration, "t", speedtest.DefaultDuration, "duration of the speed test")
+		fs.BoolVar(&speedtestArgs.runServer, "s", false, "run a speedtest server")
+		fs.BoolVar(&speedtestArgs.reverse, "r", false, "run in reverse mode (server sends, client receives)")
+		return fs
+	})(),
+	Exec: runSpeedtest,
+}
+
+var speedtestArgs struct {
+	host         string
+	testDuration time.Duration
+	runServer    bool
+	reverse      bool
+}
+
+func runSpeedtest(ctx context.Context, args []string) error {
+
+	if _, _, err := net.SplitHostPort(speedtestArgs.host); err != nil {
+		var addrErr *net.AddrError
+		if errors.As(err, &addrErr) && addrErr.Err == "missing port in address" {
+			// if no port is provided, append the default port
+			speedtestArgs.host = net.JoinHostPort(speedtestArgs.host, strconv.Itoa(speedtest.DefaultPort))
+		}
+	}
+
+	if speedtestArgs.runServer {
+		listener, err := net.Listen("tcp", speedtestArgs.host)
+		if err != nil {
+			return err
+		}
+
+		fmt.Printf("listening on %v\n", listener.Addr())
+
+		return speedtest.Serve(listener)
+	}
+
+	// Ensure the duration is within the allowed range
+	if speedtestArgs.testDuration < speedtest.MinDuration || speedtestArgs.testDuration > speedtest.MaxDuration {
+		return fmt.Errorf("test duration must be within %v and %v", speedtest.MinDuration, speedtest.MaxDuration)
+	}
+
+	dir := speedtest.Download
+	if speedtestArgs.reverse {
+		dir = speedtest.Upload
+	}
+
+	fmt.Printf("Starting a %s test with %s\n", dir, speedtestArgs.host)
+	results, err := speedtest.RunClient(dir, speedtestArgs.testDuration, speedtestArgs.host)
+	if err != nil {
+		return err
+	}
+
+	w := tabwriter.NewWriter(os.Stdout, 12, 0, 0, ' ', tabwriter.TabIndent)
+	fmt.Println("Results:")
+	fmt.Fprintln(w, "Interval\t\tTransfer\t\tBandwidth\t\t")
+	for _, r := range results {
+		if r.Total {
+			fmt.Fprintln(w, "-------------------------------------------------------------------------")
+		}
+		fmt.Fprintf(w, "%.2f-%.2f\tsec\t%.4f\tMBits\t%.4f\tMbits/sec\t\n", r.IntervalStart.Seconds(), r.IntervalEnd.Seconds(), r.MegaBits(), r.MBitsPerSecond())
+	}
+	w.Flush()
+	return nil
+}

cmd/tailscale/cli/cli_test.go

@@ -13,9 +13,11 @@ import (
 	"strings"
 	"testing"
 
+	"github.com/google/go-cmp/cmp"
 	"inet.af/netaddr"
 	"tailscale.com/ipn"
 	"tailscale.com/ipn/ipnstate"
+	"tailscale.com/types/persist"
 	"tailscale.com/types/preftype"
 )
 
@@ -440,8 +442,9 @@ func TestCheckForAccidentalSettingReverts(t *testing.T) {
 			}
 			applyImplicitPrefs(newPrefs, tt.curPrefs, tt.curUser)
 			var got string
-			if err := checkForAccidentalSettingReverts(flagSet, tt.curPrefs, newPrefs, upCheckEnv{
+			if err := checkForAccidentalSettingReverts(newPrefs, tt.curPrefs, upCheckEnv{
 				goos:          goos,
+				flagSet:       flagSet,
 				curExitNodeIP: tt.curExitNodeIP,
 			}); err != nil {
 				got = err.Error()
@@ -688,3 +691,108 @@ func TestFlagAppliesToOS(t *testing.T) {
 		})
 	}
 }
+
+func TestUpdatePrefs(t *testing.T) {
+	tests := []struct {
+		name     string
+		flags    []string // argv to be parsed into env.flagSet and env.upArgs
+		curPrefs *ipn.Prefs
+		env      upCheckEnv // empty goos means "linux"
+
+		wantSimpleUp   bool
+		wantJustEditMP *ipn.MaskedPrefs
+		wantErrSubtr   string
+	}{
+		{
+			name:  "bare_up_means_up",
+			flags: []string{},
+			curPrefs: &ipn.Prefs{
+				ControlURL:  ipn.DefaultControlURL,
+				WantRunning: false,
+				Hostname:    "foo",
+			},
+		},
+		{
+			name:  "just_up",
+			flags: []string{},
+			curPrefs: &ipn.Prefs{
+				ControlURL: ipn.DefaultControlURL,
+				Persist:    &persist.Persist{LoginName: "crawshaw.github"},
+			},
+			env: upCheckEnv{
+				backendState: "Stopped",
+			},
+			wantSimpleUp: true,
+		},
+		{
+			name:  "just_edit",
+			flags: []string{},
+			curPrefs: &ipn.Prefs{
+				ControlURL: ipn.DefaultControlURL,
+				Persist:    &persist.Persist{LoginName: "crawshaw.github"},
+			},
+			env:            upCheckEnv{backendState: "Running"},
+			wantSimpleUp:   true,
+			wantJustEditMP: &ipn.MaskedPrefs{WantRunningSet: true},
+		},
+		{
+			name:  "control_synonym",
+			flags: []string{},
+			curPrefs: &ipn.Prefs{
+				ControlURL: "https://login.tailscale.com",
+				Persist:    &persist.Persist{LoginName: "crawshaw.github"},
+			},
+			env:            upCheckEnv{backendState: "Running"},
+			wantSimpleUp:   true,
+			wantJustEditMP: &ipn.MaskedPrefs{WantRunningSet: true},
+		},
+		{
+			name:  "change_login_server",
+			flags: []string{"--login-server=https://localhost:1000"},
+			curPrefs: &ipn.Prefs{
+				ControlURL:       "https://login.tailscale.com",
+				Persist:          &persist.Persist{LoginName: "crawshaw.github"},
+				AllowSingleHosts: true,
+				CorpDNS:          true,
+				NetfilterMode:    preftype.NetfilterOn,
+			},
+			env:            upCheckEnv{backendState: "Running"},
+			wantSimpleUp:   true,
+			wantJustEditMP: &ipn.MaskedPrefs{WantRunningSet: true},
+			wantErrSubtr:   "can't change --login-server without --force-reauth",
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			if tt.env.goos == "" {
+				tt.env.goos = "linux"
+			}
+			tt.env.flagSet = newUpFlagSet(tt.env.goos, &tt.env.upArgs)
+			tt.env.flagSet.Parse(tt.flags)
+
+			newPrefs, err := prefsFromUpArgs(tt.env.upArgs, t.Logf, new(ipnstate.Status), tt.env.goos)
+			if err != nil {
+				t.Fatal(err)
+			}
+			simpleUp, justEditMP, err := updatePrefs(newPrefs, tt.curPrefs, tt.env)
+			if err != nil {
+				if tt.wantErrSubtr != "" {
+					if !strings.Contains(err.Error(), tt.wantErrSubtr) {
+						t.Fatalf("want error %q, got: %v", tt.wantErrSubtr, err)
+					}
+					return
+				}
+				t.Fatal(err)
+			}
+			if simpleUp != tt.wantSimpleUp {
+				t.Fatalf("simpleUp=%v, want %v", simpleUp, tt.wantSimpleUp)
+			}
+			if justEditMP != nil {
+				justEditMP.Prefs = ipn.Prefs{} // uninteresting
+			}
+			if !reflect.DeepEqual(justEditMP, tt.wantJustEditMP) {
+				t.Fatalf("justEditMP: %v", cmp.Diff(justEditMP, tt.wantJustEditMP))
+			}
+		})
+	}
+}

cmd/tailscale/cli/diag.go

@@ -0,0 +1,55 @@
+// Copyright (c) 2021 Tailscale Inc & AUTHORS All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+// +build linux windows darwin
+
+package cli
+
+import (
+	"fmt"
+	"path/filepath"
+	"runtime"
+	"strings"
+
+	ps "github.com/mitchellh/go-ps"
+)
+
+// fixTailscaledConnectError is called when the local tailscaled has
+// been determined unreachable due to the provided origErr value. It
+// returns either the same error or a better one to help the user
+// understand why tailscaled isn't running for their platform.
+func fixTailscaledConnectError(origErr error) error {
+	procs, err := ps.Processes()
+	if err != nil {
+		return fmt.Errorf("failed to connect to local Tailscaled process and failed to enumerate processes while looking for it")
+	}
+	found := false
+	for _, proc := range procs {
+		base := filepath.Base(proc.Executable())
+		if base == "tailscaled" {
+			found = true
+			break
+		}
+		if runtime.GOOS == "darwin" && base == "IPNExtension" {
+			found = true
+			break
+		}
+		if runtime.GOOS == "windows" && strings.EqualFold(base, "tailscaled.exe") {
+			found = true
+			break
+		}
+	}
+	if !found {
+		switch runtime.GOOS {
+		case "windows":
+			return fmt.Errorf("failed to connect to local tailscaled process; is the Tailscale service running?")
+		case "darwin":
+			return fmt.Errorf("failed to connect to local Tailscale service; is Tailscale running?")
+		case "linux":
+			return fmt.Errorf("failed to connect to local tailscaled; it doesn't appear to be running (sudo systemctl start tailscaled ?)")
+		}
+		return fmt.Errorf("failed to connect to local tailscaled process; it doesn't appear to be running")
+	}
+	return fmt.Errorf("failed to connect to local tailscaled (which appears to be running). Got error: %w", origErr)
+}

cmd/tailscale/cli/diag_other.go

@@ -0,0 +1,16 @@
+// Copyright (c) 2021 Tailscale Inc & AUTHORS All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+// +build !linux,!windows,!darwin
+
+package cli
+
+import "fmt"
+
+// The github.com/mitchellh/go-ps package doesn't work on all platforms,
+// so just don't diagnose connect failures.
+
+func fixTailscaledConnectError(origErr error) error {
+	return fmt.Errorf("failed to connect to local tailscaled process (is it running?); got: %w", origErr)
+}

cmd/tailscale/cli/netcheck.go

@@ -50,7 +50,7 @@ var netcheckArgs struct {
 func runNetcheck(ctx context.Context, args []string) error {
 	c := &netcheck.Client{
 		UDPBindAddr: os.Getenv("TS_DEBUG_NETCHECK_UDP_BIND"),
-		PortMapper:  portmapper.NewClient(logger.WithPrefix(log.Printf, "portmap: ")),
+		PortMapper:  portmapper.NewClient(logger.WithPrefix(log.Printf, "portmap: "), nil),
 	}
 	if netcheckArgs.verbose {
 		c.Logf = logger.WithPrefix(log.Printf, "netcheck: ")

cmd/tailscale/cli/status.go

@@ -23,6 +23,7 @@ import (
 	"tailscale.com/ipn"
 	"tailscale.com/ipn/ipnstate"
 	"tailscale.com/net/interfaces"
+	"tailscale.com/tstime/mono"
 	"tailscale.com/util/dnsname"
 )
 
@@ -57,7 +58,7 @@ var statusArgs struct {
 func runStatus(ctx context.Context, args []string) error {
 	st, err := tailscale.Status(ctx)
 	if err != nil {
-		return err
+		return fixTailscaledConnectError(err)
 	}
 	if statusArgs.json {
 		if statusArgs.active {
@@ -193,7 +194,7 @@ func runStatus(ctx context.Context, args []string) error {
 //
 // TODO: have the server report this bool instead.
 func peerActive(ps *ipnstate.PeerStatus) bool {
-	return !ps.LastWrite.IsZero() && time.Since(ps.LastWrite) < 2*time.Minute
+	return !ps.LastWrite.IsZero() && mono.Since(ps.LastWrite) < 2*time.Minute
 }
 
 func dnsOrQuoteHostname(st *ipnstate.Status, ps *ipnstate.PeerStatus) string {

cmd/tailscale/cli/up.go

@@ -51,7 +51,14 @@ flag is also used.
 	Exec:    runUp,
 }
 
-var upFlagSet = newUpFlagSet(runtime.GOOS, &upArgs)
+func effectiveGOOS() string {
+	if v := os.Getenv("TS_DEBUG_UP_FLAG_GOOS"); v != "" {
+		return v
+	}
+	return runtime.GOOS
+}
+
+var upFlagSet = newUpFlagSet(effectiveGOOS(), &upArgs)
 
 func newUpFlagSet(goos string, upArgs *upArgsT) *flag.FlagSet {
 	upf := flag.NewFlagSet("up", flag.ExitOnError)
@@ -63,13 +70,13 @@ func newUpFlagSet(goos string, upArgs *upArgsT) *flag.FlagSet {
 	upf.BoolVar(&upArgs.acceptRoutes, "accept-routes", false, "accept routes advertised by other Tailscale nodes")
 	upf.BoolVar(&upArgs.acceptDNS, "accept-dns", true, "accept DNS configuration from the admin panel")
 	upf.BoolVar(&upArgs.singleRoutes, "host-routes", true, "install host routes to other Tailscale nodes")
-	upf.StringVar(&upArgs.exitNodeIP, "exit-node", "", "Tailscale IP of the exit node for internet traffic")
+	upf.StringVar(&upArgs.exitNodeIP, "exit-node", "", "Tailscale IP of the exit node for internet traffic, or empty string to not use an exit node")
 	upf.BoolVar(&upArgs.exitNodeAllowLANAccess, "exit-node-allow-lan-access", false, "Allow direct access to the local network when routing traffic via an exit node")
 	upf.BoolVar(&upArgs.shieldsUp, "shields-up", false, "don't allow incoming connections")
 	upf.StringVar(&upArgs.advertiseTags, "advertise-tags", "", "comma-separated ACL tags to request; each must start with \"tag:\" (e.g. \"tag:eng,tag:montreal,tag:ssh\")")
 	upf.StringVar(&upArgs.authKey, "authkey", "", "node authorization key")
 	upf.StringVar(&upArgs.hostname, "hostname", "", "hostname to use instead of the one provided by the OS")
-	upf.StringVar(&upArgs.advertiseRoutes, "advertise-routes", "", "routes to advertise to other nodes (comma-separated, e.g. \"10.0.0.0/8,192.168.0.0/24\")")
+	upf.StringVar(&upArgs.advertiseRoutes, "advertise-routes", "", "routes to advertise to other nodes (comma-separated, e.g. \"10.0.0.0/8,192.168.0.0/24\") or empty string to not advertise routes")
 	upf.BoolVar(&upArgs.advertiseDefaultRoute, "advertise-exit-node", false, "offer to be an exit node for internet traffic for the tailnet")
 	if safesocket.GOOSUsesPeerCreds(goos) {
 		upf.StringVar(&upArgs.opUser, "operator", "", "Unix username to allow to operate on tailscaled without sudo")
@@ -240,6 +247,53 @@ func prefsFromUpArgs(upArgs upArgsT, warnf logger.Logf, st *ipnstate.Status, goo
 	return prefs, nil
 }
 
+// updatePrefs updates prefs based on curPrefs
+//
+// It returns a non-nil justEditMP if we're already running and none of
+// the flags require a restart, so we can just do an EditPrefs call and
+// change the prefs at runtime (e.g. changing hostname, changing
+// advertised tags, routes, etc).
+//
+// It returns simpleUp if we're running a simple "tailscale up" to
+// transition to running from a previously-logged-in but down state,
+// without changing any settings.
+func updatePrefs(prefs, curPrefs *ipn.Prefs, env upCheckEnv) (simpleUp bool, justEditMP *ipn.MaskedPrefs, err error) {
+	if !env.upArgs.reset {
+		applyImplicitPrefs(prefs, curPrefs, env.user)
+
+		if err := checkForAccidentalSettingReverts(prefs, curPrefs, env); err != nil {
+			return false, nil, err
+		}
+	}
+
+	controlURLChanged := curPrefs.ControlURL != prefs.ControlURL &&
+		!(ipn.IsLoginServerSynonym(curPrefs.ControlURL) && ipn.IsLoginServerSynonym(prefs.ControlURL))
+	if controlURLChanged && env.backendState == ipn.Running.String() && !env.upArgs.forceReauth {
+		return false, nil, fmt.Errorf("can't change --login-server without --force-reauth")
+	}
+
+	simpleUp = env.flagSet.NFlag() == 0 &&
+		curPrefs.Persist != nil &&
+		curPrefs.Persist.LoginName != "" &&
+		env.backendState != ipn.NeedsLogin.String()
+
+	justEdit := env.backendState == ipn.Running.String() &&
+		!env.upArgs.forceReauth &&
+		!env.upArgs.reset &&
+		env.upArgs.authKey == "" &&
+		!controlURLChanged
+	if justEdit {
+		justEditMP = new(ipn.MaskedPrefs)
+		justEditMP.WantRunningSet = true
+		justEditMP.Prefs = *prefs
+		env.flagSet.Visit(func(f *flag.Flag) {
+			updateMaskedPrefsFromUpFlag(justEditMP, f.Name)
+		})
+	}
+
+	return simpleUp, justEditMP, nil
+}
+
 func runUp(ctx context.Context, args []string) error {
 	if len(args) > 0 {
 		fatalf("too many non-flag arguments: %q", args)
@@ -247,7 +301,7 @@ func runUp(ctx context.Context, args []string) error {
 
 	st, err := tailscale.Status(ctx)
 	if err != nil {
-		fatalf("can't fetch status from tailscaled: %v", err)
+		return fixTailscaledConnectError(err)
 	}
 	origAuthURL := st.AuthURL
 
@@ -280,7 +334,7 @@ func runUp(ctx context.Context, args []string) error {
 		}
 	}
 
-	prefs, err := prefsFromUpArgs(upArgs, warnf, st, runtime.GOOS)
+	prefs, err := prefsFromUpArgs(upArgs, warnf, st, effectiveGOOS())
 	if err != nil {
 		fatalf("%s", err)
 	}
@@ -296,51 +350,23 @@ func runUp(ctx context.Context, args []string) error {
 		return err
 	}
 
-	if !upArgs.reset {
-		applyImplicitPrefs(prefs, curPrefs, os.Getenv("USER"))
-
-		if err := checkForAccidentalSettingReverts(upFlagSet, curPrefs, prefs, upCheckEnv{
-			goos:          runtime.GOOS,
-			curExitNodeIP: exitNodeIP(prefs, st),
-		}); err != nil {
-			fatalf("%s", err)
-		}
+	env := upCheckEnv{
+		goos:          effectiveGOOS(),
+		user:          os.Getenv("USER"),
+		flagSet:       upFlagSet,
+		upArgs:        upArgs,
+		backendState:  st.BackendState,
+		curExitNodeIP: exitNodeIP(prefs, st),
 	}
-
-	controlURLChanged := curPrefs.ControlURL != prefs.ControlURL
-	if controlURLChanged && st.BackendState == ipn.Running.String() && !upArgs.forceReauth {
-		fatalf("can't change --login-server without --force-reauth")
+	simpleUp, justEditMP, err := updatePrefs(prefs, curPrefs, env)
+	if err != nil {
+		fatalf("%s", err)
 	}
-
-	// If we're already running and none of the flags require a
-	// restart, we can just do an EditPrefs call and change the
-	// prefs at runtime (e.g. changing hostname, changing
-	// advertised tags, routes, etc)
-	justEdit := st.BackendState == ipn.Running.String() &&
-		!upArgs.forceReauth &&
-		!upArgs.reset &&
-		upArgs.authKey == "" &&
-		!controlURLChanged
-	if justEdit {
-		mp := new(ipn.MaskedPrefs)
-		mp.WantRunningSet = true
-		mp.Prefs = *prefs
-		upFlagSet.Visit(func(f *flag.Flag) {
-			updateMaskedPrefsFromUpFlag(mp, f.Name)
-		})
-
-		_, err := tailscale.EditPrefs(ctx, mp)
+	if justEditMP != nil {
+		_, err := tailscale.EditPrefs(ctx, justEditMP)
 		return err
 	}
 
-	// simpleUp is whether we're running a simple "tailscale up"
-	// to transition to running from a previously-logged-in but
-	// down state, without changing any settings.
-	simpleUp := upFlagSet.NFlag() == 0 &&
-		curPrefs.Persist != nil &&
-		curPrefs.Persist.LoginName != "" &&
-		st.BackendState != ipn.NeedsLogin.String()
-
 	// At this point we need to subscribe to the IPN bus to watch
 	// for state transitions and possible need to authenticate.
 	c, bc, pumpCtx, cancel := connect(ctx)
@@ -365,7 +391,7 @@ func runUp(ctx context.Context, args []string) error {
 		if n.ErrMessage != nil {
 			msg := *n.ErrMessage
 			if msg == ipn.ErrMsgPermissionDenied {
-				switch runtime.GOOS {
+				switch effectiveGOOS() {
 				case "windows":
 					msg += " (Tailscale service in use by other user?)"
 				default:
@@ -381,7 +407,7 @@ func runUp(ctx context.Context, args []string) error {
 				startLoginInteractive()
 			case ipn.NeedsMachineAuth:
 				printed = true
-				fmt.Fprintf(os.Stderr, "\nTo authorize your machine, visit (as admin):\n\n\t%s/admin/machines\n\n", upArgs.server)
+				fmt.Fprintf(os.Stderr, "\nTo authorize your machine, visit (as admin):\n\n\t%s\n\n", prefs.AdminPageURL())
 			case ipn.Starting, ipn.Running:
 				// Done full authentication process
 				if printed {
@@ -439,7 +465,7 @@ func runUp(ctx context.Context, args []string) error {
 		// Windows service (~tailscaled) is the one that computes the
 		// StateKey based on the connection identity. So for now, just
 		// do as the Windows GUI's always done:
-		if runtime.GOOS == "windows" {
+		if effectiveGOOS() == "windows" {
 			// The Windows service will set this as needed based
 			// on our connection's identity.
 			opts.StateKey = ""
@@ -538,6 +564,10 @@ const accidentalUpPrefix = "Error: changing settings via 'tailscale up' requires
 // needed by checkForAccidentalSettingReverts and friends.
 type upCheckEnv struct {
 	goos          string
+	user          string
+	flagSet       *flag.FlagSet
+	upArgs        upArgsT
+	backendState  string
 	curExitNodeIP netaddr.IP
 }
 
@@ -555,14 +585,14 @@ type upCheckEnv struct {
 //
 // mp is the mask of settings actually set, where mp.Prefs is the new
 // preferences to set, including any values set from implicit flags.
-func checkForAccidentalSettingReverts(flagSet *flag.FlagSet, curPrefs, newPrefs *ipn.Prefs, env upCheckEnv) error {
+func checkForAccidentalSettingReverts(newPrefs, curPrefs *ipn.Prefs, env upCheckEnv) error {
 	if curPrefs.ControlURL == "" {
 		// Don't validate things on initial "up" before a control URL has been set.
 		return nil
 	}
 
 	flagIsSet := map[string]bool{}
-	flagSet.Visit(func(f *flag.Flag) {
+	env.flagSet.Visit(func(f *flag.Flag) {
 		flagIsSet[f.Name] = true
 	})
 
@@ -586,7 +616,7 @@ func checkForAccidentalSettingReverts(flagSet *flag.FlagSet, curPrefs, newPrefs
 		if reflect.DeepEqual(valCur, valNew) {
 			continue
 		}
-		if flagName == "login-server" && isLoginServerSynonym(valCur) && isLoginServerSynonym(valNew) {
+		if flagName == "login-server" && ipn.IsLoginServerSynonym(valCur) && ipn.IsLoginServerSynonym(valNew) {
 			continue
 		}
 		missing = append(missing, fmtFlagValueArg(flagName, valCur))
@@ -599,7 +629,7 @@ func checkForAccidentalSettingReverts(flagSet *flag.FlagSet, curPrefs, newPrefs
 	// Compute the stringification of the explicitly provided args in flagSet
 	// to prepend to the command to run.
 	var explicit []string
-	flagSet.Visit(func(f *flag.Flag) {
+	env.flagSet.Visit(func(f *flag.Flag) {
 		type isBool interface {
 			IsBoolFlag() bool
 		}
@@ -635,10 +665,6 @@ func applyImplicitPrefs(prefs, oldPrefs *ipn.Prefs, curUser string) {
 	}
 }
 
-func isLoginServerSynonym(val interface{}) bool {
-	return val == "https://login.tailscale.com" || val == "https://controlplane.tailscale.com"
-}
-
 func flagAppliesToOS(flag, goos string) bool {
 	switch flag {
 	case "netfilter-mode", "snat-subnet-routes":

cmd/tailscale/cli/web.go

@@ -15,6 +15,7 @@ import (
 	"html/template"
 	"io/ioutil"
 	"log"
+	"net"
 	"net/http"
 	"net/http/cgi"
 	"net/url"
@@ -94,9 +95,20 @@ func runWeb(ctx context.Context, args []string) error {
 		}
 		return nil
 	}
+
+	log.Printf("web server running on: %s", urlOfListenAddr(webArgs.listen))
 	return http.ListenAndServe(webArgs.listen, http.HandlerFunc(webHandler))
 }
 
+// urlOfListenAddr parses a given listen address into a formatted URL
+func urlOfListenAddr(addr string) string {
+	host, port, _ := net.SplitHostPort(addr)
+	if host == "" {
+		host = "127.0.0.1"
+	}
+	return fmt.Sprintf("http://%s", net.JoinHostPort(host, port))
+}
+
 // authorize returns the name of the user accessing the web UI after verifying
 // whether the user has access to the web UI. The function will write the
 // error to the provided http.ResponseWriter.

cmd/tailscale/cli/web_test.go

@@ -0,0 +1,44 @@
+// Copyright (c) 2021 Tailscale Inc & AUTHORS All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package cli
+
+import "testing"
+
+func TestUrlOfListenAddr(t *testing.T) {
+	tests := []struct {
+		name     string
+		in, want string
+	}{
+		{
+			name: "TestLocalhost",
+			in:   "localhost:8088",
+			want: "http://localhost:8088",
+		},
+		{
+			name: "TestNoHost",
+			in:   ":8088",
+			want: "http://127.0.0.1:8088",
+		},
+		{
+			name: "TestExplicitHost",
+			in:   "127.0.0.2:8088",
+			want: "http://127.0.0.2:8088",
+		},
+		{
+			name: "TestIPv6",
+			in:   "[::1]:8088",
+			want: "http://[::1]:8088",
+		},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			url := urlOfListenAddr(tt.in)
+			if url != tt.want {
+				t.Errorf("expected url: %q, got: %q", tt.want, url)
+			}
+		})
+	}
+}

cmd/tailscale/depaware.txt

@@ -4,8 +4,15 @@ tailscale.com/cmd/tailscale dependencies: (generated by github.com/tailscale/dep
    W    github.com/alexbrainman/sspi/internal/common                 from github.com/alexbrainman/sspi/negotiate
    W 💣 github.com/alexbrainman/sspi/negotiate                       from tailscale.com/net/tshttpproxy
         github.com/kballard/go-shellquote                            from tailscale.com/cmd/tailscale/cli
+     💣 github.com/mitchellh/go-ps                                   from tailscale.com/cmd/tailscale/cli
         github.com/peterbourgon/ff/v2                                from github.com/peterbourgon/ff/v2/ffcli
         github.com/peterbourgon/ff/v2/ffcli                          from tailscale.com/cmd/tailscale/cli
+        github.com/tailscale/goupnp                                  from github.com/tailscale/goupnp/dcps/internetgateway2
+        github.com/tailscale/goupnp/dcps/internetgateway2            from tailscale.com/net/portmapper
+        github.com/tailscale/goupnp/httpu                            from github.com/tailscale/goupnp+
+        github.com/tailscale/goupnp/scpd                             from github.com/tailscale/goupnp
+        github.com/tailscale/goupnp/soap                             from github.com/tailscale/goupnp+
+        github.com/tailscale/goupnp/ssdp                             from github.com/tailscale/goupnp
         github.com/tcnksm/go-httpstat                                from tailscale.com/net/netcheck
         github.com/toqueteos/webbrowser                              from tailscale.com/cmd/tailscale/cli
      💣 go4.org/intern                                               from inet.af/netaddr
@@ -18,6 +25,7 @@ tailscale.com/cmd/tailscale dependencies: (generated by github.com/tailscale/dep
         tailscale.com/client/tailscale                               from tailscale.com/cmd/tailscale/cli+
         tailscale.com/client/tailscale/apitype                       from tailscale.com/client/tailscale+
         tailscale.com/cmd/tailscale/cli                              from tailscale.com/cmd/tailscale
+        tailscale.com/control/controlknobs                           from tailscale.com/net/portmapper
         tailscale.com/derp                                           from tailscale.com/derp/derphttp
         tailscale.com/derp/derphttp                                  from tailscale.com/net/netcheck
         tailscale.com/disco                                          from tailscale.com/derp
@@ -41,6 +49,8 @@ tailscale.com/cmd/tailscale dependencies: (generated by github.com/tailscale/dep
         tailscale.com/syncs                                          from tailscale.com/net/interfaces+
         tailscale.com/tailcfg                                        from tailscale.com/cmd/tailscale/cli+
    W    tailscale.com/tsconst                                        from tailscale.com/net/interfaces
+     💣 tailscale.com/tstime/mono                                    from tailscale.com/cmd/tailscale/cli+
+        tailscale.com/tstime/rate                                    from tailscale.com/wgengine/filter
         tailscale.com/types/empty                                    from tailscale.com/ipn
         tailscale.com/types/ipproto                                  from tailscale.com/net/flowtrack+
         tailscale.com/types/key                                      from tailscale.com/derp+
@@ -76,7 +86,7 @@ tailscale.com/cmd/tailscale dependencies: (generated by github.com/tailscale/dep
         golang.org/x/net/idna                                        from golang.org/x/net/http/httpguts+
         golang.org/x/net/proxy                                       from tailscale.com/net/netns
    D    golang.org/x/net/route                                       from net+
-        golang.org/x/sync/errgroup                                   from tailscale.com/derp
+        golang.org/x/sync/errgroup                                   from tailscale.com/derp+
         golang.org/x/sync/singleflight                               from tailscale.com/net/dnscache
         golang.org/x/sys/cpu                                         from golang.org/x/crypto/blake2b+
   LD    golang.org/x/sys/unix                                        from tailscale.com/net/netns+
@@ -126,7 +136,7 @@ tailscale.com/cmd/tailscale dependencies: (generated by github.com/tailscale/dep
         encoding/hex                                                 from crypto/x509+
         encoding/json                                                from expvar+
         encoding/pem                                                 from crypto/tls+
-        encoding/xml                                                 from tailscale.com/cmd/tailscale/cli
+        encoding/xml                                                 from tailscale.com/cmd/tailscale/cli+
         errors                                                       from bufio+
         expvar                                                       from tailscale.com/derp+
         flag                                                         from github.com/peterbourgon/ff/v2+

cmd/tailscaled/depaware.txt

@@ -23,6 +23,12 @@ tailscale.com/cmd/tailscaled dependencies: (generated by github.com/tailscale/de
    L 💣 github.com/mdlayher/socket                                   from github.com/mdlayher/netlink
    W    github.com/pkg/errors                                        from github.com/tailscale/certstore
    W 💣 github.com/tailscale/certstore                               from tailscale.com/control/controlclient
+        github.com/tailscale/goupnp                                  from github.com/tailscale/goupnp/dcps/internetgateway2
+        github.com/tailscale/goupnp/dcps/internetgateway2            from tailscale.com/net/portmapper
+        github.com/tailscale/goupnp/httpu                            from github.com/tailscale/goupnp+
+        github.com/tailscale/goupnp/scpd                             from github.com/tailscale/goupnp
+        github.com/tailscale/goupnp/soap                             from github.com/tailscale/goupnp+
+        github.com/tailscale/goupnp/ssdp                             from github.com/tailscale/goupnp
         github.com/tcnksm/go-httpstat                                from tailscale.com/net/netcheck
      💣 go4.org/intern                                               from inet.af/netaddr
      💣 go4.org/mem                                                  from tailscale.com/derp+
@@ -79,6 +85,7 @@ tailscale.com/cmd/tailscaled dependencies: (generated by github.com/tailscale/de
         tailscale.com/client/tailscale                               from tailscale.com/derp
         tailscale.com/client/tailscale/apitype                       from tailscale.com/ipn/ipnlocal+
         tailscale.com/control/controlclient                          from tailscale.com/ipn/ipnlocal+
+        tailscale.com/control/controlknobs                           from tailscale.com/control/controlclient+
         tailscale.com/derp                                           from tailscale.com/derp/derphttp+
         tailscale.com/derp/derphttp                                  from tailscale.com/net/netcheck+
         tailscale.com/disco                                          from tailscale.com/derp+
@@ -123,6 +130,8 @@ tailscale.com/cmd/tailscaled dependencies: (generated by github.com/tailscale/de
         tailscale.com/tailcfg                                        from tailscale.com/control/controlclient+
    W    tailscale.com/tsconst                                        from tailscale.com/net/interfaces
         tailscale.com/tstime                                         from tailscale.com/wgengine/magicsock
+     💣 tailscale.com/tstime/mono                                    from tailscale.com/net/tstun+
+        tailscale.com/tstime/rate                                    from tailscale.com/wgengine/filter
         tailscale.com/types/empty                                    from tailscale.com/control/controlclient+
         tailscale.com/types/flagtype                                 from tailscale.com/cmd/tailscaled
         tailscale.com/types/ipproto                                  from tailscale.com/net/flowtrack+
@@ -136,7 +145,7 @@ tailscale.com/cmd/tailscaled dependencies: (generated by github.com/tailscale/de
         tailscale.com/types/structs                                  from tailscale.com/control/controlclient+
         tailscale.com/types/wgkey                                    from tailscale.com/control/controlclient+
    L    tailscale.com/util/cmpver                                    from tailscale.com/net/dns
-        tailscale.com/util/deephash                                  from tailscale.com/ipn/ipnlocal+
+     💣 tailscale.com/util/deephash                                  from tailscale.com/ipn/ipnlocal+
         tailscale.com/util/dnsname                                   from tailscale.com/ipn/ipnstate+
   LW    tailscale.com/util/endian                                    from tailscale.com/net/netns+
         tailscale.com/util/groupmember                               from tailscale.com/ipn/ipnserver
@@ -182,7 +191,7 @@ tailscale.com/cmd/tailscaled dependencies: (generated by github.com/tailscale/de
         golang.org/x/net/ipv6                                        from golang.zx2c4.com/wireguard/device+
         golang.org/x/net/proxy                                       from tailscale.com/net/netns
    D    golang.org/x/net/route                                       from net+
-        golang.org/x/sync/errgroup                                   from tailscale.com/derp
+        golang.org/x/sync/errgroup                                   from tailscale.com/derp+
         golang.org/x/sync/singleflight                               from tailscale.com/net/dnscache
         golang.org/x/sys/cpu                                         from golang.org/x/crypto/blake2b+
   LD    golang.org/x/sys/unix                                        from github.com/mdlayher/netlink+
@@ -236,6 +245,7 @@ tailscale.com/cmd/tailscaled dependencies: (generated by github.com/tailscale/de
         encoding/hex                                                 from crypto/x509+
         encoding/json                                                from expvar+
         encoding/pem                                                 from crypto/tls+
+        encoding/xml                                                 from github.com/tailscale/goupnp+
         errors                                                       from bufio+
         expvar                                                       from tailscale.com/derp+
         flag                                                         from tailscale.com/cmd/tailscaled+

cmd/tailscaled/tailscaled.go

@@ -28,6 +28,7 @@ import (
 	"time"
 
 	"github.com/go-multierror/multierror"
+	"tailscale.com/ipn"
 	"tailscale.com/ipn/ipnserver"
 	"tailscale.com/logpolicy"
 	"tailscale.com/net/dns"
@@ -45,15 +46,6 @@ import (
 	"tailscale.com/wgengine/router"
 )
 
-// globalStateKey is the ipn.StateKey that tailscaled loads on
-// startup.
-//
-// We have to support multiple state keys for other OSes (Windows in
-// particular), but right now Unix daemons run with a single
-// node-global state. To keep open the option of having per-user state
-// later, the global state key doesn't look like a username.
-const globalStateKey = "_daemon"
-
 // defaultTunName returns the default tun device name for the platform.
 func defaultTunName() string {
 	switch runtime.GOOS {
@@ -167,6 +159,28 @@ func main() {
 	}
 }
 
+func ipnServerOpts() (o ipnserver.Options) {
+	// Allow changing the OS-specific IPN behavior for tests
+	// so we can e.g. test Windows-specific behaviors on Linux.
+	goos := os.Getenv("TS_DEBUG_TAILSCALED_IPN_GOOS")
+	if goos == "" {
+		goos = runtime.GOOS
+	}
+
+	o.Port = 41112
+	o.StatePath = args.statepath
+	o.SocketPath = args.socketpath // even for goos=="windows", for tests
+
+	switch goos {
+	default:
+		o.SurviveDisconnects = true
+		o.AutostartStateKey = ipn.GlobalDaemonStateKey
+	case "windows":
+		// Not those.
+	}
+	return o
+}
+
 func run() error {
 	var err error
 
@@ -271,14 +285,8 @@ func run() error {
 		}
 	}()
 
-	opts := ipnserver.Options{
-		SocketPath:         args.socketpath,
-		Port:               41112,
-		StatePath:          args.statepath,
-		AutostartStateKey:  globalStateKey,
-		SurviveDisconnects: runtime.GOOS != "windows",
-		DebugMux:           debugMux,
-	}
+	opts := ipnServerOpts()
+	opts.DebugMux = debugMux
 	err = ipnserver.Run(ctx, logf, pol.PublicID.String(), ipnserver.FixedEngine(e), opts)
 	// Cancelation is not an error: it is the only way to stop ipnserver.
 	if err != nil && err != context.Canceled {
@@ -341,7 +349,7 @@ func tryEngine(logf logger.Logf, linkMon *monitor.Mon, name string) (e wgengine.
 			return nil, false, err
 		}
 		conf.Tun = dev
-		r, err := router.New(logf, dev)
+		r, err := router.New(logf, dev, linkMon)
 		if err != nil {
 			dev.Close()
 			return nil, false, err

cmd/tailscaled/tailscaled_windows.go

@@ -168,7 +168,7 @@ func startIPNServer(ctx context.Context, logid string) error {
 		if err != nil {
 			return nil, fmt.Errorf("TUN: %w", err)
 		}
-		r, err := router.New(logf, dev)
+		r, err := router.New(logf, dev, nil)
 		if err != nil {
 			dev.Close()
 			return nil, fmt.Errorf("router: %w", err)
@@ -233,12 +233,6 @@ func startIPNServer(ctx context.Context, logid string) error {
 		}
 	}()
 
-	opts := ipnserver.Options{
-		Port:               41112,
-		SurviveDisconnects: false,
-		StatePath:          args.statepath,
-	}
-
 	// getEngine is called by ipnserver to get the engine. It's
 	// not called concurrently and is not called again once it
 	// successfully returns an engine.
@@ -263,7 +257,7 @@ func startIPNServer(ctx context.Context, logid string) error {
 			return nil, fmt.Errorf("%w\n\nlogid: %v", res.Err, logid)
 		}
 	}
-	err := ipnserver.Run(ctx, logf, logid, getEngine, opts)
+	err := ipnserver.Run(ctx, logf, logid, getEngine, ipnServerOpts())
 	if err != nil {
 		logf("ipnserver.Run: %v", err)
 	}

cmd/tsshd/tsshd.go

@@ -29,8 +29,8 @@ import (
 	"time"
 	"unsafe"
 
+	"github.com/creack/pty"
 	"github.com/gliderlabs/ssh"
-	"github.com/kr/pty"
 	gossh "golang.org/x/crypto/ssh"
 	"inet.af/netaddr"
 	"tailscale.com/net/interfaces"

control/controlclient/debug.go

@@ -11,7 +11,6 @@ import (
 	"fmt"
 	"log"
 	"net/http"
-	"regexp"
 	"runtime"
 	"strconv"
 	"time"
@@ -42,28 +41,82 @@ func dumpGoroutinesToURL(c *http.Client, targetURL string) {
 	}
 }
 
-var reHexArgs = regexp.MustCompile(`\b0x[0-9a-f]+\b`)
-
 // scrubbedGoroutineDump returns the list of all current goroutines, but with the actual
 // values of arguments scrubbed out, lest it contain some private key material.
 func scrubbedGoroutineDump() []byte {
-	buf := make([]byte, 1<<20)
-	buf = buf[:runtime.Stack(buf, true)]
+	var buf []byte
+	// Grab stacks multiple times into increasingly larger buffer sizes
+	// to minimize the risk that we blow past our iOS memory limit.
+	for size := 1 << 10; size <= 1<<20; size += 1 << 10 {
+		buf = make([]byte, size)
+		buf = buf[:runtime.Stack(buf, true)]
+		if len(buf) < size {
+			// It fit.
+			break
+		}
+	}
+	return scrubHex(buf)
+}
 
+func scrubHex(buf []byte) []byte {
 	saw := map[string][]byte{} // "0x123" => "v1%3" (unique value 1 and its value mod 8)
-	return reHexArgs.ReplaceAllFunc(buf, func(in []byte) []byte {
+
+	foreachHexAddress(buf, func(in []byte) {
 		if string(in) == "0x0" {
-			return in
+			return
 		}
 		if v, ok := saw[string(in)]; ok {
-			return v
+			for i := range in {
+				in[i] = '_'
+			}
+			copy(in, v)
+			return
 		}
+		inStr := string(in)
 		u64, err := strconv.ParseUint(string(in[2:]), 16, 64)
+		for i := range in {
+			in[i] = '_'
+		}
 		if err != nil {
-			return []byte("??")
+			in[0] = '?'
+			return
 		}
 		v := []byte(fmt.Sprintf("v%d%%%d", len(saw)+1, u64%8))
-		saw[string(in)] = v
-		return v
+		saw[inStr] = v
+		copy(in, v)
 	})
+	return buf
+}
+
+var ohx = []byte("0x")
+
+// foreachHexAddress calls f with each subslice of b that matches
+// regexp `0x[0-9a-f]*`.
+func foreachHexAddress(b []byte, f func([]byte)) {
+	for len(b) > 0 {
+		i := bytes.Index(b, ohx)
+		if i == -1 {
+			return
+		}
+		b = b[i:]
+		hx := hexPrefix(b)
+		f(hx)
+		b = b[len(hx):]
+	}
+}
+
+func hexPrefix(b []byte) []byte {
+	for i, c := range b {
+		if i < 2 {
+			continue
+		}
+		if !isHexByte(c) {
+			return b[:i]
+		}
+	}
+	return b
+}
+
+func isHexByte(b byte) bool {
+	return '0' <= b && b <= '9' || 'a' <= b && b <= 'f' || 'A' <= b && b <= 'F'
 }

control/controlclient/debug_test.go

@@ -9,3 +9,22 @@ import "testing"
 func TestScrubbedGoroutineDump(t *testing.T) {
 	t.Logf("Got:\n%s\n", scrubbedGoroutineDump())
 }
+
+func TestScrubHex(t *testing.T) {
+	tests := []struct {
+		in, want string
+	}{
+		{"foo", "foo"},
+		{"", ""},
+		{"0x", "?_"},
+		{"0x001 and same 0x001", "v1%1_ and same v1%1_"},
+		{"0x008 and same 0x008", "v1%0_ and same v1%0_"},
+		{"0x001 and diff 0x002", "v1%1_ and diff v2%2_"},
+	}
+	for _, tt := range tests {
+		got := scrubHex([]byte(tt.in))
+		if string(got) != tt.want {
+			t.Errorf("for input:\n%s\n\ngot:\n%s\n\nwant:\n%s\n", tt.in, got, tt.want)
+		}
+	}
+}

control/controlclient/direct.go

@@ -31,6 +31,7 @@ import (
 
 	"golang.org/x/crypto/nacl/box"
 	"inet.af/netaddr"
+	"tailscale.com/control/controlknobs"
 	"tailscale.com/health"
 	"tailscale.com/ipn/ipnstate"
 	"tailscale.com/log/logheap"
@@ -219,6 +220,13 @@ func packageType() string {
 		// Using tailscaled or IPNExtension?
 		exe, _ := os.Executable()
 		return filepath.Base(exe)
+	case "linux":
+		// Report whether this is in a snap.
+		// See https://snapcraft.io/docs/environment-variables
+		// We just look at two somewhat arbitrarily.
+		if os.Getenv("SNAP_NAME") != "" && os.Getenv("SNAP") != "" {
+			return "snap"
+		}
 	}
 	return ""
 }
@@ -797,7 +805,10 @@ func (c *Direct) sendMapRequest(ctx context.Context, maxPolls int, cb func(*netm
 			continue
 		}
 
-		if resp.Debug != nil {
+		hasDebug := resp.Debug != nil
+		// being conservative here, if Debug not present set to False
+		controlknobs.SetDisableUPnP(hasDebug && resp.Debug.DisableUPnP.EqualBool(true))
+		if hasDebug {
 			if resp.Debug.LogHeapPprof {
 				go logheap.LogHeap(resp.Debug.LogHeapURL)
 			}

control/controlknobs/controlknobs.go

@@ -0,0 +1,34 @@
+// Copyright (c) 2021 Tailscale Inc & AUTHORS All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+// Package controlknobs contains client options configurable from control which can be turned on
+// or off. The ability to turn options on and off is for incrementally adding features in.
+package controlknobs
+
+import (
+	"os"
+	"strconv"
+
+	"tailscale.com/syncs"
+)
+
+// disableUPnP indicates whether to attempt UPnP mapping.
+var disableUPnP syncs.AtomicBool
+
+func init() {
+	v, _ := strconv.ParseBool(os.Getenv("TS_DISABLE_UPNP"))
+	SetDisableUPnP(v)
+}
+
+// DisableUPnP reports the last reported value from control
+// whether UPnP portmapping should be disabled.
+func DisableUPnP() bool {
+	return disableUPnP.Get()
+}
+
+// SetDisableUPnP sets whether control says that UPnP should be
+// disabled.
+func SetDisableUPnP(v bool) {
+	disableUPnP.Set(v)
+}

derp/derp_client.go

@@ -29,6 +29,7 @@ type Client struct {
 	br          *bufio.Reader
 	meshKey     string
 	canAckPings bool
+	isProber    bool
 
 	wmu sync.Mutex // hold while writing to bw
 	bw  *bufio.Writer
@@ -52,6 +53,7 @@ type clientOpt struct {
 	MeshKey     string
 	ServerPub   key.Public
 	CanAckPings bool
+	IsProber    bool
 }
 
 // MeshKey returns a ClientOpt to pass to the DERP server during connect to get
@@ -60,6 +62,10 @@ type clientOpt struct {
 // An empty key means to not use a mesh key.
 func MeshKey(key string) ClientOpt { return clientOptFunc(func(o *clientOpt) { o.MeshKey = key }) }
 
+// IsProber returns a ClientOpt to pass to the DERP server during connect to
+// declare that this client is a a prober.
+func IsProber(v bool) ClientOpt { return clientOptFunc(func(o *clientOpt) { o.IsProber = v }) }
+
 // ServerPublicKey returns a ClientOpt to declare that the server's DERP public key is known.
 // If key is the zero value, the returned ClientOpt is a no-op.
 func ServerPublicKey(key key.Public) ClientOpt {
@@ -93,6 +99,7 @@ func newClient(privateKey key.Private, nc Conn, brw *bufio.ReadWriter, logf logg
 		bw:          brw.Writer,
 		meshKey:     opt.MeshKey,
 		canAckPings: opt.CanAckPings,
+		isProber:    opt.IsProber,
 	}
 	if opt.ServerPub.IsZero() {
 		if err := c.recvServerKey(); err != nil {
@@ -160,6 +167,9 @@ type clientInfo struct {
 	// CanAckPings is whether the client declares it's able to ack
 	// pings.
 	CanAckPings bool
+
+	// IsProber is whether this client is a prober.
+	IsProber bool `json:",omitempty"`
 }
 
 func (c *Client) sendClientKey() error {
@@ -171,6 +181,7 @@ func (c *Client) sendClientKey() error {
 		Version:     ProtocolVersion,
 		MeshKey:     c.meshKey,
 		CanAckPings: c.canAckPings,
+		IsProber:    c.isProber,
 	})
 	if err != nil {
 		return err

derp/derp_server.go

@@ -36,6 +36,7 @@ import (
 	"go4.org/mem"
 	"golang.org/x/crypto/nacl/box"
 	"golang.org/x/sync/errgroup"
+	"golang.org/x/time/rate"
 	"inet.af/netaddr"
 	"tailscale.com/client/tailscale"
 	"tailscale.com/disco"
@@ -97,47 +98,46 @@ type Server struct {
 	metaCert    []byte // the encoded x509 cert to send after LetsEncrypt cert+intermediate
 
 	// Counters:
-	_                        [pad32bit]byte
-	packetsSent, bytesSent   expvar.Int
-	packetsRecv, bytesRecv   expvar.Int
-	packetsRecvByKind        metrics.LabelMap
-	packetsRecvDisco         *expvar.Int
-	packetsRecvOther         *expvar.Int
-	_                        [pad32bit]byte
-	packetsDropped           expvar.Int
-	packetsDroppedReason     metrics.LabelMap
-	packetsDroppedUnknown    *expvar.Int // unknown dst pubkey
-	packetsDroppedFwdUnknown *expvar.Int // unknown dst pubkey on forward
-	packetsDroppedGone       *expvar.Int // dst conn shutting down
-	packetsDroppedQueueHead  *expvar.Int // queue full, drop head packet
-	packetsDroppedQueueTail  *expvar.Int // queue full, drop tail packet
-	packetsDroppedWrite      *expvar.Int // error writing to dst conn
-	_                        [pad32bit]byte
-	packetsForwardedOut      expvar.Int
-	packetsForwardedIn       expvar.Int
-	peerGoneFrames           expvar.Int // number of peer gone frames sent
-	accepts                  expvar.Int
-	curClients               expvar.Int
-	curHomeClients           expvar.Int // ones with preferred
-	clientsReplaced          expvar.Int
-	unknownFrames            expvar.Int
-	homeMovesIn              expvar.Int // established clients announce home server moves in
-	homeMovesOut             expvar.Int // established clients announce home server moves out
-	multiForwarderCreated    expvar.Int
-	multiForwarderDeleted    expvar.Int
-	removePktForwardOther    expvar.Int
-	avgQueueDuration         *uint64 // In milliseconds; accessed atomically
+	_                            [pad32bit]byte
+	packetsSent, bytesSent       expvar.Int
+	packetsRecv, bytesRecv       expvar.Int
+	packetsRecvByKind            metrics.LabelMap
+	packetsRecvDisco             *expvar.Int
+	packetsRecvOther             *expvar.Int
+	_                            [pad32bit]byte
+	packetsDropped               expvar.Int
+	packetsDroppedReason         metrics.LabelMap
+	packetsDroppedReasonCounters []*expvar.Int // indexed by dropReason
+	packetsDroppedType           metrics.LabelMap
+	packetsDroppedTypeDisco      *expvar.Int
+	packetsDroppedTypeOther      *expvar.Int
+	_                            [pad32bit]byte
+	packetsForwardedOut          expvar.Int
+	packetsForwardedIn           expvar.Int
+	peerGoneFrames               expvar.Int // number of peer gone frames sent
+	accepts                      expvar.Int
+	curClients                   expvar.Int
+	curHomeClients               expvar.Int // ones with preferred
+	clientsReplaced              expvar.Int
+	clientsReplaceLimited        expvar.Int
+	clientsReplaceSleeping       expvar.Int
+	unknownFrames                expvar.Int
+	homeMovesIn                  expvar.Int // established clients announce home server moves in
+	homeMovesOut                 expvar.Int // established clients announce home server moves out
+	multiForwarderCreated        expvar.Int
+	multiForwarderDeleted        expvar.Int
+	removePktForwardOther        expvar.Int
+	avgQueueDuration             *uint64 // In milliseconds; accessed atomically
 
 	// verifyClients only accepts client connections to the DERP server if the clientKey is a
 	// known peer in the network, as specified by a running tailscaled's client's local api.
 	verifyClients bool
 
-	mu          sync.Mutex
-	closed      bool
-	netConns    map[Conn]chan struct{} // chan is closed when conn closes
-	clients     map[key.Public]*sclient
-	clientsEver map[key.Public]bool // never deleted from, for stats; fine for now
-	watchers    map[*sclient]bool   // mesh peer -> true
+	mu       sync.Mutex
+	closed   bool
+	netConns map[Conn]chan struct{} // chan is closed when conn closes
+	clients  map[key.Public]*sclient
+	watchers map[*sclient]bool // mesh peer -> true
 	// clientsMesh tracks all clients in the cluster, both locally
 	// and to mesh peers.  If the value is nil, that means the
 	// peer is only local (and thus in the clients Map, but not
@@ -189,8 +189,8 @@ func NewServer(privateKey key.Private, logf logger.Logf) *Server {
 		limitedLogf:          logger.RateLimitedFn(logf, 30*time.Second, 5, 100),
 		packetsRecvByKind:    metrics.LabelMap{Label: "kind"},
 		packetsDroppedReason: metrics.LabelMap{Label: "reason"},
+		packetsDroppedType:   metrics.LabelMap{Label: "type"},
 		clients:              map[key.Public]*sclient{},
-		clientsEver:          map[key.Public]bool{},
 		clientsMesh:          map[key.Public]PacketForwarder{},
 		netConns:             map[Conn]chan struct{}{},
 		memSys0:              ms.Sys,
@@ -202,12 +202,16 @@ func NewServer(privateKey key.Private, logf logger.Logf) *Server {
 	s.initMetacert()
 	s.packetsRecvDisco = s.packetsRecvByKind.Get("disco")
 	s.packetsRecvOther = s.packetsRecvByKind.Get("other")
-	s.packetsDroppedUnknown = s.packetsDroppedReason.Get("unknown_dest")
-	s.packetsDroppedFwdUnknown = s.packetsDroppedReason.Get("unknown_dest_on_fwd")
-	s.packetsDroppedGone = s.packetsDroppedReason.Get("gone")
-	s.packetsDroppedQueueHead = s.packetsDroppedReason.Get("queue_head")
-	s.packetsDroppedQueueTail = s.packetsDroppedReason.Get("queue_tail")
-	s.packetsDroppedWrite = s.packetsDroppedReason.Get("write_error")
+	s.packetsDroppedReasonCounters = []*expvar.Int{
+		s.packetsDroppedReason.Get("unknown_dest"),
+		s.packetsDroppedReason.Get("unknown_dest_on_fwd"),
+		s.packetsDroppedReason.Get("gone"),
+		s.packetsDroppedReason.Get("queue_head"),
+		s.packetsDroppedReason.Get("queue_tail"),
+		s.packetsDroppedReason.Get("write_error"),
+	}
+	s.packetsDroppedTypeDisco = s.packetsDroppedType.Get("disco")
+	s.packetsDroppedTypeOther = s.packetsDroppedType.Get("other")
 	return s
 }
 
@@ -345,26 +349,40 @@ func (s *Server) initMetacert() {
 func (s *Server) MetaCert() []byte { return s.metaCert }
 
 // registerClient notes that client c is now authenticated and ready for packets.
-// If c's public key was already connected with a different connection, the prior one is closed.
-func (s *Server) registerClient(c *sclient) {
+//
+// If c's public key was already connected with a different
+// connection, the prior one is closed, unless it's fighting rapidly
+// with another client with the same key, in which case the returned
+// ok is false, and the caller should wait the provided duration
+// before trying again.
+func (s *Server) registerClient(c *sclient) (ok bool, d time.Duration) {
 	s.mu.Lock()
 	defer s.mu.Unlock()
 	old := s.clients[c.key]
 	if old == nil {
 		c.logf("adding connection")
 	} else {
+		// Take over the old rate limiter, discarding the one
+		// our caller just made.
+		c.replaceLimiter = old.replaceLimiter
+		if rr := c.replaceLimiter.ReserveN(timeNow(), 1); rr.OK() {
+			if d := rr.DelayFrom(timeNow()); d > 0 {
+				s.clientsReplaceLimited.Add(1)
+				return false, d
+			}
+		}
 		s.clientsReplaced.Add(1)
 		c.logf("adding connection, replacing %s", old.remoteAddr)
 		go old.nc.Close()
 	}
 	s.clients[c.key] = c
-	s.clientsEver[c.key] = true
 	if _, ok := s.clientsMesh[c.key]; !ok {
 		s.clientsMesh[c.key] = nil // just for varz of total users in cluster
 	}
 	s.keyOfAddr[c.remoteIPPort] = c.key
 	s.curClients.Add(1)
 	s.broadcastPeerStateChangeLocked(c.key, true)
+	return true, 0
 }
 
 // broadcastPeerStateChangeLocked enqueues a message to all watchers
@@ -475,21 +493,29 @@ func (s *Server) accept(nc Conn, brw *bufio.ReadWriter, remoteAddr string, connN
 	remoteIPPort, _ := netaddr.ParseIPPort(remoteAddr)
 
 	c := &sclient{
-		connNum:      connNum,
-		s:            s,
-		key:          clientKey,
-		nc:           nc,
-		br:           br,
-		bw:           bw,
-		logf:         logger.WithPrefix(s.logf, fmt.Sprintf("derp client %v/%x: ", remoteAddr, clientKey)),
-		done:         ctx.Done(),
-		remoteAddr:   remoteAddr,
-		remoteIPPort: remoteIPPort,
-		connectedAt:  time.Now(),
-		sendQueue:    make(chan pkt, perClientSendQueueDepth),
-		peerGone:     make(chan key.Public),
-		canMesh:      clientInfo.MeshKey != "" && clientInfo.MeshKey == s.meshKey,
+		connNum:        connNum,
+		s:              s,
+		key:            clientKey,
+		nc:             nc,
+		br:             br,
+		bw:             bw,
+		logf:           logger.WithPrefix(s.logf, fmt.Sprintf("derp client %v/%x: ", remoteAddr, clientKey)),
+		done:           ctx.Done(),
+		remoteAddr:     remoteAddr,
+		remoteIPPort:   remoteIPPort,
+		connectedAt:    time.Now(),
+		sendQueue:      make(chan pkt, perClientSendQueueDepth),
+		discoSendQueue: make(chan pkt, perClientSendQueueDepth),
+		peerGone:       make(chan key.Public),
+		canMesh:        clientInfo.MeshKey != "" && clientInfo.MeshKey == s.meshKey,
+
+		// Allow kicking out previous connections once a
+		// minute, with a very high burst of 100. Once a
+		// minute is less than the client's 2 minute
+		// inactivity timeout.
+		replaceLimiter: rate.NewLimiter(rate.Every(time.Minute), 100),
 	}
+
 	if c.canMesh {
 		c.meshUpdate = make(chan struct{})
 	}
@@ -497,7 +523,15 @@ func (s *Server) accept(nc Conn, brw *bufio.ReadWriter, remoteAddr string, connN
 		c.info = *clientInfo
 	}
 
-	s.registerClient(c)
+	for {
+		ok, d := s.registerClient(c)
+		if ok {
+			break
+		}
+		s.clientsReplaceSleeping.Add(1)
+		timeSleep(d)
+		s.clientsReplaceSleeping.Add(-1)
+	}
 	defer s.unregisterClient(c)
 
 	err = s.sendServerInfo(bw, clientKey)
@@ -508,6 +542,12 @@ func (s *Server) accept(nc Conn, brw *bufio.ReadWriter, remoteAddr string, connN
 	return c.run(ctx)
 }
 
+// for testing
+var (
+	timeSleep = time.Sleep
+	timeNow   = time.Now
+)
+
 // run serves the client until there's an error.
 // If the client hangs up or the server is closed, run returns nil, otherwise run returns an error.
 func (c *sclient) run(ctx context.Context) error {
@@ -631,11 +671,7 @@ func (c *sclient) handleFrameForwardPacket(ft frameType, fl uint32) error {
 	s.mu.Unlock()
 
 	if dst == nil {
-		s.packetsDropped.Add(1)
-		s.packetsDroppedFwdUnknown.Add(1)
-		if debug {
-			c.logf("dropping forwarded packet for unknown %x", dstKey)
-		}
+		s.recordDrop(contents, srcKey, dstKey, dropReasonUnknownDestOnFwd)
 		return nil
 	}
 
@@ -686,11 +722,7 @@ func (c *sclient) handleFrameSendPacket(ft frameType, fl uint32) error {
 			}
 			return nil
 		}
-		s.packetsDropped.Add(1)
-		s.packetsDroppedUnknown.Add(1)
-		if debug {
-			c.logf("dropping packet for unknown %x", dstKey)
-		}
+		s.recordDrop(contents, c.key, dstKey, dropReasonUnknownDest)
 		return nil
 	}
 
@@ -702,6 +734,41 @@ func (c *sclient) handleFrameSendPacket(ft frameType, fl uint32) error {
 	return c.sendPkt(dst, p)
 }
 
+// dropReason is why we dropped a DERP frame.
+type dropReason int
+
+//go:generate go run tailscale.com/cmd/addlicense -year 2021 -file dropreason_string.go stringer -type=dropReason -trimprefix=dropReason
+
+const (
+	dropReasonUnknownDest      dropReason = iota // unknown destination pubkey
+	dropReasonUnknownDestOnFwd                   // unknown destination pubkey on a derp-forwarded packet
+	dropReasonGone                               // destination tailscaled disconnected before we could send
+	dropReasonQueueHead                          // destination queue is full, dropped packet at queue head
+	dropReasonQueueTail                          // destination queue is full, dropped packet at queue tail
+	dropReasonWriteError                         // OS write() failed
+)
+
+func (s *Server) recordDrop(packetBytes []byte, srcKey, dstKey key.Public, reason dropReason) {
+	s.packetsDropped.Add(1)
+	s.packetsDroppedReasonCounters[reason].Add(1)
+	if disco.LooksLikeDiscoWrapper(packetBytes) {
+		s.packetsDroppedTypeDisco.Add(1)
+	} else {
+		s.packetsDroppedTypeOther.Add(1)
+	}
+	if verboseDropKeys[dstKey] {
+		// Preformat the log string prior to calling limitedLogf. The
+		// limiter acts based on the format string, and we want to
+		// rate-limit per src/dst keys, not on the generic "dropped
+		// stuff" message.
+		msg := fmt.Sprintf("drop (%s) %s -> %s", srcKey.ShortString(), reason, dstKey.ShortString())
+		s.limitedLogf(msg)
+	}
+	if debug {
+		s.logf("dropping packet reason=%s dst=%s disco=%v", reason, dstKey, disco.LooksLikeDiscoWrapper(packetBytes))
+	}
+}
+
 func (c *sclient) sendPkt(dst *sclient, p pkt) error {
 	s := c.s
 	dstKey := dst.key
@@ -709,54 +776,34 @@ func (c *sclient) sendPkt(dst *sclient, p pkt) error {
 	// Attempt to queue for sending up to 3 times. On each attempt, if
 	// the queue is full, try to drop from queue head to prioritize
 	// fresher packets.
+	sendQueue := dst.sendQueue
+	if disco.LooksLikeDiscoWrapper(p.bs) {
+		sendQueue = dst.discoSendQueue
+	}
 	for attempt := 0; attempt < 3; attempt++ {
 		select {
 		case <-dst.done:
-			s.packetsDropped.Add(1)
-			s.packetsDroppedGone.Add(1)
-			if debug {
-				c.logf("dropping packet for shutdown client %x", dstKey)
-			}
+			s.recordDrop(p.bs, c.key, dstKey, dropReasonGone)
 			return nil
 		default:
 		}
 		select {
-		case dst.sendQueue <- p:
+		case sendQueue <- p:
 			return nil
 		default:
 		}
 
 		select {
-		case pkt := <-dst.sendQueue:
-			s.packetsDropped.Add(1)
-			s.packetsDroppedQueueHead.Add(1)
-			if verboseDropKeys[dstKey] {
-				// Generate a full string including src and dst, so
-				// the limiter kicks in once per src.
-				msg := fmt.Sprintf("tail drop %s -> %s", p.src.ShortString(), dstKey.ShortString())
-				c.s.limitedLogf(msg)
-			}
+		case pkt := <-sendQueue:
+			s.recordDrop(pkt.bs, c.key, dstKey, dropReasonQueueHead)
 			c.recordQueueTime(pkt.enqueuedAt)
-			if debug {
-				c.logf("dropping packet from client %x queue head", dstKey)
-			}
 		default:
 		}
 	}
 	// Failed to make room for packet. This can happen in a heavily
 	// contended queue with racing writers. Give up and tail-drop in
 	// this case to keep reader unblocked.
-	s.packetsDropped.Add(1)
-	s.packetsDroppedQueueTail.Add(1)
-	if verboseDropKeys[dstKey] {
-		// Generate a full string including src and dst, so
-		// the limiter kicks in once per src.
-		msg := fmt.Sprintf("head drop %s -> %s", p.src.ShortString(), dstKey.ShortString())
-		c.s.limitedLogf(msg)
-	}
-	if debug {
-		c.logf("dropping packet from client %x queue tail", dstKey)
-	}
+	s.recordDrop(p.bs, c.key, dstKey, dropReasonQueueTail)
 
 	return nil
 }
@@ -789,6 +836,9 @@ func (s *Server) verifyClient(clientKey key.Public, info *clientInfo) error {
 	if err != nil {
 		return fmt.Errorf("failed to query local tailscaled status: %w", err)
 	}
+	if clientKey == status.Self.PublicKey {
+		return nil
+	}
 	if _, exists := status.Peer[clientKey]; !exists {
 		return fmt.Errorf("client %v not in set of peers", clientKey)
 	}
@@ -926,19 +976,25 @@ func (s *Server) recvForwardPacket(br *bufio.Reader, frameLen uint32) (srcKey, d
 // (The "s" prefix is to more explicitly distinguish it from Client in derp_client.go)
 type sclient struct {
 	// Static after construction.
-	connNum      int64 // process-wide unique counter, incremented each Accept
-	s            *Server
-	nc           Conn
-	key          key.Public
-	info         clientInfo
-	logf         logger.Logf
-	done         <-chan struct{} // closed when connection closes
-	remoteAddr   string          // usually ip:port from net.Conn.RemoteAddr().String()
-	remoteIPPort netaddr.IPPort  // zero if remoteAddr is not ip:port.
-	sendQueue    chan pkt        // packets queued to this client; never closed
-	peerGone     chan key.Public // write request that a previous sender has disconnected (not used by mesh peers)
-	meshUpdate   chan struct{}   // write request to write peerStateChange
-	canMesh      bool            // clientInfo had correct mesh token for inter-region routing
+	connNum        int64 // process-wide unique counter, incremented each Accept
+	s              *Server
+	nc             Conn
+	key            key.Public
+	info           clientInfo
+	logf           logger.Logf
+	done           <-chan struct{} // closed when connection closes
+	remoteAddr     string          // usually ip:port from net.Conn.RemoteAddr().String()
+	remoteIPPort   netaddr.IPPort  // zero if remoteAddr is not ip:port.
+	sendQueue      chan pkt        // packets queued to this client; never closed
+	discoSendQueue chan pkt        // important packets queued to this client; never closed
+	peerGone       chan key.Public // write request that a previous sender has disconnected (not used by mesh peers)
+	meshUpdate     chan struct{}   // write request to write peerStateChange
+	canMesh        bool            // clientInfo had correct mesh token for inter-region routing
+
+	// replaceLimiter controls how quickly two connections with
+	// the same client key can kick each other off the server by
+	// taking over ownership of a key.
+	replaceLimiter *rate.Limiter
 
 	// Owned by run, not thread-safe.
 	br          *bufio.Reader
@@ -1031,12 +1087,10 @@ func (c *sclient) sendLoop(ctx context.Context) error {
 		// Drain the send queue to count dropped packets
 		for {
 			select {
-			case <-c.sendQueue:
-				c.s.packetsDropped.Add(1)
-				c.s.packetsDroppedGone.Add(1)
-				if debug {
-					c.logf("dropping packet for shutdown %x", c.key)
-				}
+			case pkt := <-c.sendQueue:
+				c.s.recordDrop(pkt.bs, pkt.src, c.key, dropReasonGone)
+			case pkt := <-c.discoSendQueue:
+				c.s.recordDrop(pkt.bs, pkt.src, c.key, dropReasonGone)
 			default:
 				return
 			}
@@ -1067,6 +1121,10 @@ func (c *sclient) sendLoop(ctx context.Context) error {
 			werr = c.sendPacket(msg.src, msg.bs)
 			c.recordQueueTime(msg.enqueuedAt)
 			continue
+		case msg := <-c.discoSendQueue:
+			werr = c.sendPacket(msg.src, msg.bs)
+			c.recordQueueTime(msg.enqueuedAt)
+			continue
 		case <-keepAliveTick.C:
 			werr = c.sendKeepAlive()
 			continue
@@ -1090,6 +1148,9 @@ func (c *sclient) sendLoop(ctx context.Context) error {
 		case msg := <-c.sendQueue:
 			werr = c.sendPacket(msg.src, msg.bs)
 			c.recordQueueTime(msg.enqueuedAt)
+		case msg := <-c.discoSendQueue:
+			werr = c.sendPacket(msg.src, msg.bs)
+			c.recordQueueTime(msg.enqueuedAt)
 		case <-keepAliveTick.C:
 			werr = c.sendKeepAlive()
 		}
@@ -1179,11 +1240,7 @@ func (c *sclient) sendPacket(srcKey key.Public, contents []byte) (err error) {
 	defer func() {
 		// Stats update.
 		if err != nil {
-			c.s.packetsDropped.Add(1)
-			c.s.packetsDroppedWrite.Add(1)
-			if debug {
-				c.logf("dropping packet to %x: %v", c.key, err)
-			}
+			c.s.recordDrop(contents, srcKey, c.key, dropReasonWriteError)
 		} else {
 			c.s.packetsSent.Add(1)
 			c.s.bytesSent.Add(int64(len(contents)))
@@ -1329,7 +1386,6 @@ func (s *Server) expVarFunc(f func() interface{}) expvar.Func {
 // ExpVar returns an expvar variable suitable for registering with expvar.Publish.
 func (s *Server) ExpVar() expvar.Var {
 	m := new(metrics.Set)
-	m.Set("counter_unique_clients_ever", s.expVarFunc(func() interface{} { return len(s.clientsEver) }))
 	m.Set("gauge_memstats_sys0", expvar.Func(func() interface{} { return int64(s.memSys0) }))
 	m.Set("gauge_watchers", s.expVarFunc(func() interface{} { return len(s.watchers) }))
 	m.Set("gauge_current_connections", &s.curClients)
@@ -1339,10 +1395,13 @@ func (s *Server) ExpVar() expvar.Var {
 	m.Set("gauge_clients_remote", expvar.Func(func() interface{} { return len(s.clientsMesh) - len(s.clients) }))
 	m.Set("accepts", &s.accepts)
 	m.Set("clients_replaced", &s.clientsReplaced)
+	m.Set("clients_replace_limited", &s.clientsReplaceLimited)
+	m.Set("gauge_clients_replace_sleeping", &s.clientsReplaceSleeping)
 	m.Set("bytes_received", &s.bytesRecv)
 	m.Set("bytes_sent", &s.bytesSent)
 	m.Set("packets_dropped", &s.packetsDropped)
 	m.Set("counter_packets_dropped_reason", &s.packetsDroppedReason)
+	m.Set("counter_packets_dropped_type", &s.packetsDroppedType)
 	m.Set("counter_packets_received_kind", &s.packetsRecvByKind)
 	m.Set("packets_sent", &s.packetsSent)
 	m.Set("packets_received", &s.packetsRecv)

derp/derp_test.go

@@ -23,6 +23,7 @@ import (
 	"testing"
 	"time"
 
+	"golang.org/x/time/rate"
 	"tailscale.com/net/nettest"
 	"tailscale.com/types/key"
 	"tailscale.com/types/logger"
@@ -849,6 +850,136 @@ func TestClientSendPong(t *testing.T) {
 
 }
 
+func TestServerReplaceClients(t *testing.T) {
+	defer func() {
+		timeSleep = time.Sleep
+		timeNow = time.Now
+	}()
+
+	var (
+		mu     sync.Mutex
+		now    = time.Unix(123, 0)
+		sleeps int
+		slept  time.Duration
+	)
+	timeSleep = func(d time.Duration) {
+		mu.Lock()
+		defer mu.Unlock()
+		sleeps++
+		slept += d
+		now = now.Add(d)
+	}
+	timeNow = func() time.Time {
+		mu.Lock()
+		defer mu.Unlock()
+		return now
+	}
+
+	serverPrivateKey := newPrivateKey(t)
+	var logger logger.Logf = logger.Discard
+	const debug = false
+	if debug {
+		logger = t.Logf
+	}
+
+	s := NewServer(serverPrivateKey, logger)
+	defer s.Close()
+
+	priv := newPrivateKey(t)
+
+	ln, err := net.Listen("tcp", "127.0.0.1:0")
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer ln.Close()
+
+	connNum := 0
+	connect := func() *Client {
+		connNum++
+		cout, err := net.Dial("tcp", ln.Addr().String())
+		if err != nil {
+			t.Fatal(err)
+		}
+
+		cin, err := ln.Accept()
+		if err != nil {
+			t.Fatal(err)
+		}
+
+		brwServer := bufio.NewReadWriter(bufio.NewReader(cin), bufio.NewWriter(cin))
+		go s.Accept(cin, brwServer, fmt.Sprintf("test-client-%d", connNum))
+
+		brw := bufio.NewReadWriter(bufio.NewReader(cout), bufio.NewWriter(cout))
+		c, err := NewClient(priv, cout, brw, logger)
+		if err != nil {
+			t.Fatalf("client %d: %v", connNum, err)
+		}
+		return c
+	}
+
+	wantVar := func(v *expvar.Int, want int64) {
+		t.Helper()
+		if got := v.Value(); got != want {
+			t.Errorf("got %d; want %d", got, want)
+		}
+	}
+
+	wantClosed := func(c *Client) {
+		t.Helper()
+		for {
+			m, err := c.Recv()
+			if err != nil {
+				t.Logf("got expected error: %v", err)
+				return
+			}
+			switch m.(type) {
+			case ServerInfoMessage:
+				continue
+			default:
+				t.Fatalf("client got %T; wanted an error", m)
+			}
+		}
+	}
+
+	c1 := connect()
+	waitConnect(t, c1)
+	c2 := connect()
+	waitConnect(t, c2)
+	wantVar(&s.clientsReplaced, 1)
+	wantClosed(c1)
+
+	for i := 0; i < 100+5; i++ {
+		c := connect()
+		defer c.nc.Close()
+		if s.clientsReplaceLimited.Value() == 0 && i < 90 {
+			continue
+		}
+		t.Logf("for %d: replaced=%d, limited=%d, sleeping=%d", i,
+			s.clientsReplaced.Value(),
+			s.clientsReplaceLimited.Value(),
+			s.clientsReplaceSleeping.Value(),
+		)
+	}
+
+	mu.Lock()
+	defer mu.Unlock()
+	if sleeps == 0 {
+		t.Errorf("no sleeps")
+	}
+	if slept == 0 {
+		t.Errorf("total sleep duration was 0")
+	}
+}
+
+func TestLimiter(t *testing.T) {
+	rl := rate.NewLimiter(rate.Every(time.Minute), 100)
+	for i := 0; i < 200; i++ {
+		r := rl.Reserve()
+		d := r.Delay()
+		t.Logf("i=%d, allow=%v, d=%v", i, r.OK(), d)
+	}
+}
+
 func BenchmarkSendRecv(b *testing.B) {
 	for _, size := range []int{10, 100, 1000, 10000} {
 		b.Run(fmt.Sprintf("msgsize=%d", size), func(b *testing.B) { benchmarkSendRecvSize(b, size) })

derp/derphttp/derphttp_client.go

@@ -50,6 +50,7 @@ type Client struct {
 	TLSConfig *tls.Config        // optional; nil means default
 	DNSCache  *dnscache.Resolver // optional; nil means no caching
 	MeshKey   string             // optional; for trusted clients
+	IsProber  bool               // optional; for probers to optional declare themselves as such
 
 	privateKey key.Private
 	logf       logger.Logf
@@ -130,6 +131,11 @@ func (c *Client) ServerPublicKey() key.Public {
 	return c.serverPubKey
 }
 
+// SelfPublicKey returns our own public key.
+func (c *Client) SelfPublicKey() key.Public {
+	return c.privateKey.Public()
+}
+
 func urlPort(u *url.URL) string {
 	if p := u.Port(); p != "" {
 		return p
@@ -338,6 +344,7 @@ func (c *Client) connect(ctx context.Context, caller string) (client *derp.Clien
 		derp.MeshKey(c.MeshKey),
 		derp.ServerPublicKey(serverPub),
 		derp.CanAckPings(c.canAckPings),
+		derp.IsProber(c.IsProber),
 	)
 	if err != nil {
 		return nil, 0, err
@@ -363,7 +370,7 @@ func (c *Client) dialURL(ctx context.Context) (net.Conn, error) {
 	dialer := netns.NewDialer()
 
 	if c.DNSCache != nil {
-		ip, _, err := c.DNSCache.LookupIP(ctx, host)
+		ip, _, _, err := c.DNSCache.LookupIP(ctx, host)
 		if err == nil {
 			hostOrIP = ip.String()
 		}
@@ -410,9 +417,7 @@ func (c *Client) dialRegion(ctx context.Context, reg *tailcfg.DERPRegion) (net.C
 func (c *Client) tlsClient(nc net.Conn, node *tailcfg.DERPNode) *tls.Conn {
 	tlsConf := tlsdial.Config(c.tlsServerName(node), c.TLSConfig)
 	if node != nil {
-		if node.DERPTestPort != 0 {
-			tlsConf.InsecureSkipVerify = true
-		}
+		tlsConf.InsecureSkipVerify = node.InsecureForTests
 		if node.CertName != "" {
 			tlsdial.SetConfigExpectedCert(tlsConf, node.CertName)
 		}
@@ -511,8 +516,8 @@ func (c *Client) dialNode(ctx context.Context, n *tailcfg.DERPNode) (net.Conn, e
 				dst = n.HostName
 			}
 			port := "443"
-			if n.DERPTestPort != 0 {
-				port = fmt.Sprint(n.DERPTestPort)
+			if n.DERPPort != 0 {
+				port = fmt.Sprint(n.DERPPort)
 			}
 			c, err := c.dialContext(ctx, proto, net.JoinHostPort(dst, port))
 			select {

derp/dropreason_string.go

@@ -0,0 +1,32 @@
+// Copyright (c) 2021 Tailscale Inc & AUTHORS All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+// Code generated by "stringer -type=dropReason -trimprefix=dropReason"; DO NOT EDIT.
+
+package derp
+
+import "strconv"
+
+func _() {
+	// An "invalid array index" compiler error signifies that the constant values have changed.
+	// Re-run the stringer command to generate them again.
+	var x [1]struct{}
+	_ = x[dropReasonUnknownDest-0]
+	_ = x[dropReasonUnknownDestOnFwd-1]
+	_ = x[dropReasonGone-2]
+	_ = x[dropReasonQueueHead-3]
+	_ = x[dropReasonQueueTail-4]
+	_ = x[dropReasonWriteError-5]
+}
+
+const _dropReason_name = "UnknownDestUnknownDestOnFwdGoneQueueHeadQueueTailWriteError"
+
+var _dropReason_index = [...]uint8{0, 11, 27, 31, 40, 49, 59}
+
+func (i dropReason) String() string {
+	if i < 0 || i >= dropReason(len(_dropReason_index)-1) {
+		return "dropReason(" + strconv.FormatInt(int64(i), 10) + ")"
+	}
+	return _dropReason_name[_dropReason_index[i]:_dropReason_index[i+1]]
+}

go.mod

@@ -7,6 +7,8 @@ require (
 	github.com/anmitsu/go-shlex v0.0.0-20200514113438-38f4b401e2be // indirect
 	github.com/aws/aws-sdk-go v1.38.52
 	github.com/coreos/go-iptables v0.6.0
+	github.com/creack/pty v1.1.9
+	github.com/dave/jennifer v1.4.1
 	github.com/frankban/quicktest v1.13.0
 	github.com/gliderlabs/ssh v0.3.2
 	github.com/go-multierror/multierror v1.0.2
@@ -16,32 +18,35 @@ require (
 	github.com/google/goexpect v0.0.0-20210430020637-ab937bf7fd6f
 	github.com/google/uuid v1.1.2
 	github.com/goreleaser/nfpm v1.10.3
+	github.com/iancoleman/strcase v0.2.0
 	github.com/jsimonetti/rtnetlink v0.0.0-20210525051524-4cc836578190
 	github.com/kballard/go-shellquote v0.0.0-20180428030007-95032a82bc51
 	github.com/klauspost/compress v1.12.2
-	github.com/kr/pty v1.1.8
 	github.com/mdlayher/netlink v1.4.1
 	github.com/mdlayher/sdnotify v0.0.0-20210228150836-ea3ec207d697
 	github.com/miekg/dns v1.1.42
+	github.com/mitchellh/go-ps v1.0.0
 	github.com/pborman/getopt v1.1.0
 	github.com/peterbourgon/ff/v2 v2.0.0
 	github.com/pkg/sftp v1.13.0
 	github.com/tailscale/certstore v0.0.0-20210528134328-066c94b793d3
 	github.com/tailscale/depaware v0.0.0-20201214215404-77d1e9757027
+	github.com/tailscale/goupnp v1.0.1-0.20210710010003-1cf2d718bbb2
+	github.com/tailscale/hujson v0.0.0-20200924210142-dde312d0d6a2
 	github.com/tcnksm/go-httpstat v0.2.0
 	github.com/toqueteos/webbrowser v1.2.0
 	go4.org/mem v0.0.0-20201119185036-c04c5a6ff174
-	golang.org/x/crypto v0.0.0-20210513164829-c07d793c2f9a
-	golang.org/x/net v0.0.0-20210525063256-abc453219eb5
+	golang.org/x/crypto v0.0.0-20210616213533-5ff15b29337e
+	golang.org/x/net v0.0.0-20210614182718-04defd469f4e
 	golang.org/x/sync v0.0.0-20210220032951-036812b2e83c
 	golang.org/x/sys v0.0.0-20210616094352-59db8d763f22
 	golang.org/x/term v0.0.0-20210503060354-a79de5458b56
 	golang.org/x/time v0.0.0-20210611083556-38a9dc6acbc6
 	golang.org/x/tools v0.1.2
-	golang.zx2c4.com/wireguard v0.0.0-20210525143454-64cb82f2b3f5
-	golang.zx2c4.com/wireguard/windows v0.3.15-0.20210525143335-94c0476d63e3
+	golang.zx2c4.com/wireguard v0.0.0-20210624150102-15b24b6179e0
+	golang.zx2c4.com/wireguard/windows v0.3.16
 	honnef.co/go/tools v0.1.4
-	inet.af/netaddr v0.0.0-20210602152128-50f8686885e3
+	inet.af/netaddr v0.0.0-20210721214506-ce7a8ad02cc1
 	inet.af/netstack v0.0.0-20210622165351-29b14ebc044e
 	inet.af/peercred v0.0.0-20210318190834-4259e17bb763
 	inet.af/wf v0.0.0-20210516214145-a5343001b756

go.sum

@@ -82,12 +82,13 @@ github.com/coreos/go-semver v0.3.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3Ee
 github.com/coreos/go-systemd v0.0.0-20190321100706-95778dfbb74e/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4=
 github.com/coreos/pkg v0.0.0-20180928190104-399ea9e2e55f/go.mod h1:E3G3o1h8I7cfcXa63jLwjI0eiQQMgzzUDFVpN/nH/eA=
 github.com/cpuguy83/go-md2man/v2 v2.0.0/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU=
-github.com/creack/pty v1.1.7/go.mod h1:lj5s0c3V2DBrqTV7llrYr5NG6My20zk30Fl46Y7DoTY=
 github.com/creack/pty v1.1.9 h1:uDmaGzcdjhF4i/plgjmEsriH11Y0o7RKapEf/LDaM3w=
 github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E=
 github.com/daixiang0/gci v0.2.4/go.mod h1:+AV8KmHTGxxwp/pY84TLQfFKp2vuKXXJVzF3kD/hfR4=
 github.com/daixiang0/gci v0.2.7 h1:bosLNficubzJZICsVzxuyNc6oAbdz0zcqLG2G/RxtY4=
 github.com/daixiang0/gci v0.2.7/go.mod h1:+4dZ7TISfSmqfAGv59ePaHfNzgGtIkHAhhdKggP1JAc=
+github.com/dave/jennifer v1.4.1 h1:XyqG6cn5RQsTj3qlWQTKlRGAyrTcsk1kUmWdZBzRjDw=
+github.com/dave/jennifer v1.4.1/go.mod h1:7jEdnm+qBcxl8PC0zyp7vxcpSRnzXSt9r39tpTVGlwA=
 github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
 github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
@@ -296,6 +297,8 @@ github.com/hashicorp/serf v0.8.2/go.mod h1:6hOLApaqBFA1NXqRQAsxw9QxuDEvNxSQRwA/J
 github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU=
 github.com/huandu/xstrings v1.3.2 h1:L18LIDzqlW6xN2rEkpdV8+oL/IXWJ1APd+vsdYy4Wdw=
 github.com/huandu/xstrings v1.3.2/go.mod h1:y5/lhBue+AyNmUVz9RLU9xbLR0o4KIIExikq4ovT0aE=
+github.com/iancoleman/strcase v0.2.0 h1:05I4QRnGpI0m37iZQRuskXh+w77mr6Z41lwQzuHLwW0=
+github.com/iancoleman/strcase v0.2.0/go.mod h1:iwCmte+B7n89clKwxIoIXy/HfoL7AsD47ZCWhYzw7ho=
 github.com/imdario/mergo v0.3.9/go.mod h1:2EnlNZ0deacrJVfApfmtdGgDfMuh/nq6Ok1EcJh5FfA=
 github.com/imdario/mergo v0.3.11 h1:3tnifQM4i+fbajXKBHXWEH+KvNHqojZ778UH75j3bGA=
 github.com/imdario/mergo v0.3.11/go.mod h1:jmQim1M+e3UYxmgPu/WyfjB3N3VflVyUjjjwH0dnCYA=
@@ -355,8 +358,6 @@ github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORN
 github.com/kr/pretty v0.2.1 h1:Fmg33tUaq4/8ym9TJN1x7sLJnHVwhP33CNkpYV/7rwI=
 github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI=
 github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
-github.com/kr/pty v1.1.8 h1:AkaSdXYQOWeaO3neb8EM634ahkXXe3jYbVh/F9lq+GI=
-github.com/kr/pty v1.1.8/go.mod h1:O1sed60cT9XZ5uDucP5qwvh+TE3NnUj51EiZO/lmSfw=
 github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
 github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY=
 github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE=
@@ -418,6 +419,7 @@ github.com/mitchellh/copystructure v1.0.0/go.mod h1:SNtv71yrdKgLRyLFxmLdkAbkKEFW
 github.com/mitchellh/go-homedir v1.0.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0=
 github.com/mitchellh/go-homedir v1.1.0 h1:lukF9ziXFxDFPkA1vsr5zpc1XuPDn/wFntq5mG+4E0Y=
 github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0=
+github.com/mitchellh/go-ps v1.0.0 h1:i6ampVEEF4wQFF+bkYfwYgY+F/uYJDktmvLPf7qIgjc=
 github.com/mitchellh/go-ps v1.0.0/go.mod h1:J4lOc8z8yJs6vUwklHw2XEIiT4z4C40KtWVN3nvg8Pg=
 github.com/mitchellh/go-testing-interface v1.0.0/go.mod h1:kRemZodwjscx+RGhAo8eIhFbs2+BFgRtFPeD/KE+zxI=
 github.com/mitchellh/gox v0.4.0/go.mod h1:Sd9lOJ0+aimLBi73mGofS1ycjY8lL3uZM3JPS42BGNg=
@@ -579,6 +581,10 @@ github.com/tailscale/certstore v0.0.0-20210528134328-066c94b793d3 h1:fEubocuQkrl
 github.com/tailscale/certstore v0.0.0-20210528134328-066c94b793d3/go.mod h1:2P+hpOwd53e7JMX/L4f3VXkv1G+33ES6IWZSrkIeWNs=
 github.com/tailscale/depaware v0.0.0-20201214215404-77d1e9757027 h1:lK99QQdH3yBWY6aGilF+IRlQIdmhzLrsEmF6JgN+Ryw=
 github.com/tailscale/depaware v0.0.0-20201214215404-77d1e9757027/go.mod h1:p9lPsd+cx33L3H9nNoecRRxPssFKUwwI50I3pZ0yT+8=
+github.com/tailscale/goupnp v1.0.1-0.20210710010003-1cf2d718bbb2 h1:AIJ8AF9O7jBmCwilP0ydwJMIzW5dw48Us8f3hLJhYBY=
+github.com/tailscale/goupnp v1.0.1-0.20210710010003-1cf2d718bbb2/go.mod h1:PdCqy9JzfWMJf1H5UJW2ip33/d4YkoKN0r67yKH1mG8=
+github.com/tailscale/hujson v0.0.0-20200924210142-dde312d0d6a2 h1:reREUgl2FG+o7YCsrZB8XLjnuKv5hEIWtnOdAbRAXZI=
+github.com/tailscale/hujson v0.0.0-20200924210142-dde312d0d6a2/go.mod h1:STqf+YV0ADdzk4ejtXFsGqDpATP9JoL0OB+hiFQbkdE=
 github.com/tcnksm/go-httpstat v0.2.0 h1:rP7T5e5U2HfmOBmZzGgGZjBQ5/GluWUylujl0tJ04I0=
 github.com/tcnksm/go-httpstat v0.2.0/go.mod h1:s3JVJFtQxtBEBC9dwcdTTXS9xFnM3SXAZwPG41aurT8=
 github.com/tdakkota/asciicheck v0.0.0-20200416190851-d7f85be797a2/go.mod h1:yHp0ai0Z9gUljN3o0xMhYJnH/IcvkdTBOX2fmJ93JEM=
@@ -650,8 +656,8 @@ golang.org/x/crypto v0.0.0-20201016220609-9e8e0b390897/go.mod h1:LzIPMQfyMNhhGPh
 golang.org/x/crypto v0.0.0-20201208171446-5f87f3452ae9/go.mod h1:jdWPYTVW3xRLrWPugEBEK3UY2ZEsg3UU495nc5E+M+I=
 golang.org/x/crypto v0.0.0-20201221181555-eec23a3978ad/go.mod h1:jdWPYTVW3xRLrWPugEBEK3UY2ZEsg3UU495nc5E+M+I=
 golang.org/x/crypto v0.0.0-20210220033148-5ea612d1eb83/go.mod h1:jdWPYTVW3xRLrWPugEBEK3UY2ZEsg3UU495nc5E+M+I=
-golang.org/x/crypto v0.0.0-20210513164829-c07d793c2f9a h1:kr2P4QFmQr29mSLA43kwrOcgcReGTfbE9N577tCTuBc=
-golang.org/x/crypto v0.0.0-20210513164829-c07d793c2f9a/go.mod h1:P+XmwS30IXTQdn5tA2iutPOUgjI07+tq3H3K9MVA1s8=
+golang.org/x/crypto v0.0.0-20210616213533-5ff15b29337e h1:gsTQYXdTw2Gq7RBsWvlQ91b+aEQ6bXFUngBGuR8sPpI=
+golang.org/x/crypto v0.0.0-20210616213533-5ff15b29337e/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc=
 golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
 golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
 golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
@@ -716,8 +722,9 @@ golang.org/x/net v0.0.0-20201224014010-6772e930b67b/go.mod h1:m0MpNAwzfU5UDzcl9v
 golang.org/x/net v0.0.0-20210119194325-5f4716e94777/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg=
 golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg=
 golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM=
-golang.org/x/net v0.0.0-20210525063256-abc453219eb5 h1:wjuX4b5yYQnEQHzd+CBcrcC6OVR2J1CN6mUy0oSxIPo=
 golang.org/x/net v0.0.0-20210525063256-abc453219eb5/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
+golang.org/x/net v0.0.0-20210614182718-04defd469f4e h1:XpT3nA5TvE525Ne3hInMh6+GETgn27Zfm9dxsThnX2Q=
+golang.org/x/net v0.0.0-20210614182718-04defd469f4e/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
 golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
 golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
 golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
@@ -783,12 +790,12 @@ golang.org/x/sys v0.0.0-20210216163648-f7da38b97c65/go.mod h1:h1NjWce9XRLGQEsW7w
 golang.org/x/sys v0.0.0-20210301091718-77cc2087c03b/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20210303074136-134d130e1a04/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20210305230114-8fe3ee5dd75b/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20210309040221-94ec62e08169/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20210330210617-4fbd30eecc44/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20210403161142-5e06dd20ab57/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20210525143221-35b2ab0089ea/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20210616094352-59db8d763f22 h1:RqytpXGR1iVNX7psjB3ff8y7sNFinVFvkx1c8SjBkio=
 golang.org/x/sys v0.0.0-20210616094352-59db8d763f22/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw=
@@ -869,11 +876,10 @@ golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8T
 golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
 golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 h1:go1bK/D/BFZV2I8cIQd1NKEZ+0owSTG1fDTci4IqFcE=
 golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
-golang.zx2c4.com/wireguard v0.0.0-20210521230051-c27ff9b9f6f7/go.mod h1:a057zjmoc00UN7gVkaJt2sXVK523kMJcogDTEvPIasg=
-golang.zx2c4.com/wireguard v0.0.0-20210525143454-64cb82f2b3f5 h1:5D3v3AKu7ktIhDlqZhZ4+YeNKsW+dnc2+zfFAdhwa8M=
-golang.zx2c4.com/wireguard v0.0.0-20210525143454-64cb82f2b3f5/go.mod h1:laHzsbfMhGSobUmruXWAyMKKHSqvIcrqZJMyHD+/3O8=
-golang.zx2c4.com/wireguard/windows v0.3.15-0.20210525143335-94c0476d63e3 h1:Xw0ZuZcvq981iPGZoLrUXhrK2jOJAw/B6gZxc6g8FsU=
-golang.zx2c4.com/wireguard/windows v0.3.15-0.20210525143335-94c0476d63e3/go.mod h1:f/UVhQ6vXZKDodGB3Glgwu9B3djRxR14jIbcuxD8NBw=
+golang.zx2c4.com/wireguard v0.0.0-20210624150102-15b24b6179e0 h1:qINUmOnDCCF7i14oomDDkGmlda7BSDTGfge77/aqdfk=
+golang.zx2c4.com/wireguard v0.0.0-20210624150102-15b24b6179e0/go.mod h1:laHzsbfMhGSobUmruXWAyMKKHSqvIcrqZJMyHD+/3O8=
+golang.zx2c4.com/wireguard/windows v0.3.16 h1:S42i0kp3SFHZm1mMFTtiU3OnEQJ0GRVOVlMkBhSDTZI=
+golang.zx2c4.com/wireguard/windows v0.3.16/go.mod h1:f80rkFY2CKQklps1GHE15k/M4Tq78aofbr1iQM5MTVY=
 google.golang.org/api v0.4.0/go.mod h1:8k5glujaEP+g9n7WNsDg8QP6cUVNI86fCNMcbazEtwE=
 google.golang.org/api v0.7.0/go.mod h1:WtwebWUNSVBH/HAw79HIFXZNqEvBhG+Ra+ax0hx3E3M=
 google.golang.org/api v0.8.0/go.mod h1:o4eAsZoiT+ibD93RtjEohWalFOjRDx6CVaqeizhEnKg=
@@ -949,8 +955,8 @@ honnef.co/go/tools v0.0.1-2020.1.6/go.mod h1:pyyisuGw24ruLjrr1ddx39WE0y9OooInRzE
 honnef.co/go/tools v0.1.4 h1:SadWOkti5uVN1FAMgxn165+Mw00fuQKyk4Gyn/inxNQ=
 honnef.co/go/tools v0.1.4/go.mod h1:NgwopIslSNH47DimFoV78dnkksY2EFtX0ajyb3K/las=
 inet.af/netaddr v0.0.0-20210515010201-ad03edc7c841/go.mod h1:z0nx+Dh+7N7CC8V5ayHtHGpZpxLQZZxkIaaz6HN65Ls=
-inet.af/netaddr v0.0.0-20210602152128-50f8686885e3 h1:RlarOdsmOUCCvy7Xm1JchJIGuQsuKwD/Lo1bjYmfuQI=
-inet.af/netaddr v0.0.0-20210602152128-50f8686885e3/go.mod h1:z0nx+Dh+7N7CC8V5ayHtHGpZpxLQZZxkIaaz6HN65Ls=
+inet.af/netaddr v0.0.0-20210721214506-ce7a8ad02cc1 h1:mxmfTV6kjXTlFqqFETnG9FQZzNFc6AKunZVAgQ3b7WA=
+inet.af/netaddr v0.0.0-20210721214506-ce7a8ad02cc1/go.mod h1:z0nx+Dh+7N7CC8V5ayHtHGpZpxLQZZxkIaaz6HN65Ls=
 inet.af/netstack v0.0.0-20210622165351-29b14ebc044e h1:z11NK94NQcI3DA+a3pUC/2dRYTph1kPX6B0FnCaMDzk=
 inet.af/netstack v0.0.0-20210622165351-29b14ebc044e/go.mod h1:fG3G1dekmK8oDX3iVzt8c0zICLMLSN8SjdxbXVt0WjU=
 inet.af/peercred v0.0.0-20210318190834-4259e17bb763 h1:gPSJmmVzmdy4kHhlCMx912GdiUz3k/RzJGg0ADqy1dg=

hostinfo/hostinfo.go

@@ -27,6 +27,7 @@ const (
 	AWSLambda       = EnvType("lm")
 	Heroku          = EnvType("hr")
 	AzureAppService = EnvType("az")
+	AWSFargate      = EnvType("fg")
 )
 
 var envType atomic.Value // of EnvType
@@ -53,6 +54,9 @@ func getEnvType() EnvType {
 	if inAzureAppService() {
 		return AzureAppService
 	}
+	if inAWSFargate() {
+		return AWSFargate
+	}
 	return ""
 }
 
@@ -115,3 +119,10 @@ func inAzureAppService() bool {
 	}
 	return false
 }
+
+func inAWSFargate() bool {
+	if os.Getenv("AWS_EXECUTION_ENV") == "AWS_ECS_FARGATE" {
+		return true
+	}
+	return false
+}

ipn/handle.go

@@ -156,7 +156,7 @@ func (h *Handle) Expiry() time.Time {
 }
 
 func (h *Handle) AdminPageURL() string {
-	return h.prefsCache.ControlURLOrDefault() + "/admin/machines"
+	return h.prefsCache.AdminPageURL()
 }
 
 func (h *Handle) StartLoginInteractive() {

ipn/ipnlocal/local.go

@@ -95,7 +95,7 @@ type LocalBackend struct {
 	serverURL             string           // tailcontrol URL
 	newDecompressor       func() (controlclient.Decompressor, error)
 
-	filterHash string
+	filterHash deephash.Sum
 
 	// The mutex protects the following elements.
 	mu             sync.Mutex
@@ -179,6 +179,7 @@ func NewLocalBackend(logf logger.Logf, logid string, store ipn.StateStore, e wge
 		gotPortPollRes: make(chan struct{}),
 	}
 	b.statusChanged = sync.NewCond(&b.statusLock)
+	b.e.SetStatusCallback(b.setWgengineStatus)
 
 	linkMon := e.GetLinkMonitor()
 	b.prevIfState = linkMon.InterfaceState()
@@ -214,6 +215,15 @@ func (b *LocalBackend) SetDirectFileRoot(dir string) {
 	b.directFileRoot = dir
 }
 
+// b.mu must be held.
+func (b *LocalBackend) maybePauseControlClientLocked() {
+	if b.cc == nil {
+		return
+	}
+	networkUp := b.prevIfState.AnyInterfaceUp()
+	b.cc.SetPaused((b.state == ipn.Stopped && b.netMap != nil) || !networkUp)
+}
+
 // linkChange is our link monitor callback, called whenever the network changes.
 // major is whether ifst is different than earlier.
 func (b *LocalBackend) linkChange(major bool, ifst *interfaces.State) {
@@ -222,11 +232,7 @@ func (b *LocalBackend) linkChange(major bool, ifst *interfaces.State) {
 
 	hadPAC := b.prevIfState.HasPAC()
 	b.prevIfState = ifst
-
-	networkUp := ifst.AnyInterfaceUp()
-	if b.cc != nil {
-		go b.cc.SetPaused((b.state == ipn.Stopped && b.netMap != nil) || !networkUp)
-	}
+	b.maybePauseControlClientLocked()
 
 	// If the PAC-ness of the network changed, reconfig wireguard+route to
 	// add/remove subnets.
@@ -605,8 +611,8 @@ func (b *LocalBackend) setWgengineStatus(s *wgengine.Status, err error) {
 
 	if cc != nil {
 		cc.UpdateEndpoints(0, s.LocalAddrs)
+		b.stateMachine()
 	}
-	b.stateMachine()
 
 	b.statusLock.Lock()
 	b.statusChanged.Broadcast()
@@ -757,6 +763,12 @@ func (b *LocalBackend) Start(opts ipn.Options) error {
 		newPrefs := opts.UpdatePrefs
 		newPrefs.Persist = b.prefs.Persist
 		b.prefs = newPrefs
+
+		if opts.StateKey != "" {
+			if err := b.store.WriteState(opts.StateKey, b.prefs.ToBytes()); err != nil {
+				b.logf("failed to save UpdatePrefs state: %v", err)
+			}
+		}
 	}
 
 	wantRunning := b.prefs.WantRunning
@@ -857,7 +869,6 @@ func (b *LocalBackend) Start(opts ipn.Options) error {
 	}
 
 	cc.SetStatusFunc(b.setClientStatus)
-	b.e.SetStatusCallback(b.setWgengineStatus)
 	b.e.SetNetInfoCallback(b.setNetInfo)
 
 	b.mu.Lock()
@@ -933,7 +944,7 @@ func (b *LocalBackend) updateFilter(netMap *netmap.NetworkMap, prefs *ipn.Prefs)
 	localNets, _ := localNetsB.IPSet()
 	logNets, _ := logNetsB.IPSet()
 
-	changed := deephash.UpdateHash(&b.filterHash, haveNetmap, addrs, packetFilter, localNets.Ranges(), logNets.Ranges(), shieldsUp)
+	changed := deephash.Update(&b.filterHash, haveNetmap, addrs, packetFilter, localNets.Ranges(), logNets.Ranges(), shieldsUp)
 	if !changed {
 		return
 	}
@@ -973,6 +984,44 @@ var removeFromDefaultRoute = []netaddr.IPPrefix{
 	tsaddr.TailscaleULARange(),
 }
 
+// internalAndExternalInterfaces splits interface routes into "internal"
+// and "external" sets. Internal routes are those of virtual ethernet
+// network interfaces used by guest VMs and containers, such as WSL and
+// Docker.
+//
+// Given that "internal" routes don't leave the device, we choose to
+// trust them more, allowing access to them when an Exit Node is enabled.
+func internalAndExternalInterfaces() (internal, external []netaddr.IPPrefix, err error) {
+	if err := interfaces.ForeachInterfaceAddress(func(iface interfaces.Interface, pfx netaddr.IPPrefix) {
+		if tsaddr.IsTailscaleIP(pfx.IP()) {
+			return
+		}
+		if pfx.IsSingleIP() {
+			return
+		}
+		if runtime.GOOS == "windows" {
+			// Windows Hyper-V prefixes all MAC addresses with 00:15:5d.
+			// https://docs.microsoft.com/en-us/troubleshoot/windows-server/virtualization/default-limit-256-dynamic-mac-addresses
+			//
+			// This includes WSL2 vEthernet.
+			// Importantly: by default WSL2 /etc/resolv.conf points to
+			// a stub resolver running on the host vEthernet IP.
+			// So enabling exit nodes with the default tailnet
+			// configuration breaks WSL2 DNS without this.
+			mac := iface.Interface.HardwareAddr
+			if len(mac) == 6 && mac[0] == 0x00 && mac[1] == 0x15 && mac[2] == 0x5d {
+				internal = append(internal, pfx)
+				return
+			}
+		}
+		external = append(external, pfx)
+	}); err != nil {
+		return nil, nil, err
+	}
+
+	return internal, external, nil
+}
+
 func interfaceRoutes() (ips *netaddr.IPSet, hostIPs []netaddr.IP, err error) {
 	var b netaddr.IPSetBuilder
 	if err := interfaces.ForeachInterfaceAddress(func(_ interfaces.Interface, pfx netaddr.IPPrefix) {
@@ -2108,18 +2157,21 @@ func (b *LocalBackend) routerConfig(cfg *wgcfg.Config, prefs *ipn.Prefs) *router
 		if !default6 {
 			rs.Routes = append(rs.Routes, ipv6Default)
 		}
+		internalIPs, externalIPs, err := internalAndExternalInterfaces()
+		if err != nil {
+			b.logf("failed to discover interface ips: %v", err)
+		}
 		if runtime.GOOS == "linux" || runtime.GOOS == "darwin" || runtime.GOOS == "windows" {
-			// Only allow local lan access on linux machines for now.
-			ips, _, err := interfaceRoutes()
-			if err != nil {
-				b.logf("failed to discover interface ips: %v", err)
-			}
+			rs.LocalRoutes = internalIPs // unconditionally allow access to guest VM networks
 			if prefs.ExitNodeAllowLANAccess {
-				rs.LocalRoutes = ips.Prefixes()
+				rs.LocalRoutes = append(rs.LocalRoutes, externalIPs...)
+				if len(externalIPs) != 0 {
+					b.logf("allowing exit node access to internal IPs: %v", internalIPs)
+				}
 			} else {
 				// Explicitly add routes to the local network so that we do not
 				// leak any traffic.
-				rs.Routes = append(rs.Routes, ips.Prefixes()...)
+				rs.Routes = append(rs.Routes, externalIPs...)
 			}
 		}
 	}
@@ -2148,6 +2200,18 @@ func applyPrefsToHostinfo(hi *tailcfg.Hostinfo, prefs *ipn.Prefs) {
 	}
 	if v := prefs.OSVersion; v != "" {
 		hi.OSVersion = v
+
+		// The Android app annotates when Google Play Services
+		// aren't available by tacking on a string to the
+		// OSVersion. Promote that to the Hostinfo.Package
+		// field instead, rather than adding a new pref, as
+		// this applyPrefsToHostinfo mechanism is mostly
+		// abused currently. TODO(bradfitz): instead let
+		// frontends update Hostinfo, without using Prefs.
+		if runtime.GOOS == "android" && strings.HasSuffix(v, " [nogoogle]") {
+			hi.Package = "nogoogle"
+			hi.OSVersion = strings.TrimSuffix(v, " [nogoogle]")
+		}
 	}
 	if m := prefs.DeviceModel; m != "" {
 		hi.DeviceModel = m
@@ -2167,9 +2231,7 @@ func (b *LocalBackend) enterState(newState ipn.State) {
 	oldState := b.state
 	b.state = newState
 	prefs := b.prefs
-	cc := b.cc
 	netMap := b.netMap
-	networkUp := b.prevIfState.AnyInterfaceUp()
 	activeLogin := b.activeLogin
 	authURL := b.authURL
 	if newState == ipn.Running {
@@ -2179,6 +2241,7 @@ func (b *LocalBackend) enterState(newState ipn.State) {
 		// Transitioning away from running.
 		b.closePeerAPIListenersLocked()
 	}
+	b.maybePauseControlClientLocked()
 	b.mu.Unlock()
 
 	if oldState == newState {
@@ -2189,10 +2252,6 @@ func (b *LocalBackend) enterState(newState ipn.State) {
 	health.SetIPNState(newState.String(), prefs.WantRunning)
 	b.send(ipn.Notify{State: &newState})
 
-	if cc != nil {
-		cc.SetPaused((newState == ipn.Stopped && netMap != nil) || !networkUp)
-	}
-
 	switch newState {
 	case ipn.NeedsLogin:
 		systemd.Status("Needs login: %s", authURL)
@@ -2453,6 +2512,7 @@ func (b *LocalBackend) setNetMapLocked(nm *netmap.NetworkMap) {
 		b.logf("active login: %v", login)
 		b.activeLogin = login
 	}
+	b.maybePauseControlClientLocked()
 
 	// Determine if file sharing is enabled
 	fs := hasCapability(nm, tailcfg.CapabilityFileSharing)
@@ -2724,17 +2784,18 @@ func (b *LocalBackend) CheckIPForwarding() error {
 		return nil
 	}
 
+	const suffix = "\nSubnet routes won't work without IP forwarding.\nSee https://tailscale.com/kb/1104/enable-ip-forwarding/"
 	for _, key := range keys {
 		bs, err := exec.Command("sysctl", "-n", key).Output()
 		if err != nil {
-			return fmt.Errorf("couldn't check %s (%v).\nSubnet routes won't work without IP forwarding.", key, err)
+			return fmt.Errorf("couldn't check %s (%v)%s", key, err, suffix)
 		}
 		on, err := strconv.ParseBool(string(bytes.TrimSpace(bs)))
 		if err != nil {
-			return fmt.Errorf("couldn't parse %s (%v).\nSubnet routes won't work without IP forwarding.", key, err)
+			return fmt.Errorf("couldn't parse %s (%v)%s.", key, err, suffix)
 		}
 		if !on {
-			return fmt.Errorf("%s is disabled. Subnet routes won't work.", key)
+			return fmt.Errorf("%s is disabled.%s", key, suffix)
 		}
 	}
 	return nil

ipn/ipnlocal/state_test.go

@@ -14,6 +14,7 @@ import (
 
 	"tailscale.com/control/controlclient"
 	"tailscale.com/ipn"
+	"tailscale.com/syncs"
 	"tailscale.com/tailcfg"
 	"tailscale.com/types/empty"
 	"tailscale.com/types/logger"
@@ -277,7 +278,7 @@ func TestStateMachine(t *testing.T) {
 	c := qt.New(t)
 
 	logf := t.Logf
-	store := new(ipn.MemoryStore)
+	store := new(testStateStorage)
 	e, err := wgengine.NewFakeUserspaceEngine(logf, 0)
 	if err != nil {
 		t.Fatalf("NewFakeUserspaceEngine: %v", err)
@@ -353,7 +354,7 @@ func TestStateMachine(t *testing.T) {
 	c.Assert(b.Start(ipn.Options{StateKey: ipn.GlobalDaemonStateKey}), qt.IsNil)
 	{
 		// BUG: strictly, it should pause, not unpause, here, since !WantRunning.
-		c.Assert([]string{"Shutdown", "New", "unpause"}, qt.DeepEquals, cc.getCalls())
+		c.Assert([]string{"Shutdown", "unpause", "New", "unpause"}, qt.DeepEquals, cc.getCalls())
 
 		nn := notifies.drain(2)
 		c.Assert(cc.getCalls(), qt.HasLen, 0)
@@ -388,7 +389,7 @@ func TestStateMachine(t *testing.T) {
 	url1 := "http://localhost:1/1"
 	cc.send(nil, url1, false, nil)
 	{
-		c.Assert(cc.getCalls(), qt.DeepEquals, []string{})
+		c.Assert(cc.getCalls(), qt.DeepEquals, []string{"unpause"})
 
 		// ...but backend eats that notification, because the user
 		// didn't explicitly request interactive login yet, and
@@ -413,7 +414,7 @@ func TestStateMachine(t *testing.T) {
 		// We're still not logged in so there's nothing we can do
 		// with it. (And empirically, it's providing an empty list
 		// of endpoints.)
-		c.Assert([]string{"UpdateEndpoints"}, qt.DeepEquals, cc.getCalls())
+		c.Assert([]string{"UpdateEndpoints", "unpause"}, qt.DeepEquals, cc.getCalls())
 		c.Assert(nn[0].BrowseToURL, qt.Not(qt.IsNil))
 		c.Assert(url1, qt.Equals, *nn[0].BrowseToURL)
 	}
@@ -439,7 +440,7 @@ func TestStateMachine(t *testing.T) {
 	cc.send(nil, url2, false, nil)
 	{
 		// BUG: UpdateEndpoints again, this is getting silly.
-		c.Assert([]string{"UpdateEndpoints"}, qt.DeepEquals, cc.getCalls())
+		c.Assert([]string{"UpdateEndpoints", "unpause", "unpause"}, qt.DeepEquals, cc.getCalls())
 
 		// This time, backend should emit it to the UI right away,
 		// because the UI is anxiously awaiting a new URL to visit.
@@ -469,7 +470,7 @@ func TestStateMachine(t *testing.T) {
 		// wait until it gets into Starting.
 		// TODO: (Currently this test doesn't detect that bug, but
 		// it's visible in the logs)
-		c.Assert([]string{"unpause", "UpdateEndpoints"}, qt.DeepEquals, cc.getCalls())
+		c.Assert([]string{"unpause", "unpause", "UpdateEndpoints", "unpause"}, qt.DeepEquals, cc.getCalls())
 		c.Assert(nn[0].LoginFinished, qt.Not(qt.IsNil))
 		c.Assert(nn[1].Prefs, qt.Not(qt.IsNil))
 		c.Assert(nn[2].State, qt.Not(qt.IsNil))
@@ -482,7 +483,7 @@ func TestStateMachine(t *testing.T) {
 	notifies.expect(1)
 	// BUG: the real controlclient sends LoginFinished with every
 	// notification while it's in StateAuthenticated, but not StateSynced.
-	// We should send it exactly once, or every time we're authenticated,
+	// It should send it exactly once, or every time we're authenticated,
 	// but the current code is brittle.
 	// (ie. I suspect it would be better to change false->true in send()
 	// below, and do the same in the real controlclient.)
@@ -491,7 +492,7 @@ func TestStateMachine(t *testing.T) {
 	})
 	{
 		nn := notifies.drain(1)
-		c.Assert([]string{"unpause", "UpdateEndpoints"}, qt.DeepEquals, cc.getCalls())
+		c.Assert([]string{"unpause", "unpause", "UpdateEndpoints", "unpause"}, qt.DeepEquals, cc.getCalls())
 		c.Assert(nn[0].State, qt.Not(qt.IsNil))
 		c.Assert(ipn.Starting, qt.Equals, *nn[0].State)
 	}
@@ -523,6 +524,7 @@ func TestStateMachine(t *testing.T) {
 
 	// The user changes their preference to WantRunning after all.
 	t.Logf("\n\nWantRunning -> true")
+	store.awaitWrite()
 	notifies.expect(2)
 	b.EditPrefs(&ipn.MaskedPrefs{
 		WantRunningSet: true,
@@ -532,15 +534,17 @@ func TestStateMachine(t *testing.T) {
 		nn := notifies.drain(2)
 		// BUG: UpdateEndpoints isn't needed here.
 		// BUG: Login isn't needed here. We never logged out.
-		c.Assert([]string{"Login", "unpause", "UpdateEndpoints"}, qt.DeepEquals, cc.getCalls())
+		c.Assert([]string{"Login", "unpause", "UpdateEndpoints", "unpause"}, qt.DeepEquals, cc.getCalls())
 		// BUG: I would expect Prefs to change first, and state after.
 		c.Assert(nn[0].State, qt.Not(qt.IsNil))
 		c.Assert(nn[1].Prefs, qt.Not(qt.IsNil))
 		c.Assert(ipn.Starting, qt.Equals, *nn[0].State)
+		c.Assert(store.sawWrite(), qt.IsTrue)
 	}
 
 	// Test the fast-path frontend reconnection.
-	// This one is very finicky, so we have to force State==Running.
+	// This one is very finicky, so we have to force State==Running
+	// or it won't use the fast path.
 	// TODO: actually get to State==Running, rather than cheating.
 	//  That'll require spinning up a fake DERP server and putting it in
 	//  the netmap.
@@ -561,18 +565,20 @@ func TestStateMachine(t *testing.T) {
 	b.state = ipn.Starting
 
 	// User wants to logout.
+	store.awaitWrite()
 	t.Logf("\n\nLogout (async)")
 	notifies.expect(2)
 	b.Logout()
 	{
 		nn := notifies.drain(2)
-		c.Assert([]string{"pause", "StartLogout"}, qt.DeepEquals, cc.getCalls())
+		c.Assert([]string{"pause", "StartLogout", "pause"}, qt.DeepEquals, cc.getCalls())
 		c.Assert(nn[0].State, qt.Not(qt.IsNil))
 		c.Assert(nn[1].Prefs, qt.Not(qt.IsNil))
 		c.Assert(ipn.Stopped, qt.Equals, *nn[0].State)
 		c.Assert(nn[1].Prefs.LoggedOut, qt.IsTrue)
 		c.Assert(nn[1].Prefs.WantRunning, qt.IsFalse)
 		c.Assert(ipn.Stopped, qt.Equals, b.State())
+		c.Assert(store.sawWrite(), qt.IsTrue)
 	}
 
 	// Let's make the logout succeed.
@@ -582,7 +588,7 @@ func TestStateMachine(t *testing.T) {
 	cc.send(nil, "", false, nil)
 	{
 		nn := notifies.drain(1)
-		c.Assert([]string{"unpause"}, qt.DeepEquals, cc.getCalls())
+		c.Assert([]string{"unpause", "unpause"}, qt.DeepEquals, cc.getCalls())
 		c.Assert(nn[0].State, qt.Not(qt.IsNil))
 		c.Assert(ipn.NeedsLogin, qt.Equals, *nn[0].State)
 		c.Assert(b.Prefs().LoggedOut, qt.IsTrue)
@@ -598,7 +604,7 @@ func TestStateMachine(t *testing.T) {
 		notifies.drain(0)
 		// BUG: the backend has already called StartLogout, and we're
 		// still logged out. So it shouldn't call it again.
-		c.Assert([]string{"StartLogout"}, qt.DeepEquals, cc.getCalls())
+		c.Assert([]string{"StartLogout", "unpause"}, qt.DeepEquals, cc.getCalls())
 		c.Assert(cc.getCalls(), qt.HasLen, 0)
 		c.Assert(b.Prefs().LoggedOut, qt.IsTrue)
 		c.Assert(b.Prefs().WantRunning, qt.IsFalse)
@@ -612,8 +618,7 @@ func TestStateMachine(t *testing.T) {
 	cc.send(nil, "", false, nil)
 	{
 		notifies.drain(0)
-		c.Assert(cc.getCalls(), qt.HasLen, 0)
-		c.Assert(cc.getCalls(), qt.HasLen, 0)
+		c.Assert(cc.getCalls(), qt.DeepEquals, []string{"unpause", "unpause"})
 		c.Assert(b.Prefs().LoggedOut, qt.IsTrue)
 		c.Assert(b.Prefs().WantRunning, qt.IsFalse)
 		c.Assert(ipn.NeedsLogin, qt.Equals, b.State())
@@ -627,7 +632,7 @@ func TestStateMachine(t *testing.T) {
 	// I guess, since that's supposed to be synchronous.
 	{
 		notifies.drain(0)
-		c.Assert([]string{"Logout"}, qt.DeepEquals, cc.getCalls())
+		c.Assert([]string{"Logout", "unpause"}, qt.DeepEquals, cc.getCalls())
 		c.Assert(cc.getCalls(), qt.HasLen, 0)
 		c.Assert(b.Prefs().LoggedOut, qt.IsTrue)
 		c.Assert(b.Prefs().WantRunning, qt.IsFalse)
@@ -641,8 +646,7 @@ func TestStateMachine(t *testing.T) {
 	cc.send(nil, "", false, nil)
 	{
 		notifies.drain(0)
-		c.Assert(cc.getCalls(), qt.HasLen, 0)
-		c.Assert(cc.getCalls(), qt.HasLen, 0)
+		c.Assert(cc.getCalls(), qt.DeepEquals, []string{"unpause", "unpause"})
 		c.Assert(b.Prefs().LoggedOut, qt.IsTrue)
 		c.Assert(b.Prefs().WantRunning, qt.IsFalse)
 		c.Assert(ipn.NeedsLogin, qt.Equals, b.State())
@@ -673,7 +677,7 @@ func TestStateMachine(t *testing.T) {
 		// BUG: We already called Shutdown(), no need to do it again.
 		// BUG: Way too soon for UpdateEndpoints.
 		// BUG: don't unpause because we're not logged in.
-		c.Assert([]string{"Shutdown", "New", "UpdateEndpoints", "unpause"}, qt.DeepEquals, cc.getCalls())
+		c.Assert([]string{"Shutdown", "unpause", "New", "UpdateEndpoints", "unpause"}, qt.DeepEquals, cc.getCalls())
 
 		nn := notifies.drain(2)
 		c.Assert(cc.getCalls(), qt.HasLen, 0)
@@ -698,7 +702,7 @@ func TestStateMachine(t *testing.T) {
 	})
 	{
 		nn := notifies.drain(3)
-		c.Assert([]string{"unpause"}, qt.DeepEquals, cc.getCalls())
+		c.Assert([]string{"unpause", "unpause"}, qt.DeepEquals, cc.getCalls())
 		c.Assert(nn[0].LoginFinished, qt.Not(qt.IsNil))
 		c.Assert(nn[1].Prefs, qt.Not(qt.IsNil))
 		c.Assert(nn[2].State, qt.Not(qt.IsNil))
@@ -738,9 +742,8 @@ func TestStateMachine(t *testing.T) {
 		//  on startup, otherwise UIs can't show the node list, login
 		//  name, etc when in state ipn.Stopped.
 		//  Arguably they shouldn't try. But they currently do.
-		c.Assert([]string{"Shutdown", "New", "UpdateEndpoints", "Login", "unpause"}, qt.DeepEquals, cc.getCalls())
-
 		nn := notifies.drain(2)
+		c.Assert([]string{"Shutdown", "unpause", "New", "UpdateEndpoints", "Login", "unpause"}, qt.DeepEquals, cc.getCalls())
 		c.Assert(cc.getCalls(), qt.HasLen, 0)
 		c.Assert(nn[0].Prefs, qt.Not(qt.IsNil))
 		c.Assert(nn[1].State, qt.Not(qt.IsNil))
@@ -749,6 +752,25 @@ func TestStateMachine(t *testing.T) {
 		c.Assert(ipn.Stopped, qt.Equals, *nn[1].State)
 	}
 
+	// When logged in but !WantRunning, ipn leaves us unpaused to retrieve
+	// the first netmap. Simulate that netmap being received, after which
+	// it should pause us, to avoid wasting CPU retrieving unnecessarily
+	// additional netmap updates.
+	//
+	// TODO: really the various GUIs and prefs should be refactored to
+	//  not require the netmap structure at all when starting while
+	//  !WantRunning. That would remove the need for this (or contacting
+	//  the control server at all when stopped).
+	t.Logf("\n\nStart4 -> netmap")
+	notifies.expect(0)
+	cc.send(nil, "", true, &netmap.NetworkMap{
+		MachineStatus: tailcfg.MachineAuthorized,
+	})
+	{
+		notifies.drain(0)
+		c.Assert([]string{"pause", "pause"}, qt.DeepEquals, cc.getCalls())
+	}
+
 	// Request connection.
 	// The state machine didn't call Login() earlier, so now it needs to.
 	t.Logf("\n\nWantRunning4 -> true")
@@ -775,7 +797,7 @@ func TestStateMachine(t *testing.T) {
 	})
 	{
 		nn := notifies.drain(2)
-		c.Assert([]string{"unpause"}, qt.DeepEquals, cc.getCalls())
+		c.Assert([]string{"pause"}, qt.DeepEquals, cc.getCalls())
 		// BUG: I would expect Prefs to change first, and state after.
 		c.Assert(nn[0].State, qt.Not(qt.IsNil))
 		c.Assert(nn[1].Prefs, qt.Not(qt.IsNil))
@@ -785,25 +807,27 @@ func TestStateMachine(t *testing.T) {
 	// We want to try logging in as a different user, while Stopped.
 	// First, start the login process (without logging out first).
 	t.Logf("\n\nLoginDifferent")
-	notifies.expect(2)
+	notifies.expect(1)
 	b.StartLoginInteractive()
 	url3 := "http://localhost:1/3"
 	cc.send(nil, url3, false, nil)
 	{
-		nn := notifies.drain(2)
+		nn := notifies.drain(1)
 		// It might seem like WantRunning should switch to true here,
 		// but that would be risky since we already have a valid
 		// user account. It might try to reconnect to the old account
 		// before the new one is ready. So no change yet.
-		c.Assert([]string{"Login", "unpause"}, qt.DeepEquals, cc.getCalls())
+		//
+		// Because the login hasn't yet completed, the old login
+		// is still valid, so it's correct that we stay paused.
+		c.Assert([]string{"Login", "pause"}, qt.DeepEquals, cc.getCalls())
 		c.Assert(nn[0].BrowseToURL, qt.Not(qt.IsNil))
-		c.Assert(nn[1].State, qt.Not(qt.IsNil))
 		c.Assert(*nn[0].BrowseToURL, qt.Equals, url3)
-		c.Assert(ipn.NeedsLogin, qt.Equals, *nn[1].State)
 	}
 
-	// Now, let's say the interactive login completed, using a different
-	// user account than before.
+	// Now, let's complete the interactive login, using a different
+	// user account than before. WantRunning changes to true after an
+	// interactive login, so we end up unpaused.
 	t.Logf("\n\nLoginDifferent URL visited")
 	notifies.expect(3)
 	cc.persist.LoginName = "user3"
@@ -812,7 +836,13 @@ func TestStateMachine(t *testing.T) {
 	})
 	{
 		nn := notifies.drain(3)
-		c.Assert([]string{"unpause"}, qt.DeepEquals, cc.getCalls())
+		// BUG: pause() being called here is a bad sign.
+		//  It means that either the state machine ran at least once
+		//  with the old netmap, or it ran with the new login+netmap
+		//  and !WantRunning. But since it's a fresh and successful
+		//  new login, WantRunning is true, so there was never a
+		//  reason to pause().
+		c.Assert([]string{"pause", "unpause"}, qt.DeepEquals, cc.getCalls())
 		c.Assert(nn[0].LoginFinished, qt.Not(qt.IsNil))
 		c.Assert(nn[1].Prefs, qt.Not(qt.IsNil))
 		c.Assert(nn[2].State, qt.Not(qt.IsNil))
@@ -831,7 +861,7 @@ func TestStateMachine(t *testing.T) {
 	{
 		// NOTE: cc.Shutdown() is correct here, since we didn't call
 		// b.Shutdown() ourselves.
-		c.Assert([]string{"Shutdown", "New", "UpdateEndpoints", "Login"}, qt.DeepEquals, cc.getCalls())
+		c.Assert([]string{"Shutdown", "unpause", "New", "UpdateEndpoints", "Login", "unpause"}, qt.DeepEquals, cc.getCalls())
 
 		nn := notifies.drain(1)
 		c.Assert(cc.getCalls(), qt.HasLen, 0)
@@ -850,7 +880,7 @@ func TestStateMachine(t *testing.T) {
 	})
 	{
 		nn := notifies.drain(1)
-		c.Assert([]string{"unpause"}, qt.DeepEquals, cc.getCalls())
+		c.Assert([]string{"unpause", "unpause"}, qt.DeepEquals, cc.getCalls())
 		// NOTE: No LoginFinished message since no interactive
 		// login was needed.
 		c.Assert(nn[0].State, qt.Not(qt.IsNil))
@@ -861,3 +891,29 @@ func TestStateMachine(t *testing.T) {
 		c.Assert(ipn.Starting, qt.Equals, b.State())
 	}
 }
+
+type testStateStorage struct {
+	mem     ipn.MemoryStore
+	written syncs.AtomicBool
+}
+
+func (s *testStateStorage) ReadState(id ipn.StateKey) ([]byte, error) {
+	return s.mem.ReadState(id)
+}
+
+func (s *testStateStorage) WriteState(id ipn.StateKey, bs []byte) error {
+	s.written.Set(true)
+	return s.mem.WriteState(id, bs)
+}
+
+// awaitWrite clears the "I've seen writes" bit, in prep for a future
+// call to sawWrite to see if a write arrived.
+func (s *testStateStorage) awaitWrite() { s.written.Set(false) }
+
+// sawWrite reports whether there's been a WriteState call since the most
+// recent awaitWrite call.
+func (s *testStateStorage) sawWrite() bool {
+	v := s.written.Get()
+	s.awaitWrite()
+	return v
+}

ipn/ipnstate/ipnstate.go

@@ -20,6 +20,7 @@ import (
 
 	"inet.af/netaddr"
 	"tailscale.com/tailcfg"
+	"tailscale.com/tstime/mono"
 	"tailscale.com/types/key"
 	"tailscale.com/util/dnsname"
 )
@@ -90,7 +91,7 @@ type PeerStatus struct {
 	RxBytes       int64
 	TxBytes       int64
 	Created       time.Time // time registered with tailcontrol
-	LastWrite     time.Time // time last packet sent
+	LastWrite     mono.Time // time last packet sent
 	LastSeen      time.Time // last seen to tailcontrol
 	LastHandshake time.Time // with local wireguard
 	KeepAlive     bool
@@ -320,7 +321,7 @@ table tbody tr:nth-child(even) td { background-color: #f5f5f5; }
 	f("<tr><th>Peer</th><th>OS</th><th>Node</th><th>Owner</th><th>Rx</th><th>Tx</th><th>Activity</th><th>Connection</th></tr>\n")
 	f("</thead>\n<tbody>\n")
 
-	now := time.Now()
+	now := mono.Now()
 
 	var peers []*PeerStatus
 	for _, peer := range st.Peers() {
@@ -378,7 +379,7 @@ table tbody tr:nth-child(even) td { background-color: #f5f5f5; }
 		f("<td>")
 
 		// TODO: let server report this active bool instead
-		active := !ps.LastWrite.IsZero() && time.Since(ps.LastWrite) < 2*time.Minute
+		active := !ps.LastWrite.IsZero() && mono.Since(ps.LastWrite) < 2*time.Minute
 		if active {
 			if ps.Relay != "" && ps.CurAddr == "" {
 				f("relay <b>%s</b>", html.EscapeString(ps.Relay))

ipn/localapi/localapi.go

@@ -266,7 +266,7 @@ func (h *Handler) serveFiles(w http.ResponseWriter, r *http.Request) {
 		http.Error(w, "file access denied", http.StatusForbidden)
 		return
 	}
-	suffix := strings.TrimPrefix(r.URL.Path, "/localapi/v0/files/")
+	suffix := strings.TrimPrefix(r.URL.EscapedPath(), "/localapi/v0/files/")
 	if suffix == "" {
 		if r.Method != "GET" {
 			http.Error(w, "want GET to list files", 400)

ipn/prefs.go

@@ -25,11 +25,17 @@ import (
 
 //go:generate go run tailscale.com/cmd/cloner -type=Prefs -output=prefs_clone.go
 
-// DefaultControlURL returns the URL base of the control plane
+// DefaultControlURL is the URL base of the control plane
 // ("coordination server") for use when no explicit one is configured.
 // The default control plane is the hosted version run by Tailscale.com.
 const DefaultControlURL = "https://controlplane.tailscale.com"
 
+// IsLoginServerSynonym reports whether a URL is a drop-in replacement
+// for the primary Tailscale login server.
+func IsLoginServerSynonym(val interface{}) bool {
+	return val == "https://login.tailscale.com" || val == "https://controlplane.tailscale.com"
+}
+
 // Prefs are the user modifiable settings of the Tailscale node agent.
 type Prefs struct {
 	// ControlURL is the URL of the control server to use.
@@ -405,6 +411,16 @@ func (p *Prefs) ControlURLOrDefault() string {
 	return DefaultControlURL
 }
 
+// AdminPageURL returns the admin web site URL for the current ControlURL.
+func (p *Prefs) AdminPageURL() string {
+	url := p.ControlURLOrDefault()
+	if IsLoginServerSynonym(url) {
+		// TODO(crawshaw): In future release, make this https://console.tailscale.com
+		url = "https://login.tailscale.com"
+	}
+	return url + "/admin/machines"
+}
+
 // PrefsFromBytes deserializes Prefs from a JSON blob. If
 // enforceDefaults is true, Prefs.RouteAll and Prefs.AllowSingleHosts
 // are forced on.

ipn/store.go

@@ -67,9 +67,6 @@ func (s *MemoryStore) String() string { return "MemoryStore" }
 func (s *MemoryStore) ReadState(id StateKey) ([]byte, error) {
 	s.mu.Lock()
 	defer s.mu.Unlock()
-	if s.cache == nil {
-		s.cache = map[StateKey][]byte{}
-	}
 	bs, ok := s.cache[id]
 	if !ok {
 		return nil, ErrStateNotExist

logpolicy/logpolicy.go

@@ -128,6 +128,13 @@ func (l logWriter) Write(buf []byte) (int, error) {
 // logsDir returns the directory to use for log configuration and
 // buffer storage.
 func logsDir(logf logger.Logf) string {
+	if d := os.Getenv("TS_LOGS_DIR"); d != "" {
+		fi, err := os.Stat(d)
+		if err == nil && fi.IsDir() {
+			return d
+		}
+	}
+
 	// STATE_DIRECTORY is set by systemd 240+ but we support older
 	// systems-d. For example, Ubuntu 18.04 (Bionic Beaver) is 237.
 	systemdStateDir := os.Getenv("STATE_DIRECTORY")

net/dns/config.go

@@ -5,9 +5,12 @@
 package dns
 
 import (
+	"bufio"
+	"fmt"
 	"sort"
 
 	"inet.af/netaddr"
+	"tailscale.com/net/dns/resolver"
 	"tailscale.com/util/dnsname"
 )
 
@@ -38,6 +41,20 @@ type Config struct {
 	Hosts map[dnsname.FQDN][]netaddr.IP
 }
 
+// WriteToBufioWriter write a debug version of c for logs to w, omitting
+// spammy stuff like *.arpa entries and replacing it with a total count.
+func (c *Config) WriteToBufioWriter(w *bufio.Writer) {
+	w.WriteString("{DefaultResolvers:")
+	resolver.WriteIPPorts(w, c.DefaultResolvers)
+
+	w.WriteString(" Routes:")
+	resolver.WriteRoutes(w, c.Routes)
+
+	fmt.Fprintf(w, " SearchDomains:%v", c.SearchDomains)
+	fmt.Fprintf(w, " Hosts:%v", len(c.Hosts))
+	w.WriteString("}")
+}
+
 // needsAnyResolvers reports whether c requires a resolver to be set
 // at the OS level.
 func (c Config) needsOSResolver() bool {

net/dns/ini.go

@@ -2,6 +2,8 @@
 // Use of this source code is governed by a BSD-style
 // license that can be found in the LICENSE file.
 
+// +build windows
+
 package dns
 
 import (

net/dns/ini_test.go

@@ -2,6 +2,8 @@
 // Use of this source code is governed by a BSD-style
 // license that can be found in the LICENSE file.
 
+// +build windows
+
 package dns
 
 import (

net/dns/manager.go

@@ -5,6 +5,7 @@
 package dns
 
 import (
+	"bufio"
 	"runtime"
 	"time"
 
@@ -50,14 +51,18 @@ func NewManager(logf logger.Logf, oscfg OSConfigurator, linkMon *monitor.Mon, li
 }
 
 func (m *Manager) Set(cfg Config) error {
-	m.logf("Set: %+v", cfg)
+	m.logf("Set: %v", logger.ArgWriter(func(w *bufio.Writer) {
+		cfg.WriteToBufioWriter(w)
+	}))
 
 	rcfg, ocfg, err := m.compileConfig(cfg)
 	if err != nil {
 		return err
 	}
 
-	m.logf("Resolvercfg: %+v", rcfg)
+	m.logf("Resolvercfg: %v", logger.ArgWriter(func(w *bufio.Writer) {
+		rcfg.WriteToBufioWriter(w)
+	}))
 	m.logf("OScfg: %+v", ocfg)
 
 	if err := m.resolver.SetConfig(rcfg); err != nil {

net/dns/resolver/doh_test.go

@@ -0,0 +1,96 @@
+// Copyright (c) 2021 Tailscale Inc & AUTHORS All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package resolver
+
+import (
+	"context"
+	"flag"
+	"net/http"
+	"testing"
+
+	"golang.org/x/net/dns/dnsmessage"
+)
+
+var testDoH = flag.Bool("test-doh", false, "do real DoH tests against the network")
+
+const someDNSID = 123 // something non-zero as a test; in violation of spec's SHOULD of 0
+
+func someDNSQuestion(t testing.TB) []byte {
+	b := dnsmessage.NewBuilder(nil, dnsmessage.Header{
+		OpCode:           0, // query
+		RecursionDesired: true,
+		ID:               someDNSID,
+	})
+	b.StartQuestions() // err
+	b.Question(dnsmessage.Question{
+		Name:  dnsmessage.MustNewName("tailscale.com."),
+		Type:  dnsmessage.TypeA,
+		Class: dnsmessage.ClassINET,
+	})
+	msg, err := b.Finish()
+	if err != nil {
+		t.Fatal(err)
+	}
+	return msg
+}
+
+func TestDoH(t *testing.T) {
+	if !*testDoH {
+		t.Skip("skipping manual test without --test-doh flag")
+	}
+	if len(knownDoH) == 0 {
+		t.Fatal("no known DoH")
+	}
+
+	f := new(forwarder)
+
+	for ip := range knownDoH {
+		t.Run(ip.String(), func(t *testing.T) {
+			urlBase, c, ok := f.getDoHClient(ip)
+			if !ok {
+				t.Fatal("expected DoH")
+			}
+			res, err := f.sendDoH(context.Background(), urlBase, c, someDNSQuestion(t))
+			if err != nil {
+				t.Fatal(err)
+			}
+			c.Transport.(*http.Transport).CloseIdleConnections()
+
+			var p dnsmessage.Parser
+			h, err := p.Start(res)
+			if err != nil {
+				t.Fatal(err)
+			}
+			if h.ID != someDNSID {
+				t.Errorf("response DNS ID = %v; want %v", h.ID, someDNSID)
+			}
+
+			p.SkipAllQuestions()
+			aa, err := p.AllAnswers()
+			if err != nil {
+				t.Fatal(err)
+			}
+			if len(aa) == 0 {
+				t.Fatal("no answers")
+			}
+			for _, r := range aa {
+				t.Logf("got: %v", r.GoString())
+			}
+		})
+	}
+}
+
+func TestDoHV6Fallback(t *testing.T) {
+	for ip, base := range knownDoH {
+		if ip.Is4() {
+			ip6, ok := dohV6(base)
+			if !ok {
+				t.Errorf("no v6 DoH known for %v", ip)
+			} else if !ip6.Is6() {
+				t.Errorf("dohV6(%q) returned non-v6 address %v", base, ip6)
+			}
+		}
+	}
+}

net/dns/resolver/forwarder.go

@@ -9,15 +9,22 @@ import (
 	"context"
 	"encoding/binary"
 	"errors"
+	"fmt"
 	"hash/crc32"
 	"io"
+	"io/ioutil"
 	"math/rand"
 	"net"
+	"net/http"
+	"runtime"
+	"sort"
+	"strings"
 	"sync"
 	"time"
 
 	dns "golang.org/x/net/dns/dnsmessage"
 	"inet.af/netaddr"
+	"tailscale.com/net/netns"
 	"tailscale.com/types/logger"
 	"tailscale.com/util/dnsname"
 	"tailscale.com/wgengine/monitor"
@@ -29,6 +36,16 @@ const headerBytes = 12
 const (
 	// responseTimeout is the maximal amount of time to wait for a DNS response.
 	responseTimeout = 5 * time.Second
+
+	// dohTransportTimeout is how long to keep idle HTTP
+	// connections open to DNS-over-HTTPs servers. This is pretty
+	// arbitrary.
+	dohTransportTimeout = 30 * time.Second
+
+	// wellKnownHostBackupDelay is how long to artificially delay upstream
+	// DNS queries to the "fallback" DNS server IP for a known provider
+	// (e.g. how long to wait to query Google's 8.8.4.4 after 8.8.8.8).
+	wellKnownHostBackupDelay = 200 * time.Millisecond
 )
 
 var errNoUpstreams = errors.New("upstream nameservers not set")
@@ -108,6 +125,7 @@ func clampEDNSSize(packet []byte, maxSize uint16) {
 		return
 	}
 
+	// https://datatracker.ietf.org/doc/html/rfc6891#section-6.1.2
 	opt := packet[len(packet)-optFixedBytes:]
 
 	if opt[0] != 0 {
@@ -124,8 +142,8 @@ func clampEDNSSize(packet []byte, maxSize uint16) {
 		// Be conservative and don't touch unknown versions.
 		return
 	}
-	// Ignore flags in opt[7:9]
-	if binary.BigEndian.Uint16(opt[10:12]) != 0 {
+	// Ignore flags in opt[6:9]
+	if binary.BigEndian.Uint16(opt[9:11]) != 0 {
 		// RDLEN must be 0 (no variable length data). We're at the end of the
 		// packet so this should be 0 anyway)..
 		return
@@ -141,7 +159,20 @@ func clampEDNSSize(packet []byte, maxSize uint16) {
 
 type route struct {
 	Suffix    dnsname.FQDN
-	Resolvers []netaddr.IPPort
+	Resolvers []resolverAndDelay
+}
+
+// resolverAndDelay is an upstream DNS resolver and a delay for how
+// long to wait before querying it.
+type resolverAndDelay struct {
+	// ipp is the upstream resolver.
+	ipp netaddr.IPPort
+
+	// startDelay is an amount to delay this resolver at
+	// start. It's used when, say, there are four Google or
+	// Cloudflare DNS IPs (two IPv4 + two IPv6) and we don't want
+	// to race all four at once.
+	startDelay time.Duration
 }
 
 // forwarder forwards DNS packets to a number of upstream nameservers.
@@ -149,6 +180,7 @@ type forwarder struct {
 	logf    logger.Logf
 	linkMon *monitor.Mon
 	linkSel ForwardLinkSelector
+	dohSem  chan struct{}
 
 	ctx       context.Context    // good until Close
 	ctxCancel context.CancelFunc // closes ctx
@@ -158,6 +190,8 @@ type forwarder struct {
 
 	mu sync.Mutex // guards following
 
+	dohClient map[netaddr.IP]*http.Client
+
 	// routes are per-suffix resolvers to use, with
 	// the most specific routes first.
 	routes []route
@@ -168,11 +202,18 @@ func init() {
 }
 
 func newForwarder(logf logger.Logf, responses chan packet, linkMon *monitor.Mon, linkSel ForwardLinkSelector) *forwarder {
+	maxDoHInFlight := 1000 // effectively unlimited
+	if runtime.GOOS == "ios" {
+		// No HTTP/2 on iOS yet (for size reasons), so DoH is
+		// pricier.
+		maxDoHInFlight = 10
+	}
 	f := &forwarder{
 		logf:      logger.WithPrefix(logf, "forward: "),
 		linkMon:   linkMon,
 		linkSel:   linkSel,
 		responses: responses,
+		dohSem:    make(chan struct{}, maxDoHInFlight),
 	}
 	f.ctx, f.ctxCancel = context.WithCancel(context.Background())
 	return f
@@ -183,7 +224,84 @@ func (f *forwarder) Close() error {
 	return nil
 }
 
-func (f *forwarder) setRoutes(routes []route) {
+// resolversWithDelays maps from a set of DNS server ip:ports (currently
+// the port is always 53) to a slice of a type that included a
+// startDelay. So if ipps contains e.g. four Google DNS IPs (two IPv4
+// + twoIPv6), this function partition adds delays to some.
+func resolversWithDelays(ipps []netaddr.IPPort) []resolverAndDelay {
+	type hostAndFam struct {
+		host string // some arbitrary string representing DNS host (currently the DoH base)
+		bits uint8  // either 32 or 128 for IPv4 vs IPv6s address family
+	}
+
+	// Track how many of each known resolver host are in the list,
+	// per address family.
+	total := map[hostAndFam]int{}
+
+	rr := make([]resolverAndDelay, len(ipps))
+	for _, ipp := range ipps {
+		ip := ipp.IP()
+		if host, ok := knownDoH[ip]; ok {
+			total[hostAndFam{host, ip.BitLen()}]++
+		}
+	}
+
+	done := map[hostAndFam]int{}
+	for i, ipp := range ipps {
+		ip := ipp.IP()
+		var startDelay time.Duration
+		if host, ok := knownDoH[ip]; ok {
+			key4 := hostAndFam{host, 32}
+			key6 := hostAndFam{host, 128}
+			switch {
+			case ip.Is4():
+				if done[key4] > 0 {
+					startDelay += wellKnownHostBackupDelay
+				}
+			case ip.Is6():
+				total4 := total[key4]
+				if total4 >= 2 {
+					// If we have two IPv4 IPs of the same provider
+					// already in the set, delay the IPv6 queries
+					// until halfway through the timeout (so wait
+					// 2.5 seconds). Even the network is IPv6-only,
+					// the DoH dialer will fallback to IPv6
+					// immediately anyway.
+					startDelay = responseTimeout / 2
+				} else if total4 == 1 {
+					startDelay += wellKnownHostBackupDelay
+				}
+				if done[key6] > 0 {
+					startDelay += wellKnownHostBackupDelay
+				}
+			}
+			done[hostAndFam{host, ip.BitLen()}]++
+		}
+		rr[i] = resolverAndDelay{
+			ipp:        ipp,
+			startDelay: startDelay,
+		}
+	}
+	return rr
+}
+
+// setRoutes sets the routes to use for DNS forwarding. It's called by
+// Resolver.SetConfig on reconfig.
+//
+// The memory referenced by routesBySuffix should not be modified.
+func (f *forwarder) setRoutes(routesBySuffix map[dnsname.FQDN][]netaddr.IPPort) {
+	routes := make([]route, 0, len(routesBySuffix))
+	for suffix, ipps := range routesBySuffix {
+		routes = append(routes, route{
+			Suffix:    suffix,
+			Resolvers: resolversWithDelays(ipps),
+		})
+	}
+	// Sort from longest prefix to shortest.
+	sort.Slice(routes, func(i, j int) bool {
+		return routes[i].Suffix.NumLabels() > routes[j].Suffix.NumLabels()
+	})
+
 	f.mu.Lock()
 	defer f.mu.Unlock()
 	f.routes = routes
@@ -210,21 +328,104 @@ func (f *forwarder) packetListener(ip netaddr.IP) (packetListener, error) {
 	return lc, nil
 }
 
+func (f *forwarder) getDoHClient(ip netaddr.IP) (urlBase string, c *http.Client, ok bool) {
+	urlBase, ok = knownDoH[ip]
+	if !ok {
+		return
+	}
+	f.mu.Lock()
+	defer f.mu.Unlock()
+	if c, ok := f.dohClient[ip]; ok {
+		return urlBase, c, true
+	}
+	if f.dohClient == nil {
+		f.dohClient = map[netaddr.IP]*http.Client{}
+	}
+	nsDialer := netns.NewDialer()
+	c = &http.Client{
+		Transport: &http.Transport{
+			IdleConnTimeout: dohTransportTimeout,
+			DialContext: func(ctx context.Context, netw, addr string) (net.Conn, error) {
+				if !strings.HasPrefix(netw, "tcp") {
+					return nil, fmt.Errorf("unexpected network %q", netw)
+				}
+				c, err := nsDialer.DialContext(ctx, "tcp", net.JoinHostPort(ip.String(), "443"))
+				// If v4 failed, try an equivalent v6 also in the time remaining.
+				if err != nil && ctx.Err() == nil {
+					if ip6, ok := dohV6(urlBase); ok && ip.Is4() {
+						if c6, err := nsDialer.DialContext(ctx, "tcp", net.JoinHostPort(ip6.String(), "443")); err == nil {
+							return c6, nil
+						}
+					}
+				}
+				return c, err
+			},
+		},
+	}
+	f.dohClient[ip] = c
+	return urlBase, c, true
+}
+
+const dohType = "application/dns-message"
+
+func (f *forwarder) releaseDoHSem() { <-f.dohSem }
+
+func (f *forwarder) sendDoH(ctx context.Context, urlBase string, c *http.Client, packet []byte) ([]byte, error) {
+	// Bound the number of HTTP requests in flight. This primarily
+	// matters for iOS where we're very memory constrained and
+	// HTTP requests are heavier on iOS where we don't include
+	// HTTP/2 for binary size reasons (as binaries on iOS linked
+	// with Go code cost memory proportional to the binary size,
+	// for reasons not fully understood).
+	select {
+	case f.dohSem <- struct{}{}:
+	case <-ctx.Done():
+		return nil, ctx.Err()
+	}
+	defer f.releaseDoHSem()
+
+	req, err := http.NewRequestWithContext(ctx, "POST", urlBase, bytes.NewReader(packet))
+	if err != nil {
+		return nil, err
+	}
+	req.Header.Set("Content-Type", dohType)
+	// Note: we don't currently set the Accept header (which is
+	// only a SHOULD in the spec) as iOS doesn't use HTTP/2 and
+	// we'd rather save a few bytes on outgoing requests when
+	// empirically no provider cares about the Accept header's
+	// absence.
+
+	hres, err := c.Do(req)
+	if err != nil {
+		return nil, err
+	}
+	defer hres.Body.Close()
+	if hres.StatusCode != 200 {
+		return nil, errors.New(hres.Status)
+	}
+	if ct := hres.Header.Get("Content-Type"); ct != dohType {
+		return nil, fmt.Errorf("unexpected response Content-Type %q", ct)
+	}
+	return ioutil.ReadAll(hres.Body)
+}
+
 // send sends packet to dst. It is best effort.
 //
 // send expects the reply to have the same txid as txidOut.
 //
-// The provided closeOnCtxDone lets send register values to Close if
-// the caller's ctx expires. This avoids send from allocating its own
-// waiting goroutine to interrupt the ReadFrom, as memory is tight on
-// iOS and we want the number of pending DNS lookups to be bursty
-// without too much associated goroutine/memory cost.
-func (f *forwarder) send(ctx context.Context, txidOut txid, closeOnCtxDone *closePool, packet []byte, dst netaddr.IPPort) ([]byte, error) {
-	// TODO(bradfitz): if dst.IP is 8.8.8.8 or 8.8.4.4 or 1.1.1.1, etc, or
-	// something dynamically probed earlier to support DoH or DoT,
-	// do that here instead.
+func (f *forwarder) send(ctx context.Context, fq *forwardQuery, dst netaddr.IPPort) ([]byte, error) {
+	ip := dst.IP()
 
-	ln, err := f.packetListener(dst.IP())
+	// Upgrade known DNS IPs to DoH (DNS-over-HTTPs).
+	if urlBase, dc, ok := f.getDoHClient(ip); ok {
+		res, err := f.sendDoH(ctx, urlBase, dc, fq.packet)
+		if err == nil || ctx.Err() != nil {
+			return res, err
+		}
+		f.logf("DoH error from %v: %v", ip, err)
+	}
+
+	ln, err := f.packetListener(ip)
 	if err != nil {
 		return nil, err
 	}
@@ -235,10 +436,10 @@ func (f *forwarder) send(ctx context.Context, txidOut txid, closeOnCtxDone *clos
 	}
 	defer conn.Close()
 
-	closeOnCtxDone.Add(conn)
-	defer closeOnCtxDone.Remove(conn)
+	fq.closeOnCtxDone.Add(conn)
+	defer fq.closeOnCtxDone.Remove(conn)
 
-	if _, err := conn.WriteTo(packet, dst.UDPAddr()); err != nil {
+	if _, err := conn.WriteTo(fq.packet, dst.UDPAddr()); err != nil {
 		if err := ctx.Err(); err != nil {
 			return nil, err
 		}
@@ -267,7 +468,7 @@ func (f *forwarder) send(ctx context.Context, txidOut txid, closeOnCtxDone *clos
 	}
 	out = out[:n]
 	txid := getTxID(out)
-	if txid != txidOut {
+	if txid != fq.txid {
 		return nil, errors.New("txid doesn't match")
 	}
 
@@ -290,7 +491,7 @@ func (f *forwarder) send(ctx context.Context, txidOut txid, closeOnCtxDone *clos
 }
 
 // resolvers returns the resolvers to use for domain.
-func (f *forwarder) resolvers(domain dnsname.FQDN) []netaddr.IPPort {
+func (f *forwarder) resolvers(domain dnsname.FQDN) []resolverAndDelay {
 	f.mu.Lock()
 	routes := f.routes
 	f.mu.Unlock()
@@ -302,6 +503,30 @@ func (f *forwarder) resolvers(domain dnsname.FQDN) []netaddr.IPPort {
 	return nil
 }
 
+// forwardQuery is information and state about a forwarded DNS query that's
+// being sent to 1 or more upstreams.
+//
+// In the case of racing against multiple equivalent upstreams
+// (e.g. Google or CloudFlare's 4 DNS IPs: 2 IPv4 + 2 IPv6), this type
+// handles racing them more intelligently than just blasting away 4
+// queries at once.
+type forwardQuery struct {
+	txid   txid
+	packet []byte
+
+	// closeOnCtxDone lets send register values to Close if the
+	// caller's ctx expires. This avoids send from allocating its
+	// own waiting goroutine to interrupt the ReadFrom, as memory
+	// is tight on iOS and we want the number of pending DNS
+	// lookups to be bursty without too much associated
+	// goroutine/memory cost.
+	closeOnCtxDone *closePool
+
+	// TODO(bradfitz): add race delay state:
+	// mu sync.Mutex
+	// ...
+}
+
 // forward forwards the query to all upstream nameservers and returns the first response.
 func (f *forwarder) forward(query packet) error {
 	domain, err := nameFromQuery(query.bs)
@@ -309,7 +534,6 @@ func (f *forwarder) forward(query packet) error {
 		return err
 	}
 
-	txid := getTxID(query.bs)
 	clampEDNSSize(query.bs, maxResponseBytes)
 
 	resolvers := f.resolvers(domain)
@@ -317,8 +541,12 @@ func (f *forwarder) forward(query packet) error {
 		return errNoUpstreams
 	}
 
-	closeOnCtxDone := new(closePool)
-	defer closeOnCtxDone.Close()
+	fq := &forwardQuery{
+		txid:           getTxID(query.bs),
+		packet:         query.bs,
+		closeOnCtxDone: new(closePool),
+	}
+	defer fq.closeOnCtxDone.Close()
 
 	ctx, cancel := context.WithTimeout(f.ctx, responseTimeout)
 	defer cancel()
@@ -329,9 +557,18 @@ func (f *forwarder) forward(query packet) error {
 		firstErr error
 	)
 
-	for _, ipp := range resolvers {
-		go func(ipp netaddr.IPPort) {
-			resb, err := f.send(ctx, txid, closeOnCtxDone, query.bs, ipp)
+	for _, rr := range resolvers {
+		go func(rr resolverAndDelay) {
+			if rr.startDelay > 0 {
+				timer := time.NewTimer(rr.startDelay)
+				select {
+				case <-timer.C:
+				case <-ctx.Done():
+					timer.Stop()
+					return
+				}
+			}
+			resb, err := f.send(ctx, fq, rr.ipp)
 			if err != nil {
 				mu.Lock()
 				defer mu.Unlock()
@@ -344,7 +581,7 @@ func (f *forwarder) forward(query packet) error {
 			case resc <- resb:
 			default:
 			}
-		}(ipp)
+		}(rr)
 	}
 
 	select {
@@ -432,3 +669,63 @@ func (p *closePool) Close() error {
 	}
 	return nil
 }
+
+var knownDoH = map[netaddr.IP]string{}
+
+var dohIPsOfBase = map[string][]netaddr.IP{}
+
+func addDoH(ipStr, base string) {
+	ip := netaddr.MustParseIP(ipStr)
+	knownDoH[ip] = base
+	dohIPsOfBase[base] = append(dohIPsOfBase[base], ip)
+}
+
+func dohV6(base string) (ip netaddr.IP, ok bool) {
+	for _, ip := range dohIPsOfBase[base] {
+		if ip.Is6() {
+			return ip, true
+		}
+	}
+	return ip, false
+}
+
+func init() {
+	// Cloudflare
+	addDoH("1.1.1.1", "https://cloudflare-dns.com/dns-query")
+	addDoH("1.0.0.1", "https://cloudflare-dns.com/dns-query")
+	addDoH("2606:4700:4700::1111", "https://cloudflare-dns.com/dns-query")
+	addDoH("2606:4700:4700::1001", "https://cloudflare-dns.com/dns-query")
+
+	// Cloudflare -Malware
+	addDoH("1.1.1.2", "https://security.cloudflare-dns.com/dns-query")
+	addDoH("1.0.0.2", "https://security.cloudflare-dns.com/dns-query")
+	addDoH("2606:4700:4700::1112", "https://security.cloudflare-dns.com/dns-query")
+	addDoH("2606:4700:4700::1002", "https://security.cloudflare-dns.com/dns-query")
+
+	// Cloudflare -Malware -Adult
+	addDoH("1.1.1.3", "https://family.cloudflare-dns.com/dns-query")
+	addDoH("1.0.0.3", "https://family.cloudflare-dns.com/dns-query")
+	addDoH("2606:4700:4700::1113", "https://family.cloudflare-dns.com/dns-query")
+	addDoH("2606:4700:4700::1003", "https://family.cloudflare-dns.com/dns-query")
+
+	// Google
+	addDoH("8.8.8.8", "https://dns.google/dns-query")
+	addDoH("8.8.4.4", "https://dns.google/dns-query")
+	addDoH("2001:4860:4860::8888", "https://dns.google/dns-query")
+	addDoH("2001:4860:4860::8844", "https://dns.google/dns-query")
+
+	// OpenDNS
+	// TODO(bradfitz): OpenDNS is unique amongst this current set in that
+	// its DoH DNS names resolve to different IPs than its normal DNS
+	// IPs. Support that later. For now we assume that they're the same.
+	// addDoH("208.67.222.222", "https://doh.opendns.com/dns-query")
+	// addDoH("208.67.220.220", "https://doh.opendns.com/dns-query")
+	// addDoH("208.67.222.123", "https://doh.familyshield.opendns.com/dns-query")
+	// addDoH("208.67.220.123", "https://doh.familyshield.opendns.com/dns-query")
+
+	// Quad9
+	addDoH("9.9.9.9", "https://dns.quad9.net/dns-query")
+	addDoH("149.112.112.112", "https://dns.quad9.net/dns-query")
+	addDoH("2620:fe::fe", "https://dns.quad9.net/dns-query")
+	addDoH("2620:fe::fe:9", "https://dns.quad9.net/dns-query")
+}

net/dns/resolver/tsdns.go

@@ -7,8 +7,10 @@
 package resolver
 
 import (
+	"bufio"
 	"encoding/hex"
 	"errors"
+	"fmt"
 	"runtime"
 	"sort"
 	"strings"
@@ -79,6 +81,74 @@ type Config struct {
 	LocalDomains []dnsname.FQDN
 }
 
+// WriteToBufioWriter write a debug version of c for logs to w, omitting
+// spammy stuff like *.arpa entries and replacing it with a total count.
+func (c *Config) WriteToBufioWriter(w *bufio.Writer) {
+	w.WriteString("{Routes:")
+	WriteRoutes(w, c.Routes)
+	fmt.Fprintf(w, " Hosts:%v LocalDomains:[", len(c.Hosts))
+	space := false
+	arpa := 0
+	for _, d := range c.LocalDomains {
+		if strings.HasSuffix(string(d), ".arpa.") {
+			arpa++
+			continue
+		}
+		if space {
+			w.WriteByte(' ')
+		}
+		w.WriteString(string(d))
+		space = true
+	}
+	w.WriteString("]")
+	if arpa > 0 {
+		fmt.Fprintf(w, "+%darpa", arpa)
+	}
+	w.WriteString("}")
+}
+
+// WriteIPPorts writes vv to w.
+func WriteIPPorts(w *bufio.Writer, vv []netaddr.IPPort) {
+	w.WriteByte('[')
+	var b []byte
+	for i, v := range vv {
+		if i > 0 {
+			w.WriteByte(' ')
+		}
+		b = v.AppendTo(b[:0])
+		w.Write(b)
+	}
+	w.WriteByte(']')
+}
+
+// WriteRoutes writes routes to w, omitting *.arpa routes and instead
+// summarizing how many of them there were.
+func WriteRoutes(w *bufio.Writer, routes map[dnsname.FQDN][]netaddr.IPPort) {
+	var kk []dnsname.FQDN
+	arpa := 0
+	for k := range routes {
+		if strings.HasSuffix(string(k), ".arpa.") {
+			arpa++
+			continue
+		}
+		kk = append(kk, k)
+	}
+	sort.Slice(kk, func(i, j int) bool { return kk[i] < kk[j] })
+	w.WriteByte('{')
+	for i, k := range kk {
+		if i > 0 {
+			w.WriteByte(' ')
+		}
+		w.WriteString(string(k))
+		w.WriteByte(':')
+		WriteIPPorts(w, routes[k])
+	}
+	w.WriteByte('}')
+	if arpa > 0 {
+		fmt.Fprintf(w, "+%darpa", arpa)
+	}
+}
+
 // Resolver is a DNS resolver for nodes on the Tailscale network,
 // associating them with domain names of the form <mynode>.<mydomain>.<root>.
 // If it is asked to resolve a domain that is not of that form,
@@ -138,7 +208,6 @@ func (r *Resolver) SetConfig(cfg Config) error {
 		r.saveConfigForTests(cfg)
 	}
 
-	routes := make([]route, 0, len(cfg.Routes))
 	reverse := make(map[netaddr.IP]dnsname.FQDN, len(cfg.Hosts))
 
 	for host, ips := range cfg.Hosts {
@@ -147,18 +216,7 @@ func (r *Resolver) SetConfig(cfg Config) error {
 		}
 	}
 
-	for suffix, ips := range cfg.Routes {
-		routes = append(routes, route{
-			Suffix:    suffix,
-			Resolvers: ips,
-		})
-	}
-	// Sort from longest prefix to shortest.
-	sort.Slice(routes, func(i, j int) bool {
-		return routes[i].Suffix.NumLabels() > routes[j].Suffix.NumLabels()
-	})
-
-	r.forwarder.setRoutes(routes)
+	r.forwarder.setRoutes(cfg.Routes)
 
 	r.mu.Lock()
 	defer r.mu.Unlock()

net/dns/resolver/tsdns_test.go

@@ -931,8 +931,11 @@ func TestAllocs(t *testing.T) {
 		query []byte
 		want  int
 	}{
-		// Name lowercasing and response slice created by dns.NewBuilder.
-		{"forward", dnspacket("test1.ipn.dev.", dns.TypeA, noEdns), 2},
+		// Name lowercasing, response slice created by dns.NewBuilder,
+		// and closure allocation from go call.
+		// (Closure allocation only happens when using new register ABI,
+		// which is amd64 with Go 1.17, and probably more platforms later.)
+		{"forward", dnspacket("test1.ipn.dev.", dns.TypeA, noEdns), 3},
 		// 3 extra allocs in rdnsNameToIPv4 and one in marshalPTRRecord (dns.NewName).
 		{"reverse", dnspacket("4.3.2.1.in-addr.arpa.", dns.TypePTR, noEdns), 5},
 	}

net/dnscache/dnscache.go

@@ -2,6 +2,8 @@
 // Use of this source code is governed by a BSD-style
 // license that can be found in the LICENSE file.
 
+// TODO(bradfitz): update this code to use netaddr more
+
 // Package dnscache contains a minimal DNS cache that makes a bunch of
 // assumptions that are only valid for us. Not recommended for general use.
 package dnscache
@@ -78,8 +80,9 @@ type Resolver struct {
 }
 
 type ipCacheEntry struct {
-	ip      net.IP // either v4 or v6
-	ip6     net.IP // nil if no v4 or no v6
+	ip      net.IP       // either v4 or v6
+	ip6     net.IP       // nil if no v4 or no v6
+	allIPs  []net.IPAddr // 1+ v4 and/or v6
 	expires time.Time
 }
 
@@ -105,81 +108,82 @@ var debug, _ = strconv.ParseBool(os.Getenv("TS_DEBUG_DNS_CACHE"))
 //
 // If err is nil, ip will be non-nil. The v6 address may be nil even
 // with a nil error.
-func (r *Resolver) LookupIP(ctx context.Context, host string) (ip, v6 net.IP, err error) {
+func (r *Resolver) LookupIP(ctx context.Context, host string) (ip, v6 net.IP, allIPs []net.IPAddr, err error) {
 	if ip := net.ParseIP(host); ip != nil {
 		if ip4 := ip.To4(); ip4 != nil {
-			return ip4, nil, nil
+			return ip4, nil, []net.IPAddr{{IP: ip4}}, nil
 		}
 		if debug {
 			log.Printf("dnscache: %q is an IP", host)
 		}
-		return ip, nil, nil
+		return ip, nil, []net.IPAddr{{IP: ip}}, nil
 	}
 
-	if ip, ip6, ok := r.lookupIPCache(host); ok {
+	if ip, ip6, allIPs, ok := r.lookupIPCache(host); ok {
 		if debug {
 			log.Printf("dnscache: %q = %v (cached)", host, ip)
 		}
-		return ip, ip6, nil
+		return ip, ip6, allIPs, nil
 	}
 
-	type ipPair struct {
+	type ipRes struct {
 		ip, ip6 net.IP
+		allIPs  []net.IPAddr
 	}
 	ch := r.sf.DoChan(host, func() (interface{}, error) {
-		ip, ip6, err := r.lookupIP(host)
+		ip, ip6, allIPs, err := r.lookupIP(host)
 		if err != nil {
 			return nil, err
 		}
-		return ipPair{ip, ip6}, nil
+		return ipRes{ip, ip6, allIPs}, nil
 	})
 	select {
 	case res := <-ch:
 		if res.Err != nil {
 			if r.UseLastGood {
-				if ip, ip6, ok := r.lookupIPCacheExpired(host); ok {
+				if ip, ip6, allIPs, ok := r.lookupIPCacheExpired(host); ok {
 					if debug {
 						log.Printf("dnscache: %q using %v after error", host, ip)
 					}
-					return ip, ip6, nil
+					return ip, ip6, allIPs, nil
 				}
 			}
 			if debug {
 				log.Printf("dnscache: error resolving %q: %v", host, res.Err)
 			}
-			return nil, nil, res.Err
+			return nil, nil, nil, res.Err
 		}
-		pair := res.Val.(ipPair)
-		return pair.ip, pair.ip6, nil
+		r := res.Val.(ipRes)
+		return r.ip, r.ip6, r.allIPs, nil
 	case <-ctx.Done():
 		if debug {
 			log.Printf("dnscache: context done while resolving %q: %v", host, ctx.Err())
 		}
-		return nil, nil, ctx.Err()
+		return nil, nil, nil, ctx.Err()
 	}
 }
 
-func (r *Resolver) lookupIPCache(host string) (ip, ip6 net.IP, ok bool) {
+func (r *Resolver) lookupIPCache(host string) (ip, ip6 net.IP, allIPs []net.IPAddr, ok bool) {
 	r.mu.Lock()
 	defer r.mu.Unlock()
 	if ent, ok := r.ipCache[host]; ok && ent.expires.After(time.Now()) {
-		return ent.ip, ent.ip6, true
+		return ent.ip, ent.ip6, ent.allIPs, true
 	}
-	return nil, nil, false
+	return nil, nil, nil, false
 }
 
-func (r *Resolver) lookupIPCacheExpired(host string) (ip, ip6 net.IP, ok bool) {
+func (r *Resolver) lookupIPCacheExpired(host string) (ip, ip6 net.IP, allIPs []net.IPAddr, ok bool) {
 	r.mu.Lock()
 	defer r.mu.Unlock()
 	if ent, ok := r.ipCache[host]; ok {
-		return ent.ip, ent.ip6, true
+		return ent.ip, ent.ip6, ent.allIPs, true
 	}
-	return nil, nil, false
+	return nil, nil, nil, false
 }
 
 func (r *Resolver) lookupTimeoutForHost(host string) time.Duration {
 	if r.UseLastGood {
-		if _, _, ok := r.lookupIPCacheExpired(host); ok {
+		if _, _, _, ok := r.lookupIPCacheExpired(host); ok {
 			// If we have some previous good value for this host,
 			// don't give this DNS lookup much time. If we're in a
 			// situation where the user's DNS server is unreachable
@@ -194,12 +198,12 @@ func (r *Resolver) lookupTimeoutForHost(host string) time.Duration {
 	return 10 * time.Second
 }
 
-func (r *Resolver) lookupIP(host string) (ip, ip6 net.IP, err error) {
-	if ip, ip6, ok := r.lookupIPCache(host); ok {
+func (r *Resolver) lookupIP(host string) (ip, ip6 net.IP, allIPs []net.IPAddr, err error) {
+	if ip, ip6, allIPs, ok := r.lookupIPCache(host); ok {
 		if debug {
 			log.Printf("dnscache: %q found in cache as %v", host, ip)
 		}
-		return ip, ip6, nil
+		return ip, ip6, allIPs, nil
 	}
 
 	ctx, cancel := context.WithTimeout(context.Background(), r.lookupTimeoutForHost(host))
@@ -218,10 +222,10 @@ func (r *Resolver) lookupIP(host string) (ip, ip6 net.IP, err error) {
 		}
 	}
 	if err != nil {
-		return nil, nil, err
+		return nil, nil, nil, err
 	}
 	if len(ips) == 0 {
-		return nil, nil, fmt.Errorf("no IPs for %q found", host)
+		return nil, nil, nil, fmt.Errorf("no IPs for %q found", host)
 	}
 
 	have4 := false
@@ -240,12 +244,12 @@ func (r *Resolver) lookupIP(host string) (ip, ip6 net.IP, err error) {
 			}
 		}
 	}
-	r.addIPCache(host, ip, ip6, r.ttl())
-	return ip, ip6, nil
+	r.addIPCache(host, ip, ip6, ips, r.ttl())
+	return ip, ip6, ips, nil
 }
 
-func (r *Resolver) addIPCache(host string, ip, ip6 net.IP, d time.Duration) {
-	if isPrivateIP(ip) {
+func (r *Resolver) addIPCache(host string, ip, ip6 net.IP, allIPs []net.IPAddr, d time.Duration) {
+	if naIP, _ := netaddr.FromStdIP(ip); naIP.IsPrivate() {
 		// Don't cache obviously wrong entries from captive portals.
 		// TODO: use DoH or DoT for the forwarding resolver?
 		if debug {
@@ -263,27 +267,14 @@ func (r *Resolver) addIPCache(host string, ip, ip6 net.IP, d time.Duration) {
 	if r.ipCache == nil {
 		r.ipCache = make(map[string]ipCacheEntry)
 	}
-	r.ipCache[host] = ipCacheEntry{ip: ip, ip6: ip6, expires: time.Now().Add(d)}
-}
-
-func mustCIDR(s string) *net.IPNet {
-	_, ipNet, err := net.ParseCIDR(s)
-	if err != nil {
-		panic(err)
+	r.ipCache[host] = ipCacheEntry{
+		ip:      ip,
+		ip6:     ip6,
+		allIPs:  allIPs,
+		expires: time.Now().Add(d),
 	}
-	return ipNet
 }
 
-func isPrivateIP(ip net.IP) bool {
-	return private1.Contains(ip) || private2.Contains(ip) || private3.Contains(ip)
-}
-
-var (
-	private1 = mustCIDR("10.0.0.0/8")
-	private2 = mustCIDR("172.16.0.0/12")
-	private3 = mustCIDR("192.168.0.0/16")
-)
-
 type DialContextFunc func(ctx context.Context, network, address string) (net.Conn, error)
 
 // Dialer returns a wrapped DialContext func that uses the provided dnsCache.
@@ -305,41 +296,131 @@ func Dialer(fwd DialContextFunc, dnsCache *Resolver) DialContextFunc {
 				// Return with original error
 				return
 			}
-			for _, ip := range ips {
-				dst := net.JoinHostPort(ip.String(), port)
-				if c, err := fwd(ctx, network, dst); err == nil {
-					retConn = c
-					ret = nil
-					return
-				}
+			if c, err := raceDial(ctx, fwd, network, ips, port); err == nil {
+				retConn = c
+				ret = nil
+				return
 			}
 		}()
 
-		ip, ip6, err := dnsCache.LookupIP(ctx, host)
+		ip, ip6, allIPs, err := dnsCache.LookupIP(ctx, host)
 		if err != nil {
 			return nil, fmt.Errorf("failed to resolve %q: %w", host, err)
 		}
-		dst := net.JoinHostPort(ip.String(), port)
-		if debug {
-			log.Printf("dnscache: dialing %s, %s for %s", network, dst, address)
+		i4s := v4addrs(allIPs)
+		if len(i4s) < 2 {
+			dst := net.JoinHostPort(ip.String(), port)
+			if debug {
+				log.Printf("dnscache: dialing %s, %s for %s", network, dst, address)
+			}
+			c, err := fwd(ctx, network, dst)
+			if err == nil || ctx.Err() != nil || ip6 == nil {
+				return c, err
+			}
+			// Fall back to trying IPv6.
+			dst = net.JoinHostPort(ip6.String(), port)
+			return fwd(ctx, network, dst)
 		}
-		c, err := fwd(ctx, network, dst)
-		if err == nil || ctx.Err() != nil || ip6 == nil {
-			return c, err
-		}
-		// Fall back to trying IPv6.
-		// TODO(bradfitz): this is a primarily for IPv6-only
-		// hosts; it's not supposed to be a real Happy
-		// Eyeballs implementation. We should use the net
-		// package's implementation of that by plumbing this
-		// dnscache impl into net.Dialer.Resolver.Dial and
-		// unmarshal/marshal DNS queries/responses to the net
-		// package. This works for v6-only hosts for now.
-		dst = net.JoinHostPort(ip6.String(), port)
-		return fwd(ctx, network, dst)
+
+		// Multiple IPv4 candidates, and 0+ IPv6.
+		ipsToTry := append(i4s, v6addrs(allIPs)...)
+		return raceDial(ctx, fwd, network, ipsToTry, port)
 	}
 }
 
+// fallbackDelay is how long to wait between trying subsequent
+// addresses when multiple options are available.
+// 300ms is the same as Go's Happy Eyeballs fallbackDelay value.
+const fallbackDelay = 300 * time.Millisecond
+
+// raceDial tries to dial port on each ip in ips, starting a new race
+// dial every fallbackDelay apart, returning whichever completes first.
+func raceDial(ctx context.Context, fwd DialContextFunc, network string, ips []netaddr.IP, port string) (net.Conn, error) {
+	ctx, cancel := context.WithCancel(ctx)
+	defer cancel()
+
+	type res struct {
+		c   net.Conn
+		err error
+	}
+	resc := make(chan res)           // must be unbuffered
+	failBoost := make(chan struct{}) // best effort send on dial failure
+
+	go func() {
+		for i, ip := range ips {
+			if i != 0 {
+				timer := time.NewTimer(fallbackDelay)
+				select {
+				case <-timer.C:
+				case <-failBoost:
+					timer.Stop()
+				case <-ctx.Done():
+					timer.Stop()
+					return
+				}
+			}
+			go func(ip netaddr.IP) {
+				c, err := fwd(ctx, network, net.JoinHostPort(ip.String(), port))
+				if err != nil {
+					// Best effort wake-up a pending dial.
+					// e.g. IPv4 dials failing quickly on an IPv6-only system.
+					// In that case we don't want to wait 300ms per IPv4 before
+					// we get to the IPv6 addresses.
+					select {
+					case failBoost <- struct{}{}:
+					default:
+					}
+				}
+				select {
+				case resc <- res{c, err}:
+				case <-ctx.Done():
+					if c != nil {
+						c.Close()
+					}
+				}
+			}(ip)
+		}
+	}()
+
+	var firstErr error
+	var fails int
+	for {
+		select {
+		case r := <-resc:
+			if r.c != nil {
+				return r.c, nil
+			}
+			fails++
+			if firstErr == nil {
+				firstErr = r.err
+			}
+			if fails == len(ips) {
+				return nil, firstErr
+			}
+		case <-ctx.Done():
+			return nil, ctx.Err()
+		}
+	}
+}
+
+func v4addrs(aa []net.IPAddr) (ret []netaddr.IP) {
+	for _, a := range aa {
+		if ip, ok := netaddr.FromStdIP(a.IP); ok && ip.Is4() {
+			ret = append(ret, ip)
+		}
+	}
+	return ret
+}
+
+func v6addrs(aa []net.IPAddr) (ret []netaddr.IP) {
+	for _, a := range aa {
+		if ip, ok := netaddr.FromStdIP(a.IP); ok && ip.Is6() {
+			ret = append(ret, ip)
+		}
+	}
+	return ret
+}
+
 var errTLSHandshakeTimeout = errors.New("timeout doing TLS handshake")
 
 // TLSDialer is like Dialer but returns a func suitable for using with net/http.Transport.DialTLSContext.

net/dnscache/dnscache_test.go

@@ -5,24 +5,29 @@
 package dnscache
 
 import (
+	"context"
+	"flag"
 	"net"
 	"testing"
+	"time"
 )
 
-func TestIsPrivateIP(t *testing.T) {
-	tests := []struct {
-		ip   string
-		want bool
-	}{
-		{"10.1.2.3", true},
-		{"172.16.1.100", true},
-		{"192.168.1.1", true},
-		{"1.2.3.4", false},
-	}
+var dialTest = flag.String("dial-test", "", "if non-empty, addr:port to test dial")
 
-	for _, test := range tests {
-		if got := isPrivateIP(net.ParseIP(test.ip)); got != test.want {
-			t.Errorf("isPrivateIP(%q)=%v, want %v", test.ip, got, test.want)
-		}
+func TestDialer(t *testing.T) {
+	if *dialTest == "" {
+		t.Skip("skipping; --dial-test is blank")
 	}
+	r := new(Resolver)
+	var std net.Dialer
+	dialer := Dialer(std.DialContext, r)
+	t0 := time.Now()
+	ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
+	defer cancel()
+	c, err := dialer(ctx, "tcp", *dialTest)
+	if err != nil {
+		t.Fatal(err)
+	}
+	t.Logf("dialed in %v", time.Since(t0))
+	c.Close()
 }

net/dnsfallback/dns-fallback-servers.json

@@ -102,6 +102,27 @@
 					"HostName": "derp4b.tailscale.com",
 					"IPv4": "157.230.25.0",
 					"IPv6": "2a03:b0c0:3:e0::58f:3001"
+				},
+				{
+					"Name": "4c",
+					"RegionID": 4,
+					"HostName": "derp4c.tailscale.com",
+					"IPv4": "134.122.77.138",
+					"IPv6": "2a03:b0c0:3:d0::1501:6001"
+				},
+				{
+					"Name": "4d",
+					"RegionID": 4,
+					"HostName": "derp4d.tailscale.com",
+					"IPv4": "134.122.94.167",
+					"IPv6": "2a03:b0c0:3:d0::1501:b001"
+				},
+				{
+					"Name": "4e",
+					"RegionID": 4,
+					"HostName": "derp4e.tailscale.com",
+					"IPv4": "134.122.74.153",
+					"IPv6": "2a03:b0c0:3:d0::29:9001"
 				}
 			]
 		},
@@ -158,6 +179,13 @@
 					"HostName": "derp8.tailscale.com",
 					"IPv4": "167.71.139.179",
 					"IPv6": "2a03:b0c0:1:e0::3cc:e001"
+				},
+				{
+					"Name": "8b",
+					"RegionID": 8,
+					"HostName": "derp8b.tailscale.com",
+					"IPv4": "46.101.74.201",
+					"IPv6": "2a03:b0c0:1:d0::ec1:e001"
 				}
 			]
 		},

net/interfaces/interfaces.go

@@ -479,7 +479,7 @@ func HTTPOfListener(ln net.Listener) string {
 	var privateIP string
 	ForeachInterfaceAddress(func(i Interface, pfx netaddr.IPPrefix) {
 		ip := pfx.IP()
-		if isPrivateIP(ip) {
+		if ip.IsPrivate() {
 			if privateIP == "" {
 				privateIP = ip.String()
 			}
@@ -519,21 +519,15 @@ func LikelyHomeRouterIP() (gateway, myIP netaddr.IP, ok bool) {
 		if !i.IsUp() || ip.IsZero() || !myIP.IsZero() {
 			return
 		}
-		for _, prefix := range privatev4s {
-			if prefix.Contains(gateway) && prefix.Contains(ip) {
-				myIP = ip
-				ok = true
-				return
-			}
+		if gateway.IsPrivate() && ip.IsPrivate() {
+			myIP = ip
+			ok = true
+			return
 		}
 	})
 	return gateway, myIP, !myIP.IsZero()
 }
 
-func isPrivateIP(ip netaddr.IP) bool {
-	return private1.Contains(ip) || private2.Contains(ip) || private3.Contains(ip)
-}
-
 // isUsableV4 reports whether ip is a usable IPv4 address which could
 // conceivably be used to get Internet connectivity. Globally routable and
 // private IPv4 addresses are always Usable, and link local 169.254.x.x
@@ -557,20 +551,8 @@ func isUsableV6(ip netaddr.IP) bool {
 		(tsaddr.IsULA(ip) && !tsaddr.TailscaleULARange().Contains(ip))
 }
 
-func mustCIDR(s string) netaddr.IPPrefix {
-	prefix, err := netaddr.ParseIPPrefix(s)
-	if err != nil {
-		panic(err)
-	}
-	return prefix
-}
-
 var (
-	private1   = mustCIDR("10.0.0.0/8")
-	private2   = mustCIDR("172.16.0.0/12")
-	private3   = mustCIDR("192.168.0.0/16")
-	privatev4s = []netaddr.IPPrefix{private1, private2, private3}
-	v6Global1  = mustCIDR("2000::/3")
+	v6Global1 = netaddr.MustParseIPPrefix("2000::/3")
 )
 
 // anyInterestingIP reports whether pfxs contains any IP that matches

net/interfaces/interfaces_darwin_test.go

@@ -73,7 +73,7 @@ func likelyHomeRouterIPDarwinExec() (ret netaddr.IP, ok bool) {
 			return nil
 		}
 		ip, err := netaddr.ParseIP(string(mem.Append(nil, ipm)))
-		if err == nil && isPrivateIP(ip) {
+		if err == nil && ip.IsPrivate() {
 			ret = ip
 			// We've found what we're looking for.
 			return errStopReadingNetstatTable

net/interfaces/interfaces_linux.go

@@ -72,7 +72,7 @@ func likelyHomeRouterIPLinux() (ret netaddr.IP, ok bool) {
 			return nil // ignore error, skip line and keep going
 		}
 		ip := netaddr.IPv4(byte(ipu32), byte(ipu32>>8), byte(ipu32>>16), byte(ipu32>>24))
-		if isPrivateIP(ip) {
+		if ip.IsPrivate() {
 			ret = ip
 		}
 		return nil

net/interfaces/interfaces_linux_test.go

@@ -73,6 +73,32 @@ func TestExtremelyLongProcNetRoute(t *testing.T) {
 	}
 }
 
+// test the specific /proc/net/route path as found on AWS App Runner instances
+func TestAwsAppRunnerDefaultRouteInterface(t *testing.T) {
+	dir := t.TempDir()
+	savedProcNetRoutePath := procNetRoutePath
+	defer func() { procNetRoutePath = savedProcNetRoutePath }()
+	procNetRoutePath = filepath.Join(dir, "CloudRun")
+	buf := []byte("Iface\tDestination\tGateway\tFlags\tRefCnt\tUse\tMetric\tMask\tMTU\tWindow\tIRTT\n" +
+		"eth0\t00000000\tF9AFFEA9\t0003\t0\t0\t0\t00000000\t0\t0\t0\n" +
+		"*\tFEA9FEA9\t00000000\t0005\t0\t0\t0\tFFFFFFFF\t0\t0\t0\n" +
+		"ecs-eth0\t02AAFEA9\t01ACFEA9\t0007\t0\t0\t0\tFFFFFFFF\t0\t0\t0\n" +
+		"ecs-eth0\t00ACFEA9\t00000000\t0001\t0\t0\t0\t00FFFFFF\t0\t0\t0\n" +
+		"eth0\t00AFFEA9\t00000000\t0001\t0\t0\t0\t00FFFFFF\t0\t0\t0\n")
+	err := ioutil.WriteFile(procNetRoutePath, buf, 0644)
+	if err != nil {
+		t.Fatal(err)
+	}
+	got, err := DefaultRouteInterface()
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	if got != "eth0" {
+		t.Fatalf("got %s, want eth0", got)
+	}
+}
+
 func BenchmarkDefaultRouteInterface(b *testing.B) {
 	b.ReportAllocs()
 	for i := 0; i < b.N; i++ {

net/interfaces/interfaces_windows.go

@@ -93,7 +93,7 @@ func likelyHomeRouterIPWindows() (ret netaddr.IP, ok bool) {
 		}
 	}
 
-	if !ret.IsZero() && !isPrivateIP(ret) {
+	if !ret.IsZero() && !ret.IsPrivate() {
 		// Default route has a non-private gateway
 		return netaddr.IP{}, false
 	}

net/netns/netns.go

@@ -17,6 +17,8 @@ package netns
 import (
 	"context"
 	"net"
+
+	"inet.af/netaddr"
 )
 
 // Listener returns a new net.Listener with its Control hook func
@@ -66,3 +68,19 @@ type Dialer interface {
 	Dial(network, address string) (net.Conn, error)
 	DialContext(ctx context.Context, network, address string) (net.Conn, error)
 }
+
+func isLocalhost(addr string) bool {
+	host, _, err := net.SplitHostPort(addr)
+	if err != nil {
+		// error means the string didn't contain a port number, so use the string directly
+		host = addr
+	}
+
+	// localhost6 == RedHat /etc/hosts for ::1, ip6-loopback & ip6-localhost == Debian /etc/hosts for ::1
+	if host == "localhost" || host == "localhost6" || host == "ip6-loopback" || host == "ip6-localhost" {
+		return true
+	}
+
+	ip, _ := netaddr.ParseIP(host)
+	return ip.IsLoopback()
+}

net/netns/netns_linux.go

@@ -7,8 +7,8 @@
 package netns
 
 import (
-	"flag"
 	"fmt"
+	"net"
 	"os"
 	"os/exec"
 	"sync"
@@ -26,32 +26,54 @@ import (
 // wgengine/router/router_linux.go.
 const tailscaleBypassMark = 0x80000
 
-// ipRuleOnce is the sync.Once & cached value for ipRuleAvailable.
-var ipRuleOnce struct {
+// socketMarkWorksOnce is the sync.Once & cached value for useSocketMark.
+var socketMarkWorksOnce struct {
 	sync.Once
 	v bool
 }
 
-// ipRuleAvailable reports whether the 'ip rule' command works.
-// If it doesn't, we have to use SO_BINDTODEVICE on our sockets instead.
-func ipRuleAvailable() bool {
-	ipRuleOnce.Do(func() {
-		ipRuleOnce.v = exec.Command("ip", "rule").Run() == nil
+// socketMarkWorks returns whether SO_MARK works.
+func socketMarkWorks() bool {
+	addr, err := net.ResolveUDPAddr("udp", "127.0.0.1:1")
+	if err != nil {
+		return true // unsure, returning true does the least harm.
+	}
+
+	sConn, err := net.DialUDP("udp", nil, addr)
+	if err != nil {
+		return true // unsure, return true
+	}
+	defer sConn.Close()
+
+	rConn, err := sConn.SyscallConn()
+	if err != nil {
+		return true // unsure, return true
+	}
+
+	var sockErr error
+	err = rConn.Control(func(fd uintptr) {
+		sockErr = setBypassMark(fd)
 	})
-	return ipRuleOnce.v
+	if err != nil || sockErr != nil {
+		return false
+	}
+
+	return true
+}
+
+// useSocketMark reports whether SO_MARK works.
+// If it doesn't, we have to use SO_BINDTODEVICE on our sockets instead.
+func useSocketMark() bool {
+	socketMarkWorksOnce.Do(func() {
+		ipRuleWorks := exec.Command("ip", "rule").Run() == nil
+		socketMarkWorksOnce.v = ipRuleWorks && socketMarkWorks()
+	})
+	return socketMarkWorksOnce.v
 }
 
 // ignoreErrors returns true if we should ignore setsocketopt errors in
 // this instance.
 func ignoreErrors() bool {
-	// If we're in a test, ignore errors. Assume the test knows
-	// what it's doing and will do its own skips or permission
-	// checks if it's setting up a world that needs netns to work.
-	// But by default, assume that tests don't need netns and it's
-	// harmless to ignore the sockopts failing.
-	if flag.CommandLine.Lookup("test.v") != nil {
-		return true
-	}
 	if os.Getuid() != 0 {
 		// only root can manipulate these socket flags
 		return true
@@ -64,9 +86,14 @@ func ignoreErrors() bool {
 // It's intentionally the same signature as net.Dialer.Control
 // and net.ListenConfig.Control.
 func control(network, address string, c syscall.RawConn) error {
+	if isLocalhost(address) {
+		// Don't bind to an interface for localhost connections.
+		return nil
+	}
+
 	var sockErr error
 	err := c.Control(func(fd uintptr) {
-		if ipRuleAvailable() {
+		if useSocketMark() {
 			sockErr = setBypassMark(fd)
 		} else {
 			sockErr = bindToDevice(fd)

net/netns/netns_linux_test.go

@@ -49,3 +49,9 @@ func TestBypassMarkInSync(t *testing.T) {
 	}
 	t.Errorf("tailscaleBypassMark not found in router_linux.go")
 }
+
+func TestSocketMarkWorks(t *testing.T) {
+	_ = socketMarkWorks()
+	// we cannot actually assert whether the test runner has SO_MARK available
+	// or not, as we don't know. We're just checking that it doesn't panic.
+}

net/netns/netns_test.go

@@ -40,3 +40,40 @@ func TestDial(t *testing.T) {
 	defer c.Close()
 	t.Logf("got addr %v", c.RemoteAddr())
 }
+
+func TestIsLocalhost(t *testing.T) {
+	tests := []struct {
+		name string
+		host string
+		want bool
+	}{
+		{"IPv4 loopback", "127.0.0.1", true},
+		{"IPv4 !loopback", "192.168.0.1", false},
+		{"IPv4 loopback with port", "127.0.0.1:1", true},
+		{"IPv4 !loopback with port", "192.168.0.1:1", false},
+		{"IPv4 unspecified", "0.0.0.0", false},
+		{"IPv4 unspecified with port", "0.0.0.0:1", false},
+		{"IPv6 loopback", "::1", true},
+		{"IPv6 !loopback", "2001:4860:4860::8888", false},
+		{"IPv6 loopback with port", "[::1]:1", true},
+		{"IPv6 !loopback with port", "[2001:4860:4860::8888]:1", false},
+		{"IPv6 unspecified", "::", false},
+		{"IPv6 unspecified with port", "[::]:1", false},
+		{"empty", "", false},
+		{"hostname", "example.com", false},
+		{"localhost", "localhost", true},
+		{"localhost6", "localhost6", true},
+		{"localhost with port", "localhost:1", true},
+		{"localhost6 with port", "localhost6:1", true},
+		{"ip6-localhost", "ip6-localhost", true},
+		{"ip6-localhost with port", "ip6-localhost:1", true},
+		{"ip6-loopback", "ip6-loopback", true},
+		{"ip6-loopback with port", "ip6-loopback:1", true},
+	}
+
+	for _, test := range tests {
+		if got := isLocalhost(test.host); got != test.want {
+			t.Errorf("isLocalhost(%q) = %v, want %v", test.name, got, test.want)
+		}
+	}
+}

net/portmapper/disabled_stubs.go

@@ -0,0 +1,29 @@
+// Copyright (c) 2021 Tailscale Inc & AUTHORS All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+// +build ios
+// (https://github.com/tailscale/tailscale/issues/2495)
+
+package portmapper
+
+import (
+	"context"
+
+	"inet.af/netaddr"
+)
+
+type upnpClient interface{}
+
+func getUPnPClient(ctx context.Context, gw netaddr.IP) (upnpClient, error) {
+	return nil, nil
+}
+
+func (c *Client) getUPnPPortMapping(
+	ctx context.Context,
+	gw netaddr.IP,
+	internal netaddr.IPPort,
+	prevPort uint16,
+) (external netaddr.IPPort, ok bool) {
+	return netaddr.IPPort{}, false
+}

net/portmapper/portmapper.go

@@ -2,8 +2,8 @@
 // Use of this source code is governed by a BSD-style
 // license that can be found in the LICENSE file.
 
-// Package portmapper is a UDP port mapping client. It currently only does
-// NAT-PMP, but will likely do UPnP and perhaps PCP later.
+// Package portmapper is a UDP port mapping client. It currently allows for mapping over
+// NAT-PMP and UPnP, but will perhaps do PCP later.
 package portmapper
 
 import (
@@ -17,7 +17,6 @@ import (
 	"sync"
 	"time"
 
-	"go4.org/mem"
 	"inet.af/netaddr"
 	"tailscale.com/net/interfaces"
 	"tailscale.com/net/netns"
@@ -44,9 +43,15 @@ const trustServiceStillAvailableDuration = 10 * time.Minute
 type Client struct {
 	logf         logger.Logf
 	ipAndGateway func() (gw, ip netaddr.IP, ok bool)
+	onChange     func() // or nil
 
 	mu sync.Mutex // guards following, and all fields thereof
 
+	// runningCreate is whether we're currently working on creating
+	// a port mapping (whether GetCachedMappingOrStartCreatingOne kicked
+	// off a createMapping goroutine).
+	runningCreate bool
+
 	lastMyIP netaddr.IP
 	lastGW   netaddr.IP
 	closed   bool
@@ -60,26 +65,46 @@ type Client struct {
 	pcpSawTime  time.Time // time we last saw PCP was available
 	uPnPSawTime time.Time // time we last saw UPnP was available
 
-	localPort  uint16
-	pmpMapping *pmpMapping // non-nil if we have a PMP mapping
+	localPort uint16
+
+	mapping mapping // non-nil if we have a mapping
+}
+
+// mapping represents a created port-mapping over some protocol.  It specifies a lease duration,
+// how to release the mapping, and whether the map is still valid.
+//
+// After a mapping is created, it should be immutable, and thus reads should be safe across
+// concurrent goroutines.
+type mapping interface {
+	// Release will attempt to unmap the established port mapping. It will block until completion,
+	// but can be called asynchronously. Release should be idempotent, and thus even if called
+	// multiple times should not cause additional side-effects.
+	Release(context.Context)
+	// goodUntil will return the lease time that the mapping is valid for.
+	GoodUntil() time.Time
+	// renewAfter returns the earliest time that the mapping should be renewed.
+	RenewAfter() time.Time
+	// externalIPPort indicates what port the mapping can be reached from on the outside.
+	External() netaddr.IPPort
 }
 
 // HaveMapping reports whether we have a current valid mapping.
 func (c *Client) HaveMapping() bool {
 	c.mu.Lock()
 	defer c.mu.Unlock()
-	return c.pmpMapping != nil && c.pmpMapping.useUntil.After(time.Now())
+	return c.mapping != nil && c.mapping.GoodUntil().After(time.Now())
 }
 
 // pmpMapping is an already-created PMP mapping.
 //
 // All fields are immutable once created.
 type pmpMapping struct {
-	gw       netaddr.IP
-	external netaddr.IPPort
-	internal netaddr.IPPort
-	useUntil time.Time // the mapping's lifetime minus renewal interval
-	epoch    uint32
+	gw         netaddr.IP
+	external   netaddr.IPPort
+	internal   netaddr.IPPort
+	renewAfter time.Time // the time at which we want to renew the mapping
+	goodUntil  time.Time // the mapping's total lifetime
+	epoch      uint32
 }
 
 // externalValid reports whether m.external is valid, with both its IP and Port populated.
@@ -87,9 +112,13 @@ func (m *pmpMapping) externalValid() bool {
 	return !m.external.IP().IsZero() && m.external.Port() != 0
 }
 
-// release does a best effort fire-and-forget release of the PMP mapping m.
-func (m *pmpMapping) release() {
-	uc, err := netns.Listener().ListenPacket(context.Background(), "udp4", ":0")
+func (p *pmpMapping) GoodUntil() time.Time     { return p.goodUntil }
+func (p *pmpMapping) RenewAfter() time.Time    { return p.renewAfter }
+func (p *pmpMapping) External() netaddr.IPPort { return p.external }
+
+// Release does a best effort fire-and-forget release of the PMP mapping m.
+func (m *pmpMapping) Release(ctx context.Context) {
+	uc, err := netns.Listener().ListenPacket(ctx, "udp4", ":0")
 	if err != nil {
 		return
 	}
@@ -99,10 +128,15 @@ func (m *pmpMapping) release() {
 }
 
 // NewClient returns a new portmapping client.
-func NewClient(logf logger.Logf) *Client {
+//
+// The optional onChange argument specifies a func to run in a new
+// goroutine whenever the port mapping status has changed. If nil,
+// it doesn't make a callback.
+func NewClient(logf logger.Logf, onChange func()) *Client {
 	return &Client{
 		logf:         logf,
 		ipAndGateway: interfaces.LikelyHomeRouterIP,
+		onChange:     onChange,
 	}
 }
 
@@ -154,7 +188,6 @@ func (c *Client) gatewayAndSelfIP() (gw, myIP netaddr.IP, ok bool) {
 		gw = netaddr.IP{}
 		myIP = netaddr.IP{}
 	}
-
 	c.mu.Lock()
 	defer c.mu.Unlock()
 
@@ -167,11 +200,11 @@ func (c *Client) gatewayAndSelfIP() (gw, myIP netaddr.IP, ok bool) {
 }
 
 func (c *Client) invalidateMappingsLocked(releaseOld bool) {
-	if c.pmpMapping != nil {
+	if c.mapping != nil {
 		if releaseOld {
-			c.pmpMapping.release()
+			c.mapping.Release(context.Background())
 		}
-		c.pmpMapping = nil
+		c.mapping = nil
 	}
 	c.pmpPubIP = netaddr.IP{}
 	c.pmpPubIPTime = time.Time{}
@@ -221,8 +254,7 @@ func closeCloserOnContextDone(ctx context.Context, c io.Closer) (stop func()) {
 	return func() { close(stopWaitDone) }
 }
 
-// NoMappingError is returned by CreateOrGetMapping when no NAT
-// mapping could be returned.
+// NoMappingError is returned when no NAT mapping could be done.
 type NoMappingError struct {
 	err error
 }
@@ -238,25 +270,76 @@ func IsNoMappingError(err error) bool {
 
 var (
 	ErrNoPortMappingServices = errors.New("no port mapping services were found")
-	ErrGatewayNotFound       = errors.New("failed to look up gateway address")
+	ErrGatewayRange          = errors.New("skipping portmap; gateway range likely lacks support")
 )
 
-// CreateOrGetMapping either creates a new mapping or returns a cached
+// GetCachedMappingOrStartCreatingOne quickly returns with our current cached portmapping, if any.
+// If there's not one, it starts up a background goroutine to create one.
+// If the background goroutine ends up creating one, the onChange hook registered with the
+// NewClient constructor (if any) will fire.
+func (c *Client) GetCachedMappingOrStartCreatingOne() (external netaddr.IPPort, ok bool) {
+	c.mu.Lock()
+	defer c.mu.Unlock()
+
+	// Do we have an existing mapping that's valid?
+	now := time.Now()
+	if m := c.mapping; m != nil {
+		if now.Before(m.GoodUntil()) {
+			if now.After(m.RenewAfter()) {
+				c.maybeStartMappingLocked()
+			}
+			return m.External(), true
+		}
+	}
+
+	c.maybeStartMappingLocked()
+	return netaddr.IPPort{}, false
+}
+
+// maybeStartMappingLocked starts a createMapping goroutine up, if one isn't already running.
+//
+// c.mu must be held.
+func (c *Client) maybeStartMappingLocked() {
+	if !c.runningCreate {
+		c.runningCreate = true
+		go c.createMapping()
+	}
+}
+
+func (c *Client) createMapping() {
+	ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
+	defer cancel()
+
+	defer func() {
+		c.mu.Lock()
+		defer c.mu.Unlock()
+		c.runningCreate = false
+	}()
+
+	if _, err := c.createOrGetMapping(ctx); err == nil && c.onChange != nil {
+		go c.onChange()
+	} else if err != nil && !IsNoMappingError(err) {
+		c.logf("createOrGetMapping: %v", err)
+	}
+}
+
+// createOrGetMapping either creates a new mapping or returns a cached
 // valid one.
 //
 // If no mapping is available, the error will be of type
 // NoMappingError; see IsNoMappingError.
-func (c *Client) CreateOrGetMapping(ctx context.Context) (external netaddr.IPPort, err error) {
+func (c *Client) createOrGetMapping(ctx context.Context) (external netaddr.IPPort, err error) {
 	gw, myIP, ok := c.gatewayAndSelfIP()
 	if !ok {
-		return netaddr.IPPort{}, NoMappingError{ErrGatewayNotFound}
+		return netaddr.IPPort{}, NoMappingError{ErrGatewayRange}
 	}
 
 	c.mu.Lock()
 	localPort := c.localPort
+	internalAddr := netaddr.IPPortFrom(myIP, localPort)
 	m := &pmpMapping{
 		gw:       gw,
-		internal: netaddr.IPPortFrom(myIP, localPort),
+		internal: internalAddr,
 	}
 
 	// prevPort is the port we had most previously, if any. We try
@@ -265,13 +348,13 @@ func (c *Client) CreateOrGetMapping(ctx context.Context) (external netaddr.IPPor
 
 	// Do we have an existing mapping that's valid?
 	now := time.Now()
-	if m := c.pmpMapping; m != nil {
-		if now.Before(m.useUntil) {
+	if m := c.mapping; m != nil {
+		if now.Before(m.RenewAfter()) {
 			defer c.mu.Unlock()
-			return m.external, nil
+			return m.External(), nil
 		}
 		// The mapping might still be valid, so just try to renew it.
-		prevPort = m.external.Port()
+		prevPort = m.External().Port()
 	}
 
 	// If we just did a Probe (e.g. via netchecker) but didn't
@@ -283,6 +366,10 @@ func (c *Client) CreateOrGetMapping(ctx context.Context) (external netaddr.IPPor
 	}
 	if c.lastProbe.After(now.Add(-5*time.Second)) && !haveRecentPMP {
 		c.mu.Unlock()
+		// fallback to UPnP portmapping
+		if mapping, ok := c.getUPnPPortMapping(ctx, gw, internalAddr, prevPort); ok {
+			return mapping, nil
+		}
 		return netaddr.IPPort{}, NoMappingError{ErrNoPortMappingServices}
 	}
 
@@ -320,6 +407,10 @@ func (c *Client) CreateOrGetMapping(ctx context.Context) (external netaddr.IPPor
 			if ctx.Err() == context.Canceled {
 				return netaddr.IPPort{}, err
 			}
+			// fallback to UPnP portmapping
+			if mapping, ok := c.getUPnPPortMapping(ctx, gw, internalAddr, prevPort); ok {
+				return mapping, nil
+			}
 			return netaddr.IPPort{}, NoMappingError{ErrNoPortMappingServices}
 		}
 		srcu := srci.(*net.UDPAddr)
@@ -342,8 +433,9 @@ func (c *Client) CreateOrGetMapping(ctx context.Context) (external netaddr.IPPor
 			if pres.OpCode == pmpOpReply|pmpOpMapUDP {
 				m.external = m.external.WithPort(pres.ExternalPort)
 				d := time.Duration(pres.MappingValidSeconds) * time.Second
-				d /= 2 // renew in half the time
-				m.useUntil = time.Now().Add(d)
+				now := time.Now()
+				m.goodUntil = now.Add(d)
+				m.renewAfter = now.Add(d / 2) // renew in half the time
 				m.epoch = pres.SecondsSinceEpoch
 			}
 		}
@@ -351,7 +443,7 @@ func (c *Client) CreateOrGetMapping(ctx context.Context) (external netaddr.IPPor
 		if m.externalValid() {
 			c.mu.Lock()
 			defer c.mu.Unlock()
-			c.pmpMapping = m
+			c.mapping = m
 			return m.external, nil
 		}
 	}
@@ -448,7 +540,7 @@ type ProbeResult struct {
 func (c *Client) Probe(ctx context.Context) (res ProbeResult, err error) {
 	gw, myIP, ok := c.gatewayAndSelfIP()
 	if !ok {
-		return res, ErrGatewayNotFound
+		return res, ErrGatewayRange
 	}
 	defer func() {
 		if err == nil {
@@ -468,9 +560,27 @@ func (c *Client) Probe(ctx context.Context) (res ProbeResult, err error) {
 	defer cancel()
 	defer closeCloserOnContextDone(ctx, uc)()
 
+	if c.sawUPnPRecently() {
+		res.UPnP = true
+	} else {
+		hasUPnP := make(chan bool, 1)
+		defer func() {
+			res.UPnP = <-hasUPnP
+		}()
+		go func() {
+			client, err := getUPnPClient(ctx, gw)
+			if err == nil && client != nil {
+				hasUPnP <- true
+				c.mu.Lock()
+				c.uPnPSawTime = time.Now()
+				c.mu.Unlock()
+			}
+			close(hasUPnP)
+		}()
+	}
+
 	pcpAddr := netaddr.IPPortFrom(gw, pcpPort).UDPAddr()
 	pmpAddr := netaddr.IPPortFrom(gw, pmpPort).UDPAddr()
-	upnpAddr := netaddr.IPPortFrom(gw, upnpPort).UDPAddr()
 
 	// Don't send probes to services that we recently learned (for
 	// the same gw/myIP) are available. See
@@ -485,16 +595,11 @@ func (c *Client) Probe(ctx context.Context) (res ProbeResult, err error) {
 	} else {
 		uc.WriteTo(pcpAnnounceRequest(myIP), pcpAddr)
 	}
-	if c.sawUPnPRecently() {
-		res.UPnP = true
-	} else {
-		uc.WriteTo(uPnPPacket, upnpAddr)
-	}
 
 	buf := make([]byte, 1500)
 	pcpHeard := false // true when we get any PCP response
 	for {
-		if pcpHeard && res.PMP && res.UPnP {
+		if pcpHeard && res.PMP {
 			// Nothing more to discover.
 			return res, nil
 		}
@@ -507,13 +612,6 @@ func (c *Client) Probe(ctx context.Context) (res ProbeResult, err error) {
 		}
 		port := addr.(*net.UDPAddr).Port
 		switch port {
-		case upnpPort:
-			if mem.Contains(mem.B(buf[:n]), mem.S(":InternetGatewayDevice:")) {
-				res.UPnP = true
-				c.mu.Lock()
-				c.uPnPSawTime = time.Now()
-				c.mu.Unlock()
-			}
 		case pcpPort: // same as pmpPort
 			if pres, ok := parsePCPResponse(buf[:n]); ok {
 				if pres.OpCode == pcpOpReply|pcpOpAnnounce {
@@ -625,14 +723,4 @@ func parsePCPResponse(b []byte) (res pcpResponse, ok bool) {
 	return res, true
 }
 
-const (
-	upnpPort = 1900
-)
-
-var uPnPPacket = []byte("M-SEARCH * HTTP/1.1\r\n" +
-	"HOST: 239.255.255.250:1900\r\n" +
-	"ST: ssdp:all\r\n" +
-	"MAN: \"ssdp:discover\"\r\n" +
-	"MX: 2\r\n\r\n")
-
 var pmpReqExternalAddrPacket = []byte{0, 0} // version 0, opcode 0 = "Public address request"

net/portmapper/portmapper_test.go

@@ -16,13 +16,14 @@ func TestCreateOrGetMapping(t *testing.T) {
 	if v, _ := strconv.ParseBool(os.Getenv("HIT_NETWORK")); !v {
 		t.Skip("skipping test without HIT_NETWORK=1")
 	}
-	c := NewClient(t.Logf)
+	c := NewClient(t.Logf, nil)
+	defer c.Close()
 	c.SetLocalPort(1234)
 	for i := 0; i < 2; i++ {
 		if i > 0 {
 			time.Sleep(100 * time.Millisecond)
 		}
-		ext, err := c.CreateOrGetMapping(context.Background())
+		ext, err := c.createOrGetMapping(context.Background())
 		t.Logf("Got: %v, %v", ext, err)
 	}
 }
@@ -31,13 +32,14 @@ func TestClientProbe(t *testing.T) {
 	if v, _ := strconv.ParseBool(os.Getenv("HIT_NETWORK")); !v {
 		t.Skip("skipping test without HIT_NETWORK=1")
 	}
-	c := NewClient(t.Logf)
-	for i := 0; i < 2; i++ {
+	c := NewClient(t.Logf, nil)
+	defer c.Close()
+	for i := 0; i < 3; i++ {
 		if i > 0 {
 			time.Sleep(100 * time.Millisecond)
 		}
 		res, err := c.Probe(context.Background())
-		t.Logf("Got: %+v, %v", res, err)
+		t.Logf("Got(t=%dms): %+v, %v", i*100, res, err)
 	}
 }
 
@@ -45,10 +47,11 @@ func TestClientProbeThenMap(t *testing.T) {
 	if v, _ := strconv.ParseBool(os.Getenv("HIT_NETWORK")); !v {
 		t.Skip("skipping test without HIT_NETWORK=1")
 	}
-	c := NewClient(t.Logf)
+	c := NewClient(t.Logf, nil)
+	defer c.Close()
 	c.SetLocalPort(1234)
 	res, err := c.Probe(context.Background())
 	t.Logf("Probe: %+v, %v", res, err)
-	ext, err := c.CreateOrGetMapping(context.Background())
-	t.Logf("CreateOrGetMapping: %v, %v", ext, err)
+	ext, err := c.createOrGetMapping(context.Background())
+	t.Logf("createOrGetMapping: %v, %v", ext, err)
 }

net/portmapper/upnp.go

@@ -0,0 +1,248 @@
+// Copyright (c) 2021 Tailscale Inc & AUTHORS All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+// +build !ios
+// (https://github.com/tailscale/tailscale/issues/2495)
+
+package portmapper
+
+import (
+	"context"
+	"fmt"
+	"math/rand"
+	"net/url"
+	"time"
+
+	"github.com/tailscale/goupnp/dcps/internetgateway2"
+	"inet.af/netaddr"
+	"tailscale.com/control/controlknobs"
+)
+
+// References:
+//
+// WANIP Connection v2: http://upnp.org/specs/gw/UPnP-gw-WANIPConnection-v2-Service.pdf
+
+// upnpMapping is a port mapping over the upnp protocol. After being created it is immutable,
+// but the client field may be shared across mapping instances.
+type upnpMapping struct {
+	gw         netaddr.IP
+	external   netaddr.IPPort
+	internal   netaddr.IPPort
+	goodUntil  time.Time
+	renewAfter time.Time
+
+	// client is a connection to a upnp device, and may be reused across different UPnP mappings.
+	client upnpClient
+}
+
+func (u *upnpMapping) GoodUntil() time.Time     { return u.goodUntil }
+func (u *upnpMapping) RenewAfter() time.Time    { return u.renewAfter }
+func (u *upnpMapping) External() netaddr.IPPort { return u.external }
+func (u *upnpMapping) Release(ctx context.Context) {
+	u.client.DeletePortMapping(ctx, "", u.external.Port(), "udp")
+}
+
+// upnpClient is an interface over the multiple different clients exported by goupnp,
+// exposing the functions we need for portmapping. They are auto-generated from XML-specs.
+type upnpClient interface {
+	AddPortMapping(
+		ctx context.Context,
+
+		// remoteHost is the remote device sending packets to this device, in the format of x.x.x.x.
+		// The empty string, "", means any host out on the internet can send packets in.
+		remoteHost string,
+
+		// externalPort is the exposed port of this port mapping. Visible during NAT operations.
+		// 0 will let the router select the port, but there is an additional call,
+		// `AddAnyPortMapping`, which is available on 1 of the 3 possible protocols,
+		// which should be used if available. See `addAnyPortMapping` below, which calls this if
+		// `AddAnyPortMapping` is not supported.
+		externalPort uint16,
+
+		// protocol is whether this is over TCP or UDP. Either "tcp" or "udp".
+		protocol string,
+
+		// internalPort is the port that the gateway device forwards the traffic to.
+		internalPort uint16,
+		// internalClient is the IP address that packets will be forwarded to for this mapping.
+		// Internal client is of the form "x.x.x.x".
+		internalClient string,
+
+		// enabled is whether this portmapping should be enabled or disabled.
+		enabled bool,
+		// portMappingDescription is a user-readable description of this portmapping.
+		portMappingDescription string,
+		// leaseDurationSec is the duration of this portmapping. The value of this argument must be
+		// greater than 0. From the spec, it appears if it is set to 0, it will switch to using
+		// 604800 seconds, but not sure why this is desired. The recommended time is 3600 seconds.
+		leaseDurationSec uint32,
+	) (err error)
+
+	DeletePortMapping(ctx context.Context, remoteHost string, externalPort uint16, protocol string) error
+	GetExternalIPAddress(ctx context.Context) (externalIPAddress string, err error)
+}
+
+// tsPortMappingDesc gets sent to UPnP clients as a human-readable label for the portmapping.
+// It is not used for anything other than labelling.
+const tsPortMappingDesc = "tailscale-portmap"
+
+// addAnyPortMapping abstracts over different UPnP client connections, calling the available
+// AddAnyPortMapping call if available for WAN IP connection v2, otherwise defaulting to the old
+// behavior of calling AddPortMapping with port = 0 to specify a wildcard port.
+// It returns the new external port (which may not be identical to the external port specified),
+// or an error.
+func addAnyPortMapping(
+	ctx context.Context,
+	upnp upnpClient,
+	externalPort uint16,
+	internalPort uint16,
+	internalClient string,
+	leaseDuration time.Duration,
+) (newPort uint16, err error) {
+	if upnp, ok := upnp.(*internetgateway2.WANIPConnection2); ok {
+		return upnp.AddAnyPortMapping(
+			ctx,
+			"",
+			externalPort,
+			"udp",
+			internalPort,
+			internalClient,
+			true,
+			tsPortMappingDesc,
+			uint32(leaseDuration.Seconds()),
+		)
+	}
+	for externalPort == 0 {
+		externalPort = uint16(rand.Intn(65535))
+	}
+	err = upnp.AddPortMapping(
+		ctx,
+		"",
+		externalPort,
+		"udp",
+		internalPort,
+		internalClient,
+		true,
+		tsPortMappingDesc,
+		uint32(leaseDuration.Seconds()),
+	)
+	return externalPort, err
+}
+
+// getUPnPClients gets a client for interfacing with UPnP, ignoring the underlying protocol for
+// now.
+// Adapted from https://github.com/huin/goupnp/blob/master/GUIDE.md.
+func getUPnPClient(ctx context.Context, gw netaddr.IP) (upnpClient, error) {
+	if controlknobs.DisableUPnP() {
+		return nil, nil
+	}
+	ctx, cancel := context.WithTimeout(ctx, 250*time.Millisecond)
+	defer cancel()
+	// Attempt to connect over the multiple available connection types concurrently,
+	// returning the fastest.
+
+	// TODO(jknodt): this url seems super brittle? maybe discovery is better but this is faster
+	u, err := url.Parse(fmt.Sprintf("http://%s:5000/rootDesc.xml", gw))
+	if err != nil {
+		return nil, err
+	}
+
+	clients := make(chan upnpClient, 3)
+	go func() {
+		var err error
+		ip1Clients, err := internetgateway2.NewWANIPConnection1ClientsByURL(ctx, u)
+		if err == nil && len(ip1Clients) > 0 {
+			clients <- ip1Clients[0]
+		}
+	}()
+	go func() {
+		ip2Clients, err := internetgateway2.NewWANIPConnection2ClientsByURL(ctx, u)
+		if err == nil && len(ip2Clients) > 0 {
+			clients <- ip2Clients[0]
+		}
+	}()
+	go func() {
+		ppp1Clients, err := internetgateway2.NewWANPPPConnection1ClientsByURL(ctx, u)
+		if err == nil && len(ppp1Clients) > 0 {
+			clients <- ppp1Clients[0]
+		}
+	}()
+
+	select {
+	case client := <-clients:
+		return client, nil
+	case <-ctx.Done():
+		return nil, ctx.Err()
+	}
+}
+
+// getUPnPPortMapping attempts to create a port-mapping over the UPnP protocol. On success,
+// it will return the externally exposed IP and port. Otherwise, it will return a zeroed IP and
+// port and an error.
+func (c *Client) getUPnPPortMapping(
+	ctx context.Context,
+	gw netaddr.IP,
+	internal netaddr.IPPort,
+	prevPort uint16,
+) (external netaddr.IPPort, ok bool) {
+	if controlknobs.DisableUPnP() {
+		return netaddr.IPPort{}, false
+	}
+	now := time.Now()
+	upnp := &upnpMapping{
+		gw:       gw,
+		internal: internal,
+	}
+
+	var client upnpClient
+	var err error
+	c.mu.Lock()
+	oldMapping, ok := c.mapping.(*upnpMapping)
+	c.mu.Unlock()
+	if ok && oldMapping != nil {
+		client = oldMapping.client
+	} else {
+		client, err = getUPnPClient(ctx, gw)
+		if err != nil {
+			return netaddr.IPPort{}, false
+		}
+	}
+	if client == nil {
+		return netaddr.IPPort{}, false
+	}
+
+	var newPort uint16
+	newPort, err = addAnyPortMapping(
+		ctx,
+		client,
+		prevPort,
+		internal.Port(),
+		internal.IP().String(),
+		time.Second*pmpMapLifetimeSec,
+	)
+	if err != nil {
+		return netaddr.IPPort{}, false
+	}
+	// TODO cache this ip somewhere?
+	extIP, err := client.GetExternalIPAddress(ctx)
+	if err != nil {
+		// TODO this doesn't seem right
+		return netaddr.IPPort{}, false
+	}
+	externalIP, err := netaddr.ParseIP(extIP)
+	if err != nil {
+		return netaddr.IPPort{}, false
+	}
+
+	upnp.external = netaddr.IPPortFrom(externalIP, newPort)
+	d := time.Duration(pmpMapLifetimeSec) * time.Second
+	upnp.goodUntil = now.Add(d)
+	upnp.renewAfter = now.Add(d / 2)
+	upnp.client = client
+	c.mu.Lock()
+	defer c.mu.Unlock()
+	c.mapping = upnp
+	c.localPort = newPort
+	return upnp.external, true
+}

net/speedtest/speedtest.go

@@ -0,0 +1,88 @@
+// Copyright (c) 2021 Tailscale Inc & AUTHORS All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+// Package speedtest contains both server and client code for
+// running speedtests between tailscale nodes.
+package speedtest
+
+import (
+	"time"
+)
+
+const (
+	blockSize       = 32000                 // size of the block of data to send
+	MinDuration     = 5 * time.Second       // minimum duration for a test
+	DefaultDuration = MinDuration           // default duration for a test
+	MaxDuration     = 30 * time.Second      // maximum duration for a test
+	version         = 1                     // value used when comparing client and server versions
+	increment       = time.Second           // increment to display results for, in seconds
+	minInterval     = 10 * time.Millisecond // minimum interval length for a result to be included
+	DefaultPort     = 20333
+)
+
+// config is the initial message sent to the server, that contains information on how to
+// conduct the test.
+type config struct {
+	Version      int           `json:"version"`
+	TestDuration time.Duration `json:"time"`
+	Direction    Direction     `json:"direction"`
+}
+
+// configResponse is the response to the testConfig message. If the server has an
+// error with the config, the Error variable will hold that error value.
+type configResponse struct {
+	Error string `json:"error,omitempty"`
+}
+
+// This represents the Result of a speedtest within a specific interval
+type Result struct {
+	Bytes         int           // number of bytes sent/received during the interval
+	IntervalStart time.Duration // duration between the start of the interval and the start of the test
+	IntervalEnd   time.Duration // duration between the end of the interval and the start of the test
+	Total         bool          // if true, this result struct represents the entire test, rather than a segment of the test
+}
+
+func (r Result) MBitsPerSecond() float64 {
+	return r.MegaBits() / (r.IntervalEnd - r.IntervalStart).Seconds()
+}
+
+func (r Result) MegaBytes() float64 {
+	return float64(r.Bytes) / 1000000.0
+}
+
+func (r Result) MegaBits() float64 {
+	return r.MegaBytes() * 8.0
+}
+
+func (r Result) Interval() time.Duration {
+	return r.IntervalEnd - r.IntervalStart
+}
+
+type Direction int
+
+const (
+	Download Direction = iota
+	Upload
+)
+
+func (d Direction) String() string {
+	switch d {
+	case Upload:
+		return "upload"
+	case Download:
+		return "download"
+	default:
+		return ""
+	}
+}
+
+func (d *Direction) Reverse() {
+	switch *d {
+	case Upload:
+		*d = Download
+	case Download:
+		*d = Upload
+	default:
+	}
+}

net/speedtest/speedtest_client.go

@@ -0,0 +1,42 @@
+// Copyright (c) 2021 Tailscale Inc & AUTHORS All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package speedtest
+
+import (
+	"encoding/json"
+	"errors"
+	"net"
+	"time"
+)
+
+// RunClient dials the given address and starts a speedtest.
+// It returns any errors that come up in the tests.
+// If there are no errors in the test, it returns a slice of results.
+func RunClient(direction Direction, duration time.Duration, host string) ([]Result, error) {
+	conn, err := net.Dial("tcp", host)
+	if err != nil {
+		return nil, err
+	}
+
+	conf := config{TestDuration: duration, Version: version, Direction: direction}
+
+	defer conn.Close()
+	encoder := json.NewEncoder(conn)
+
+	if err = encoder.Encode(conf); err != nil {
+		return nil, err
+	}
+
+	var response configResponse
+	decoder := json.NewDecoder(conn)
+	if err = decoder.Decode(&response); err != nil {
+		return nil, err
+	}
+	if response.Error != "" {
+		return nil, errors.New(response.Error)
+	}
+
+	return doTest(conn, conf)
+}

net/speedtest/speedtest_server.go

@@ -0,0 +1,158 @@
+// Copyright (c) 2021 Tailscale Inc & AUTHORS All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package speedtest
+
+import (
+	"crypto/rand"
+	"encoding/json"
+	"errors"
+	"fmt"
+	"io"
+	"net"
+	"time"
+)
+
+// Serve starts up the server on a given host and port pair. It starts to listen for
+// connections and handles each one in a goroutine. Because it runs in an infinite loop,
+// this function only returns if any of the speedtests return with errors, or if the
+// listener is closed.
+func Serve(l net.Listener) error {
+	for {
+		conn, err := l.Accept()
+		if errors.Is(err, net.ErrClosed) {
+			return nil
+		}
+		if err != nil {
+			return err
+		}
+		err = handleConnection(conn)
+		if err != nil {
+			return err
+		}
+	}
+}
+
+// handleConnection handles the initial exchange between the server and the client.
+// It reads the testconfig message into a config struct. If any errors occur with
+// the testconfig (specifically, if there is a version mismatch), it will return those
+// errors to the client with a configResponse. After the exchange, it will start
+// the speed test.
+func handleConnection(conn net.Conn) error {
+	defer conn.Close()
+	var conf config
+
+	decoder := json.NewDecoder(conn)
+	err := decoder.Decode(&conf)
+	encoder := json.NewEncoder(conn)
+
+	// Both return and encode errors that occurred before the test started.
+	if err != nil {
+		encoder.Encode(configResponse{Error: err.Error()})
+		return err
+	}
+
+	// The server should always be doing the opposite of what the client is doing.
+	conf.Direction.Reverse()
+
+	if conf.Version != version {
+		err = fmt.Errorf("version mismatch! Server is version %d, client is version %d", version, conf.Version)
+		encoder.Encode(configResponse{Error: err.Error()})
+		return err
+	}
+
+	// Start the test
+	encoder.Encode(configResponse{})
+	_, err = doTest(conn, conf)
+	return err
+}
+
+// TODO include code to detect whether the code is direct vs DERP
+
+// doTest contains the code to run both the upload and download speedtest.
+// the direction value in the config parameter determines which test to run.
+func doTest(conn net.Conn, conf config) ([]Result, error) {
+	bufferData := make([]byte, blockSize)
+
+	intervalBytes := 0
+	totalBytes := 0
+
+	var currentTime time.Time
+	var results []Result
+
+	startTime := time.Now()
+	lastCalculated := startTime
+
+	if conf.Direction == Download {
+		conn.SetReadDeadline(time.Now().Add(conf.TestDuration).Add(5 * time.Second))
+	} else {
+		_, err := rand.Read(bufferData)
+		if err != nil {
+			return nil, err
+		}
+
+	}
+
+SpeedTestLoop:
+	for {
+		var n int
+		var err error
+
+		if conf.Direction == Download {
+			n, err = io.ReadFull(conn, bufferData)
+			switch err {
+			case io.EOF, io.ErrUnexpectedEOF:
+				break SpeedTestLoop
+			case nil:
+				// successful read
+			default:
+				return nil, fmt.Errorf("unexpected error has occured: %w", err)
+			}
+		} else {
+			// Need to change the data a little bit, to avoid any compression.
+			for i := range bufferData {
+				bufferData[i]++
+			}
+			n, err = conn.Write(bufferData)
+			if err != nil {
+				// If the write failed, there is most likely something wrong with the connection.
+				return nil, fmt.Errorf("upload failed: %w", err)
+			}
+		}
+		currentTime = time.Now()
+		intervalBytes += n
+
+		// checks if the current time is more or equal to the lastCalculated time plus the increment
+		if currentTime.After(lastCalculated.Add(increment)) {
+			intervalStart := lastCalculated.Sub(startTime)
+			intervalEnd := currentTime.Sub(startTime)
+			if (intervalEnd - intervalStart) > minInterval {
+				results = append(results, Result{Bytes: intervalBytes, IntervalStart: intervalStart, IntervalEnd: intervalEnd, Total: false})
+			}
+			lastCalculated = currentTime
+			totalBytes += intervalBytes
+			intervalBytes = 0
+		}
+
+		if conf.Direction == Upload && time.Since(startTime) > conf.TestDuration {
+			break SpeedTestLoop
+		}
+	}
+
+	// get last segment
+	intervalStart := lastCalculated.Sub(startTime)
+	intervalEnd := currentTime.Sub(startTime)
+	if (intervalEnd - intervalStart) > minInterval {
+		results = append(results, Result{Bytes: intervalBytes, IntervalStart: intervalStart, IntervalEnd: intervalEnd, Total: false})
+	}
+
+	// get total
+	totalBytes += intervalBytes
+	intervalEnd = currentTime.Sub(startTime)
+	if intervalEnd > minInterval {
+		results = append(results, Result{Bytes: totalBytes, IntervalStart: 0, IntervalEnd: intervalEnd, Total: true})
+	}
+
+	return results, nil
+}

net/speedtest/speedtest_test.go

@@ -0,0 +1,81 @@
+// Copyright (c) 2021 Tailscale Inc & AUTHORS All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package speedtest
+
+import (
+	"net"
+	"testing"
+)
+
+func TestDownload(t *testing.T) {
+	// start a listener and find the port where the server will be listening.
+	l, err := net.Listen("tcp", ":0")
+	if err != nil {
+		t.Fatal(err)
+	}
+	t.Cleanup(func() { l.Close() })
+
+	serverIP := l.Addr().String()
+	t.Log("server IP found:", serverIP)
+
+	type state struct {
+		err error
+	}
+	displayResult := func(t *testing.T, r Result) {
+		t.Helper()
+		t.Logf("{ Megabytes: %.2f, Start: %.1f, End: %.1f, Total: %t }", r.MegaBytes(), r.IntervalStart.Seconds(), r.IntervalEnd.Seconds(), r.Total)
+	}
+	stateChan := make(chan state, 1)
+
+	go func() {
+		err := Serve(l)
+		stateChan <- state{err: err}
+	}()
+
+	// ensure that the test returns an appropriate number of Result structs
+	expectedLen := int(DefaultDuration.Seconds()) + 1
+
+	t.Run("download test", func(t *testing.T) {
+		// conduct a download test
+		results, err := RunClient(Download, DefaultDuration, serverIP)
+
+		if err != nil {
+			t.Fatal("download test failed:", err)
+		}
+
+		if len(results) < expectedLen {
+			t.Fatalf("download results: expected length: %d, actual length: %d", expectedLen, len(results))
+		}
+
+		for _, result := range results {
+			displayResult(t, result)
+		}
+	})
+
+	t.Run("upload test", func(t *testing.T) {
+		// conduct an upload test
+		results, err := RunClient(Upload, DefaultDuration, serverIP)
+
+		if err != nil {
+			t.Fatal("upload test failed:", err)
+		}
+
+		if len(results) < expectedLen {
+			t.Fatalf("upload results: expected length: %d, actual length: %d", expectedLen, len(results))
+		}
+
+		for _, result := range results {
+			displayResult(t, result)
+		}
+	})
+
+	// causes the server goroutine to finish
+	l.Close()
+
+	testState := <-stateChan
+	if testState.err != nil {
+		t.Error("server error:", err)
+	}
+}

net/tstun/wrap.go

@@ -18,6 +18,7 @@ import (
 	"golang.zx2c4.com/wireguard/tun"
 	"inet.af/netaddr"
 	"tailscale.com/net/packet"
+	"tailscale.com/tstime/mono"
 	"tailscale.com/types/ipproto"
 	"tailscale.com/types/logger"
 	"tailscale.com/wgengine/filter"
@@ -64,7 +65,7 @@ type Wrapper struct {
 
 	closeOnce sync.Once
 
-	lastActivityAtomic int64 // unix seconds of last send or receive
+	lastActivityAtomic mono.Time // time of last send or receive
 
 	destIPActivity atomic.Value // of map[netaddr.IP]func()
 
@@ -153,9 +154,10 @@ func Wrap(logf logger.Logf, tdev tun.Device) *Wrapper {
 		// a goroutine should not block when setting it, even with no listeners.
 		bufferConsumed: make(chan struct{}, 1),
 		closed:         make(chan struct{}),
-		outbound:       make(chan tunReadResult),
-		eventsUpDown:   make(chan tun.Event),
-		eventsOther:    make(chan tun.Event),
+		// outbound can be unbuffered; the buffer is an optimization.
+		outbound:     make(chan tunReadResult, 1),
+		eventsUpDown: make(chan tun.Event),
+		eventsOther:  make(chan tun.Event),
 		// TODO(dmytro): (highly rate-limited) hexdumps should happen on unknown packets.
 		filterFlags: filter.LogAccepts | filter.LogDrops,
 	}
@@ -164,6 +166,7 @@ func Wrap(logf logger.Logf, tdev tun.Device) *Wrapper {
 	go tun.pumpEvents()
 	// The buffer starts out consumed.
 	tun.bufferConsumed <- struct{}{}
+	tun.noteActivity()
 
 	return tun
 }
@@ -363,16 +366,15 @@ func (t *Wrapper) filterOut(p *packet.Parsed) filter.Response {
 
 // noteActivity records that there was a read or write at the current time.
 func (t *Wrapper) noteActivity() {
-	atomic.StoreInt64(&t.lastActivityAtomic, time.Now().Unix())
+	t.lastActivityAtomic.StoreAtomic(mono.Now())
 }
 
 // IdleDuration reports how long it's been since the last read or write to this device.
 //
-// Its value is only accurate to roughly second granularity.
-// If there's never been activity, the duration is since 1970.
+// Its value should only be presumed accurate to roughly 10ms granularity.
+// If there's never been activity, the duration is since the wrapper was created.
 func (t *Wrapper) IdleDuration() time.Duration {
-	sec := atomic.LoadInt64(&t.lastActivityAtomic)
-	return time.Since(time.Unix(sec, 0))
+	return mono.Since(t.lastActivityAtomic.LoadAtomic())
 }
 
 func (t *Wrapper) Read(buf []byte, offset int) (int, error) {

net/tstun/wrap_test.go

@@ -10,13 +10,13 @@ import (
 	"fmt"
 	"strconv"
 	"strings"
-	"sync/atomic"
 	"testing"
 	"unsafe"
 
 	"golang.zx2c4.com/wireguard/tun/tuntest"
 	"inet.af/netaddr"
 	"tailscale.com/net/packet"
+	"tailscale.com/tstime/mono"
 	"tailscale.com/types/ipproto"
 	"tailscale.com/types/logger"
 	"tailscale.com/wgengine/filter"
@@ -335,9 +335,9 @@ func TestFilter(t *testing.T) {
 				// data was actually filtered.
 				// If it stays zero, nothing made it through
 				// to the wrapped TUN.
-				atomic.StoreInt64(&tun.lastActivityAtomic, 0)
+				tun.lastActivityAtomic.StoreAtomic(0)
 				_, err = tun.Write(tt.data, 0)
-				filtered = atomic.LoadInt64(&tun.lastActivityAtomic) == 0
+				filtered = tun.lastActivityAtomic.LoadAtomic() == 0
 			} else {
 				chtun.Outbound <- tt.data
 				n, err = tun.Read(buf[:], 0)
@@ -416,7 +416,7 @@ func TestAtomic64Alignment(t *testing.T) {
 	}
 
 	c := new(Wrapper)
-	atomic.StoreInt64(&c.lastActivityAtomic, 123)
+	c.lastActivityAtomic.StoreAtomic(mono.Now())
 }
 
 func TestPeerAPIBypass(t *testing.T) {

safesocket/safesocket_darwin.go

@@ -53,15 +53,16 @@ func localTCPPortAndTokenDarwin() (port int, token string, err error) {
 	// The current process is running outside the sandbox, so use
 	// lsof to find the IPNExtension:
 
-	out, err := exec.Command("lsof",
+	cmd := exec.Command("lsof",
 		"-n",                             // numeric sockets; don't do DNS lookups, etc
 		"-a",                             // logical AND remaining options
 		fmt.Sprintf("-u%d", os.Getuid()), // process of same user only
 		"-c", "IPNExtension",             // starting with IPNExtension
 		"-F", // machine-readable output
-	).Output()
+	)
+	out, err := cmd.Output()
 	if err != nil {
-		return 0, "", fmt.Errorf("failed to run lsof looking for IPNExtension: %w", err)
+		return 0, "", fmt.Errorf("failed to run '%s' looking for IPNExtension: %w", cmd, err)
 	}
 	bs := bufio.NewScanner(bytes.NewReader(out))
 	subStr := []byte(".tailscale.ipn.macos/sameuserproof-")

safesocket/unixsocket.go

@@ -28,16 +28,15 @@ func connect(path string, port uint16) (net.Conn, error) {
 	pipe, err := net.Dial("unix", path)
 	if err != nil {
 		if runtime.GOOS == "darwin" {
-			extConn, err := connectMacOSAppSandbox()
-			if err != nil {
-				log.Printf("safesocket: failed to connect to Tailscale IPNExtension: %v", err)
-			} else {
-				return extConn, nil
+			extConn, extErr := connectMacOSAppSandbox()
+			if extErr != nil {
+				return nil, fmt.Errorf("safesocket: failed to connect to %v: %v; failed to connect to Tailscale IPNExtension: %v", path, err, extErr)
 			}
+			return extConn, nil
 		}
 		return nil, err
 	}
-	return pipe, err
+	return pipe, nil
 }
 
 // TODO(apenwarr): handle magic cookie auth

scripts/check_license_headers.sh

@@ -37,7 +37,7 @@ for file in $(find $1 -name '*.go' -not -path '*/.git/*'); do
         ;;
         $1/wgengine/router/ifconfig_windows.go)
             # WireGuard copyright.
-            ;;
+        ;;
         *)
             header="$(head -3 $file)"
             if ! check_file "$header"; then

syncs/syncs.go

@@ -79,6 +79,16 @@ func (b *AtomicBool) Set(v bool) {
 	atomic.StoreInt32((*int32)(b), n)
 }
 
+// Swap sets b to v and reports whether it changed.
+func (b *AtomicBool) Swap(v bool) (changed bool) {
+	var n int32
+	if v {
+		n = 1
+	}
+	old := atomic.SwapInt32((*int32)(b), n)
+	return old != n
+}
+
 func (b *AtomicBool) Get() bool {
 	return atomic.LoadInt32((*int32)(b)) != 0
 }

tailcfg/derpmap.go

@@ -130,10 +130,15 @@ type DERPNode struct {
 	// server.
 	STUNOnly bool `json:",omitempty"`
 
-	// DERPTestPort is used in tests to override the port, instead
-	// of using the default port of 443. If non-zero, TLS
-	// verification is skipped.
-	DERPTestPort int `json:",omitempty"`
+	// DERPPort optionally provides an alternate TLS port number
+	// for the DERP HTTPS server.
+	//
+	// If zero, 443 is used.
+	DERPPort int `json:",omitempty"`
+
+	// InsecureForTests is used by unit tests to disable TLS verification.
+	// It should not be set by users.
+	InsecureForTests bool `json:",omitempty"`
 
 	// STUNTestIP is used in tests to override the STUN server's IP.
 	// If empty, it's assumed to be the same as the DERP server.

tailcfg/tailcfg.go

@@ -164,6 +164,12 @@ type Node struct {
 	Hostinfo   Hostinfo
 	Created    time.Time
 
+	// PrimaryRoutes are the routes from AllowedIPs that this node
+	// is currently the primary subnet router for, as determined
+	// by the control plane. It does not include the self address
+	// values from Addresses that are in AllowedIPs.
+	PrimaryRoutes []netaddr.IPPrefix `json:",omitempty"`
+
 	// LastSeen is when the node was last online. It is not
 	// updated when Online is true. It is nil if the current
 	// node doesn't have permission to know, or the node
@@ -1060,6 +1066,13 @@ type Debug struct {
 	// :0 to get a random local port, ignoring any configured
 	// fixed port.
 	RandomizeClientPort bool `json:",omitempty"`
+
+	/// DisableUPnP is whether the client will attempt to perform a UPnP portmapping.
+	// By default, we want to enable it to see if it works on more clients.
+	//
+	// If UPnP catastrophically fails for people, this should be set to True to kill
+	// new attempts at UPnP connections.
+	DisableUPnP opt.Bool `json:",omitempty"`
 }
 
 func (k MachineKey) String() string                   { return fmt.Sprintf("mkey:%x", k[:]) }
@@ -1135,6 +1148,7 @@ func (n *Node) Equal(n2 *Node) bool {
 		eqBoolPtr(n.Online, n2.Online) &&
 		eqCIDRs(n.Addresses, n2.Addresses) &&
 		eqCIDRs(n.AllowedIPs, n2.AllowedIPs) &&
+		eqCIDRs(n.PrimaryRoutes, n2.PrimaryRoutes) &&
 		eqStrings(n.Endpoints, n2.Endpoints) &&
 		n.DERP == n2.DERP &&
 		n.Hostinfo.Equal(&n2.Hostinfo) &&

tailcfg/tailcfg_clone.go

@@ -49,6 +49,7 @@ func (src *Node) Clone() *Node {
 	dst.AllowedIPs = append(src.AllowedIPs[:0:0], src.AllowedIPs...)
 	dst.Endpoints = append(src.Endpoints[:0:0], src.Endpoints...)
 	dst.Hostinfo = *src.Hostinfo.Clone()
+	dst.PrimaryRoutes = append(src.PrimaryRoutes[:0:0], src.PrimaryRoutes...)
 	if dst.LastSeen != nil {
 		dst.LastSeen = new(time.Time)
 		*dst.LastSeen = *src.LastSeen
@@ -79,6 +80,7 @@ var _NodeNeedsRegeneration = Node(struct {
 	DERP                    string
 	Hostinfo                Hostinfo
 	Created                 time.Time
+	PrimaryRoutes           []netaddr.IPPrefix
 	LastSeen                *time.Time
 	Online                  *bool
 	KeepAlive               bool
@@ -327,16 +329,17 @@ func (src *DERPNode) Clone() *DERPNode {
 // A compilation failure here means this code must be regenerated, with command:
 //   tailscale.com/cmd/cloner -type User,Node,Hostinfo,NetInfo,Login,DNSConfig,DNSResolver,RegisterResponse,DERPRegion,DERPMap,DERPNode
 var _DERPNodeNeedsRegeneration = DERPNode(struct {
-	Name         string
-	RegionID     int
-	HostName     string
-	CertName     string
-	IPv4         string
-	IPv6         string
-	STUNPort     int
-	STUNOnly     bool
-	DERPTestPort int
-	STUNTestIP   string
+	Name             string
+	RegionID         int
+	HostName         string
+	CertName         string
+	IPv4             string
+	IPv6             string
+	STUNPort         int
+	STUNOnly         bool
+	DERPPort         int
+	InsecureForTests bool
+	STUNTestIP       string
 }{})
 
 // Clone duplicates src into dst and reports whether it succeeded.

tailcfg/tailcfg_test.go

@@ -194,7 +194,8 @@ func TestNodeEqual(t *testing.T) {
 		"ID", "StableID", "Name", "User", "Sharer",
 		"Key", "KeyExpiry", "Machine", "DiscoKey",
 		"Addresses", "AllowedIPs", "Endpoints", "DERP", "Hostinfo",
-		"Created", "LastSeen", "Online", "KeepAlive", "MachineAuthorized",
+		"Created", "PrimaryRoutes",
+		"LastSeen", "Online", "KeepAlive", "MachineAuthorized",
 		"Capabilities",
 		"ComputedName", "computedHostIfDifferent", "ComputedNameWithHost",
 	}

tempfork/wireguard-windows/firewall/zsyscall_windows.go

@@ -1,5 +1,3 @@
-// +build windows
-
 // Code generated by 'go generate'; DO NOT EDIT.
 
 package firewall

tstest/integration/gen_deps.go

@@ -12,16 +12,25 @@ import (
 	"fmt"
 	"io/ioutil"
 	"log"
+	"os"
 	"os/exec"
 )
 
 func main() {
+	for _, goos := range []string{"windows", "linux", "darwin", "freebsd", "openbsd"} {
+		generate(goos)
+	}
+}
+
+func generate(goos string) {
 	var x struct {
 		Imports []string
 	}
-	j, err := exec.Command("go", "list", "-json", "tailscale.com/cmd/tailscaled").Output()
+	cmd := exec.Command("go", "list", "-json", "tailscale.com/cmd/tailscaled")
+	cmd.Env = append(os.Environ(), "GOOS="+goos, "GOARCH=amd64")
+	j, err := cmd.Output()
 	if err != nil {
-		log.Fatal(err)
+		log.Fatalf("GOOS=%s GOARCH=amd64 %s: %v", goos, cmd, err)
 	}
 	if err := json.Unmarshal(j, &x); err != nil {
 		log.Fatal(err)
@@ -46,7 +55,8 @@ import (
 	}
 	fmt.Fprintf(&out, ")\n")
 
-	err = ioutil.WriteFile("tailscaled_deps_test.go", out.Bytes(), 0644)
+	filename := fmt.Sprintf("tailscaled_deps_test_%s.go", goos)
+	err = ioutil.WriteFile(filename, out.Bytes(), 0644)
 	if err != nil {
 		log.Fatal(err)
 	}

tstest/integration/integration.go

@@ -145,14 +145,15 @@ func RunDERPAndSTUN(t testing.TB, logf logger.Logf, ipAddress string) (derpMap *
 				RegionCode: "test",
 				Nodes: []*tailcfg.DERPNode{
 					{
-						Name:         "t1",
-						RegionID:     1,
-						HostName:     ipAddress,
-						IPv4:         ipAddress,
-						IPv6:         "none",
-						STUNPort:     stunAddr.Port,
-						DERPTestPort: httpsrv.Listener.Addr().(*net.TCPAddr).Port,
-						STUNTestIP:   stunAddr.IP.String(),
+						Name:             "t1",
+						RegionID:         1,
+						HostName:         ipAddress,
+						IPv4:             ipAddress,
+						IPv6:             "none",
+						STUNPort:         stunAddr.Port,
+						DERPPort:         httpsrv.Listener.Addr().(*net.TCPAddr).Port,
+						InsecureForTests: true,
+						STUNTestIP:       stunAddr.IP.String(),
 					},
 				},
 			},
@@ -204,6 +205,13 @@ func (lc *LogCatcher) logsString() string {
 	return lc.buf.String()
 }
 
+// Reset clears the buffered logs from memory.
+func (lc *LogCatcher) Reset() {
+	lc.mu.Lock()
+	defer lc.mu.Unlock()
+	lc.buf.Reset()
+}
+
 func (lc *LogCatcher) ServeHTTP(w http.ResponseWriter, r *http.Request) {
 	var body io.Reader = r.Body
 	if r.Header.Get("Content-Encoding") == "zstd" {

tstest/integration/integration_test.go

@@ -23,12 +23,15 @@ import (
 	"path/filepath"
 	"regexp"
 	"runtime"
+	"strings"
 	"sync"
 	"sync/atomic"
 	"testing"
 	"time"
 
 	"go4.org/mem"
+	"inet.af/netaddr"
+	"tailscale.com/ipn"
 	"tailscale.com/ipn/ipnstate"
 	"tailscale.com/safesocket"
 	"tailscale.com/tailcfg"
@@ -39,11 +42,14 @@ import (
 
 var (
 	verboseTailscaled = flag.Bool("verbose-tailscaled", false, "verbose tailscaled logging")
+	verboseTailscale  = flag.Bool("verbose-tailscale", false, "verbose tailscale CLI logging")
 )
 
 var mainError atomic.Value // of error
 
 func TestMain(m *testing.M) {
+	// Have to disable UPnP which hits the network, otherwise it fails due to HTTP proxy.
+	os.Setenv("TS_DISABLE_UPNP", "true")
 	flag.Parse()
 	v := m.Run()
 	if v != 0 {
@@ -67,29 +73,9 @@ func TestOneNodeUp_NoAuth(t *testing.T) {
 
 	d1 := n1.StartDaemon(t)
 	defer d1.Kill()
-
-	n1.AwaitListening(t)
-
-	st := n1.MustStatus(t)
-	t.Logf("Status: %s", st.BackendState)
-
-	if err := tstest.WaitFor(20*time.Second, func() error {
-		const sub = `Program starting: `
-		if !env.LogCatcher.logsContains(mem.S(sub)) {
-			return fmt.Errorf("log catcher didn't see %#q; got %s", sub, env.LogCatcher.logsString())
-		}
-		return nil
-	}); err != nil {
-		t.Error(err)
-	}
-
+	n1.AwaitResponding(t)
 	n1.MustUp()
 
-	if d, _ := time.ParseDuration(os.Getenv("TS_POST_UP_SLEEP")); d > 0 {
-		t.Logf("Sleeping for %v to give 'up' time to misbehave (https://github.com/tailscale/tailscale/issues/1840) ...", d)
-		time.Sleep(d)
-	}
-
 	t.Logf("Got IP: %v", n1.AwaitIP(t))
 	n1.AwaitRunning(t)
 
@@ -98,6 +84,48 @@ func TestOneNodeUp_NoAuth(t *testing.T) {
 	t.Logf("number of HTTP logcatcher requests: %v", env.LogCatcher.numRequests())
 }
 
+// test Issue 2321: Start with UpdatePrefs should save prefs to disk
+func TestStateSavedOnStart(t *testing.T) {
+	t.Parallel()
+	bins := BuildTestBinaries(t)
+
+	env := newTestEnv(t, bins)
+	defer env.Close()
+
+	n1 := newTestNode(t, env)
+
+	d1 := n1.StartDaemon(t)
+	defer d1.Kill()
+	n1.AwaitResponding(t)
+	n1.MustUp()
+
+	t.Logf("Got IP: %v", n1.AwaitIP(t))
+	n1.AwaitRunning(t)
+
+	p1 := n1.diskPrefs(t)
+	t.Logf("Prefs1: %v", p1.Pretty())
+
+	// Bring it down, to prevent an EditPrefs call in the
+	// subsequent "up", as we want to test the bug when
+	// cmd/tailscale implements "up" via LocalBackend.Start.
+	n1.MustDown()
+
+	// And change the hostname to something:
+	if err := n1.Tailscale("up", "--login-server="+n1.env.ControlServer.URL, "--hostname=foo").Run(); err != nil {
+		t.Fatalf("up: %v", err)
+	}
+
+	p2 := n1.diskPrefs(t)
+	if pretty := p1.Pretty(); pretty == p2.Pretty() {
+		t.Errorf("Prefs didn't change on disk after 'up', still: %s", pretty)
+	}
+	if p2.Hostname != "foo" {
+		t.Errorf("Prefs.Hostname = %q; want foo", p2.Hostname)
+	}
+
+	d1.MustCleanShutdown(t)
+}
+
 func TestOneNodeUp_Auth(t *testing.T) {
 	t.Parallel()
 	bins := BuildTestBinaries(t)
@@ -284,6 +312,76 @@ func TestAddPingRequest(t *testing.T) {
 	t.Error("all ping attempts failed")
 }
 
+// Issue 2434: when "down" (WantRunning false), tailscaled shouldn't
+// be connected to control.
+func TestNoControlConnWhenDown(t *testing.T) {
+	t.Parallel()
+	bins := BuildTestBinaries(t)
+
+	env := newTestEnv(t, bins)
+	defer env.Close()
+
+	n1 := newTestNode(t, env)
+
+	d1 := n1.StartDaemon(t)
+	defer d1.Kill()
+	n1.AwaitResponding(t)
+
+	// Come up the first time.
+	n1.MustUp()
+	ip1 := n1.AwaitIP(t)
+	n1.AwaitRunning(t)
+
+	// Then bring it down and stop the daemon.
+	n1.MustDown()
+	d1.MustCleanShutdown(t)
+
+	env.LogCatcher.Reset()
+	d2 := n1.StartDaemon(t)
+	defer d2.Kill()
+	n1.AwaitResponding(t)
+
+	st := n1.MustStatus(t)
+	if got, want := st.BackendState, "Stopped"; got != want {
+		t.Fatalf("after restart, state = %q; want %q", got, want)
+	}
+
+	ip2 := n1.AwaitIP(t)
+	if ip1 != ip2 {
+		t.Errorf("IPs different: %q vs %q", ip1, ip2)
+	}
+
+	// The real test: verify our daemon doesn't have an HTTP request open.:
+	if n := env.Control.InServeMap(); n != 0 {
+		t.Errorf("in serve map = %d; want 0", n)
+	}
+
+	d2.MustCleanShutdown(t)
+}
+
+// Issue 2137: make sure Windows tailscaled works with the CLI alone,
+// without the GUI to kick off a Start.
+func TestOneNodeUpWindowsStyle(t *testing.T) {
+	t.Parallel()
+	bins := BuildTestBinaries(t)
+
+	env := newTestEnv(t, bins)
+	defer env.Close()
+
+	n1 := newTestNode(t, env)
+	n1.upFlagGOOS = "windows"
+
+	d1 := n1.StartDaemonAsIPNGOOS(t, "windows")
+	defer d1.Kill()
+	n1.AwaitResponding(t)
+	n1.MustUp("--unattended")
+
+	t.Logf("Got IP: %v", n1.AwaitIP(t))
+	n1.AwaitRunning(t)
+
+	d1.MustCleanShutdown(t)
+}
+
 // testEnv contains the test environment (set of servers) used by one
 // or more nodes.
 type testEnv struct {
@@ -360,9 +458,10 @@ func (e *testEnv) Close() error {
 type testNode struct {
 	env *testEnv
 
-	dir       string // temp dir for sock & state
-	sockFile  string
-	stateFile string
+	dir        string // temp dir for sock & state
+	sockFile   string
+	stateFile  string
+	upFlagGOOS string // if non-empty, sets TS_DEBUG_UP_FLAG_GOOS for cmd/tailscale CLI
 
 	mu        sync.Mutex
 	onLogLine []func([]byte)
@@ -372,14 +471,58 @@ type testNode struct {
 // The node is not started automatically.
 func newTestNode(t *testing.T, env *testEnv) *testNode {
 	dir := t.TempDir()
+	sockFile := filepath.Join(dir, "tailscale.sock")
+	if len(sockFile) >= 104 {
+		t.Fatalf("sockFile path %q (len %v) is too long, must be < 104", sockFile, len(sockFile))
+	}
 	return &testNode{
 		env:       env,
 		dir:       dir,
-		sockFile:  filepath.Join(dir, "tailscale.sock"),
+		sockFile:  sockFile,
 		stateFile: filepath.Join(dir, "tailscale.state"),
 	}
 }
 
+func (n *testNode) diskPrefs(t testing.TB) *ipn.Prefs {
+	t.Helper()
+	if _, err := ioutil.ReadFile(n.stateFile); err != nil {
+		t.Fatalf("reading prefs: %v", err)
+	}
+	fs, err := ipn.NewFileStore(n.stateFile)
+	if err != nil {
+		t.Fatalf("reading prefs, NewFileStore: %v", err)
+	}
+	prefBytes, err := fs.ReadState(ipn.GlobalDaemonStateKey)
+	if err != nil {
+		t.Fatalf("reading prefs, ReadState: %v", err)
+	}
+	p := new(ipn.Prefs)
+	if err := json.Unmarshal(prefBytes, p); err != nil {
+		t.Fatalf("reading prefs, JSON unmarshal: %v", err)
+	}
+	return p
+}
+
+// AwaitResponding waits for n's tailscaled to be up enough to be
+// responding, but doesn't wait for any particular state.
+func (n *testNode) AwaitResponding(t testing.TB) {
+	t.Helper()
+	n.AwaitListening(t)
+
+	st := n.MustStatus(t)
+	t.Logf("Status: %s", st.BackendState)
+
+	if err := tstest.WaitFor(20*time.Second, func() error {
+		const sub = `Program starting: `
+		if !n.env.LogCatcher.logsContains(mem.S(sub)) {
+			return fmt.Errorf("log catcher didn't see %#q; got %s", sub, n.env.LogCatcher.logsString())
+		}
+		return nil
+	}); err != nil {
+		t.Fatal(err)
+	}
+}
+
 // addLogLineHook registers a hook f to be called on each tailscaled
 // log line output.
 func (n *testNode) addLogLineHook(f func([]byte)) {
@@ -481,6 +624,10 @@ func (d *Daemon) MustCleanShutdown(t testing.TB) {
 // StartDaemon starts the node's tailscaled, failing if it fails to
 // start.
 func (n *testNode) StartDaemon(t testing.TB) *Daemon {
+	return n.StartDaemonAsIPNGOOS(t, runtime.GOOS)
+}
+
+func (n *testNode) StartDaemonAsIPNGOOS(t testing.TB, ipnGOOS string) *Daemon {
 	cmd := exec.Command(n.env.Binaries.Daemon,
 		"--tun=userspace-networking",
 		"--state="+n.stateFile,
@@ -491,6 +638,8 @@ func (n *testNode) StartDaemon(t testing.TB) *Daemon {
 		"TS_LOG_TARGET="+n.env.LogCatcherServer.URL,
 		"HTTP_PROXY="+n.env.TrafficTrapServer.URL,
 		"HTTPS_PROXY="+n.env.TrafficTrapServer.URL,
+		"TS_DEBUG_TAILSCALED_IPN_GOOS="+ipnGOOS,
+		"TS_LOGS_DIR="+t.TempDir(),
 	)
 	cmd.Stderr = &nodeOutputParser{n: n}
 	if *verboseTailscaled {
@@ -505,14 +654,27 @@ func (n *testNode) StartDaemon(t testing.TB) *Daemon {
 	}
 }
 
-func (n *testNode) MustUp() {
+func (n *testNode) MustUp(extraArgs ...string) {
 	t := n.env.t
-	t.Logf("Running up --login-server=%s ...", n.env.ControlServer.URL)
-	if err := n.Tailscale("up", "--login-server="+n.env.ControlServer.URL).Run(); err != nil {
+	args := []string{
+		"up",
+		"--login-server=" + n.env.ControlServer.URL,
+	}
+	args = append(args, extraArgs...)
+	t.Logf("Running %v ...", args)
+	if err := n.Tailscale(args...).Run(); err != nil {
 		t.Fatalf("up: %v", err)
 	}
 }
 
+func (n *testNode) MustDown() {
+	t := n.env.t
+	t.Logf("Running down ...")
+	if err := n.Tailscale("down").Run(); err != nil {
+		t.Fatalf("down: %v", err)
+	}
+}
+
 // AwaitListening waits for the tailscaled to be serving local clients
 // over its localhost IPC mechanism. (Unix socket, etc)
 func (n *testNode) AwaitListening(t testing.TB) {
@@ -528,24 +690,46 @@ func (n *testNode) AwaitListening(t testing.TB) {
 	}
 }
 
-func (n *testNode) AwaitIP(t testing.TB) (ips string) {
+func (n *testNode) AwaitIPs(t testing.TB) []netaddr.IP {
 	t.Helper()
+	var addrs []netaddr.IP
 	if err := tstest.WaitFor(20*time.Second, func() error {
-		out, err := n.Tailscale("ip").Output()
+		cmd := n.Tailscale("ip")
+		cmd.Stdout = nil // in case --verbose-tailscale was set
+		cmd.Stderr = nil // in case --verbose-tailscale was set
+		out, err := cmd.Output()
 		if err != nil {
 			return err
 		}
-		ips = string(out)
+		ips := string(out)
+		ipslice := strings.Fields(ips)
+		addrs = make([]netaddr.IP, len(ipslice))
+
+		for i, ip := range ipslice {
+			netIP, err := netaddr.ParseIP(ip)
+			if err != nil {
+				t.Fatal(err)
+			}
+			addrs[i] = netIP
+		}
 		return nil
 	}); err != nil {
 		t.Fatalf("awaiting an IP address: %v", err)
 	}
-	if ips == "" {
+	if len(addrs) == 0 {
 		t.Fatalf("returned IP address was blank")
 	}
-	return ips
+	return addrs
 }
 
+// AwaitIP returns the IP address of n.
+func (n *testNode) AwaitIP(t testing.TB) netaddr.IP {
+	t.Helper()
+	ips := n.AwaitIPs(t)
+	return ips[0]
+}
+
+// AwaitRunning waits for n to reach the IPN state "Running".
 func (n *testNode) AwaitRunning(t testing.TB) {
 	t.Helper()
 	if err := tstest.WaitFor(20*time.Second, func() error {
@@ -568,11 +752,22 @@ func (n *testNode) Tailscale(arg ...string) *exec.Cmd {
 	cmd := exec.Command(n.env.Binaries.CLI, "--socket="+n.sockFile)
 	cmd.Args = append(cmd.Args, arg...)
 	cmd.Dir = n.dir
+	cmd.Env = append(os.Environ(),
+		"TS_DEBUG_UP_FLAG_GOOS="+n.upFlagGOOS,
+		"TS_LOGS_DIR="+n.env.t.TempDir(),
+	)
+	if *verboseTailscale {
+		cmd.Stdout = os.Stdout
+		cmd.Stderr = os.Stderr
+	}
 	return cmd
 }
 
 func (n *testNode) Status() (*ipnstate.Status, error) {
-	out, err := n.Tailscale("status", "--json").CombinedOutput()
+	cmd := n.Tailscale("status", "--json")
+	cmd.Stdout = nil // in case --verbose-tailscale was set
+	cmd.Stderr = nil // in case --verbose-tailscale was set
+	out, err := cmd.CombinedOutput()
 	if err != nil {
 		return nil, fmt.Errorf("running tailscale status: %v, %s", err, out)
 	}

tstest/integration/tailscaled_deps_test_darwin.go

@@ -0,0 +1,60 @@
+// Copyright (c) 2021 Tailscale Inc & AUTHORS All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+// Code generated by gen_deps.go; DO NOT EDIT.
+
+package integration
+
+import (
+	// And depend on a bunch of tailscaled innards, for Go's test caching.
+	// Otherwise cmd/go never sees that we depend on these packages'
+	// transitive deps when we run "go install tailscaled" in a child
+	// process and can cache a prior success when a dependency changes.
+	_ "context"
+	_ "crypto/tls"
+	_ "encoding/json"
+	_ "errors"
+	_ "flag"
+	_ "fmt"
+	_ "github.com/go-multierror/multierror"
+	_ "io"
+	_ "io/ioutil"
+	_ "log"
+	_ "net"
+	_ "net/http"
+	_ "net/http/httptrace"
+	_ "net/http/pprof"
+	_ "net/url"
+	_ "os"
+	_ "os/exec"
+	_ "os/signal"
+	_ "path/filepath"
+	_ "runtime"
+	_ "runtime/debug"
+	_ "strconv"
+	_ "strings"
+	_ "syscall"
+	_ "tailscale.com/derp/derphttp"
+	_ "tailscale.com/ipn"
+	_ "tailscale.com/ipn/ipnserver"
+	_ "tailscale.com/logpolicy"
+	_ "tailscale.com/net/dns"
+	_ "tailscale.com/net/interfaces"
+	_ "tailscale.com/net/socks5/tssocks"
+	_ "tailscale.com/net/tshttpproxy"
+	_ "tailscale.com/net/tstun"
+	_ "tailscale.com/paths"
+	_ "tailscale.com/tailcfg"
+	_ "tailscale.com/types/flagtype"
+	_ "tailscale.com/types/key"
+	_ "tailscale.com/types/logger"
+	_ "tailscale.com/util/osshare"
+	_ "tailscale.com/version"
+	_ "tailscale.com/version/distro"
+	_ "tailscale.com/wgengine"
+	_ "tailscale.com/wgengine/monitor"
+	_ "tailscale.com/wgengine/netstack"
+	_ "tailscale.com/wgengine/router"
+	_ "time"
+)

tstest/integration/tailscaled_deps_test_linux.go

@@ -0,0 +1,58 @@
+// Copyright (c) 2021 Tailscale Inc & AUTHORS All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+// Code generated by gen_deps.go; DO NOT EDIT.
+
+package integration
+
+import (
+	// And depend on a bunch of tailscaled innards, for Go's test caching.
+	// Otherwise cmd/go never sees that we depend on these packages'
+	// transitive deps when we run "go install tailscaled" in a child
+	// process and can cache a prior success when a dependency changes.
+	_ "context"
+	_ "crypto/tls"
+	_ "encoding/json"
+	_ "errors"
+	_ "flag"
+	_ "fmt"
+	_ "github.com/go-multierror/multierror"
+	_ "io"
+	_ "io/ioutil"
+	_ "log"
+	_ "net"
+	_ "net/http"
+	_ "net/http/httptrace"
+	_ "net/http/pprof"
+	_ "net/url"
+	_ "os"
+	_ "os/signal"
+	_ "runtime"
+	_ "runtime/debug"
+	_ "strconv"
+	_ "strings"
+	_ "syscall"
+	_ "tailscale.com/derp/derphttp"
+	_ "tailscale.com/ipn"
+	_ "tailscale.com/ipn/ipnserver"
+	_ "tailscale.com/logpolicy"
+	_ "tailscale.com/net/dns"
+	_ "tailscale.com/net/interfaces"
+	_ "tailscale.com/net/socks5/tssocks"
+	_ "tailscale.com/net/tshttpproxy"
+	_ "tailscale.com/net/tstun"
+	_ "tailscale.com/paths"
+	_ "tailscale.com/tailcfg"
+	_ "tailscale.com/types/flagtype"
+	_ "tailscale.com/types/key"
+	_ "tailscale.com/types/logger"
+	_ "tailscale.com/util/osshare"
+	_ "tailscale.com/version"
+	_ "tailscale.com/version/distro"
+	_ "tailscale.com/wgengine"
+	_ "tailscale.com/wgengine/monitor"
+	_ "tailscale.com/wgengine/netstack"
+	_ "tailscale.com/wgengine/router"
+	_ "time"
+)

tstest/integration/tailscaled_deps_test_openbsd.go

@@ -0,0 +1,58 @@
+// Copyright (c) 2021 Tailscale Inc & AUTHORS All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+// Code generated by gen_deps.go; DO NOT EDIT.
+
+package integration
+
+import (
+	// And depend on a bunch of tailscaled innards, for Go's test caching.
+	// Otherwise cmd/go never sees that we depend on these packages'
+	// transitive deps when we run "go install tailscaled" in a child
+	// process and can cache a prior success when a dependency changes.
+	_ "context"
+	_ "crypto/tls"
+	_ "encoding/json"
+	_ "errors"
+	_ "flag"
+	_ "fmt"
+	_ "github.com/go-multierror/multierror"
+	_ "io"
+	_ "io/ioutil"
+	_ "log"
+	_ "net"
+	_ "net/http"
+	_ "net/http/httptrace"
+	_ "net/http/pprof"
+	_ "net/url"
+	_ "os"
+	_ "os/signal"
+	_ "runtime"
+	_ "runtime/debug"
+	_ "strconv"
+	_ "strings"
+	_ "syscall"
+	_ "tailscale.com/derp/derphttp"
+	_ "tailscale.com/ipn"
+	_ "tailscale.com/ipn/ipnserver"
+	_ "tailscale.com/logpolicy"
+	_ "tailscale.com/net/dns"
+	_ "tailscale.com/net/interfaces"
+	_ "tailscale.com/net/socks5/tssocks"
+	_ "tailscale.com/net/tshttpproxy"
+	_ "tailscale.com/net/tstun"
+	_ "tailscale.com/paths"
+	_ "tailscale.com/tailcfg"
+	_ "tailscale.com/types/flagtype"
+	_ "tailscale.com/types/key"
+	_ "tailscale.com/types/logger"
+	_ "tailscale.com/util/osshare"
+	_ "tailscale.com/version"
+	_ "tailscale.com/version/distro"
+	_ "tailscale.com/wgengine"
+	_ "tailscale.com/wgengine/monitor"
+	_ "tailscale.com/wgengine/netstack"
+	_ "tailscale.com/wgengine/router"
+	_ "time"
+)

tstest/integration/tailscaled_deps_test_windows.go

@@ -0,0 +1,65 @@
+// Copyright (c) 2021 Tailscale Inc & AUTHORS All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+// Code generated by gen_deps.go; DO NOT EDIT.
+
+package integration
+
+import (
+	// And depend on a bunch of tailscaled innards, for Go's test caching.
+	// Otherwise cmd/go never sees that we depend on these packages'
+	// transitive deps when we run "go install tailscaled" in a child
+	// process and can cache a prior success when a dependency changes.
+	_ "context"
+	_ "crypto/tls"
+	_ "encoding/json"
+	_ "errors"
+	_ "flag"
+	_ "fmt"
+	_ "github.com/go-multierror/multierror"
+	_ "golang.org/x/sys/windows"
+	_ "golang.org/x/sys/windows/svc"
+	_ "golang.org/x/sys/windows/svc/mgr"
+	_ "golang.zx2c4.com/wireguard/windows/tunnel/winipcfg"
+	_ "inet.af/netaddr"
+	_ "io"
+	_ "io/ioutil"
+	_ "log"
+	_ "net"
+	_ "net/http"
+	_ "net/http/httptrace"
+	_ "net/http/pprof"
+	_ "net/url"
+	_ "os"
+	_ "os/signal"
+	_ "runtime"
+	_ "runtime/debug"
+	_ "strconv"
+	_ "strings"
+	_ "syscall"
+	_ "tailscale.com/derp/derphttp"
+	_ "tailscale.com/ipn"
+	_ "tailscale.com/ipn/ipnserver"
+	_ "tailscale.com/logpolicy"
+	_ "tailscale.com/logtail/backoff"
+	_ "tailscale.com/net/dns"
+	_ "tailscale.com/net/interfaces"
+	_ "tailscale.com/net/socks5/tssocks"
+	_ "tailscale.com/net/tshttpproxy"
+	_ "tailscale.com/net/tstun"
+	_ "tailscale.com/paths"
+	_ "tailscale.com/tailcfg"
+	_ "tailscale.com/types/flagtype"
+	_ "tailscale.com/types/key"
+	_ "tailscale.com/types/logger"
+	_ "tailscale.com/util/osshare"
+	_ "tailscale.com/version"
+	_ "tailscale.com/version/distro"
+	_ "tailscale.com/wf"
+	_ "tailscale.com/wgengine"
+	_ "tailscale.com/wgengine/monitor"
+	_ "tailscale.com/wgengine/netstack"
+	_ "tailscale.com/wgengine/router"
+	_ "time"
+)

tstest/integration/testcontrol/testcontrol.go

@@ -28,6 +28,7 @@ import (
 	"github.com/klauspost/compress/zstd"
 	"golang.org/x/crypto/nacl/box"
 	"inet.af/netaddr"
+	"tailscale.com/net/tsaddr"
 	"tailscale.com/smallzstd"
 	"tailscale.com/tailcfg"
 	"tailscale.com/types/logger"
@@ -50,6 +51,7 @@ type Server struct {
 	mux         *http.ServeMux
 
 	mu            sync.Mutex
+	inServeMap    int
 	cond          *sync.Cond // lazily initialized by condLocked
 	pubKey        wgkey.Key
 	privKey       wgkey.Private
@@ -406,8 +408,12 @@ func (s *Server) serveRegister(w http.ResponseWriter, r *http.Request, mkey tail
 
 	machineAuthorized := true // TODO: add Server.RequireMachineAuth
 
+	v4Prefix := netaddr.IPPrefixFrom(netaddr.IPv4(100, 64, uint8(tailcfg.NodeID(user.ID)>>8), uint8(tailcfg.NodeID(user.ID))), 32)
+	v6Prefix := netaddr.IPPrefixFrom(tsaddr.Tailscale4To6(v4Prefix.IP()), 128)
+
 	allowedIPs := []netaddr.IPPrefix{
-		netaddr.MustParseIPPrefix(fmt.Sprintf("100.64.%d.%d/32", uint8(tailcfg.NodeID(user.ID)>>8), uint8(tailcfg.NodeID(user.ID)))),
+		v4Prefix,
+		v6Prefix,
 	}
 
 	s.nodes[req.NodeKey] = &tailcfg.Node{
@@ -504,7 +510,22 @@ func (s *Server) UpdateNode(n *tailcfg.Node) (peersToUpdate []tailcfg.NodeID) {
 	return peersToUpdate
 }
 
+func (s *Server) incrInServeMap(delta int) {
+	s.mu.Lock()
+	defer s.mu.Unlock()
+	s.inServeMap += delta
+}
+
+// InServeMap returns the number of clients currently in a MapRequest HTTP handler.
+func (s *Server) InServeMap() int {
+	s.mu.Lock()
+	defer s.mu.Unlock()
+	return s.inServeMap
+}
+
 func (s *Server) serveMap(w http.ResponseWriter, r *http.Request, mkey tailcfg.MachineKey) {
+	s.incrInServeMap(1)
+	defer s.incrInServeMap(-1)
 	ctx := r.Context()
 
 	req := new(tailcfg.MapRequest)
@@ -631,6 +652,9 @@ func (s *Server) MapResponse(req *tailcfg.MapRequest) (res *tailcfg.MapResponse,
 		Domain:          string(user.Domain),
 		CollectServices: "true",
 		PacketFilter:    tailcfg.FilterAllowAll,
+		Debug: &tailcfg.Debug{
+			DisableUPnP: "true",
+		},
 	}
 	for _, p := range s.AllNodes() {
 		if p.StableID != node.StableID {
@@ -638,8 +662,12 @@ func (s *Server) MapResponse(req *tailcfg.MapRequest) (res *tailcfg.MapResponse,
 		}
 	}
 
+	v4Prefix := netaddr.IPPrefixFrom(netaddr.IPv4(100, 64, uint8(tailcfg.NodeID(user.ID)>>8), uint8(tailcfg.NodeID(user.ID))), 32)
+	v6Prefix := netaddr.IPPrefixFrom(tsaddr.Tailscale4To6(v4Prefix.IP()), 128)
+
 	res.Node.Addresses = []netaddr.IPPrefix{
-		netaddr.MustParseIPPrefix(fmt.Sprintf("100.64.%d.%d/32", uint8(node.ID>>8), uint8(node.ID))),
+		v4Prefix,
+		v6Prefix,
 	}
 	res.Node.AllowedIPs = res.Node.Addresses
 

tstest/integration/vms/derive_bindhost_test.go

@@ -0,0 +1,50 @@
+// Copyright (c) 2021 Tailscale Inc & AUTHORS All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+// +build linux
+
+package vms
+
+import (
+	"io"
+	"testing"
+
+	"inet.af/netaddr"
+	"tailscale.com/net/interfaces"
+)
+
+func deriveBindhost(t *testing.T) string {
+	t.Helper()
+
+	ifName, err := interfaces.DefaultRouteInterface()
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	var ret string
+	err = interfaces.ForeachInterfaceAddress(func(i interfaces.Interface, prefix netaddr.IPPrefix) {
+		if ret != "" || i.Name != ifName {
+			return
+		}
+		ret = prefix.IP().String()
+	})
+	if ret != "" {
+		return ret
+	}
+	if err != nil {
+		t.Fatal(err)
+	}
+	t.Fatal("can't find a bindhost")
+	return "unreachable"
+}
+
+func TestDeriveBindhost(t *testing.T) {
+	t.Log(deriveBindhost(t))
+}
+
+type nopWriteCloser struct {
+	io.Writer
+}
+
+func (nwc nopWriteCloser) Close() error { return nil }