tailcfg: ping request notes WIP

The earlier eb06ec172f fixed
the flaky SSH issue (tailscale/corp#1725) by making sure that packets
addressed to Tailscale IPs in hybrid netstack mode weren't delivered
to netstack, but another issue remained:

All traffic handled by netstack was also potentially being handled by
the host networking stack, as the filter hook returned "Accept", which
made it keep processing. This could lead to various random racey chaos
as a function of OS/firewalls/routes/etc.

Instead, once we inject into netstack, stop our caller's packet
processing.

Signed-off-by: Brad Fitzpatrick <bradfitz@tailscale.com>

.github/workflows/coverage.yml

@@ -1,48 +0,0 @@
-name: Code Coverage
-
-on:
-  push:
-    branches:
-      - main
-  pull_request:
-    branches:
-      - '*'
-
-jobs:
-  build:
-    runs-on: ubuntu-latest
-
-    if: "!contains(github.event.head_commit.message, '[ci skip]')"
-
-    steps:
-
-    - name: Set up Go
-      uses: actions/setup-go@v1
-      with:
-        go-version: 1.16
-      id: go
-
-    - name: Check out code into the Go module directory
-      uses: actions/checkout@v1
-
-      # https://markphelps.me/2019/11/speed-up-your-go-builds-with-actions-cache/
-    - name: Restore Cache
-      uses: actions/cache@preview
-      id: cache
-      with:
-        path: ~/go/pkg/mod
-        key: ${{ runner.os }}-${{ hashFiles('**/go.sum') }}
-
-    - name: Basic build
-      run: go build ./cmd/...
-
-    - name: Run tests on linux with coverage data
-      run: go test -race -coverprofile=coverage.txt -bench=. -benchtime=1x ./...
-
-    - name: coveralls.io
-      uses: shogo82148/actions-goveralls@v1
-      env:
-        COVERALLS_TOKEN: ${{ secrets.COVERALLS_TOKEN }}
-        GITHUB_TOKEN: ${{ secrets.COVERALLS_BOT_PUBLIC_REPO_TOKEN }}
-      with:
-        path-to-profile: ./coverage.txt

.github/workflows/linux-race.yml

@@ -28,8 +28,8 @@ jobs:
     - name: Basic build
       run: go build ./cmd/...
 
-    - name: Run tests with -race flag on linux
-      run: go test -race ./...
+    - name: Run tests and benchmarks with -race flag on linux
+      run: go test -race -bench=. -benchtime=1x ./...
 
     - uses: k0kubun/action-slack@v2.0.0
       with:

.github/workflows/linux.yml

@@ -29,7 +29,7 @@ jobs:
       run: go build ./cmd/...
 
     - name: Run tests on linux
-      run: go test ./...
+      run: go test -bench=. -benchtime=1x ./...
 
     - uses: k0kubun/action-slack@v2.0.0
       with:

.github/workflows/linux32.yml

@@ -29,7 +29,7 @@ jobs:
       run: GOARCH=386 go build ./cmd/...
 
     - name: Run tests on linux
-      run: GOARCH=386 go test ./...
+      run: GOARCH=386 go test -bench=. -benchtime=1x ./...
 
     - uses: k0kubun/action-slack@v2.0.0
       with:

.github/workflows/staticcheck.yml

@@ -24,11 +24,23 @@ jobs:
     - name: Run go vet
       run: go vet ./...
 
-    - name: Print staticcheck version
-      run: go run honnef.co/go/tools/cmd/staticcheck -version
+    - name: Install staticcheck
+      run: "GOBIN=~/.local/bin go install honnef.co/go/tools/cmd/staticcheck"
 
-    - name: Run staticcheck
-      run: "go run honnef.co/go/tools/cmd/staticcheck -- $(go list ./... | grep -v tempfork)"
+    - name: Print staticcheck version
+      run: "staticcheck -version"
+
+    - name: Run staticcheck (linux/amd64)
+      run: "GOOS=linux GOARCH=amd64 staticcheck -- $(go list ./... | grep -v tempfork)"
+
+    - name: Run staticcheck (darwin/amd64)
+      run: "GOOS=darwin GOARCH=amd64 staticcheck -- $(go list ./... | grep -v tempfork)"
+
+    - name: Run staticcheck (windows/amd64)
+      run: "GOOS=windows GOARCH=amd64 staticcheck -- $(go list ./... | grep -v tempfork)"
+
+    - name: Run staticcheck (windows/386)
+      run: "GOOS=windows GOARCH=386 staticcheck -- $(go list ./... | grep -v tempfork)"
 
     - uses: k0kubun/action-slack@v2.0.0
       with:

.github/workflows/windows-race.yml

@@ -33,7 +33,10 @@ jobs:
           ${{ runner.os }}-go-
 
     - name: Test with -race flag
-      run: go test -race ./...
+      # Don't use -bench=. -benchtime=1x.
+      # Somewhere in the layers (powershell?)
+      # the equals signs cause great confusion.
+      run: go test -race -bench . -benchtime 1x ./...
 
     - uses: k0kubun/action-slack@v2.0.0
       with:

.github/workflows/windows.yml

@@ -33,7 +33,10 @@ jobs:
           ${{ runner.os }}-go-
 
     - name: Test
-      run: go test ./...
+      # Don't use -bench=. -benchtime=1x.
+      # Somewhere in the layers (powershell?)
+      # the equals signs cause great confusion.
+      run: go test -bench . -benchtime 1x ./...
 
     - uses: k0kubun/action-slack@v2.0.0
       with:

cmd/tailscale/cli/cli.go

@@ -8,8 +8,10 @@ package cli
 
 import (
 	"context"
+	"errors"
 	"flag"
 	"fmt"
+	"io"
 	"log"
 	"net"
 	"os"
@@ -49,6 +51,14 @@ func ActLikeCLI() bool {
 		return false
 	}
 
+	// Xcode adds the -NSDocumentRevisionsDebugMode flag on execution.
+	// If present, we are almost certainly being run as a GUI.
+	for _, arg := range os.Args {
+		if arg == "-NSDocumentRevisionsDebugMode" {
+			return false
+		}
+	}
+
 	// Looking at the environment of the GUI Tailscale app (ps eww
 	// $PID), empirically none of these environment variables are
 	// present. But all or some of these should be present with
@@ -95,7 +105,7 @@ change in the future.
 			pingCmd,
 			versionCmd,
 			webCmd,
-			pushCmd,
+			fileCmd,
 			bugReportCmd,
 		},
 		FlagSet:   rootfs,
@@ -169,21 +179,22 @@ func connect(ctx context.Context) (net.Conn, *ipn.BackendClient, context.Context
 }
 
 // pump receives backend messages on conn and pushes them into bc.
-func pump(ctx context.Context, bc *ipn.BackendClient, conn net.Conn) {
+func pump(ctx context.Context, bc *ipn.BackendClient, conn net.Conn) error {
 	defer conn.Close()
 	for ctx.Err() == nil {
 		msg, err := ipn.ReadMsg(conn)
 		if err != nil {
 			if ctx.Err() != nil {
-				return
+				return ctx.Err()
 			}
-			if !gotSignal.Get() {
-				log.Printf("ReadMsg: %v\n", err)
+			if errors.Is(err, io.EOF) || errors.Is(err, net.ErrClosed) {
+				return fmt.Errorf("%w (tailscaled stopped running?)", err)
 			}
-			break
+			return err
 		}
 		bc.GotNotifyMsg(msg)
 	}
+	return ctx.Err()
 }
 
 func strSliceContains(ss []string, s string) bool {

cmd/tailscale/cli/cli_test.go

@@ -9,6 +9,7 @@ import (
 	"encoding/json"
 	"flag"
 	"fmt"
+	"reflect"
 	"strings"
 	"testing"
 
@@ -40,6 +41,8 @@ func TestCheckForAccidentalSettingReverts(t *testing.T) {
 		name     string
 		flagSet  map[string]bool
 		curPrefs *ipn.Prefs
+		curUser  string // os.Getenv("USER") on the client side
+		goos     string // empty means "linux"
 		mp       *ipn.MaskedPrefs
 		want     string
 	}{
@@ -78,7 +81,7 @@ func TestCheckForAccidentalSettingReverts(t *testing.T) {
 				WantRunningSet: true,
 				CorpDNSSet:     true,
 			},
-			want: `'tailscale up' without --reset requires all preferences with changing values to be explicitly mentioned; --hostname is not specified but its default value of "" differs from current value "foo"`,
+			want: accidentalUpPrefix + " --accept-dns --hostname=foo",
 		},
 		{
 			name:    "hostname_changing_explicitly",
@@ -143,32 +146,352 @@ func TestCheckForAccidentalSettingReverts(t *testing.T) {
 			flagSet:  f("authkey"),
 			curPrefs: ipn.NewPrefs(),
 			mp: &ipn.MaskedPrefs{
-				Prefs:          *defaultPrefsFromUpArgs(t),
+				Prefs:          *defaultPrefsFromUpArgs(t, "linux"),
 				WantRunningSet: true,
 			},
 			want: "",
 		},
+		{
+			name:    "implicit_operator_change",
+			flagSet: f("hostname"),
+			curPrefs: &ipn.Prefs{
+				ControlURL:   ipn.DefaultControlURL,
+				OperatorUser: "alice",
+			},
+			curUser: "eve",
+			mp: &ipn.MaskedPrefs{
+				Prefs: ipn.Prefs{
+					ControlURL: ipn.DefaultControlURL,
+				},
+				ControlURLSet: true,
+			},
+			want: accidentalUpPrefix + " --hostname= --operator=alice",
+		},
+		{
+			name:    "implicit_operator_matches_shell_user",
+			flagSet: f("hostname"),
+			curPrefs: &ipn.Prefs{
+				ControlURL:   ipn.DefaultControlURL,
+				OperatorUser: "alice",
+			},
+			curUser: "alice",
+			mp: &ipn.MaskedPrefs{
+				Prefs: ipn.Prefs{
+					ControlURL: ipn.DefaultControlURL,
+				},
+				ControlURLSet: true,
+			},
+			want: "",
+		},
+		{
+			name:    "error_advertised_routes_exit_node_removed",
+			flagSet: f("advertise-routes"),
+			curPrefs: &ipn.Prefs{
+				ControlURL: ipn.DefaultControlURL,
+				AdvertiseRoutes: []netaddr.IPPrefix{
+					netaddr.MustParseIPPrefix("10.0.42.0/24"),
+					netaddr.MustParseIPPrefix("0.0.0.0/0"),
+					netaddr.MustParseIPPrefix("::/0"),
+				},
+			},
+			mp: &ipn.MaskedPrefs{
+				Prefs: ipn.Prefs{
+					ControlURL: ipn.DefaultControlURL,
+					AdvertiseRoutes: []netaddr.IPPrefix{
+						netaddr.MustParseIPPrefix("10.0.42.0/24"),
+					},
+				},
+				AdvertiseRoutesSet: true,
+			},
+			want: accidentalUpPrefix + " --advertise-routes=10.0.42.0/24 --advertise-exit-node",
+		},
+		{
+			name:    "advertised_routes_exit_node_removed",
+			flagSet: f("advertise-routes", "advertise-exit-node"),
+			curPrefs: &ipn.Prefs{
+				ControlURL: ipn.DefaultControlURL,
+				AdvertiseRoutes: []netaddr.IPPrefix{
+					netaddr.MustParseIPPrefix("10.0.42.0/24"),
+					netaddr.MustParseIPPrefix("0.0.0.0/0"),
+					netaddr.MustParseIPPrefix("::/0"),
+				},
+			},
+			mp: &ipn.MaskedPrefs{
+				Prefs: ipn.Prefs{
+					ControlURL: ipn.DefaultControlURL,
+					AdvertiseRoutes: []netaddr.IPPrefix{
+						netaddr.MustParseIPPrefix("10.0.42.0/24"),
+					},
+				},
+				AdvertiseRoutesSet: true,
+			},
+			want: "",
+		},
+		{
+			name:    "advertised_routes_includes_the_0_routes", // but no --advertise-exit-node
+			flagSet: f("advertise-routes"),
+			curPrefs: &ipn.Prefs{
+				ControlURL: ipn.DefaultControlURL,
+				AdvertiseRoutes: []netaddr.IPPrefix{
+					netaddr.MustParseIPPrefix("10.0.42.0/24"),
+					netaddr.MustParseIPPrefix("0.0.0.0/0"),
+					netaddr.MustParseIPPrefix("::/0"),
+				},
+			},
+			mp: &ipn.MaskedPrefs{
+				Prefs: ipn.Prefs{
+					ControlURL: ipn.DefaultControlURL,
+					AdvertiseRoutes: []netaddr.IPPrefix{
+						netaddr.MustParseIPPrefix("11.1.43.0/24"),
+						netaddr.MustParseIPPrefix("0.0.0.0/0"),
+						netaddr.MustParseIPPrefix("::/0"),
+					},
+				},
+				AdvertiseRoutesSet: true,
+			},
+			want: "",
+		},
+		{
+			name:    "advertised_routes_includes_only_one_0_route", // and no --advertise-exit-node
+			flagSet: f("advertise-routes"),
+			curPrefs: &ipn.Prefs{
+				ControlURL: ipn.DefaultControlURL,
+				AdvertiseRoutes: []netaddr.IPPrefix{
+					netaddr.MustParseIPPrefix("10.0.42.0/24"),
+					netaddr.MustParseIPPrefix("0.0.0.0/0"),
+					netaddr.MustParseIPPrefix("::/0"),
+				},
+			},
+			mp: &ipn.MaskedPrefs{
+				Prefs: ipn.Prefs{
+					ControlURL: ipn.DefaultControlURL,
+					AdvertiseRoutes: []netaddr.IPPrefix{
+						netaddr.MustParseIPPrefix("11.1.43.0/24"),
+						netaddr.MustParseIPPrefix("0.0.0.0/0"),
+					},
+				},
+				AdvertiseRoutesSet: true,
+			},
+			want: accidentalUpPrefix + " --advertise-routes=11.1.43.0/24,0.0.0.0/0 --advertise-exit-node",
+		},
+		{
+			name:    "advertise_exit_node", // Issue 1859
+			flagSet: f("advertise-exit-node"),
+			curPrefs: &ipn.Prefs{
+				ControlURL: ipn.DefaultControlURL,
+			},
+			mp: &ipn.MaskedPrefs{
+				Prefs: ipn.Prefs{
+					ControlURL: ipn.DefaultControlURL,
+					AdvertiseRoutes: []netaddr.IPPrefix{
+						netaddr.MustParseIPPrefix("0.0.0.0/0"),
+						netaddr.MustParseIPPrefix("::/0"),
+					},
+				},
+				// Note: without setting "AdvertiseRoutesSet", as
+				// updateMaskedPrefsFromUpFlag doesn't set that.
+			},
+			want: "",
+		},
+		{
+			name:    "advertise_exit_node_over_existing_routes",
+			flagSet: f("advertise-exit-node"),
+			curPrefs: &ipn.Prefs{
+				ControlURL: ipn.DefaultControlURL,
+				AdvertiseRoutes: []netaddr.IPPrefix{
+					netaddr.MustParseIPPrefix("1.2.0.0/16"),
+				},
+			},
+			mp: &ipn.MaskedPrefs{
+				Prefs: ipn.Prefs{
+					ControlURL: ipn.DefaultControlURL,
+					AdvertiseRoutes: []netaddr.IPPrefix{
+						netaddr.MustParseIPPrefix("0.0.0.0/0"),
+						netaddr.MustParseIPPrefix("::/0"),
+					},
+				},
+				// Note: without setting "AdvertiseRoutesSet", as
+				// updateMaskedPrefsFromUpFlag doesn't set that.
+			},
+			want: accidentalUpPrefix + " --advertise-exit-node --advertise-routes=1.2.0.0/16",
+		},
+		{
+			name:    "advertise_exit_node_over_existing_routes_and_exit_node",
+			flagSet: f("advertise-exit-node"),
+			curPrefs: &ipn.Prefs{
+				ControlURL: ipn.DefaultControlURL,
+				AdvertiseRoutes: []netaddr.IPPrefix{
+					netaddr.MustParseIPPrefix("0.0.0.0/0"),
+					netaddr.MustParseIPPrefix("::/0"),
+					netaddr.MustParseIPPrefix("1.2.0.0/16"),
+				},
+			},
+			mp: &ipn.MaskedPrefs{
+				Prefs: ipn.Prefs{
+					ControlURL: ipn.DefaultControlURL,
+					AdvertiseRoutes: []netaddr.IPPrefix{
+						netaddr.MustParseIPPrefix("0.0.0.0/0"),
+						netaddr.MustParseIPPrefix("::/0"),
+					},
+				},
+				// Note: without setting "AdvertiseRoutesSet", as
+				// updateMaskedPrefsFromUpFlag doesn't set that.
+			},
+			want: accidentalUpPrefix + " --advertise-exit-node --advertise-routes=1.2.0.0/16",
+		},
+		{
+			name:    "exit_node_clearing", // Issue 1777
+			flagSet: f("exit-node"),
+			curPrefs: &ipn.Prefs{
+				ControlURL: ipn.DefaultControlURL,
+				ExitNodeID: "fooID",
+			},
+			mp: &ipn.MaskedPrefs{
+				Prefs: ipn.Prefs{
+					ControlURL: ipn.DefaultControlURL,
+					ExitNodeIP: netaddr.IP{},
+				},
+				ExitNodeIPSet: true,
+			},
+			want: "",
+		},
+		{
+			name:    "remove_all_implicit",
+			flagSet: f("force-reauth"),
+			curPrefs: &ipn.Prefs{
+				WantRunning:      true,
+				ControlURL:       ipn.DefaultControlURL,
+				RouteAll:         true,
+				AllowSingleHosts: false,
+				ExitNodeIP:       netaddr.MustParseIP("100.64.5.6"),
+				CorpDNS:          true,
+				ShieldsUp:        true,
+				AdvertiseTags:    []string{"tag:foo", "tag:bar"},
+				Hostname:         "myhostname",
+				ForceDaemon:      true,
+				AdvertiseRoutes: []netaddr.IPPrefix{
+					netaddr.MustParseIPPrefix("10.0.0.0/16"),
+				},
+				NetfilterMode: preftype.NetfilterNoDivert,
+				OperatorUser:  "alice",
+			},
+			curUser: "eve",
+			mp: &ipn.MaskedPrefs{
+				Prefs: ipn.Prefs{
+					ControlURL:  ipn.DefaultControlURL,
+					WantRunning: true,
+				},
+			},
+			want: accidentalUpPrefix + " --force-reauth --accept-routes --exit-node=100.64.5.6 --accept-dns --shields-up --advertise-tags=tag:foo,tag:bar --hostname=myhostname --unattended --advertise-routes=10.0.0.0/16 --netfilter-mode=nodivert --operator=alice",
+		},
+		{
+			name:    "remove_all_implicit_except_hostname",
+			flagSet: f("hostname"),
+			curPrefs: &ipn.Prefs{
+				WantRunning:      true,
+				ControlURL:       ipn.DefaultControlURL,
+				RouteAll:         true,
+				AllowSingleHosts: false,
+				ExitNodeIP:       netaddr.MustParseIP("100.64.5.6"),
+				CorpDNS:          true,
+				ShieldsUp:        true,
+				AdvertiseTags:    []string{"tag:foo", "tag:bar"},
+				Hostname:         "myhostname",
+				ForceDaemon:      true,
+				AdvertiseRoutes: []netaddr.IPPrefix{
+					netaddr.MustParseIPPrefix("10.0.0.0/16"),
+				},
+				NetfilterMode: preftype.NetfilterNoDivert,
+				OperatorUser:  "alice",
+			},
+			curUser: "eve",
+			mp: &ipn.MaskedPrefs{
+				Prefs: ipn.Prefs{
+					ControlURL:  ipn.DefaultControlURL,
+					WantRunning: true,
+					Hostname:    "newhostname",
+				},
+				HostnameSet: true,
+			},
+			want: accidentalUpPrefix + " --hostname=newhostname --accept-routes --exit-node=100.64.5.6 --accept-dns --shields-up --advertise-tags=tag:foo,tag:bar --unattended --advertise-routes=10.0.0.0/16 --netfilter-mode=nodivert --operator=alice",
+		},
+		{
+			name:    "loggedout_is_implicit",
+			flagSet: f("advertise-exit-node"),
+			curPrefs: &ipn.Prefs{
+				ControlURL: ipn.DefaultControlURL,
+				LoggedOut:  true,
+			},
+			mp: &ipn.MaskedPrefs{
+				Prefs: ipn.Prefs{
+					ControlURL: ipn.DefaultControlURL,
+					AdvertiseRoutes: []netaddr.IPPrefix{
+						netaddr.MustParseIPPrefix("0.0.0.0/0"),
+					},
+				},
+				AdvertiseRoutesSet: true,
+			},
+			// not an error. LoggedOut is implicit.
+			want: "",
+		},
+		{
+			// Test that a pre-1.8 version of Tailscale with bogus NoSNAT pref
+			// values is able to enable exit nodes without warnings.
+			name:    "make_windows_exit_node",
+			flagSet: f("advertise-exit-node"),
+			curPrefs: &ipn.Prefs{
+				ControlURL: ipn.DefaultControlURL,
+				NoSNAT:     true, // assume this no-op accidental pre-1.8 value
+			},
+			goos: "windows",
+			mp: &ipn.MaskedPrefs{
+				Prefs: ipn.Prefs{
+					ControlURL: ipn.DefaultControlURL,
+					AdvertiseRoutes: []netaddr.IPPrefix{
+						netaddr.MustParseIPPrefix("192.168.0.0/16"),
+					},
+				},
+				AdvertiseRoutesSet: true,
+			},
+			want: "", // not an error
+		},
+		{
+			name:    "ignore_netfilter_change_non_linux",
+			flagSet: f("accept-dns"),
+			curPrefs: &ipn.Prefs{
+				ControlURL:    ipn.DefaultControlURL,
+				NetfilterMode: preftype.NetfilterNoDivert, // we never had this bug, but pretend it got set non-zero on Windows somehow
+			},
+			goos: "windows",
+			mp: &ipn.MaskedPrefs{
+				Prefs: ipn.Prefs{
+					ControlURL: ipn.DefaultControlURL,
+					CorpDNS:    false,
+				},
+				CorpDNSSet: true,
+			},
+			want: "", // not an error
+		},
 	}
 	for _, tt := range tests {
 		t.Run(tt.name, func(t *testing.T) {
+			goos := "linux"
+			if tt.goos != "" {
+				goos = tt.goos
+			}
 			var got string
-			if err := checkForAccidentalSettingReverts(tt.flagSet, tt.curPrefs, tt.mp); err != nil {
+			if err := checkForAccidentalSettingReverts(tt.flagSet, tt.curPrefs, tt.mp, goos, tt.curUser); err != nil {
 				got = err.Error()
 			}
-			if got != tt.want {
+			if strings.TrimSpace(got) != tt.want {
 				t.Errorf("unexpected result\n got: %s\nwant: %s\n", got, tt.want)
 			}
 		})
 	}
 }
 
-func defaultPrefsFromUpArgs(t testing.TB) *ipn.Prefs {
-	upFlagSet.Parse(nil) // populates upArgs
-	if upFlagSet.Lookup("netfilter-mode") == nil && upArgs.netfilterMode == "" {
-		// This flag is not compiled on on-Linux platforms,
-		// but prefsFromUpArgs requires it be populated.
-		upArgs.netfilterMode = defaultNetfilterMode()
-	}
+func defaultPrefsFromUpArgs(t testing.TB, goos string) *ipn.Prefs {
+	upArgs := upArgsFromOSArgs(goos)
 	prefs, err := prefsFromUpArgs(upArgs, logger.Discard, new(ipnstate.Status), "linux")
 	if err != nil {
 		t.Fatalf("defaultPrefsFromUpArgs: %v", err)
@@ -177,6 +500,12 @@ func defaultPrefsFromUpArgs(t testing.TB) *ipn.Prefs {
 	return prefs
 }
 
+func upArgsFromOSArgs(goos string, flagArgs ...string) (args upArgsT) {
+	fs := newUpFlagSet(goos, &args)
+	fs.Parse(flagArgs) // populates args
+	return
+}
+
 func TestPrefsFromUpArgs(t *testing.T) {
 	tests := []struct {
 		name     string
@@ -188,13 +517,43 @@ func TestPrefsFromUpArgs(t *testing.T) {
 		wantWarn string
 	}{
 		{
-			name: "zero",
-			goos: "windows",
-			args: upArgsT{},
+			name: "default_linux",
+			goos: "linux",
+			args: upArgsFromOSArgs("linux"),
 			want: &ipn.Prefs{
-				WantRunning:   true,
-				NoSNAT:        true,
-				NetfilterMode: preftype.NetfilterOn, // silly, but default from ipn.NewPref currently
+				ControlURL:       ipn.DefaultControlURL,
+				WantRunning:      true,
+				NoSNAT:           false,
+				NetfilterMode:    preftype.NetfilterOn,
+				CorpDNS:          true,
+				AllowSingleHosts: true,
+			},
+		},
+		{
+			name: "default_windows",
+			goos: "windows",
+			args: upArgsFromOSArgs("windows"),
+			want: &ipn.Prefs{
+				ControlURL:       ipn.DefaultControlURL,
+				WantRunning:      true,
+				CorpDNS:          true,
+				AllowSingleHosts: true,
+				NetfilterMode:    preftype.NetfilterOn,
+			},
+		},
+		{
+			name: "advertise_default_route",
+			args: upArgsFromOSArgs("linux", "--advertise-exit-node"),
+			want: &ipn.Prefs{
+				ControlURL:       ipn.DefaultControlURL,
+				WantRunning:      true,
+				AllowSingleHosts: true,
+				CorpDNS:          true,
+				AdvertiseRoutes: []netaddr.IPPrefix{
+					netaddr.MustParseIPPrefix("0.0.0.0/0"),
+					netaddr.MustParseIPPrefix("::/0"),
+				},
+				NetfilterMode: preftype.NetfilterOn,
 			},
 		},
 		{
@@ -334,3 +693,26 @@ func TestPrefsFromUpArgs(t *testing.T) {
 	}
 
 }
+
+func TestPrefFlagMapping(t *testing.T) {
+	prefType := reflect.TypeOf(ipn.Prefs{})
+	for i := 0; i < prefType.NumField(); i++ {
+		prefName := prefType.Field(i).Name
+		if _, ok := flagForPref[prefName]; ok {
+			continue
+		}
+		switch prefName {
+		case "WantRunning", "Persist", "LoggedOut":
+			// All explicitly handled (ignored) by checkForAccidentalSettingReverts.
+			continue
+		case "OSVersion", "DeviceModel":
+			// Only used by Android, which doesn't have a CLI mode anyway, so
+			// fine to not map.
+			continue
+		case "NotepadURLs":
+			// TODO(bradfitz): https://github.com/tailscale/tailscale/issues/1830
+			continue
+		}
+		t.Errorf("unexpected new ipn.Pref field %q is not handled by up.go (see addPrefFlagMapping and checkForAccidentalSettingReverts)", prefName)
+	}
+}

cmd/tailscale/cli/file.go

@@ -0,0 +1,444 @@
+// Copyright (c) 2021 Tailscale Inc & AUTHORS All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package cli
+
+import (
+	"bytes"
+	"context"
+	"errors"
+	"flag"
+	"fmt"
+	"io"
+	"io/ioutil"
+	"log"
+	"mime"
+	"net/http"
+	"net/url"
+	"os"
+	"path/filepath"
+	"strconv"
+	"strings"
+	"time"
+	"unicode/utf8"
+
+	"github.com/peterbourgon/ff/v2/ffcli"
+	"golang.org/x/time/rate"
+	"inet.af/netaddr"
+	"tailscale.com/client/tailscale"
+	"tailscale.com/client/tailscale/apitype"
+	"tailscale.com/ipn"
+	"tailscale.com/net/tsaddr"
+	"tailscale.com/version"
+)
+
+var fileCmd = &ffcli.Command{
+	Name:       "file",
+	ShortUsage: "file <cp|get> ...",
+	ShortHelp:  "Send or receive files",
+	Subcommands: []*ffcli.Command{
+		fileCpCmd,
+		fileGetCmd,
+	},
+	Exec: func(context.Context, []string) error {
+		// TODO(bradfitz): is there a better ffcli way to
+		// annotate subcommand-required commands that don't
+		// have an exec body of their own?
+		return errors.New("file subcommand required; run 'tailscale file -h' for details")
+	},
+}
+
+var fileCpCmd = &ffcli.Command{
+	Name:       "cp",
+	ShortUsage: "file cp <files...> <target>:",
+	ShortHelp:  "Copy file(s) to a host",
+	Exec:       runCp,
+	FlagSet: (func() *flag.FlagSet {
+		fs := flag.NewFlagSet("cp", flag.ExitOnError)
+		fs.StringVar(&cpArgs.name, "name", "", "alternate filename to use, especially useful when <file> is \"-\" (stdin)")
+		fs.BoolVar(&cpArgs.verbose, "verbose", false, "verbose output")
+		fs.BoolVar(&cpArgs.targets, "targets", false, "list possible file cp targets")
+		return fs
+	})(),
+}
+
+var cpArgs struct {
+	name    string
+	verbose bool
+	targets bool
+}
+
+func runCp(ctx context.Context, args []string) error {
+	if cpArgs.targets {
+		return runCpTargets(ctx, args)
+	}
+	if len(args) < 2 {
+		//lint:ignore ST1005 no sorry need that colon at the end
+		return errors.New("usage: tailscale file cp <files...> <target>:")
+	}
+	files, target := args[:len(args)-1], args[len(args)-1]
+	if !strings.HasSuffix(target, ":") {
+		return fmt.Errorf("final argument to 'tailscale file cp' must end in colon")
+	}
+	target = strings.TrimSuffix(target, ":")
+	hadBrackets := false
+	if strings.HasPrefix(target, "[") && strings.HasSuffix(target, "]") {
+		hadBrackets = true
+		target = strings.TrimSuffix(strings.TrimPrefix(target, "["), "]")
+	}
+	if ip, err := netaddr.ParseIP(target); err == nil && ip.Is6() && !hadBrackets {
+		return fmt.Errorf("an IPv6 literal must be written as [%s]", ip)
+	} else if hadBrackets && (err != nil || !ip.Is6()) {
+		return errors.New("unexpected brackets around target")
+	}
+	ip, err := tailscaleIPFromArg(ctx, target)
+	if err != nil {
+		return err
+	}
+
+	peerAPIBase, lastSeen, isOffline, err := discoverPeerAPIBase(ctx, ip)
+	if err != nil {
+		return fmt.Errorf("can't send to %s: %v", target, err)
+	}
+	if isOffline {
+		fmt.Fprintf(os.Stderr, "# warning: %s is offline\n", target)
+	} else if !lastSeen.IsZero() && time.Since(lastSeen) > lastSeenOld {
+		fmt.Fprintf(os.Stderr, "# warning: %s last seen %v ago\n", target, time.Since(lastSeen).Round(time.Minute))
+	}
+
+	if len(files) > 1 {
+		if cpArgs.name != "" {
+			return errors.New("can't use --name= with multiple files")
+		}
+		for _, fileArg := range files {
+			if fileArg == "-" {
+				return errors.New("can't use '-' as STDIN file when providing filename arguments")
+			}
+		}
+	}
+
+	for _, fileArg := range files {
+		var fileContents io.Reader
+		var name = cpArgs.name
+		var contentLength int64 = -1
+		if fileArg == "-" {
+			fileContents = os.Stdin
+			if name == "" {
+				name, fileContents, err = pickStdinFilename()
+				if err != nil {
+					return err
+				}
+			}
+		} else {
+			f, err := os.Open(fileArg)
+			if err != nil {
+				if version.IsSandboxedMacOS() {
+					return errors.New("the GUI version of Tailscale on macOS runs in a macOS sandbox that can't read files")
+				}
+				return err
+			}
+			defer f.Close()
+			fi, err := f.Stat()
+			if err != nil {
+				return err
+			}
+			if fi.IsDir() {
+				return errors.New("directories not supported")
+			}
+			contentLength = fi.Size()
+			fileContents = io.LimitReader(f, contentLength)
+			if name == "" {
+				name = filepath.Base(fileArg)
+			}
+
+			if slow, _ := strconv.ParseBool(os.Getenv("TS_DEBUG_SLOW_PUSH")); slow {
+				fileContents = &slowReader{r: fileContents}
+			}
+		}
+
+		dstURL := peerAPIBase + "/v0/put/" + url.PathEscape(name)
+		req, err := http.NewRequestWithContext(ctx, "PUT", dstURL, fileContents)
+		if err != nil {
+			return err
+		}
+		req.ContentLength = contentLength
+		if cpArgs.verbose {
+			log.Printf("sending to %v ...", dstURL)
+		}
+		res, err := http.DefaultClient.Do(req)
+		if err != nil {
+			return err
+		}
+		if res.StatusCode == 200 {
+			io.Copy(ioutil.Discard, res.Body)
+			res.Body.Close()
+			continue
+		}
+		io.Copy(os.Stdout, res.Body)
+		res.Body.Close()
+		return errors.New(res.Status)
+	}
+	return nil
+}
+
+func discoverPeerAPIBase(ctx context.Context, ipStr string) (base string, lastSeen time.Time, isOffline bool, err error) {
+	ip, err := netaddr.ParseIP(ipStr)
+	if err != nil {
+		return "", time.Time{}, false, err
+	}
+	fts, err := tailscale.FileTargets(ctx)
+	if err != nil {
+		return "", time.Time{}, false, err
+	}
+	for _, ft := range fts {
+		n := ft.Node
+		for _, a := range n.Addresses {
+			if a.IP != ip {
+				continue
+			}
+			if n.LastSeen != nil {
+				lastSeen = *n.LastSeen
+			}
+			isOffline = n.Online != nil && !*n.Online
+			return ft.PeerAPIURL, lastSeen, isOffline, nil
+		}
+	}
+	return "", time.Time{}, false, fileTargetErrorDetail(ctx, ip)
+}
+
+// fileTargetErrorDetail returns a non-nil error saying why ip is an
+// invalid file sharing target.
+func fileTargetErrorDetail(ctx context.Context, ip netaddr.IP) error {
+	found := false
+	if st, err := tailscale.Status(ctx); err == nil && st.Self != nil {
+		for _, peer := range st.Peer {
+			for _, pip := range peer.TailscaleIPs {
+				if pip == ip {
+					found = true
+					if peer.UserID != st.Self.UserID {
+						return errors.New("owned by different user; can only send files to your own devices")
+					}
+				}
+			}
+		}
+	}
+	if found {
+		return errors.New("target seems to be running an old Tailscale version")
+	}
+	if !tsaddr.IsTailscaleIP(ip) {
+		return fmt.Errorf("unknown target; %v is not a Tailscale IP address", ip)
+	}
+	return errors.New("unknown target; not in your Tailnet")
+}
+
+const maxSniff = 4 << 20
+
+func ext(b []byte) string {
+	if len(b) < maxSniff && utf8.Valid(b) {
+		return ".txt"
+	}
+	if exts, _ := mime.ExtensionsByType(http.DetectContentType(b)); len(exts) > 0 {
+		return exts[0]
+	}
+	return ""
+}
+
+// pickStdinFilename reads a bit of stdin to return a good filename
+// for its contents. The returned Reader is the concatenation of the
+// read and unread bits.
+func pickStdinFilename() (name string, r io.Reader, err error) {
+	sniff, err := io.ReadAll(io.LimitReader(os.Stdin, maxSniff))
+	if err != nil {
+		return "", nil, err
+	}
+	return "stdin" + ext(sniff), io.MultiReader(bytes.NewReader(sniff), os.Stdin), nil
+}
+
+type slowReader struct {
+	r  io.Reader
+	rl *rate.Limiter
+}
+
+func (r *slowReader) Read(p []byte) (n int, err error) {
+	const burst = 4 << 10
+	plen := len(p)
+	if plen > burst {
+		plen = burst
+	}
+	if r.rl == nil {
+		r.rl = rate.NewLimiter(rate.Limit(1<<10), burst)
+	}
+	n, err = r.r.Read(p[:plen])
+	r.rl.WaitN(context.Background(), n)
+	return
+}
+
+const lastSeenOld = 20 * time.Minute
+
+func runCpTargets(ctx context.Context, args []string) error {
+	if len(args) > 0 {
+		return errors.New("invalid arguments with --targets")
+	}
+	fts, err := tailscale.FileTargets(ctx)
+	if err != nil {
+		return err
+	}
+	for _, ft := range fts {
+		n := ft.Node
+		var detail string
+		if n.Online != nil {
+			if !*n.Online {
+				detail = "offline"
+			}
+		} else {
+			detail = "unknown-status"
+		}
+		if detail != "" && n.LastSeen != nil {
+			d := time.Since(*n.LastSeen)
+			detail += fmt.Sprintf("; last seen %v ago", d.Round(time.Minute))
+		}
+		if detail != "" {
+			detail = "\t" + detail
+		}
+		fmt.Printf("%s\t%s%s\n", n.Addresses[0].IP, n.ComputedName, detail)
+	}
+	return nil
+}
+
+var fileGetCmd = &ffcli.Command{
+	Name:       "get",
+	ShortUsage: "file get [--wait] [--verbose] <target-directory>",
+	ShortHelp:  "Move files out of the Tailscale file inbox",
+	Exec:       runFileGet,
+	FlagSet: (func() *flag.FlagSet {
+		fs := flag.NewFlagSet("get", flag.ExitOnError)
+		fs.BoolVar(&getArgs.wait, "wait", false, "wait for a file to arrive if inbox is empty")
+		fs.BoolVar(&getArgs.verbose, "verbose", false, "verbose output")
+		return fs
+	})(),
+}
+
+var getArgs struct {
+	wait    bool
+	verbose bool
+}
+
+func runFileGet(ctx context.Context, args []string) error {
+	if len(args) != 1 {
+		return errors.New("usage: file get <target-directory>")
+	}
+	log.SetFlags(0)
+
+	dir := args[0]
+	if dir == "/dev/null" {
+		return wipeInbox(ctx)
+	}
+
+	if fi, err := os.Stat(dir); err != nil || !fi.IsDir() {
+		return fmt.Errorf("%q is not a directory", dir)
+	}
+
+	var wfs []apitype.WaitingFile
+	var err error
+	for {
+		wfs, err = tailscale.WaitingFiles(ctx)
+		if err != nil {
+			return fmt.Errorf("getting WaitingFiles: %v", err)
+		}
+		if len(wfs) != 0 || !getArgs.wait {
+			break
+		}
+		if getArgs.verbose {
+			log.Printf("waiting for file...")
+		}
+		if err := waitForFile(ctx); err != nil {
+			return err
+		}
+	}
+
+	deleted := 0
+	for _, wf := range wfs {
+		rc, size, err := tailscale.GetWaitingFile(ctx, wf.Name)
+		if err != nil {
+			return fmt.Errorf("opening inbox file %q: %v", wf.Name, err)
+		}
+		targetFile := filepath.Join(dir, wf.Name)
+		of, err := os.OpenFile(targetFile, os.O_RDWR|os.O_CREATE|os.O_EXCL, 0644)
+		if err != nil {
+			if _, err := os.Stat(targetFile); err == nil {
+				return fmt.Errorf("refusing to overwrite %v", targetFile)
+			}
+			return err
+		}
+		_, err = io.Copy(of, rc)
+		rc.Close()
+		if err != nil {
+			return fmt.Errorf("failed to write %v: %v", targetFile, err)
+		}
+		if err := of.Close(); err != nil {
+			return err
+		}
+		if getArgs.verbose {
+			log.Printf("wrote %v (%d bytes)", wf.Name, size)
+		}
+		if err := tailscale.DeleteWaitingFile(ctx, wf.Name); err != nil {
+			return fmt.Errorf("deleting %q from inbox: %v", wf.Name, err)
+		}
+		deleted++
+	}
+	if getArgs.verbose {
+		log.Printf("moved %d files", deleted)
+	}
+	return nil
+}
+
+func wipeInbox(ctx context.Context) error {
+	if getArgs.wait {
+		return errors.New("can't use --wait with /dev/null target")
+	}
+	wfs, err := tailscale.WaitingFiles(ctx)
+	if err != nil {
+		return fmt.Errorf("getting WaitingFiles: %v", err)
+	}
+	deleted := 0
+	for _, wf := range wfs {
+		if getArgs.verbose {
+			log.Printf("deleting %v ...", wf.Name)
+		}
+		if err := tailscale.DeleteWaitingFile(ctx, wf.Name); err != nil {
+			return fmt.Errorf("deleting %q: %v", wf.Name, err)
+		}
+		deleted++
+	}
+	if getArgs.verbose {
+		log.Printf("deleted %d files", deleted)
+	}
+	return nil
+}
+
+func waitForFile(ctx context.Context) error {
+	c, bc, pumpCtx, cancel := connect(ctx)
+	defer cancel()
+	fileWaiting := make(chan bool, 1)
+	bc.SetNotifyCallback(func(n ipn.Notify) {
+		if n.ErrMessage != nil {
+			log.Fatal(*n.ErrMessage)
+		}
+		if n.FilesWaiting != nil {
+			select {
+			case fileWaiting <- true:
+			default:
+			}
+		}
+	})
+	go pump(pumpCtx, bc, c)
+	select {
+	case <-fileWaiting:
+		return nil
+	case <-pumpCtx.Done():
+		return pumpCtx.Err()
+	case <-ctx.Done():
+		return ctx.Err()
+	}
+}

cmd/tailscale/cli/ping.go

@@ -80,7 +80,8 @@ func runPing(ctx context.Context, args []string) error {
 			prc <- pr
 		}
 	})
-	go pump(ctx, bc, c)
+	pumpErr := make(chan error, 1)
+	go func() { pumpErr <- pump(ctx, bc, c) }()
 
 	hostOrIP := args[0]
 	ip, err := tailscaleIPFromArg(ctx, hostOrIP)
@@ -101,6 +102,8 @@ func runPing(ctx context.Context, args []string) error {
 		select {
 		case <-timer.C:
 			fmt.Printf("timeout waiting for ping reply\n")
+		case err := <-pumpErr:
+			return err
 		case pr := <-prc:
 			timer.Stop()
 			if pr.Err != "" {

cmd/tailscale/cli/push.go

@@ -1,218 +0,0 @@
-// Copyright (c) 2021 Tailscale Inc & AUTHORS All rights reserved.
-// Use of this source code is governed by a BSD-style
-// license that can be found in the LICENSE file.
-
-package cli
-
-import (
-	"bytes"
-	"context"
-	"errors"
-	"flag"
-	"fmt"
-	"io"
-	"log"
-	"mime"
-	"net/http"
-	"net/url"
-	"os"
-	"strconv"
-	"time"
-	"unicode/utf8"
-
-	"github.com/peterbourgon/ff/v2/ffcli"
-	"golang.org/x/time/rate"
-	"inet.af/netaddr"
-	"tailscale.com/client/tailscale"
-)
-
-var pushCmd = &ffcli.Command{
-	Name:       "push",
-	ShortUsage: "push [--flags] <hostname-or-IP> <file>",
-	ShortHelp:  "Push a file to a host",
-	Exec:       runPush,
-	FlagSet: (func() *flag.FlagSet {
-		fs := flag.NewFlagSet("push", flag.ExitOnError)
-		fs.StringVar(&pushArgs.name, "name", "", "alternate filename to use, especially useful when <file> is \"-\" (stdin)")
-		fs.BoolVar(&pushArgs.verbose, "verbose", false, "verbose output")
-		fs.BoolVar(&pushArgs.targets, "targets", false, "list possible push targets")
-		return fs
-	})(),
-}
-
-var pushArgs struct {
-	name    string
-	verbose bool
-	targets bool
-}
-
-func runPush(ctx context.Context, args []string) error {
-	if pushArgs.targets {
-		return runPushTargets(ctx, args)
-	}
-	if len(args) != 2 || args[0] == "" {
-		return errors.New("usage: push <hostname-or-IP> <file>\n       push --targets")
-	}
-	var ip string
-
-	hostOrIP, fileArg := args[0], args[1]
-	ip, err := tailscaleIPFromArg(ctx, hostOrIP)
-	if err != nil {
-		return err
-	}
-
-	peerAPIBase, lastSeen, err := discoverPeerAPIBase(ctx, ip)
-	if err != nil {
-		return err
-	}
-
-	if !lastSeen.IsZero() && time.Since(lastSeen) > lastSeenOld {
-		fmt.Fprintf(os.Stderr, "# warning: %s last seen %v ago\n", hostOrIP, time.Since(lastSeen).Round(time.Minute))
-	}
-
-	var fileContents io.Reader
-	var name = pushArgs.name
-	var contentLength int64 = -1
-	if fileArg == "-" {
-		fileContents = os.Stdin
-		if name == "" {
-			name, fileContents, err = pickStdinFilename()
-			if err != nil {
-				return err
-			}
-		}
-	} else {
-		f, err := os.Open(fileArg)
-		if err != nil {
-			return err
-		}
-		defer f.Close()
-		fi, err := f.Stat()
-		if err != nil {
-			return err
-		}
-		if fi.IsDir() {
-			return errors.New("directories not supported")
-		}
-		contentLength = fi.Size()
-		fileContents = io.LimitReader(f, contentLength)
-		if name == "" {
-			name = fileArg
-		}
-
-		if slow, _ := strconv.ParseBool(os.Getenv("TS_DEBUG_SLOW_PUSH")); slow {
-			fileContents = &slowReader{r: fileContents}
-		}
-	}
-
-	dstURL := peerAPIBase + "/v0/put/" + url.PathEscape(name)
-	req, err := http.NewRequestWithContext(ctx, "PUT", dstURL, fileContents)
-	if err != nil {
-		return err
-	}
-	req.ContentLength = contentLength
-	if pushArgs.verbose {
-		log.Printf("sending to %v ...", dstURL)
-	}
-	res, err := http.DefaultClient.Do(req)
-	if err != nil {
-		return err
-	}
-	defer res.Body.Close()
-	if res.StatusCode == 200 {
-		return nil
-	}
-	io.Copy(os.Stdout, res.Body)
-	return errors.New(res.Status)
-}
-
-func discoverPeerAPIBase(ctx context.Context, ipStr string) (base string, lastSeen time.Time, err error) {
-	ip, err := netaddr.ParseIP(ipStr)
-	if err != nil {
-		return "", time.Time{}, err
-	}
-	fts, err := tailscale.FileTargets(ctx)
-	if err != nil {
-		return "", time.Time{}, err
-	}
-	for _, ft := range fts {
-		n := ft.Node
-		for _, a := range n.Addresses {
-			if a.IP != ip {
-				continue
-			}
-			if n.LastSeen != nil {
-				lastSeen = *n.LastSeen
-			}
-			return ft.PeerAPIURL, lastSeen, nil
-		}
-	}
-	return "", time.Time{}, errors.New("target seems to be running an old Tailscale version")
-}
-
-const maxSniff = 4 << 20
-
-func ext(b []byte) string {
-	if len(b) < maxSniff && utf8.Valid(b) {
-		return ".txt"
-	}
-	if exts, _ := mime.ExtensionsByType(http.DetectContentType(b)); len(exts) > 0 {
-		return exts[0]
-	}
-	return ""
-}
-
-// pickStdinFilename reads a bit of stdin to return a good filename
-// for its contents. The returned Reader is the concatenation of the
-// read and unread bits.
-func pickStdinFilename() (name string, r io.Reader, err error) {
-	sniff, err := io.ReadAll(io.LimitReader(os.Stdin, maxSniff))
-	if err != nil {
-		return "", nil, err
-	}
-	return "stdin" + ext(sniff), io.MultiReader(bytes.NewReader(sniff), os.Stdin), nil
-}
-
-type slowReader struct {
-	r  io.Reader
-	rl *rate.Limiter
-}
-
-func (r *slowReader) Read(p []byte) (n int, err error) {
-	const burst = 4 << 10
-	plen := len(p)
-	if plen > burst {
-		plen = burst
-	}
-	if r.rl == nil {
-		r.rl = rate.NewLimiter(rate.Limit(1<<10), burst)
-	}
-	n, err = r.r.Read(p[:plen])
-	r.rl.WaitN(context.Background(), n)
-	return
-}
-
-const lastSeenOld = 20 * time.Minute
-
-func runPushTargets(ctx context.Context, args []string) error {
-	if len(args) > 0 {
-		return errors.New("invalid arguments with --targets")
-	}
-	fts, err := tailscale.FileTargets(ctx)
-	if err != nil {
-		return err
-	}
-	for _, ft := range fts {
-		n := ft.Node
-		var ago string
-		if n.LastSeen == nil {
-			ago = "\tnode never seen"
-		} else {
-			if d := time.Since(*n.LastSeen); d > lastSeenOld {
-				ago = fmt.Sprintf("\tlast seen %v ago", d.Round(time.Minute))
-			}
-		}
-		fmt.Printf("%s\t%s%s\n", n.Addresses[0].IP, n.ComputedName, ago)
-	}
-	return nil
-}

cmd/tailscale/cli/up.go

@@ -17,7 +17,7 @@ import (
 	"strings"
 	"sync"
 
-	"github.com/go-multierror/multierror"
+	shellquote "github.com/kballard/go-shellquote"
 	"github.com/peterbourgon/ff/v2/ffcli"
 	"inet.af/netaddr"
 	"tailscale.com/client/tailscale"
@@ -52,7 +52,9 @@ flag is also used.
 	Exec:    runUp,
 }
 
-var upFlagSet = (func() *flag.FlagSet {
+var upFlagSet = newUpFlagSet(runtime.GOOS, &upArgs)
+
+func newUpFlagSet(goos string, upArgs *upArgsT) *flag.FlagSet {
 	upf := flag.NewFlagSet("up", flag.ExitOnError)
 
 	upf.BoolVar(&upArgs.forceReauth, "force-reauth", false, "force reauthentication")
@@ -70,18 +72,18 @@ var upFlagSet = (func() *flag.FlagSet {
 	upf.StringVar(&upArgs.hostname, "hostname", "", "hostname to use instead of the one provided by the OS")
 	upf.StringVar(&upArgs.advertiseRoutes, "advertise-routes", "", "routes to advertise to other nodes (comma-separated, e.g. \"10.0.0.0/8,192.168.0.0/24\")")
 	upf.BoolVar(&upArgs.advertiseDefaultRoute, "advertise-exit-node", false, "offer to be an exit node for internet traffic for the tailnet")
-	if safesocket.PlatformUsesPeerCreds() {
+	if safesocket.GOOSUsesPeerCreds(goos) {
 		upf.StringVar(&upArgs.opUser, "operator", "", "Unix username to allow to operate on tailscaled without sudo")
 	}
-	if runtime.GOOS == "linux" {
+	switch goos {
+	case "linux":
 		upf.BoolVar(&upArgs.snat, "snat-subnet-routes", true, "source NAT traffic to local routes advertised with --advertise-routes")
 		upf.StringVar(&upArgs.netfilterMode, "netfilter-mode", defaultNetfilterMode(), "netfilter mode (one of on, nodivert, off)")
-	}
-	if runtime.GOOS == "windows" {
+	case "windows":
 		upf.BoolVar(&upArgs.forceDaemon, "unattended", false, "run in \"Unattended Mode\" where Tailscale keeps running even after the current GUI user logs out (Windows-only)")
 	}
 	return upf
-})()
+}
 
 func defaultNetfilterMode() string {
 	if distro.Get() == distro.Synology {
@@ -214,12 +216,13 @@ func prefsFromUpArgs(upArgs upArgsT, warnf logger.Logf, st *ipnstate.Status, goo
 	prefs.ShieldsUp = upArgs.shieldsUp
 	prefs.AdvertiseRoutes = routes
 	prefs.AdvertiseTags = tags
-	prefs.NoSNAT = !upArgs.snat
 	prefs.Hostname = upArgs.hostname
 	prefs.ForceDaemon = upArgs.forceDaemon
 	prefs.OperatorUser = upArgs.opUser
 
 	if goos == "linux" {
+		prefs.NoSNAT = !upArgs.snat
+
 		switch upArgs.netfilterMode {
 		case "on":
 			prefs.NetfilterMode = preftype.NetfilterOn
@@ -245,6 +248,23 @@ func runUp(ctx context.Context, args []string) error {
 	if err != nil {
 		fatalf("can't fetch status from tailscaled: %v", err)
 	}
+	origAuthURL := st.AuthURL
+
+	// printAuthURL reports whether we should print out the
+	// provided auth URL from an IPN notify.
+	printAuthURL := func(url string) bool {
+		if upArgs.authKey != "" {
+			// Issue 1755: when using an authkey, don't
+			// show an authURL that might still be pending
+			// from a previous non-completed interactive
+			// login.
+			return false
+		}
+		if upArgs.forceReauth && url == origAuthURL {
+			return false
+		}
+		return true
+	}
 
 	if distro.Get() == distro.Synology {
 		notSupported := "not yet supported on Synology; see https://github.com/tailscale/tailscale/issues/451"
@@ -285,7 +305,7 @@ func runUp(ctx context.Context, args []string) error {
 	})
 
 	if !upArgs.reset {
-		if err := checkForAccidentalSettingReverts(flagSet, curPrefs, mp); err != nil {
+		if err := checkForAccidentalSettingReverts(flagSet, curPrefs, mp, runtime.GOOS, os.Getenv("USER")); err != nil {
 			fatalf("%s", err)
 		}
 	}
@@ -312,7 +332,10 @@ func runUp(ctx context.Context, args []string) error {
 	// simpleUp is whether we're running a simple "tailscale up"
 	// to transition to running from a previously-logged-in but
 	// down state, without changing any settings.
-	simpleUp := len(flagSet) == 0 && curPrefs.Persist != nil && curPrefs.Persist.LoginName != ""
+	simpleUp := len(flagSet) == 0 &&
+		curPrefs.Persist != nil &&
+		curPrefs.Persist.LoginName != "" &&
+		st.BackendState != ipn.NeedsLogin.String()
 
 	// At this point we need to subscribe to the IPN bus to watch
 	// for state transitions and possible need to authenticate.
@@ -321,7 +344,8 @@ func runUp(ctx context.Context, args []string) error {
 
 	startingOrRunning := make(chan bool, 1) // gets value once starting or running
 	gotEngineUpdate := make(chan bool, 1)   // gets value upon an engine update
-	go pump(pumpCtx, bc, c)
+	pumpErr := make(chan error, 1)
+	go func() { pumpErr <- pump(pumpCtx, bc, c) }()
 
 	printed := !simpleUp
 	var loginOnce sync.Once
@@ -367,7 +391,7 @@ func runUp(ctx context.Context, args []string) error {
 				cancel()
 			}
 		}
-		if url := n.BrowseToURL; url != nil {
+		if url := n.BrowseToURL; url != nil && printAuthURL(*url) {
 			printed = true
 			fmt.Fprintf(os.Stderr, "\nTo authenticate, visit:\n\n\t%s\n\n", *url)
 		}
@@ -381,6 +405,8 @@ func runUp(ctx context.Context, args []string) error {
 	case <-gotEngineUpdate:
 	case <-pumpCtx.Done():
 		return pumpCtx.Err()
+	case err := <-pumpErr:
+		return err
 	}
 
 	// Special case: bare "tailscale up" means to just start
@@ -396,11 +422,10 @@ func runUp(ctx context.Context, args []string) error {
 			return err
 		}
 	} else {
-		bc.SetPrefs(prefs)
-
 		opts := ipn.Options{
-			StateKey: ipn.GlobalDaemonStateKey,
-			AuthKey:  upArgs.authKey,
+			StateKey:    ipn.GlobalDaemonStateKey,
+			AuthKey:     upArgs.authKey,
+			UpdatePrefs: prefs,
 		}
 		// On Windows, we still run in mostly the "legacy" way that
 		// predated the server's StateStore. That is, we send an empty
@@ -418,7 +443,9 @@ func runUp(ctx context.Context, args []string) error {
 		}
 
 		bc.Start(opts)
-		startLoginInteractive()
+		if upArgs.forceReauth {
+			startLoginInteractive()
+		}
 	}
 
 	select {
@@ -431,6 +458,8 @@ func runUp(ctx context.Context, args []string) error {
 		default:
 		}
 		return pumpCtx.Err()
+	case err := <-pumpErr:
+		return err
 	}
 }
 
@@ -450,7 +479,7 @@ func init() {
 	addPrefFlagMapping("netfilter-mode", "NetfilterMode")
 	addPrefFlagMapping("shields-up", "ShieldsUp")
 	addPrefFlagMapping("snat-subnet-routes", "NoSNAT")
-	addPrefFlagMapping("exit-node", "ExitNodeIP", "ExitNodeIP")
+	addPrefFlagMapping("exit-node", "ExitNodeIP", "ExitNodeID")
 	addPrefFlagMapping("exit-node-allow-lan-access", "ExitNodeAllowLANAccess")
 	addPrefFlagMapping("unattended", "ForceDaemon")
 	addPrefFlagMapping("operator", "OperatorUser")
@@ -486,6 +515,12 @@ func updateMaskedPrefsFromUpFlag(mp *ipn.MaskedPrefs, flagName string) {
 	}
 }
 
+const accidentalUpPrefix = "Error: changing settings via 'tailscale up' requires mentioning all\n" +
+	"non-default flags. To proceed, either re-run your command with --reset or\n" +
+	"use the command below to explicitly mention the current value of\n" +
+	"all non-default settings:\n\n" +
+	"\ttailscale up"
+
 // checkForAccidentalSettingReverts checks for people running
 // "tailscale up" with a subset of the flags they originally ran it
 // with.
@@ -500,7 +535,7 @@ func updateMaskedPrefsFromUpFlag(mp *ipn.MaskedPrefs, flagName string) {
 //
 // mp is the mask of settings actually set, where mp.Prefs is the new
 // preferences to set, including any values set from implicit flags.
-func checkForAccidentalSettingReverts(flagSet map[string]bool, curPrefs *ipn.Prefs, mp *ipn.MaskedPrefs) error {
+func checkForAccidentalSettingReverts(flagSet map[string]bool, curPrefs *ipn.Prefs, mp *ipn.MaskedPrefs, goos, curUser string) error {
 	if len(flagSet) == 0 {
 		// A bare "tailscale up" is a special case to just
 		// mean bringing the network up without any changes.
@@ -519,17 +554,49 @@ func checkForAccidentalSettingReverts(flagSet map[string]bool, curPrefs *ipn.Pre
 	ev := reflect.ValueOf(curWithExplicitEdits).Elem()
 	// Implicit values (what we'd get if we replaced everything with flag defaults):
 	iv := reflect.ValueOf(&mp.Prefs).Elem()
-	var errs []error
-	var didExitNodeErr bool
+
+	var missing []string
+	flagExplicitValue := map[string]interface{}{} // e.g. "accept-dns" => true (from flagSet)
 	for i := 0; i < prefType.NumField(); i++ {
 		prefName := prefType.Field(i).Name
+		// Persist is a legacy field used for storing keys, which
+		// probably should never have been part of Prefs. It's
+		// likely to migrate elsewhere eventually.
 		if prefName == "Persist" {
 			continue
 		}
-		flagName, hasFlag := flagForPref[prefName]
-		if hasFlag && flagSet[flagName] {
+		// LoggedOut is a preference, but running the "up" command
+		// always implies that the user now prefers LoggedOut->false.
+		if prefName == "LoggedOut" {
 			continue
 		}
+		flagName, hasFlag := flagForPref[prefName]
+
+		// Special case for advertise-exit-node; which is a
+		// flag but doesn't have a corresponding pref.  The
+		// flag augments advertise-routes, so we have to infer
+		// the imaginary pref's current value from the routes.
+		if prefName == "AdvertiseRoutes" &&
+			hasExitNodeRoutes(curPrefs.AdvertiseRoutes) &&
+			!hasExitNodeRoutes(curWithExplicitEdits.AdvertiseRoutes) &&
+			!flagSet["advertise-exit-node"] {
+			missing = append(missing, "--advertise-exit-node")
+		}
+
+		if hasFlag && flagSet[flagName] {
+			flagExplicitValue[flagName] = ev.Field(i).Interface()
+			continue
+		}
+
+		if prefName == "AdvertiseRoutes" &&
+			(len(curPrefs.AdvertiseRoutes) == 0 ||
+				hasExitNodeRoutes(curPrefs.AdvertiseRoutes) && len(curPrefs.AdvertiseRoutes) == 2) &&
+			hasExitNodeRoutes(mp.Prefs.AdvertiseRoutes) &&
+			len(mp.Prefs.AdvertiseRoutes) == 2 &&
+			flagSet["advertise-exit-node"] {
+			continue
+		}
+
 		// Get explicit value and implicit value
 		ex, im := ev.Field(i), iv.Field(i)
 		switch ex.Kind() {
@@ -540,33 +607,125 @@ func checkForAccidentalSettingReverts(flagSet map[string]bool, curPrefs *ipn.Pre
 			}
 		}
 		exi, imi := ex.Interface(), im.Interface()
+
 		if reflect.DeepEqual(exi, imi) {
 			continue
 		}
 		switch flagName {
-		case "":
-			errs = append(errs, fmt.Errorf("'tailscale up' without --reset requires all preferences with changing values to be explicitly mentioned; this command would change the value of flagless pref %q", prefName))
-		case "exit-node":
-			if !didExitNodeErr {
-				didExitNodeErr = true
-				errs = append(errs, errors.New("'tailscale up' without --reset requires all preferences with changing values to be explicitly mentioned; --exit-node is not specified but an exit node is currently configured"))
+		case "operator":
+			if imi == "" && exi == curUser {
+				// Don't require setting operator if the current user matches
+				// the configured operator.
+				continue
 			}
+		case "snat-subnet-routes", "netfilter-mode":
+			if goos != "linux" {
+				// Issue 1833: we used to accidentally set the NoSNAT
+				// pref for non-Linux nodes. It only affects Linux, so
+				// ignore it if it changes. Likewise, ignore
+				// Linux-only netfilter-mode on non-Linux.
+				continue
+			}
+		}
+		switch flagName {
+		case "":
+			return fmt.Errorf("'tailscale up' without --reset requires all preferences with changing values to be explicitly mentioned; this command would change the value of flagless pref %q", prefName)
+		case "exit-node":
+			if prefName == "ExitNodeIP" {
+				missing = append(missing, fmtFlagValueArg("exit-node", fmtSettingVal(exi)))
+			}
+		case "advertise-routes":
+			routes := withoutExitNodes(exi.([]netaddr.IPPrefix))
+			missing = append(missing, fmtFlagValueArg("advertise-routes", fmtSettingVal(routes)))
 		default:
-			errs = append(errs, fmt.Errorf("'tailscale up' without --reset requires all preferences with changing values to be explicitly mentioned; --%s is not specified but its default value of %v differs from current value %v",
-				flagName, fmtSettingVal(imi), fmtSettingVal(exi)))
+			missing = append(missing, fmtFlagValueArg(flagName, fmtSettingVal(exi)))
 		}
 	}
-	return multierror.New(errs)
+	if len(missing) == 0 {
+		return nil
+	}
+	var sb strings.Builder
+	sb.WriteString(accidentalUpPrefix)
+
+	var flagSetSorted []string
+	for f := range flagSet {
+		flagSetSorted = append(flagSetSorted, f)
+	}
+	sort.Strings(flagSetSorted)
+	for _, flagName := range flagSetSorted {
+		if ev, ok := flagExplicitValue[flagName]; ok {
+			fmt.Fprintf(&sb, " %s", fmtFlagValueArg(flagName, fmtSettingVal(ev)))
+		} else {
+			fmt.Fprintf(&sb, " --%s", flagName)
+		}
+	}
+	for _, a := range missing {
+		fmt.Fprintf(&sb, " %s", a)
+	}
+	sb.WriteString("\n\n")
+	return errors.New(sb.String())
+}
+
+func fmtFlagValueArg(flagName, val string) string {
+	if val == "true" {
+		// TODO: check flagName's type to see if its Pref is of type bool
+		return "--" + flagName
+	}
+	if val == "" {
+		return "--" + flagName + "="
+	}
+	return fmt.Sprintf("--%s=%v", flagName, shellquote.Join(val))
 }
 
 func fmtSettingVal(v interface{}) string {
 	switch v := v.(type) {
 	case bool:
 		return strconv.FormatBool(v)
-	case string, preftype.NetfilterMode:
-		return fmt.Sprintf("%q", v)
+	case string:
+		return v
+	case preftype.NetfilterMode:
+		return v.String()
 	case []string:
 		return strings.Join(v, ",")
+	case []netaddr.IPPrefix:
+		var sb strings.Builder
+		for i, r := range v {
+			if i > 0 {
+				sb.WriteByte(',')
+			}
+			sb.WriteString(r.String())
+		}
+		return sb.String()
 	}
 	return fmt.Sprint(v)
 }
+
+func hasExitNodeRoutes(rr []netaddr.IPPrefix) bool {
+	var v4, v6 bool
+	for _, r := range rr {
+		if r.Bits == 0 {
+			if r.IP.Is4() {
+				v4 = true
+			} else if r.IP.Is6() {
+				v6 = true
+			}
+		}
+	}
+	return v4 && v6
+}
+
+// withoutExitNodes returns rr unchanged if it has only 1 or 0 /0
+// routes. If it has both IPv4 and IPv6 /0 routes, then it returns
+// a copy with all /0 routes removed.
+func withoutExitNodes(rr []netaddr.IPPrefix) []netaddr.IPPrefix {
+	if !hasExitNodeRoutes(rr) {
+		return rr
+	}
+	var out []netaddr.IPPrefix
+	for _, r := range rr {
+		if r.Bits > 0 {
+			out = append(out, r)
+		}
+	}
+	return out
+}

cmd/tailscale/cli/web.go

@@ -73,7 +73,11 @@ func runWeb(ctx context.Context, args []string) error {
 	}
 
 	if webArgs.cgi {
-		return cgi.Serve(http.HandlerFunc(webHandler))
+		if err := cgi.Serve(http.HandlerFunc(webHandler)); err != nil {
+			log.Printf("tailscale.cgi: %v", err)
+			return err
+		}
+		return nil
 	}
 	return http.ListenAndServe(webArgs.listen, http.HandlerFunc(webHandler))
 }
@@ -208,7 +212,7 @@ func webHandler(w http.ResponseWriter, r *http.Request) {
 	if r.Method == "POST" {
 		type mi map[string]interface{}
 		w.Header().Set("Content-Type", "application/json")
-		url, err := tailscaleUp(r.Context())
+		url, err := tailscaleUpForceReauth(r.Context())
 		if err != nil {
 			json.NewEncoder(w).Encode(mi{"error": err})
 			return
@@ -244,7 +248,7 @@ func webHandler(w http.ResponseWriter, r *http.Request) {
 }
 
 // TODO(crawshaw): some of this is very similar to the code in 'tailscale up', can we share anything?
-func tailscaleUp(ctx context.Context) (authURL string, retErr error) {
+func tailscaleUpForceReauth(ctx context.Context) (authURL string, retErr error) {
 	prefs := ipn.NewPrefs()
 	prefs.ControlURL = ipn.DefaultControlURL
 	prefs.WantRunning = true
@@ -256,10 +260,31 @@ func tailscaleUp(ctx context.Context) (authURL string, retErr error) {
 		prefs.NetfilterMode = preftype.NetfilterOff
 	}
 
-	c, bc, ctx, cancel := connect(ctx)
+	st, err := tailscale.Status(ctx)
+	if err != nil {
+		return "", fmt.Errorf("can't fetch status: %v", err)
+	}
+	origAuthURL := st.AuthURL
+
+	// printAuthURL reports whether we should print out the
+	// provided auth URL from an IPN notify.
+	printAuthURL := func(url string) bool {
+		return url != origAuthURL
+	}
+
+	c, bc, pumpCtx, cancel := connect(ctx)
 	defer cancel()
 
+	gotEngineUpdate := make(chan bool, 1) // gets value upon an engine update
+	go pump(pumpCtx, bc, c)
+
 	bc.SetNotifyCallback(func(n ipn.Notify) {
+		if n.Engine != nil {
+			select {
+			case gotEngineUpdate <- true:
+			default:
+			}
+		}
 		if n.ErrMessage != nil {
 			msg := *n.ErrMessage
 			if msg == ipn.ErrMsgPermissionDenied {
@@ -272,11 +297,21 @@ func tailscaleUp(ctx context.Context) (authURL string, retErr error) {
 			}
 			retErr = fmt.Errorf("backend error: %v", msg)
 			cancel()
-		} else if url := n.BrowseToURL; url != nil {
+		} else if url := n.BrowseToURL; url != nil && printAuthURL(*url) {
 			authURL = *url
 			cancel()
 		}
 	})
+	// Wait for backend client to be connected so we know
+	// we're subscribed to updates. Otherwise we can miss
+	// an update upon its transition to running. Do so by causing some traffic
+	// back to the bus that we then wait on.
+	bc.RequestEngineStatus()
+	select {
+	case <-gotEngineUpdate:
+	case <-pumpCtx.Done():
+		return authURL, pumpCtx.Err()
+	}
 
 	bc.SetPrefs(prefs)
 
@@ -284,7 +319,6 @@ func tailscaleUp(ctx context.Context) (authURL string, retErr error) {
 		StateKey: ipn.GlobalDaemonStateKey,
 	})
 	bc.StartLoginInteractive()
-	pump(ctx, bc, c)
 
 	if authURL == "" && retErr == nil {
 		return "", fmt.Errorf("login failed with no backend error message")

cmd/tailscale/cli/web.html

@@ -11,140 +11,133 @@
 </head>
 
 <body class="py-14">
-	<main class="container max-w-lg mx-auto py-6 px-8 bg-white rounded-md shadow-2xl" style="width: 95%">
-		<header class="flex justify-between items-center min-width-0 py-2 mb-8">
-			<svg width="26" height="26" viewBox="0 0 23 23" title="Tailscale" fill="none" xmlns="http://www.w3.org/2000/svg"
-				class="flex-shrink-0 mr-4">
-				<circle opacity="0.2" cx="3.4" cy="3.25" r="2.7" fill="currentColor"></circle>
-				<circle cx="3.4" cy="11.3" r="2.7" fill="currentColor"></circle>
-				<circle opacity="0.2" cx="3.4" cy="19.5" r="2.7" fill="currentColor"></circle>
-				<circle cx="11.5" cy="11.3" r="2.7" fill="currentColor"></circle>
-				<circle cx="11.5" cy="19.5" r="2.7" fill="currentColor"></circle>
-				<circle opacity="0.2" cx="11.5" cy="3.25" r="2.7" fill="currentColor"></circle>
-				<circle opacity="0.2" cx="19.5" cy="3.25" r="2.7" fill="currentColor"></circle>
-				<circle cx="19.5" cy="11.3" r="2.7" fill="currentColor"></circle>
-				<circle opacity="0.2" cx="19.5" cy="19.5" r="2.7" fill="currentColor"></circle>
-			</svg>
-			<div class="flex items-center justify-end space-x-2 w-2/3">
-				{{ with .Profile.LoginName }}
-				<div class="text-right truncate leading-4">
-					<h4 class="truncate">{{.}}</h4>
-					<a href="#" class="text-xs text-gray-500 hover:text-gray-700 js-loginButton">Switch account</a>
-				</div>
+<main class="container max-w-lg mx-auto py-6 px-8 bg-white rounded-md shadow-2xl" style="width: 95%">
+	<header class="flex justify-between items-center min-width-0 py-2 mb-8">
+		<svg width="26" height="26" viewBox="0 0 23 23" title="Tailscale" fill="none" xmlns="http://www.w3.org/2000/svg"
+			class="flex-shrink-0 mr-4">
+			<circle opacity="0.2" cx="3.4" cy="3.25" r="2.7" fill="currentColor"></circle>
+			<circle cx="3.4" cy="11.3" r="2.7" fill="currentColor"></circle>
+			<circle opacity="0.2" cx="3.4" cy="19.5" r="2.7" fill="currentColor"></circle>
+			<circle cx="11.5" cy="11.3" r="2.7" fill="currentColor"></circle>
+			<circle cx="11.5" cy="19.5" r="2.7" fill="currentColor"></circle>
+			<circle opacity="0.2" cx="11.5" cy="3.25" r="2.7" fill="currentColor"></circle>
+			<circle opacity="0.2" cx="19.5" cy="3.25" r="2.7" fill="currentColor"></circle>
+			<circle cx="19.5" cy="11.3" r="2.7" fill="currentColor"></circle>
+			<circle opacity="0.2" cx="19.5" cy="19.5" r="2.7" fill="currentColor"></circle>
+		</svg>
+		<div class="flex items-center justify-end space-x-2 w-2/3">
+			{{ with .Profile.LoginName }}
+			<div class="text-right truncate leading-4">
+				<h4 class="truncate">{{.}}</h4>
+				<a href="#" class="text-xs text-gray-500 hover:text-gray-700 js-loginButton">Switch account</a>
+			</div>
+			{{ end }}
+			<div class="relative flex-shrink-0 w-8 h-8 rounded-full overflow-hidden">
+				{{ with .Profile.ProfilePicURL }}
+				<div class="w-8 h-8 flex pointer-events-none rounded-full bg-gray-200"
+					style="background-image: url('{{.}}'); background-size: cover;"></div>
+				{{ else }}
+				<div class="w-8 h-8 flex pointer-events-none rounded-full border border-gray-400 border-dashed"></div>
 				{{ end }}
-				<div class="relative flex-shrink-0 w-8 h-8 rounded-full overflow-hidden">
-					{{ with .Profile.ProfilePicURL }}
-					<div class="w-8 h-8 flex pointer-events-none rounded-full bg-gray-200"
-						style="background-image: url('{{.}}'); background-size: cover;"></div>
-					{{ else }}
-					<div class="w-8 h-8 flex pointer-events-none rounded-full border border-gray-400 border-dashed"></div>
-					{{ end }}
-				</div>
 			</div>
-		</header>
-		{{ if .IP }}
-		<div
-			class="border border-gray-200 bg-gray-0 rounded-lg p-2 pl-3 pr-3 mb-8 width-full flex items-center justify-between">
-			<div class="flex items-center min-width-0">
-				<svg class="flex-shrink-0 text-gray-600 mr-3 ml-1" xmlns="http://www.w3.org/2000/svg" width="20" height="20"
-					viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round"
-					stroke-linejoin="round">
-					<rect x="2" y="2" width="20" height="8" rx="2" ry="2"></rect>
-					<rect x="2" y="14" width="20" height="8" rx="2" ry="2"></rect>
-					<line x1="6" y1="6" x2="6.01" y2="6"></line>
-					<line x1="6" y1="18" x2="6.01" y2="18"></line>
-				</svg>
-				<h4 class="font-semibold truncate mr-2">{{.DeviceName}}</h4>
-			</div>
-			<h5>{{.IP}}</h5>
 		</div>
-		{{ end }}
-		{{ if or (eq .Status "NeedsLogin") (eq .Status "NoState") }}
-		{{ if .IP }}
-		<div class="mb-6">
-			<p class="text-gray-700">Your device's key has expired. Reauthenticate this device by logging in again, or <a
-					href="https://tailscale.com/kb/1028/key-expiry" class="link" target="_blank">learn more</a>.</p>
+	</header>
+	{{ if .IP }}
+	<div
+		class="border border-gray-200 bg-gray-0 rounded-lg p-2 pl-3 pr-3 mb-8 width-full flex items-center justify-between">
+		<div class="flex items-center min-width-0">
+			<svg class="flex-shrink-0 text-gray-600 mr-3 ml-1" xmlns="http://www.w3.org/2000/svg" width="20" height="20"
+				viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round"
+				stroke-linejoin="round">
+				<rect x="2" y="2" width="20" height="8" rx="2" ry="2"></rect>
+				<rect x="2" y="14" width="20" height="8" rx="2" ry="2"></rect>
+				<line x1="6" y1="6" x2="6.01" y2="6"></line>
+				<line x1="6" y1="18" x2="6.01" y2="18"></line>
+			</svg>
+			<h4 class="font-semibold truncate mr-2">{{.DeviceName}}</h4>
 		</div>
-		<a href="#" class="mb-4 js-loginButton" target="_blank">
-			<button class="button button-blue w-full">Reauthenticate</button>
-		</a>
-		{{ else }}
-		<div class="mb-6">
-			<h3 class="text-3xl font-semibold mb-3">Log in</h3>
-			<p class="text-gray-700">Get started by logging in to your Tailscale network. Or,&nbsp;learn&nbsp;more at <a
-					href="https://tailscale.com/" class="link" target="_blank">tailscale.com</a>.</p>
-		</div>
-		<a href="#" class="mb-4 js-loginButton" target="_blank">
-			<button class="button button-blue w-full">Log In</button>
-		</a>
-		{{ end }}
-		{{ else if eq .Status "NeedsMachineAuth" }}
-		<div class="mb-4">
-			This device is authorized, but needs approval from a network admin before it can connect to the network.
-		</div>
-		{{ else }}
-		<div class="mb-4">
-			<p>You are connected! Access this device over Tailscale using the device name or IP address above.</p>
-		</div>
-		<a href="#" class="mb-4 link font-medium js-loginButton" target="_blank">Reauthenticate</a>
-		{{ end }}
-	</main>
-	<script>
-		(function () {
-			let loginButtons = document.querySelectorAll(".js-loginButton");
-			let fetchingUrl = false;
+		<h5>{{.IP}}</h5>
+	</div>
+	{{ end }}
+	{{ if or (eq .Status "NeedsLogin") (eq .Status "NoState") }}
+	{{ if .IP }}
+	<div class="mb-6">
+		<p class="text-gray-700">Your device's key has expired. Reauthenticate this device by logging in again, or <a
+				href="https://tailscale.com/kb/1028/key-expiry" class="link" target="_blank">learn more</a>.</p>
+	</div>
+	<a href="#" class="mb-4 js-loginButton" target="_blank">
+		<button class="button button-blue w-full">Reauthenticate</button>
+	</a>
+	{{ else }}
+	<div class="mb-6">
+		<h3 class="text-3xl font-semibold mb-3">Log in</h3>
+		<p class="text-gray-700">Get started by logging in to your Tailscale network. Or,&nbsp;learn&nbsp;more at <a
+				href="https://tailscale.com/" class="link" target="_blank">tailscale.com</a>.</p>
+	</div>
+	<a href="#" class="mb-4 js-loginButton" target="_blank">
+		<button class="button button-blue w-full">Log In</button>
+	</a>
+	{{ end }}
+	{{ else if eq .Status "NeedsMachineAuth" }}
+	<div class="mb-4">
+		This device is authorized, but needs approval from a network admin before it can connect to the network.
+	</div>
+	{{ else }}
+	<div class="mb-4">
+		<p>You are connected! Access this device over Tailscale using the device name or IP address above.</p>
+	</div>
+	<a href="#" class="mb-4 link font-medium js-loginButton" target="_blank">Reauthenticate</a>
+	{{ end }}
+</main>
+<script>(function () {
+let loginButtons = document.querySelectorAll(".js-loginButton");
+let fetchingUrl = false;
 
-			function handleClick(e) {
-				e.preventDefault();
+function handleClick(e) {
+	e.preventDefault();
 
-				if (fetchingUrl) {
-					return;
-				}
+	if (fetchingUrl) {
+		return;
+	}
 
-				fetchingUrl = true;
-				const urlParams = new URLSearchParams(window.location.search);
-				const token = urlParams.get("SynoToken");
-				const nextParams = new URLSearchParams({ up: true });
-				if (token) {
-					nextParams.set("SynoToken", token)
-				}
-				const nextUrl = new URL(window.location);
-				nextUrl.search = nextParams.toString()
-				const url = nextUrl.toString();
+	fetchingUrl = true;
+	const urlParams = new URLSearchParams(window.location.search);
+	const token = urlParams.get("SynoToken");
+	const nextParams = new URLSearchParams({ up: true });
+	if (token) {
+		nextParams.set("SynoToken", token)
+	}
+	const nextUrl = new URL(window.location);
+	nextUrl.search = nextParams.toString()
+	const url = nextUrl.toString();
 
-				const tab = window.open("/redirect", "_blank");
+	fetch(url, {
+		method: "POST",
+		headers: {
+			"Accept": "application/json",
+			"Content-Type": "application/json",
+		}
+	}).then(res => res.json()).then(res => {
+		fetchingUrl = false;
+		const err = res["error"];
+		if (err) {
+			throw new Error(err);
+		}
+		const url = res["url"];
+		if (url) {
+			document.location.href = url;
+		} else {
+			location.reload();
+		}
+	}).catch(err => {
+		alert("Failed to log in: " + err.message);
+	});
+}
 
-				fetch(url, {
-					method: "POST",
-					headers: {
-						"Accept": "application/json",
-						"Content-Type": "application/json",
-					}
-				}).then(res => res.json()).then(res => {
-					fetchingUrl = false;
-					const err = res["error"];
-					if (err) {
-						throw new Error(err);
-					}
-					const url = res["url"];
-					if (url) {
-						authUrl = url;
-						tab.location = url;
-						tab.focus();
-					} else {
-						location.reload();
-					}
-				}).catch(err => {
-					tab.close();
-					alert("Failed to log in: " + err.message);
-				});
-			}
-
-			Array.from(loginButtons).forEach(el => {
-				el.addEventListener("click", handleClick);
-			})
-		})();
-	</script>
+Array.from(loginButtons).forEach(el => {
+	el.addEventListener("click", handleClick);
+})
+})();</script>
 </body>
 
 </html>

cmd/tailscale/depaware.txt

@@ -2,7 +2,7 @@ tailscale.com/cmd/tailscale dependencies: (generated by github.com/tailscale/dep
 
    W 💣 github.com/alexbrainman/sspi                                 from github.com/alexbrainman/sspi/negotiate
    W 💣 github.com/alexbrainman/sspi/negotiate                       from tailscale.com/net/tshttpproxy
-        github.com/go-multierror/multierror                          from tailscale.com/cmd/tailscale/cli
+        github.com/kballard/go-shellquote                            from tailscale.com/cmd/tailscale/cli
         github.com/peterbourgon/ff/v2                                from github.com/peterbourgon/ff/v2/ffcli
         github.com/peterbourgon/ff/v2/ffcli                          from tailscale.com/cmd/tailscale/cli
         github.com/tcnksm/go-httpstat                                from tailscale.com/net/netcheck
@@ -15,7 +15,7 @@ tailscale.com/cmd/tailscale dependencies: (generated by github.com/tailscale/dep
         rsc.io/goversion/version                                     from tailscale.com/version
         tailscale.com/atomicfile                                     from tailscale.com/ipn
         tailscale.com/client/tailscale                               from tailscale.com/cmd/tailscale/cli
-        tailscale.com/client/tailscale/apitype                       from tailscale.com/client/tailscale
+        tailscale.com/client/tailscale/apitype                       from tailscale.com/client/tailscale+
         tailscale.com/cmd/tailscale/cli                              from tailscale.com/cmd/tailscale
         tailscale.com/derp                                           from tailscale.com/derp/derphttp
         tailscale.com/derp/derphttp                                  from tailscale.com/net/netcheck
@@ -33,7 +33,7 @@ tailscale.com/cmd/tailscale dependencies: (generated by github.com/tailscale/dep
         tailscale.com/net/portmapper                                 from tailscale.com/net/netcheck+
         tailscale.com/net/stun                                       from tailscale.com/net/netcheck
         tailscale.com/net/tlsdial                                    from tailscale.com/derp/derphttp
-        tailscale.com/net/tsaddr                                     from tailscale.com/net/interfaces
+        tailscale.com/net/tsaddr                                     from tailscale.com/net/interfaces+
      💣 tailscale.com/net/tshttpproxy                                from tailscale.com/derp/derphttp+
         tailscale.com/paths                                          from tailscale.com/cmd/tailscale/cli+
         tailscale.com/safesocket                                     from tailscale.com/cmd/tailscale/cli+
@@ -85,7 +85,7 @@ tailscale.com/cmd/tailscale dependencies: (generated by github.com/tailscale/dep
         golang.org/x/text/transform                                  from golang.org/x/text/secure/bidirule+
         golang.org/x/text/unicode/bidi                               from golang.org/x/net/idna+
         golang.org/x/text/unicode/norm                               from golang.org/x/net/idna
-        golang.org/x/time/rate                                       from tailscale.com/types/logger+
+        golang.org/x/time/rate                                       from tailscale.com/cmd/tailscale/cli+
         bufio                                                        from compress/flate+
         bytes                                                        from bufio+
         compress/flate                                               from compress/gzip+

cmd/tailscaled/depaware.txt

@@ -131,18 +131,21 @@ tailscale.com/cmd/tailscaled dependencies: (generated by github.com/tailscale/de
         tailscale.com/types/strbuilder                               from tailscale.com/net/packet
         tailscale.com/types/structs                                  from tailscale.com/control/controlclient+
         tailscale.com/types/wgkey                                    from tailscale.com/control/controlclient+
+   L    tailscale.com/util/cmpver                                    from tailscale.com/net/dns
         tailscale.com/util/dnsname                                   from tailscale.com/ipn/ipnstate+
   LW    tailscale.com/util/endian                                    from tailscale.com/net/netns+
    L    tailscale.com/util/lineread                                  from tailscale.com/control/controlclient+
+        tailscale.com/util/osshare                                   from tailscale.com/cmd/tailscaled+
         tailscale.com/util/pidowner                                  from tailscale.com/ipn/ipnserver
         tailscale.com/util/racebuild                                 from tailscale.com/logpolicy
         tailscale.com/util/systemd                                   from tailscale.com/control/controlclient+
+        tailscale.com/util/uniq                                      from tailscale.com/wgengine/magicsock
         tailscale.com/util/winutil                                   from tailscale.com/logpolicy+
         tailscale.com/version                                        from tailscale.com/cmd/tailscaled+
         tailscale.com/version/distro                                 from tailscale.com/control/controlclient+
         tailscale.com/wgengine                                       from tailscale.com/cmd/tailscaled+
         tailscale.com/wgengine/filter                                from tailscale.com/control/controlclient+
-        tailscale.com/wgengine/magicsock                             from tailscale.com/cmd/tailscaled+
+        tailscale.com/wgengine/magicsock                             from tailscale.com/wgengine+
         tailscale.com/wgengine/monitor                               from tailscale.com/wgengine+
         tailscale.com/wgengine/netstack                              from tailscale.com/cmd/tailscaled
         tailscale.com/wgengine/router                                from tailscale.com/cmd/tailscaled+
@@ -185,7 +188,7 @@ tailscale.com/cmd/tailscaled dependencies: (generated by github.com/tailscale/de
         golang.org/x/text/transform                                  from golang.org/x/text/secure/bidirule+
         golang.org/x/text/unicode/bidi                               from golang.org/x/net/idna+
         golang.org/x/text/unicode/norm                               from golang.org/x/net/idna
-        golang.org/x/time/rate                                       from tailscale.com/types/logger+
+        golang.org/x/time/rate                                       from inet.af/netstack/tcpip/stack+
         bufio                                                        from compress/flate+
         bytes                                                        from bufio+
         compress/flate                                               from compress/gzip+
@@ -218,7 +221,7 @@ tailscale.com/cmd/tailscaled dependencies: (generated by github.com/tailscale/de
         debug/elf                                                    from rsc.io/goversion/version
         debug/macho                                                  from rsc.io/goversion/version
         debug/pe                                                     from rsc.io/goversion/version
-        embed                                                        from tailscale.com/net/dns
+   L    embed                                                        from tailscale.com/net/dns
         encoding                                                     from encoding/json+
         encoding/asn1                                                from crypto/x509+
         encoding/base64                                              from encoding/json+

cmd/tailscaled/install_windows.go

@@ -16,6 +16,7 @@ import (
 	"golang.org/x/sys/windows/svc/mgr"
 	"tailscale.com/logtail/backoff"
 	"tailscale.com/types/logger"
+	"tailscale.com/util/osshare"
 )
 
 func init() {
@@ -79,6 +80,9 @@ func installSystemDaemonWindows(args []string) (err error) {
 }
 
 func uninstallSystemDaemonWindows(args []string) (ret error) {
+	// Remove file sharing from Windows shell (noop in non-windows)
+	osshare.SetFileSharingEnabled(false, logger.Discard)
+
 	m, err := mgr.Connect()
 	if err != nil {
 		return fmt.Errorf("failed to connect to Windows service manager: %v", err)

cmd/tailscaled/tailscaled.go

@@ -40,10 +40,10 @@ import (
 	"tailscale.com/types/flagtype"
 	"tailscale.com/types/logger"
 	"tailscale.com/types/netmap"
+	"tailscale.com/util/osshare"
 	"tailscale.com/version"
 	"tailscale.com/version/distro"
 	"tailscale.com/wgengine"
-	"tailscale.com/wgengine/magicsock"
 	"tailscale.com/wgengine/monitor"
 	"tailscale.com/wgengine/netstack"
 	"tailscale.com/wgengine/router"
@@ -116,7 +116,7 @@ func main() {
 	flag.StringVar(&args.debug, "debug", "", "listen address ([ip]:port) of optional debug server")
 	flag.StringVar(&args.socksAddr, "socks5-server", "", `optional [ip]:port to run a SOCK5 server (e.g. "localhost:1080")`)
 	flag.StringVar(&args.tunname, "tun", defaultTunName(), `tunnel interface name; use "userspace-networking" (beta) to not use TUN`)
-	flag.Var(flagtype.PortValue(&args.port, magicsock.DefaultPort), "port", "UDP port to listen on for WireGuard and peer-to-peer traffic; 0 means automatically select")
+	flag.Var(flagtype.PortValue(&args.port, 0), "port", "UDP port to listen on for WireGuard and peer-to-peer traffic; 0 means automatically select")
 	flag.StringVar(&args.statepath, "state", paths.DefaultTailscaledStateFile(), "path of state file")
 	flag.StringVar(&args.socketpath, "socket", paths.DefaultTailscaledSocket(), "path of the service unix socket")
 	flag.BoolVar(&printVersion, "version", false, "print version information and exit")
@@ -160,7 +160,12 @@ func main() {
 		log.Fatalf("--socket is required")
 	}
 
-	if err := run(); err != nil {
+	err := run()
+
+	// Remove file sharing from Windows shell (noop in non-windows)
+	osshare.SetFileSharingEnabled(false, logger.Discard)
+
+	if err != nil {
 		// No need to log; the func already did
 		os.Exit(1)
 	}

cmd/tailscaled/tailscaled_windows.go

@@ -168,7 +168,7 @@ func startIPNServer(ctx context.Context, logid string) error {
 		r, err := router.New(logf, dev)
 		if err != nil {
 			dev.Close()
-			return nil, fmt.Errorf("Router: %w", err)
+			return nil, fmt.Errorf("router: %w", err)
 		}
 		if wrapNetstack {
 			r = netstack.NewSubnetRouterWrapper(r)
@@ -188,7 +188,7 @@ func startIPNServer(ctx context.Context, logid string) error {
 		if err != nil {
 			r.Close()
 			dev.Close()
-			return nil, fmt.Errorf("Engine: %w", err)
+			return nil, fmt.Errorf("engine: %w", err)
 		}
 		onlySubnets := true
 		if wrapNetstack {

cmd/tsshd/tsshd.go

@@ -59,11 +59,11 @@ func main() {
 
 	warned := false
 	for {
-		addr, iface, err := interfaces.Tailscale()
+		addrs, iface, err := interfaces.Tailscale()
 		if err != nil {
 			log.Fatalf("listing interfaces: %v", err)
 		}
-		if addr == nil {
+		if len(addrs) == 0 {
 			if !warned {
 				log.Printf("no tailscale interface found; polling until one is available")
 				warned = true
@@ -75,6 +75,13 @@ func main() {
 			continue
 		}
 		warned = false
+		var addr netaddr.IP
+		for _, a := range addrs {
+			if a.Is4() {
+				addr = a
+				break
+			}
+		}
 		listen := net.JoinHostPort(addr.String(), fmt.Sprint(*port))
 		log.Printf("tailscale ssh server listening on %v, %v", iface.Name, listen)
 		s := &ssh.Server{

control/controlclient/auto.go

@@ -2,18 +2,11 @@
 // Use of this source code is governed by a BSD-style
 // license that can be found in the LICENSE file.
 
-// Package controlclient implements the client for the Tailscale
-// control plane.
-//
-// It handles authentication, port picking, and collects the local
-// network configuration.
 package controlclient
 
 import (
 	"context"
-	"encoding/json"
 	"fmt"
-	"reflect"
 	"sync"
 	"time"
 
@@ -28,77 +21,6 @@ import (
 	"tailscale.com/types/wgkey"
 )
 
-// State is the high-level state of the client. It is used only in
-// unit tests for proper sequencing, don't depend on it anywhere else.
-// TODO(apenwarr): eliminate 'state', as it's now obsolete.
-type State int
-
-const (
-	StateNew = State(iota)
-	StateNotAuthenticated
-	StateAuthenticating
-	StateURLVisitRequired
-	StateAuthenticated
-	StateSynchronized // connected and received map update
-)
-
-func (s State) MarshalText() ([]byte, error) {
-	return []byte(s.String()), nil
-}
-
-func (s State) String() string {
-	switch s {
-	case StateNew:
-		return "state:new"
-	case StateNotAuthenticated:
-		return "state:not-authenticated"
-	case StateAuthenticating:
-		return "state:authenticating"
-	case StateURLVisitRequired:
-		return "state:url-visit-required"
-	case StateAuthenticated:
-		return "state:authenticated"
-	case StateSynchronized:
-		return "state:synchronized"
-	default:
-		return fmt.Sprintf("state:unknown:%d", int(s))
-	}
-}
-
-type Status struct {
-	_             structs.Incomparable
-	LoginFinished *empty.Message
-	Err           string
-	URL           string
-	Persist       *persist.Persist   // locally persisted configuration
-	NetMap        *netmap.NetworkMap // server-pushed configuration
-	Hostinfo      *tailcfg.Hostinfo  // current Hostinfo data
-	State         State
-}
-
-// Equal reports whether s and s2 are equal.
-func (s *Status) Equal(s2 *Status) bool {
-	if s == nil && s2 == nil {
-		return true
-	}
-	return s != nil && s2 != nil &&
-		(s.LoginFinished == nil) == (s2.LoginFinished == nil) &&
-		s.Err == s2.Err &&
-		s.URL == s2.URL &&
-		reflect.DeepEqual(s.Persist, s2.Persist) &&
-		reflect.DeepEqual(s.NetMap, s2.NetMap) &&
-		reflect.DeepEqual(s.Hostinfo, s2.Hostinfo) &&
-		s.State == s2.State
-}
-
-func (s Status) String() string {
-	b, err := json.MarshalIndent(s, "", "\t")
-	if err != nil {
-		panic(err)
-	}
-	return s.State.String() + " " + string(b)
-}
-
 type LoginGoal struct {
 	_               structs.Incomparable
 	wantLoggedIn    bool                 // true if we *want* to be logged in
@@ -118,8 +40,9 @@ func (g *LoginGoal) sendLogoutError(err error) {
 	}
 }
 
-// Client connects to a tailcontrol server for a node.
-type Client struct {
+// Auto connects to a tailcontrol server for a node.
+// It's a concrete implementation of the Client interface.
+type Auto struct {
 	direct   *Direct // our interface to the server APIs
 	timeNow  func() time.Time
 	logf     logger.Logf
@@ -152,8 +75,8 @@ type Client struct {
 	mapDone    chan struct{}   // when closed, map goroutine is done
 }
 
-// New creates and starts a new Client.
-func New(opts Options) (*Client, error) {
+// New creates and starts a new Auto.
+func New(opts Options) (*Auto, error) {
 	c, err := NewNoStart(opts)
 	if c != nil {
 		c.Start()
@@ -161,8 +84,8 @@ func New(opts Options) (*Client, error) {
 	return c, err
 }
 
-// NewNoStart creates a new Client, but without calling Start on it.
-func NewNoStart(opts Options) (*Client, error) {
+// NewNoStart creates a new Auto, but without calling Start on it.
+func NewNoStart(opts Options) (*Auto, error) {
 	direct, err := NewDirect(opts)
 	if err != nil {
 		return nil, err
@@ -173,7 +96,7 @@ func NewNoStart(opts Options) (*Client, error) {
 	if opts.TimeNow == nil {
 		opts.TimeNow = time.Now
 	}
-	c := &Client{
+	c := &Auto{
 		direct:   direct,
 		timeNow:  opts.TimeNow,
 		logf:     opts.Logf,
@@ -189,7 +112,7 @@ func NewNoStart(opts Options) (*Client, error) {
 
 }
 
-func (c *Client) onHealthChange(sys health.Subsystem, err error) {
+func (c *Auto) onHealthChange(sys health.Subsystem, err error) {
 	if sys == health.SysOverall {
 		return
 	}
@@ -200,12 +123,13 @@ func (c *Client) onHealthChange(sys health.Subsystem, err error) {
 // SetPaused controls whether HTTP activity should be paused.
 //
 // The client can be paused and unpaused repeatedly, unlike Start and Shutdown, which can only be used once.
-func (c *Client) SetPaused(paused bool) {
+func (c *Auto) SetPaused(paused bool) {
 	c.mu.Lock()
 	defer c.mu.Unlock()
 	if paused == c.paused {
 		return
 	}
+	c.logf("setPaused(%v)", paused)
 	c.paused = paused
 	if paused {
 		// Only cancel the map routine. (The auth routine isn't expensive
@@ -222,7 +146,7 @@ func (c *Client) SetPaused(paused bool) {
 // Start starts the client's goroutines.
 //
 // It should only be called for clients created by NewNoStart.
-func (c *Client) Start() {
+func (c *Auto) Start() {
 	go c.authRoutine()
 	go c.mapRoutine()
 }
@@ -232,7 +156,7 @@ func (c *Client) Start() {
 // streaming response open), or start a new streaming one if necessary.
 //
 // It should be called whenever there's something new to tell the server.
-func (c *Client) sendNewMapRequest() {
+func (c *Auto) sendNewMapRequest() {
 	c.mu.Lock()
 
 	// If we're not already streaming a netmap, or if we're already stuck
@@ -271,7 +195,7 @@ func (c *Client) sendNewMapRequest() {
 	}()
 }
 
-func (c *Client) cancelAuth() {
+func (c *Auto) cancelAuth() {
 	c.mu.Lock()
 	if c.authCancel != nil {
 		c.authCancel()
@@ -282,7 +206,7 @@ func (c *Client) cancelAuth() {
 	c.mu.Unlock()
 }
 
-func (c *Client) cancelMapLocked() {
+func (c *Auto) cancelMapLocked() {
 	if c.mapCancel != nil {
 		c.mapCancel()
 	}
@@ -291,13 +215,13 @@ func (c *Client) cancelMapLocked() {
 	}
 }
 
-func (c *Client) cancelMapUnsafely() {
+func (c *Auto) cancelMapUnsafely() {
 	c.mu.Lock()
 	c.cancelMapLocked()
 	c.mu.Unlock()
 }
 
-func (c *Client) cancelMapSafely() {
+func (c *Auto) cancelMapSafely() {
 	c.mu.Lock()
 	defer c.mu.Unlock()
 
@@ -333,7 +257,7 @@ func (c *Client) cancelMapSafely() {
 	}
 }
 
-func (c *Client) authRoutine() {
+func (c *Auto) authRoutine() {
 	defer close(c.authDone)
 	bo := backoff.NewBackoff("authRoutine", c.logf, 30*time.Second)
 
@@ -344,7 +268,7 @@ func (c *Client) authRoutine() {
 		if goal != nil {
 			c.logf("authRoutine: %s; wantLoggedIn=%v", c.state, goal.wantLoggedIn)
 		} else {
-			c.logf("authRoutine: %s; goal=nil", c.state)
+			c.logf("authRoutine: %s; goal=nil paused=%v", c.state, c.paused)
 		}
 		c.mu.Unlock()
 
@@ -452,7 +376,7 @@ func (c *Client) authRoutine() {
 
 // Expiry returns the credential expiration time, or the zero time if
 // the expiration time isn't known. Used in tests only.
-func (c *Client) Expiry() *time.Time {
+func (c *Auto) Expiry() *time.Time {
 	c.mu.Lock()
 	defer c.mu.Unlock()
 	return c.expiry
@@ -460,21 +384,21 @@ func (c *Client) Expiry() *time.Time {
 
 // Direct returns the underlying direct client object. Used in tests
 // only.
-func (c *Client) Direct() *Direct {
+func (c *Auto) Direct() *Direct {
 	return c.direct
 }
 
 // unpausedChanLocked returns a new channel that is closed when the
-// current Client pause is unpaused.
+// current Auto pause is unpaused.
 //
 // c.mu must be held
-func (c *Client) unpausedChanLocked() <-chan struct{} {
+func (c *Auto) unpausedChanLocked() <-chan struct{} {
 	unpaused := make(chan struct{})
 	c.unpauseWaiters = append(c.unpauseWaiters, unpaused)
 	return unpaused
 }
 
-func (c *Client) mapRoutine() {
+func (c *Auto) mapRoutine() {
 	defer close(c.mapDone)
 	bo := backoff.NewBackoff("mapRoutine", c.logf, 30*time.Second)
 
@@ -596,7 +520,10 @@ func (c *Client) mapRoutine() {
 	}
 }
 
-func (c *Client) AuthCantContinue() bool {
+func (c *Auto) AuthCantContinue() bool {
+	if c == nil {
+		return true
+	}
 	c.mu.Lock()
 	defer c.mu.Unlock()
 
@@ -604,13 +531,13 @@ func (c *Client) AuthCantContinue() bool {
 }
 
 // SetStatusFunc sets fn as the callback to run on any status change.
-func (c *Client) SetStatusFunc(fn func(Status)) {
+func (c *Auto) SetStatusFunc(fn func(Status)) {
 	c.mu.Lock()
 	c.statusFunc = fn
 	c.mu.Unlock()
 }
 
-func (c *Client) SetHostinfo(hi *tailcfg.Hostinfo) {
+func (c *Auto) SetHostinfo(hi *tailcfg.Hostinfo) {
 	if hi == nil {
 		panic("nil Hostinfo")
 	}
@@ -623,7 +550,7 @@ func (c *Client) SetHostinfo(hi *tailcfg.Hostinfo) {
 	c.sendNewMapRequest()
 }
 
-func (c *Client) SetNetInfo(ni *tailcfg.NetInfo) {
+func (c *Auto) SetNetInfo(ni *tailcfg.NetInfo) {
 	if ni == nil {
 		panic("nil NetInfo")
 	}
@@ -636,7 +563,7 @@ func (c *Client) SetNetInfo(ni *tailcfg.NetInfo) {
 	c.sendNewMapRequest()
 }
 
-func (c *Client) sendStatus(who string, err error, url string, nm *netmap.NetworkMap) {
+func (c *Auto) sendStatus(who string, err error, url string, nm *netmap.NetworkMap) {
 	c.mu.Lock()
 	state := c.state
 	loggedIn := c.loggedIn
@@ -681,7 +608,7 @@ func (c *Client) sendStatus(who string, err error, url string, nm *netmap.Networ
 	c.mu.Unlock()
 }
 
-func (c *Client) Login(t *tailcfg.Oauth2Token, flags LoginFlags) {
+func (c *Auto) Login(t *tailcfg.Oauth2Token, flags LoginFlags) {
 	c.logf("client.Login(%v, %v)", t != nil, flags)
 
 	c.mu.Lock()
@@ -695,7 +622,7 @@ func (c *Client) Login(t *tailcfg.Oauth2Token, flags LoginFlags) {
 	c.cancelAuth()
 }
 
-func (c *Client) StartLogout() {
+func (c *Auto) StartLogout() {
 	c.logf("client.StartLogout()")
 
 	c.mu.Lock()
@@ -706,7 +633,7 @@ func (c *Client) StartLogout() {
 	c.cancelAuth()
 }
 
-func (c *Client) Logout(ctx context.Context) error {
+func (c *Auto) Logout(ctx context.Context) error {
 	c.logf("client.Logout()")
 
 	errc := make(chan error, 1)
@@ -738,14 +665,14 @@ func (c *Client) Logout(ctx context.Context) error {
 //
 // The localPort field is unused except for integration tests in
 // another repo.
-func (c *Client) UpdateEndpoints(localPort uint16, endpoints []tailcfg.Endpoint) {
+func (c *Auto) UpdateEndpoints(localPort uint16, endpoints []tailcfg.Endpoint) {
 	changed := c.direct.SetEndpoints(localPort, endpoints)
 	if changed {
 		c.sendNewMapRequest()
 	}
 }
 
-func (c *Client) Shutdown() {
+func (c *Auto) Shutdown() {
 	c.logf("client.Shutdown()")
 
 	c.mu.Lock()
@@ -771,17 +698,17 @@ func (c *Client) Shutdown() {
 
 // NodePublicKey returns the node public key currently in use. This is
 // used exclusively in tests.
-func (c *Client) TestOnlyNodePublicKey() wgkey.Key {
+func (c *Auto) TestOnlyNodePublicKey() wgkey.Key {
 	priv := c.direct.GetPersist()
 	return priv.PrivateNodeKey.Public()
 }
 
-func (c *Client) TestOnlySetAuthKey(authkey string) {
+func (c *Auto) TestOnlySetAuthKey(authkey string) {
 	c.direct.mu.Lock()
 	defer c.direct.mu.Unlock()
 	c.direct.authKey = authkey
 }
 
-func (c *Client) TestOnlyTimeNow() time.Time {
+func (c *Auto) TestOnlyTimeNow() time.Time {
 	return c.timeNow()
 }

control/controlclient/client.go

@@ -0,0 +1,77 @@
+// Copyright (c) 2020 Tailscale Inc & AUTHORS All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+// Package controlclient implements the client for the Tailscale
+// control plane.
+//
+// It handles authentication, port picking, and collects the local
+// network configuration.
+package controlclient
+
+import (
+	"context"
+
+	"tailscale.com/tailcfg"
+)
+
+type LoginFlags int
+
+const (
+	LoginDefault     = LoginFlags(0)
+	LoginInteractive = LoginFlags(1 << iota) // force user login and key refresh
+)
+
+// Client represents a client connection to the control server.
+// Currently this is done through a pair of polling https requests in
+// the Auto client, but that might change eventually.
+type Client interface {
+	// SetStatusFunc provides a callback to call when control sends us
+	// a message.
+	SetStatusFunc(func(Status))
+	// Shutdown closes this session, which should not be used any further
+	// afterwards.
+	Shutdown()
+	// Login begins an interactive or non-interactive login process.
+	// Client will eventually call the Status callback with either a
+	// LoginFinished flag (on success) or an auth URL (if further
+	// interaction is needed).
+	Login(*tailcfg.Oauth2Token, LoginFlags)
+	// StartLogout starts an asynchronous logout process.
+	// When it finishes, the Status callback will be called while
+	// AuthCantContinue()==true.
+	StartLogout()
+	// Logout starts a synchronous logout process. It doesn't return
+	// until the logout operation has been completed.
+	Logout(context.Context) error
+	// SetPaused pauses or unpauses the controlclient activity as much
+	// as possible, without losing its internal state, to minimize
+	// unnecessary network activity.
+	// TODO: It might be better to simply shutdown the controlclient and
+	// make a new one when it's time to unpause.
+	SetPaused(bool)
+	// AuthCantContinue returns whether authentication is blocked. If it
+	// is, you either need to visit the auth URL (previously sent in a
+	// Status callback) or call the Login function appropriately.
+	// TODO: this probably belongs in the Status itself instead.
+	AuthCantContinue() bool
+	// SetHostinfo changes the Hostinfo structure that will be sent in
+	// subsequent node registration requests.
+	// TODO: a server-side change would let us simply upload this
+	// in a separate http request. It has nothing to do with the rest of
+	// the state machine.
+	SetHostinfo(*tailcfg.Hostinfo)
+	// SetNetinfo changes the NetIinfo structure that will be sent in
+	// subsequent node registration requests.
+	// TODO: a server-side change would let us simply upload this
+	// in a separate http request. It has nothing to do with the rest of
+	// the state machine.
+	SetNetInfo(*tailcfg.NetInfo)
+	// UpdateEndpoints changes the Endpoint structure that will be sent
+	// in subsequent node registration requests.
+	// TODO: localPort seems to be obsolete, remove it.
+	// TODO: a server-side change would let us simply upload this
+	// in a separate http request. It has nothing to do with the rest of
+	// the state machine.
+	UpdateEndpoints(localPort uint16, endpoints []tailcfg.Endpoint)
+}

control/controlclient/controlclient_test.go

@@ -22,7 +22,7 @@ func fieldsOf(t reflect.Type) (fields []string) {
 
 func TestStatusEqual(t *testing.T) {
 	// Verify that the Equal method stays in sync with reality
-	equalHandles := []string{"LoginFinished", "Err", "URL", "Persist", "NetMap", "Hostinfo", "State"}
+	equalHandles := []string{"LoginFinished", "Err", "URL", "NetMap", "State", "Persist", "Hostinfo"}
 	if have := fieldsOf(reflect.TypeOf(Status{})); !reflect.DeepEqual(have, equalHandles) {
 		t.Errorf("Status.Equal check might be out of sync\nfields: %q\nhandled: %q\n",
 			have, equalHandles)

control/controlclient/direct.go

@@ -251,13 +251,6 @@ func (c *Direct) GetPersist() persist.Persist {
 	return c.persist
 }
 
-type LoginFlags int
-
-const (
-	LoginDefault     = LoginFlags(0)
-	LoginInteractive = LoginFlags(1 << iota) // force user login and key refresh
-)
-
 func (c *Direct) TryLogout(ctx context.Context) error {
 	c.logf("direct.TryLogout()")
 
@@ -413,7 +406,7 @@ func (c *Direct) doLogin(ctx context.Context, opt loginOpt) (mustRegen bool, new
 
 		// Don't log the common error types. Signatures are not usually enabled,
 		// so these are expected.
-		if err != errCertificateNotConfigured && err != errNoCertStore {
+		if !errors.Is(err, errCertificateNotConfigured) && !errors.Is(err, errNoCertStore) {
 			c.logf("RegisterReq sign error: %v", err)
 		}
 	}
@@ -570,6 +563,11 @@ func (c *Direct) SendLiteMapUpdate(ctx context.Context) error {
 	return c.sendMapRequest(ctx, 1, nil)
 }
 
+// If we go more than pollTimeout without hearing from the server,
+// end the long poll. We should be receiving a keep alive ping
+// every minute.
+const pollTimeout = 120 * time.Second
+
 // cb nil means to omit peers.
 func (c *Direct) sendMapRequest(ctx context.Context, maxPolls int, cb func(*netmap.NetworkMap)) error {
 	c.mu.Lock()
@@ -694,10 +692,6 @@ func (c *Direct) sendMapRequest(ctx context.Context, maxPolls int, cb func(*netm
 		return nil
 	}
 
-	// If we go more than pollTimeout without hearing from the server,
-	// end the long poll. We should be receiving a keep alive ping
-	// every minute.
-	const pollTimeout = 120 * time.Second
 	timeout := time.NewTimer(pollTimeout)
 	timeoutReset := make(chan struct{})
 	pollDone := make(chan struct{})
@@ -795,6 +789,11 @@ func (c *Direct) sendMapRequest(ctx context.Context, maxPolls int, cb func(*netm
 			}
 			setControlAtomic(&controlUseDERPRoute, resp.Debug.DERPRoute)
 			setControlAtomic(&controlTrimWGConfig, resp.Debug.TrimWGConfig)
+			if sleep := time.Duration(resp.Debug.SleepSeconds * float64(time.Second)); sleep > 0 {
+				if err := sleepAsRequested(ctx, c.logf, timeoutReset, sleep); err != nil {
+					return err
+				}
+			}
 		}
 
 		nm := sess.netmapForResponse(&resp)
@@ -1181,3 +1180,34 @@ func answerPing(logf logger.Logf, c *http.Client, pr *tailcfg.PingRequest) {
 		logf("answerPing complete to %v (after %v)", pr.URL, d)
 	}
 }
+
+func sleepAsRequested(ctx context.Context, logf logger.Logf, timeoutReset chan<- struct{}, d time.Duration) error {
+	const maxSleep = 5 * time.Minute
+	if d > maxSleep {
+		logf("sleeping for %v, capped from server-requested %v ...", maxSleep, d)
+		d = maxSleep
+	} else {
+		logf("sleeping for server-requested %v ...", d)
+	}
+
+	ticker := time.NewTicker(pollTimeout / 2)
+	defer ticker.Stop()
+	timer := time.NewTimer(d)
+	defer timer.Stop()
+	for {
+		select {
+		case <-ctx.Done():
+			return ctx.Err()
+		case <-timer.C:
+			return nil
+		case <-ticker.C:
+			select {
+			case timeoutReset <- struct{}{}:
+			case <-timer.C:
+				return nil
+			case <-ctx.Done():
+				return ctx.Err()
+			}
+		}
+	}
+}

control/controlclient/status.go

@@ -0,0 +1,103 @@
+// Copyright (c) 2020 Tailscale Inc & AUTHORS All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package controlclient
+
+import (
+	"encoding/json"
+	"fmt"
+	"reflect"
+
+	"tailscale.com/tailcfg"
+	"tailscale.com/types/empty"
+	"tailscale.com/types/netmap"
+	"tailscale.com/types/persist"
+	"tailscale.com/types/structs"
+)
+
+// State is the high-level state of the client. It is used only in
+// unit tests for proper sequencing, don't depend on it anywhere else.
+//
+// TODO(apenwarr): eliminate the state, as it's now obsolete.
+//
+// apenwarr: Historical note: controlclient.Auto was originally
+// intended to be the state machine for the whole tailscale client, but that
+// turned out to not be the right abstraction layer, and it moved to
+// ipn.Backend. Since ipn.Backend now has a state machine, it would be
+// much better if controlclient could be a simple stateless API. But the
+// current server-side API (two interlocking polling https calls) makes that
+// very hard to implement. A server side API change could untangle this and
+// remove all the statefulness.
+type State int
+
+const (
+	StateNew = State(iota)
+	StateNotAuthenticated
+	StateAuthenticating
+	StateURLVisitRequired
+	StateAuthenticated
+	StateSynchronized // connected and received map update
+)
+
+func (s State) MarshalText() ([]byte, error) {
+	return []byte(s.String()), nil
+}
+
+func (s State) String() string {
+	switch s {
+	case StateNew:
+		return "state:new"
+	case StateNotAuthenticated:
+		return "state:not-authenticated"
+	case StateAuthenticating:
+		return "state:authenticating"
+	case StateURLVisitRequired:
+		return "state:url-visit-required"
+	case StateAuthenticated:
+		return "state:authenticated"
+	case StateSynchronized:
+		return "state:synchronized"
+	default:
+		return fmt.Sprintf("state:unknown:%d", int(s))
+	}
+}
+
+type Status struct {
+	_             structs.Incomparable
+	LoginFinished *empty.Message // nonempty when login finishes
+	Err           string
+	URL           string             // interactive URL to visit to finish logging in
+	NetMap        *netmap.NetworkMap // server-pushed configuration
+
+	// The internal state should not be exposed outside this
+	// package, but we have some automated tests elsewhere that need to
+	// use them. Please don't use these fields.
+	// TODO(apenwarr): Unexport or remove these.
+	State    State
+	Persist  *persist.Persist  // locally persisted configuration
+	Hostinfo *tailcfg.Hostinfo // current Hostinfo data
+}
+
+// Equal reports whether s and s2 are equal.
+func (s *Status) Equal(s2 *Status) bool {
+	if s == nil && s2 == nil {
+		return true
+	}
+	return s != nil && s2 != nil &&
+		(s.LoginFinished == nil) == (s2.LoginFinished == nil) &&
+		s.Err == s2.Err &&
+		s.URL == s2.URL &&
+		reflect.DeepEqual(s.Persist, s2.Persist) &&
+		reflect.DeepEqual(s.NetMap, s2.NetMap) &&
+		reflect.DeepEqual(s.Hostinfo, s2.Hostinfo) &&
+		s.State == s2.State
+}
+
+func (s Status) String() string {
+	b, err := json.MarshalIndent(s, "", "\t")
+	if err != nil {
+		panic(err)
+	}
+	return s.State.String() + " " + string(b)
+}

derp/derpmap/derpmap.go

@@ -83,6 +83,9 @@ func Prod() *tailcfg.DERPMap {
 			10: derpRegion(10, "sea", "Seattle",
 				derpNode("a", "137.220.36.168", "2001:19f0:8001:2d9:5400:2ff:feef:bbb1"),
 			),
+			11: derpRegion(11, "sao", "São Paulo",
+				derpNode("a", "18.230.97.74", "2600:1f1e:ee4:5611:ec5c:1736:d43b:a454"),
+			),
 		},
 	}
 }

go.mod

@@ -7,14 +7,16 @@ require (
 	github.com/anmitsu/go-shlex v0.0.0-20161002113705-648efa622239 // indirect
 	github.com/coreos/go-iptables v0.4.5
 	github.com/flynn/go-shlex v0.0.0-20150515145356-3f9db97f8568 // indirect
+	github.com/frankban/quicktest v1.12.1
 	github.com/github/certstore v0.1.0
 	github.com/gliderlabs/ssh v0.2.2
 	github.com/go-multierror/multierror v1.0.2
 	github.com/go-ole/go-ole v1.2.4
 	github.com/godbus/dbus/v5 v5.0.3
-	github.com/google/go-cmp v0.5.4
+	github.com/google/go-cmp v0.5.5
 	github.com/goreleaser/nfpm v1.1.10
 	github.com/jsimonetti/rtnetlink v0.0.0-20210212075122-66c871082f2b
+	github.com/kballard/go-shellquote v0.0.0-20180428030007-95032a82bc51
 	github.com/klauspost/compress v1.10.10
 	github.com/kr/pty v1.1.8
 	github.com/mdlayher/netlink v1.3.2
@@ -24,7 +26,7 @@ require (
 	github.com/peterbourgon/ff/v2 v2.0.0
 	github.com/pkg/errors v0.9.1 // indirect
 	github.com/tailscale/depaware v0.0.0-20201214215404-77d1e9757027
-	github.com/tailscale/wireguard-go v0.0.0-20210419202603-b32acd8f0292
+	github.com/tailscale/wireguard-go v0.0.0-20210429195722-6cd106ab1339
 	github.com/tcnksm/go-httpstat v0.2.0
 	github.com/toqueteos/webbrowser v1.2.0
 	go4.org/mem v0.0.0-20201119185036-c04c5a6ff174

go.sum

@@ -25,6 +25,8 @@ github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSs
 github.com/dvyukov/go-fuzz v0.0.0-20201127111758-49e582c6c23d/go.mod h1:11Gm+ccJnvAhCNLlf5+cS9KjtbaD5I5zaZpFMsTHWTw=
 github.com/flynn/go-shlex v0.0.0-20150515145356-3f9db97f8568 h1:BHsljHzVlRcyQhjrss6TZTdY2VfCqZPbv5k3iBFa2ZQ=
 github.com/flynn/go-shlex v0.0.0-20150515145356-3f9db97f8568/go.mod h1:xEzjJPgXI435gkrCt3MPfRiAkVrwSbHsst4LCFVfpJc=
+github.com/frankban/quicktest v1.12.1 h1:P6vQcHwZYgVGIpUzKB5DXzkEeYJppJOStPLuh9aB89c=
+github.com/frankban/quicktest v1.12.1/go.mod h1:qLE0fzW0VuyUAJgPU19zByoIr0HtCHN/r/VLSOOIySU=
 github.com/github/fakeca v0.1.0 h1:Km/MVOFvclqxPM9dZBC4+QE564nU4gz4iZ0D9pMw28I=
 github.com/github/fakeca v0.1.0/go.mod h1:+bormgoGMMuamOscx7N91aOuUST7wdaJ2rNjeohylyo=
 github.com/gliderlabs/ssh v0.2.2 h1:6zsha5zo/TWhRhwqCD3+EarCAgZ2yN28ipRnGPnwkI0=
@@ -44,6 +46,8 @@ github.com/google/go-cmp v0.5.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/
 github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
 github.com/google/go-cmp v0.5.4 h1:L8R9j+yAqZuZjsqh/z+F1NCffTKKLShY6zXTItVIZ8M=
 github.com/google/go-cmp v0.5.4/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
+github.com/google/go-cmp v0.5.5 h1:Khx7svrCpmxxtHBq5j2mp/xVjsi8hQMfNLvJFAlrGgU=
+github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
 github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI=
 github.com/google/rpmpack v0.0.0-20191226140753-aa36bfddb3a0 h1:BW6OvS3kpT5UEPbCZ+KyX/OB4Ks9/MNMhWjqPPkZxsE=
 github.com/google/rpmpack v0.0.0-20191226140753-aa36bfddb3a0/go.mod h1:RaTPr0KUf2K7fnZYLNDrr8rxAamWs3iNywJLtQ2AzBg=
@@ -61,11 +65,15 @@ github.com/jsimonetti/rtnetlink v0.0.0-20201220180245-69540ac93943/go.mod h1:z4c
 github.com/jsimonetti/rtnetlink v0.0.0-20210122163228-8d122574c736/go.mod h1:ZXpIyOK59ZnN7J0BV99cZUPmsqDRZ3eq5X+st7u/oSA=
 github.com/jsimonetti/rtnetlink v0.0.0-20210212075122-66c871082f2b h1:c3NTyLNozICy8B4mlMXemD3z/gXgQzVXZS/HqT+i3do=
 github.com/jsimonetti/rtnetlink v0.0.0-20210212075122-66c871082f2b/go.mod h1:8w9Rh8m+aHZIG69YPGGem1i5VzoyRC8nw2kA8B+ik5U=
+github.com/kballard/go-shellquote v0.0.0-20180428030007-95032a82bc51 h1:Z9n2FFNUXsshfwJMBgNA0RU6/i7WVaAegv3PtuIHPMs=
+github.com/kballard/go-shellquote v0.0.0-20180428030007-95032a82bc51/go.mod h1:CzGEWj7cYgsdH8dAjBGEr58BoE7ScuLd+fwFZ44+/x8=
 github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck=
 github.com/klauspost/compress v1.10.10 h1:a/y8CglcM7gLGYmlbP/stPE5sR3hbhFRUjCBfd/0B3I=
 github.com/klauspost/compress v1.10.10/go.mod h1:aoV0uJVorq1K+umq18yTdKaF57EivdYsUV+/s2qKfXs=
 github.com/kr/pretty v0.1.0 h1:L/CwN0zerZDmRFUapSPitk6f+Q3+0za1rQkzVuMiMFI=
 github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo=
+github.com/kr/pretty v0.2.1 h1:Fmg33tUaq4/8ym9TJN1x7sLJnHVwhP33CNkpYV/7rwI=
+github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI=
 github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
 github.com/kr/pty v1.1.8 h1:AkaSdXYQOWeaO3neb8EM634ahkXXe3jYbVh/F9lq+GI=
 github.com/kr/pty v1.1.8/go.mod h1:O1sed60cT9XZ5uDucP5qwvh+TE3NnUj51EiZO/lmSfw=
@@ -117,30 +125,8 @@ github.com/stretchr/testify v1.4.0 h1:2E4SXV/wtOkTonXsotYi4li6zVWxYlZuYNCXe9XRJy
 github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4=
 github.com/tailscale/depaware v0.0.0-20201214215404-77d1e9757027 h1:lK99QQdH3yBWY6aGilF+IRlQIdmhzLrsEmF6JgN+Ryw=
 github.com/tailscale/depaware v0.0.0-20201214215404-77d1e9757027/go.mod h1:p9lPsd+cx33L3H9nNoecRRxPssFKUwwI50I3pZ0yT+8=
-github.com/tailscale/wireguard-go v0.0.0-20210210202228-3cc76ed5f222 h1:VzTS7LIwCH8jlxwrZguU0TsCLV/MDOunoNIDJdFajyM=
-github.com/tailscale/wireguard-go v0.0.0-20210210202228-3cc76ed5f222/go.mod h1:6t0OVdJwFOKFnvaHaVMKG6GznWaHqkmiR2n3kH0t924=
-github.com/tailscale/wireguard-go v0.0.0-20210324165952-2963b66bc23a h1:tQ7Y0ALSe5109GMFB7TVtfNBsVcAuM422hVSJrXWMTE=
-github.com/tailscale/wireguard-go v0.0.0-20210324165952-2963b66bc23a/go.mod h1:6t0OVdJwFOKFnvaHaVMKG6GznWaHqkmiR2n3kH0t924=
-github.com/tailscale/wireguard-go v0.0.0-20210327173134-f6a42a1646a0 h1:7KFBvUmm3TW/K+bAN22D7M6xSSoY/39s+PajaNBGrLw=
-github.com/tailscale/wireguard-go v0.0.0-20210327173134-f6a42a1646a0/go.mod h1:6t0OVdJwFOKFnvaHaVMKG6GznWaHqkmiR2n3kH0t924=
-github.com/tailscale/wireguard-go v0.0.0-20210330185929-1689f2635004 h1:GNEPNdNHsYe5zhoR/0z2Pl/a9zXbr0IySmHV6PhCrzI=
-github.com/tailscale/wireguard-go v0.0.0-20210330185929-1689f2635004/go.mod h1:6t0OVdJwFOKFnvaHaVMKG6GznWaHqkmiR2n3kH0t924=
-github.com/tailscale/wireguard-go v0.0.0-20210330200845-4914b4a944c4 h1:7Y0H5NzrV3fwHeDrUXDFcTy8QNbAEDwr+qHyOfX4VyE=
-github.com/tailscale/wireguard-go v0.0.0-20210330200845-4914b4a944c4/go.mod h1:6t0OVdJwFOKFnvaHaVMKG6GznWaHqkmiR2n3kH0t924=
-github.com/tailscale/wireguard-go v0.0.0-20210401164443-2d6878b6b30d h1:zbDBqtYvc492gcRL5BB7AO5Aed+aVht2jbYg8SKoMYs=
-github.com/tailscale/wireguard-go v0.0.0-20210401164443-2d6878b6b30d/go.mod h1:6t0OVdJwFOKFnvaHaVMKG6GznWaHqkmiR2n3kH0t924=
-github.com/tailscale/wireguard-go v0.0.0-20210401172819-1aca620a8afb h1:6TGRROCOrjTKbt1ucBTZaDMBeScG6yVEXEjuabOiBzU=
-github.com/tailscale/wireguard-go v0.0.0-20210401172819-1aca620a8afb/go.mod h1:jy12FSeiDLRvS7VQvSoiaqH9WtpapbrC8YSzyZ7fUAk=
-github.com/tailscale/wireguard-go v0.0.0-20210401194826-bb7bc2f24083 h1:e3k65apTVs7NM6mhQ1c94XISLe+2gdizPfRdsImNL8Y=
-github.com/tailscale/wireguard-go v0.0.0-20210401194826-bb7bc2f24083/go.mod h1:jy12FSeiDLRvS7VQvSoiaqH9WtpapbrC8YSzyZ7fUAk=
-github.com/tailscale/wireguard-go v0.0.0-20210402173217-0a47c6e64d15 h1:13GZsTKbCmPGwDBurcSXT+ssYID2IfcX0MfsvhaaagY=
-github.com/tailscale/wireguard-go v0.0.0-20210402173217-0a47c6e64d15/go.mod h1:jy12FSeiDLRvS7VQvSoiaqH9WtpapbrC8YSzyZ7fUAk=
-github.com/tailscale/wireguard-go v0.0.0-20210402193818-fc309421dd43 h1:SRUknVD6AHsxfghv0By9SFjQ8dhn8K8gIFwxf3OEPyU=
-github.com/tailscale/wireguard-go v0.0.0-20210402193818-fc309421dd43/go.mod h1:g3WdWX37upLnDT8STKFWhvA34Gwrt4hIpnWR3HGufpM=
-github.com/tailscale/wireguard-go v0.0.0-20210403171604-17614717a9b5 h1:FegsXWjtyhCxpB8bBSL1kLzagtV+e7BaX07phMM8uQM=
-github.com/tailscale/wireguard-go v0.0.0-20210403171604-17614717a9b5/go.mod h1:ys4yUmhKncXy1jWP34qUHKipRjl322VVhxoh1Rkfo7c=
-github.com/tailscale/wireguard-go v0.0.0-20210419202603-b32acd8f0292 h1:rKgYi0k3TNqEz5f7sc6zNeufZcnxm1Efd6bb39cGGkY=
-github.com/tailscale/wireguard-go v0.0.0-20210419202603-b32acd8f0292/go.mod h1:ys4yUmhKncXy1jWP34qUHKipRjl322VVhxoh1Rkfo7c=
+github.com/tailscale/wireguard-go v0.0.0-20210429195722-6cd106ab1339 h1:OjLaZ57xeWJUUBAJN5KmsgjsaUABTZhcvgO/lKtZ8sQ=
+github.com/tailscale/wireguard-go v0.0.0-20210429195722-6cd106ab1339/go.mod h1:ys4yUmhKncXy1jWP34qUHKipRjl322VVhxoh1Rkfo7c=
 github.com/tcnksm/go-httpstat v0.2.0 h1:rP7T5e5U2HfmOBmZzGgGZjBQ5/GluWUylujl0tJ04I0=
 github.com/tcnksm/go-httpstat v0.2.0/go.mod h1:s3JVJFtQxtBEBC9dwcdTTXS9xFnM3SXAZwPG41aurT8=
 github.com/toqueteos/webbrowser v1.2.0 h1:tVP/gpK69Fx+qMJKsLE7TD8LuGWPnEV71wBN9rrstGQ=
@@ -164,8 +150,6 @@ golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8U
 golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
 golang.org/x/crypto v0.0.0-20201016220609-9e8e0b390897/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
 golang.org/x/crypto v0.0.0-20201112155050-0c6587e931a9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
-golang.org/x/crypto v0.0.0-20201117144127-c1f2f97bffc9/go.mod h1:jdWPYTVW3xRLrWPugEBEK3UY2ZEsg3UU495nc5E+M+I=
-golang.org/x/crypto v0.0.0-20201221181555-eec23a3978ad/go.mod h1:jdWPYTVW3xRLrWPugEBEK3UY2ZEsg3UU495nc5E+M+I=
 golang.org/x/crypto v0.0.0-20210220033148-5ea612d1eb83/go.mod h1:jdWPYTVW3xRLrWPugEBEK3UY2ZEsg3UU495nc5E+M+I=
 golang.org/x/crypto v0.0.0-20210317152858-513c2a44f670 h1:gzMM0EjIYiRmJI3+jBdFuoynZlpxa2JQZsolKu09BXo=
 golang.org/x/crypto v0.0.0-20210317152858-513c2a44f670/go.mod h1:T9bdIzuCu7OtxOm1hfPfRQxPLYneinmdGuTeoZ9dtd4=
@@ -210,22 +194,15 @@ golang.org/x/sys v0.0.0-20201009025420-dfb3f7c4e634/go.mod h1:h1NjWce9XRLGQEsW7w
 golang.org/x/sys v0.0.0-20201018230417-eeed37f84f13/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20201107080550-4d91cf3a1aaf/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20201112073958-5cba982894dd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20201117222635-ba5294a509c7/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20201118182958-a01c418693c7/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20201218084310-7d0127a74742/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20210105210732-16f7687f5001/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20210110051926-789bb1bd4061/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20210123111255-9b0068b26619/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20210124154548-22da62e12c0c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20210216163648-f7da38b97c65/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20210301091718-77cc2087c03b/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20210309040221-94ec62e08169/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20210316164454-77fc1eacc6aa/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20210317225723-c4fcb01b228e h1:XNp2Flc/1eWQGk5BLzqTAN7fQIwIbfyVTuVxXxZh73M=
-golang.org/x/sys v0.0.0-20210317225723-c4fcb01b228e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20210402192133-700132347e07 h1:4k6HsQjxj6hVMsI2Vf0yKlzt5lXxZsMW1q0zaq2k8zY=
-golang.org/x/sys v0.0.0-20210402192133-700132347e07/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20210403161142-5e06dd20ab57 h1:F5Gozwx4I1xtr/sr/8CFbb57iKi3297KFs0QDbGN60A=
 golang.org/x/sys v0.0.0-20210403161142-5e06dd20ab57/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw=
@@ -251,7 +228,6 @@ golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 h1:go1bK/D/BFZV2I8cIQd1N
 golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
 golang.zx2c4.com/wireguard v0.0.20200321-0.20201111175144-60b3766b89b9 h1:qowcZ56hhpeoESmWzI4Exhx4Y78TpCyXUJur4/c0CoE=
 golang.zx2c4.com/wireguard v0.0.20200321-0.20201111175144-60b3766b89b9/go.mod h1:LMeNfjlcPZTrBC1juwgbQyA4Zy2XVcsrdO/fIJxwyuA=
-golang.zx2c4.com/wireguard v0.0.20201118/go.mod h1:Dz+cq5bnrai9EpgYj4GDof/+qaGzbRWbeaAOs1bUYa0=
 golang.zx2c4.com/wireguard/windows v0.1.2-0.20201113162609-9b85be97fdf8 h1:nlXPqGA98n+qcq1pwZ28KjM5EsFQvamKS00A+VUeVjs=
 golang.zx2c4.com/wireguard/windows v0.1.2-0.20201113162609-9b85be97fdf8/go.mod h1:psva4yDnAHLuh7lUzOK7J7bLYxNFfo0iKWz+mi9gzkA=
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=

health/health.go

@@ -11,6 +11,7 @@ import (
 	"fmt"
 	"sort"
 	"sync"
+	"sync/atomic"
 	"time"
 
 	"github.com/go-multierror/multierror"
@@ -36,6 +37,7 @@ var (
 	ipnState                string
 	ipnWantRunning          bool
 	anyInterfaceUp          = true // until told otherwise
+	udp4Unbound             bool
 )
 
 // Subsystem is the name of a subsystem whose health can be monitored.
@@ -46,7 +48,7 @@ const (
 	// the system, rather than one particular subsystem.
 	SysOverall = Subsystem("overall")
 
-	// SysRouter is the name the wgengine/router subsystem.
+	// SysRouter is the name of the wgengine/router subsystem.
 	SysRouter = Subsystem("router")
 
 	// SysDNS is the name of the net/dns subsystem.
@@ -213,9 +215,18 @@ func SetAnyInterfaceUp(up bool) {
 	selfCheckLocked()
 }
 
+// SetUDP4Unbound sets whether the udp4 bind failed completely.
+func SetUDP4Unbound(unbound bool) {
+	mu.Lock()
+	defer mu.Unlock()
+	udp4Unbound = unbound
+	selfCheckLocked()
+}
+
 func timerSelfCheck() {
 	mu.Lock()
 	defer mu.Unlock()
+	checkReceiveFuncs()
 	selfCheckLocked()
 	if timer != nil {
 		timer.Reset(time.Minute)
@@ -255,6 +266,9 @@ func overallErrorLocked() error {
 	if d := now.Sub(derpRegionLastFrame[rid]).Round(time.Second); d > tooIdle {
 		return fmt.Errorf("haven't heard from home DERP region %v in %v", rid, d)
 	}
+	if udp4Unbound {
+		return errors.New("no udp4 bind")
+	}
 
 	// TODO: use
 	_ = inMapPollSince
@@ -263,6 +277,11 @@ func overallErrorLocked() error {
 	_ = lastMapRequestHeard
 
 	var errs []error
+	for _, recv := range receiveFuncs {
+		if recv.missing {
+			errs = append(errs, fmt.Errorf("%s is not running", recv.name))
+		}
+	}
 	for sys, err := range sysErr {
 		if err == nil || sys == SysOverall {
 			continue
@@ -275,3 +294,58 @@ func overallErrorLocked() error {
 	})
 	return multierror.New(errs)
 }
+
+var (
+	ReceiveIPv4 = ReceiveFuncStats{name: "ReceiveIPv4"}
+	ReceiveIPv6 = ReceiveFuncStats{name: "ReceiveIPv6"}
+	ReceiveDERP = ReceiveFuncStats{name: "ReceiveDERP"}
+
+	receiveFuncs = []*ReceiveFuncStats{&ReceiveIPv4, &ReceiveIPv6, &ReceiveDERP}
+)
+
+// ReceiveFuncStats tracks the calls made to a wireguard-go receive func.
+type ReceiveFuncStats struct {
+	// name is the name of the receive func.
+	name string
+	// numCalls is the number of times the receive func has ever been called.
+	// It is required because it is possible for a receive func's wireguard-go goroutine
+	// to be active even though the receive func isn't.
+	// The wireguard-go goroutine alternates between calling the receive func and
+	// processing what the func returned.
+	numCalls uint64 // accessed atomically
+	// prevNumCalls is the value of numCalls last time the health check examined it.
+	prevNumCalls uint64
+	// inCall indicates whether the receive func is currently running.
+	inCall uint32 // bool, accessed atomically
+	// missing indicates whether the receive func is not running.
+	missing bool
+}
+
+func (s *ReceiveFuncStats) Enter() {
+	atomic.AddUint64(&s.numCalls, 1)
+	atomic.StoreUint32(&s.inCall, 1)
+}
+
+func (s *ReceiveFuncStats) Exit() {
+	atomic.StoreUint32(&s.inCall, 0)
+}
+
+func checkReceiveFuncs() {
+	for _, recv := range receiveFuncs {
+		recv.missing = false
+		prev := recv.prevNumCalls
+		numCalls := atomic.LoadUint64(&recv.numCalls)
+		recv.prevNumCalls = numCalls
+		if numCalls > prev {
+			// OK: the function has gotten called since last we checked
+			continue
+		}
+		if atomic.LoadUint32(&recv.inCall) == 1 {
+			// OK: the function is active, probably blocked due to inactivity
+			continue
+		}
+		// Not OK: The function is not active, and not accumulating new calls.
+		// It is probably MIA.
+		recv.missing = true
+	}
+}

ipn/backend.go

@@ -5,6 +5,8 @@
 package ipn
 
 import (
+	"fmt"
+	"strings"
 	"time"
 
 	"tailscale.com/ipn/ipnstate"
@@ -55,17 +57,21 @@ type EngineStatus struct {
 // that they have not changed.
 // They are JSON-encoded on the wire, despite the lack of struct tags.
 type Notify struct {
-	_             structs.Incomparable
-	Version       string             // version number of IPN backend
-	ErrMessage    *string            // critical error message, if any; for InUseOtherUser, the details
-	LoginFinished *empty.Message     // event: non-nil when login process succeeded
-	State         *State             // current IPN state has changed
-	Prefs         *Prefs             // preferences were changed
-	NetMap        *netmap.NetworkMap // new netmap received
-	Engine        *EngineStatus      // wireguard engine stats
-	BrowseToURL   *string            // UI should open a browser right now
-	BackendLogID  *string            // public logtail id used by backend
-	PingResult    *ipnstate.PingResult
+	_       structs.Incomparable
+	Version string // version number of IPN backend
+
+	// ErrMessage, if non-nil, contains a critical error message.
+	// For State InUseOtherUser, ErrMessage is not critical and just contains the details.
+	ErrMessage *string
+
+	LoginFinished *empty.Message       // non-nil when/if the login process succeeded
+	State         *State               // if non-nil, the new or current IPN state
+	Prefs         *Prefs               // if non-nil, the new or current preferences
+	NetMap        *netmap.NetworkMap   // if non-nil, the new or current netmap
+	Engine        *EngineStatus        // if non-nil, the new or urrent wireguard stats
+	BrowseToURL   *string              // if non-nil, UI should open a browser right now
+	BackendLogID  *string              // if non-nil, the public logtail ID used by backend
+	PingResult    *ipnstate.PingResult // if non-nil, a ping response arrived
 
 	// FilesWaiting if non-nil means that files are buffered in
 	// the Tailscale daemon and ready for local transfer to the
@@ -88,6 +94,49 @@ type Notify struct {
 	// type is mirrored in xcode/Shared/IPN.swift
 }
 
+func (n Notify) String() string {
+	var sb strings.Builder
+	sb.WriteString("Notify{")
+	if n.ErrMessage != nil {
+		fmt.Fprintf(&sb, "err=%q ", *n.ErrMessage)
+	}
+	if n.LoginFinished != nil {
+		sb.WriteString("LoginFinished ")
+	}
+	if n.State != nil {
+		fmt.Fprintf(&sb, "state=%v ", *n.State)
+	}
+	if n.Prefs != nil {
+		fmt.Fprintf(&sb, "%v ", n.Prefs.Pretty())
+	}
+	if n.NetMap != nil {
+		sb.WriteString("NetMap{...} ")
+	}
+	if n.Engine != nil {
+		fmt.Fprintf(&sb, "wg=%v ", *n.Engine)
+	}
+	if n.BrowseToURL != nil {
+		sb.WriteString("URL=<...> ")
+	}
+	if n.BackendLogID != nil {
+		sb.WriteString("BackendLogID ")
+	}
+	if n.PingResult != nil {
+		fmt.Fprintf(&sb, "ping=%v ", *n.PingResult)
+	}
+	if n.FilesWaiting != nil {
+		sb.WriteString("FilesWaiting ")
+	}
+	if len(n.IncomingFiles) != 0 {
+		sb.WriteString("IncomingFiles ")
+	}
+	if n.LocalTCPPort != nil {
+		fmt.Fprintf(&sb, "tcpport=%v ", n.LocalTCPPort)
+	}
+	s := sb.String()
+	return s[0:len(s)-1] + "}"
+}
+
 // PartialFile represents an in-progress file transfer.
 type PartialFile struct {
 	Name         string    // e.g. "foo.jpg"
@@ -136,8 +185,30 @@ type Options struct {
 	//    state and use/update that.
 	//  - StateKey!="" && Prefs!=nil: like the previous case, but do
 	//    an initial overwrite of backend state with Prefs.
+	//
+	// NOTE(apenwarr): The above means that this Prefs field does not do
+	// what you probably think it does. It will overwrite your encryption
+	// keys. Do not use unless you know what you're doing.
 	StateKey StateKey
 	Prefs    *Prefs
+	// UpdatePrefs, if provided, overrides Options.Prefs *and* the Prefs
+	// already stored in the backend state, *except* for the Persist
+	// Persist member. If you just want to provide prefs, this is
+	// probably what you want.
+	//
+	// UpdatePrefs.Persist is always ignored. Prefs.Persist will still
+	// be used even if UpdatePrefs is provided. Other than Persist,
+	// UpdatePrefs takes precedence over Prefs.
+	//
+	// This is intended as a purely temporary workaround for the
+	// currently unexpected behaviour of Options.Prefs.
+	//
+	// TODO(apenwarr): Remove this, or rename Prefs to something else
+	//   and rename this to Prefs. Or, move Prefs.Persist elsewhere
+	//   entirely (as it always should have been), and then we wouldn't
+	//   need two separate fields at all. Or, move the fancy state
+	//   migration stuff out of Start().
+	UpdatePrefs *Prefs
 	// AuthKey is an optional node auth key used to authorize a
 	// new node key without user interaction.
 	AuthKey string

ipn/ipnlocal/local.go

@@ -46,6 +46,7 @@ import (
 	"tailscale.com/types/persist"
 	"tailscale.com/types/wgkey"
 	"tailscale.com/util/dnsname"
+	"tailscale.com/util/osshare"
 	"tailscale.com/util/systemd"
 	"tailscale.com/version"
 	"tailscale.com/wgengine"
@@ -97,14 +98,16 @@ type LocalBackend struct {
 	// The mutex protects the following elements.
 	mu             sync.Mutex
 	httpTestClient *http.Client // for controlclient. nil by default, used by tests.
+	ccGen          clientGen    // function for producing controlclient; lazily populated
 	notify         func(ipn.Notify)
-	cc             *controlclient.Client
+	cc             controlclient.Client
 	stateKey       ipn.StateKey // computed in part from user-provided value
 	userID         string       // current controlling user ID (for Windows, primarily)
 	prefs          *ipn.Prefs
 	inServerMode   bool
 	machinePrivKey wgkey.Private
 	state          ipn.State
+	capFileSharing bool // whether netMap contains the file sharing capability
 	// hostinfo is mutated in-place while mu is held.
 	hostinfo *tailcfg.Hostinfo
 	// netMap is not mutated in-place once set.
@@ -114,7 +117,8 @@ type LocalBackend struct {
 	engineStatus     ipn.EngineStatus
 	endpoints        []tailcfg.Endpoint
 	blocked          bool
-	authURL          string
+	authURL          string // cleared on Notify
+	authURLSticky    string // not cleared on Notify
 	interact         bool
 	prevIfState      *interfaces.State
 	peerAPIServer    *peerAPIServer // or nil
@@ -137,6 +141,10 @@ type LocalBackend struct {
 	statusChanged *sync.Cond
 }
 
+// clientGen is a func that creates a control plane client.
+// It's the type used by LocalBackend.SetControlClientGetterForTesting.
+type clientGen func(controlclient.Options) (controlclient.Client, error)
+
 // NewLocalBackend returns a new LocalBackend that is ready to run,
 // but is not actually running.
 func NewLocalBackend(logf logger.Logf, logid string, store ipn.StateStore, e wgengine.Engine) (*LocalBackend, error) {
@@ -144,6 +152,8 @@ func NewLocalBackend(logf logger.Logf, logid string, store ipn.StateStore, e wge
 		panic("ipn.NewLocalBackend: wgengine must not be nil")
 	}
 
+	osshare.SetFileSharingEnabled(false, logf)
+
 	// Default filter blocks everything and logs nothing, until Start() is called.
 	e.SetFilter(filter.NewAllowNone(logf, &netaddr.IPSet{}))
 
@@ -213,7 +223,7 @@ func (b *LocalBackend) linkChange(major bool, ifst *interfaces.State) {
 
 	networkUp := ifst.AnyInterfaceUp()
 	if b.cc != nil {
-		go b.cc.SetPaused(b.state == ipn.Stopped || !networkUp)
+		go b.cc.SetPaused((b.state == ipn.Stopped && b.netMap != nil) || !networkUp)
 	}
 
 	// If the PAC-ness of the network changed, reconfig wireguard+route to
@@ -310,7 +320,7 @@ func (b *LocalBackend) updateStatus(sb *ipnstate.StatusBuilder, extraLocked func
 	sb.MutateStatus(func(s *ipnstate.Status) {
 		s.Version = version.Long
 		s.BackendState = b.state.String()
-		s.AuthURL = b.authURL
+		s.AuthURL = b.authURLSticky
 		if b.netMap != nil {
 			s.MagicDNSSuffix = b.netMap.MagicDNSSuffix()
 		}
@@ -424,6 +434,10 @@ func (b *LocalBackend) setClientStatus(st controlclient.Status) {
 		// Auth completed, unblock the engine
 		b.blockEngineUpdates(false)
 		b.authReconfig()
+		b.EditPrefs(&ipn.MaskedPrefs{
+			LoggedOutSet: true,
+			Prefs:        ipn.Prefs{LoggedOut: false},
+		})
 		b.send(ipn.Notify{LoginFinished: &empty.Message{}})
 	}
 
@@ -460,6 +474,7 @@ func (b *LocalBackend) setClientStatus(st controlclient.Status) {
 	}
 	if st.URL != "" {
 		b.authURL = st.URL
+		b.authURLSticky = st.URL
 	}
 	if b.state == ipn.NeedsLogin {
 		if !b.prefs.WantRunning {
@@ -588,6 +603,51 @@ func (b *LocalBackend) SetHTTPTestClient(c *http.Client) {
 	b.httpTestClient = c
 }
 
+// SetControlClientGetterForTesting sets the func that creates a
+// control plane client. It can be called at most once, before Start.
+func (b *LocalBackend) SetControlClientGetterForTesting(newControlClient func(controlclient.Options) (controlclient.Client, error)) {
+	b.mu.Lock()
+	defer b.mu.Unlock()
+	if b.ccGen != nil {
+		panic("invalid use of SetControlClientGetterForTesting after Start")
+	}
+	b.ccGen = newControlClient
+}
+
+func (b *LocalBackend) getNewControlClientFunc() clientGen {
+	b.mu.Lock()
+	defer b.mu.Unlock()
+	if b.ccGen == nil {
+		// Initialize it rather than just returning the
+		// default to make any future call to
+		// SetControlClientGetterForTesting panic.
+		b.ccGen = func(opts controlclient.Options) (controlclient.Client, error) {
+			return controlclient.New(opts)
+		}
+	}
+	return b.ccGen
+}
+
+// startIsNoopLocked reports whether a Start call on this LocalBackend
+// with the provided Start Options would be a useless no-op.
+//
+// b.mu must be held.
+func (b *LocalBackend) startIsNoopLocked(opts ipn.Options) bool {
+	// Options has 5 fields; check all of them:
+	//   * FrontendLogID
+	//   * StateKey
+	//   * Prefs
+	//   * UpdatePrefs
+	//   * AuthKey
+	return b.state == ipn.Running &&
+		b.hostinfo != nil &&
+		b.hostinfo.FrontendLogID == opts.FrontendLogID &&
+		b.stateKey == opts.StateKey &&
+		opts.Prefs == nil &&
+		opts.UpdatePrefs == nil &&
+		opts.AuthKey == ""
+}
+
 // Start applies the configuration specified in opts, and starts the
 // state machine.
 //
@@ -609,12 +669,30 @@ func (b *LocalBackend) Start(opts ipn.Options) error {
 		b.logf("Start")
 	}
 
+	b.mu.Lock()
+
+	// The iOS client sends a "Start" whenever its UI screen comes
+	// up, just because it wants a netmap. That should be fixed,
+	// but meanwhile we can make Start cheaper here for such a
+	// case and not restart the world (which takes a few seconds).
+	// Instead, just send a notify with the state that iOS needs.
+	if b.startIsNoopLocked(opts) {
+		b.logf("Start: already running; sending notify")
+		nm := b.netMap
+		state := b.state
+		b.mu.Unlock()
+		b.send(ipn.Notify{
+			State:         &state,
+			NetMap:        nm,
+			LoginFinished: new(empty.Message),
+		})
+		return nil
+	}
+
 	hostinfo := controlclient.NewHostinfo()
 	hostinfo.BackendLogID = b.backendLogID
 	hostinfo.FrontendLogID = opts.FrontendLogID
 
-	b.mu.Lock()
-
 	if b.cc != nil {
 		// TODO(apenwarr): avoid the need to reinit controlclient.
 		// This will trigger a full relogin/reconfigure cycle every
@@ -623,7 +701,9 @@ func (b *LocalBackend) Start(opts ipn.Options) error {
 		// into sync with the minimal changes. But that's not how it
 		// is right now, which is a sign that the code is still too
 		// complicated.
+		b.mu.Unlock()
 		b.cc.Shutdown()
+		b.mu.Lock()
 	}
 	httpTestClient := b.httpTestClient
 
@@ -639,6 +719,12 @@ func (b *LocalBackend) Start(opts ipn.Options) error {
 		return fmt.Errorf("loading requested state: %v", err)
 	}
 
+	if opts.UpdatePrefs != nil {
+		newPrefs := opts.UpdatePrefs
+		newPrefs.Persist = b.prefs.Persist
+		b.prefs = newPrefs
+	}
+
 	wantRunning := b.prefs.WantRunning
 	if wantRunning {
 		if err := b.initMachineKeyLocked(); err != nil {
@@ -646,6 +732,8 @@ func (b *LocalBackend) Start(opts ipn.Options) error {
 		}
 	}
 
+	loggedOut := b.prefs.LoggedOut
+
 	b.inServerMode = b.prefs.ForceDaemon
 	b.serverURL = b.prefs.ControlURLOrDefault()
 	hostinfo.RoutableIPs = append(hostinfo.RoutableIPs, b.prefs.AdvertiseRoutes...)
@@ -699,7 +787,11 @@ func (b *LocalBackend) Start(opts ipn.Options) error {
 		debugFlags = append([]string{"netstack"}, debugFlags...)
 	}
 
-	cc, err := controlclient.New(controlclient.Options{
+	// TODO(apenwarr): The only way to change the ServerURL is to
+	// re-run b.Start(), because this is the only place we create a
+	// new controlclient. SetPrefs() allows you to overwrite ServerURL,
+	// but it won't take effect until the next Start().
+	cc, err := b.getNewControlClientFunc()(controlclient.Options{
 		GetMachinePrivateKey: b.createGetMachinePrivateKeyFunc(),
 		Logf:                 logger.WithPrefix(b.logf, "control: "),
 		Persist:              *persistv,
@@ -743,9 +835,13 @@ func (b *LocalBackend) Start(opts ipn.Options) error {
 	b.send(ipn.Notify{BackendLogID: &blid})
 	b.send(ipn.Notify{Prefs: prefs})
 
-	if wantRunning {
+	if !loggedOut && b.hasNodeKey() {
+		// Even if !WantRunning, we should verify our key, if there
+		// is one. If you want tailscaled to be completely idle,
+		// use logout instead.
 		cc.Login(nil, controlclient.LoginDefault)
 	}
+	b.stateMachine()
 	return nil
 }
 
@@ -1030,7 +1126,7 @@ func (b *LocalBackend) popBrowserAuthNow() {
 	b.mu.Lock()
 	url := b.authURL
 	b.interact = false
-	b.authURL = ""
+	b.authURL = "" // but NOT clearing authURLSticky
 	b.mu.Unlock()
 
 	b.logf("popBrowserAuthNow: url=%v", url != "")
@@ -1364,14 +1460,12 @@ func (b *LocalBackend) EditPrefs(mp *ipn.MaskedPrefs) (*ipn.Prefs, error) {
 		b.mu.Unlock()
 		return p1, nil
 	}
-	cc := b.cc
 	b.logf("EditPrefs: %v", mp.Pretty())
 	b.setPrefsLockedOnEntry("EditPrefs", p1) // does a b.mu.Unlock
 
-	if !p0.WantRunning && p1.WantRunning {
-		b.logf("EditPrefs: transitioning to running; doing Login...")
-		cc.Login(nil, controlclient.LoginDefault)
-	}
+	// Note: don't perform any actions for the new prefs here. Not
+	// every prefs change goes through EditPrefs. Put your actions
+	// in setPrefsLocksOnEntry instead.
 	return p1, nil
 }
 
@@ -1405,6 +1499,7 @@ func (b *LocalBackend) setPrefsLockedOnEntry(caller string, newp *ipn.Prefs) {
 	b.hostinfo = newHi
 	hostInfoChanged := !oldHi.Equal(newHi)
 	userID := b.userID
+	cc := b.cc
 
 	b.mu.Unlock()
 
@@ -1444,6 +1539,11 @@ func (b *LocalBackend) setPrefsLockedOnEntry(caller string, newp *ipn.Prefs) {
 		b.e.SetDERPMap(netMap.DERPMap)
 	}
 
+	if !oldp.WantRunning && newp.WantRunning {
+		b.logf("transitioning to running; doing Login...")
+		cc.Login(nil, controlclient.LoginDefault)
+	}
+
 	if oldp.WantRunning != newp.WantRunning {
 		b.stateMachine()
 	} else {
@@ -1627,6 +1727,16 @@ func (b *LocalBackend) authReconfig() {
 			}
 			var ips []netaddr.IP
 			for _, addr := range addrs {
+				// Remove IPv6 addresses for now, as we don't
+				// guarantee that the peer node actually can speak
+				// IPv6 correctly.
+				//
+				// https://github.com/tailscale/tailscale/issues/1152
+				// tracks adding the right capability reporting to
+				// enable AAAA in MagicDNS.
+				if addr.IP.Is6() {
+					continue
+				}
 				ips = append(ips, addr.IP)
 			}
 			dcfg.Hosts[fqdn] = ips
@@ -1647,15 +1757,21 @@ func (b *LocalBackend) authReconfig() {
 		switch {
 		case len(dcfg.DefaultResolvers) != 0:
 			// Default resolvers already set.
-		case len(dcfg.Routes) == 0 && len(dcfg.Hosts) == 0 && len(dcfg.AuthoritativeSuffixes) == 0:
-			// No settings requiring split DNS, no problem.
-		case (version.OS() == "iOS" || version.OS() == "macOS") && !uc.ExitNodeID.IsZero():
-			// On Apple OSes, if your NetworkExtension provides a
-			// default route, underlying primary resolvers are
-			// automatically removed, so we MUST provide a set of
-			// resolvers capable of resolving the entire world.
+		case !uc.ExitNodeID.IsZero():
+			// When using exit nodes, it's very likely the LAN
+			// resolvers will become unreachable. So, force use of the
+			// fallback resolvers until we implement DNS forwarding to
+			// exit nodes.
+			//
+			// This is especially important on Apple OSes, where
+			// adding the default route to the tunnel interface makes
+			// it "primary", and we MUST provide VPN-sourced DNS
+			// settings or we break all DNS resolution.
+			//
 			// https://github.com/tailscale/tailscale/issues/1713
 			addDefault(nm.DNS.FallbackResolvers)
+		case len(dcfg.Routes) == 0 && len(dcfg.Hosts) == 0 && len(dcfg.AuthoritativeSuffixes) == 0:
+			// No settings requiring split DNS, no problem.
 		case version.OS() == "android":
 			// We don't support split DNS at all on Android yet.
 			addDefault(nm.DNS.FallbackResolvers)
@@ -1716,6 +1832,20 @@ func (b *LocalBackend) fileRootLocked(uid tailcfg.UserID) string {
 	return dir
 }
 
+// closePeerAPIListenersLocked closes any existing peer API listeners
+// and clears out the peer API server state.
+//
+// It does not kick off any Hostinfo update with new services.
+//
+// b.mu must be held.
+func (b *LocalBackend) closePeerAPIListenersLocked() {
+	b.peerAPIServer = nil
+	for _, pln := range b.peerAPIListeners {
+		pln.Close()
+	}
+	b.peerAPIListeners = nil
+}
+
 func (b *LocalBackend) initPeerAPIListener() {
 	b.mu.Lock()
 	defer b.mu.Unlock()
@@ -1734,11 +1864,7 @@ func (b *LocalBackend) initPeerAPIListener() {
 		}
 	}
 
-	b.peerAPIServer = nil
-	for _, pln := range b.peerAPIListeners {
-		pln.Close()
-	}
-	b.peerAPIListeners = nil
+	b.closePeerAPIListenersLocked()
 
 	selfNode := b.netMap.SelfNode
 	if len(b.netMap.Addresses) == 0 || selfNode == nil {
@@ -1774,6 +1900,12 @@ func (b *LocalBackend) initPeerAPIListener() {
 		if !skipListen {
 			ln, err = ps.listen(a.IP, b.prevIfState)
 			if err != nil {
+				if runtime.GOOS == "windows" {
+					// Expected for now. See Issue 1620.
+					// But we fix it later in linkChange
+					// ("peerAPIListeners too low").
+					continue
+				}
 				b.logf("[unexpected] peerapi listen(%q) error: %v", a.IP, err)
 				continue
 			}
@@ -1957,25 +2089,33 @@ func applyPrefsToHostinfo(hi *tailcfg.Hostinfo, prefs *ipn.Prefs) {
 // happen".
 func (b *LocalBackend) enterState(newState ipn.State) {
 	b.mu.Lock()
-	state := b.state
+	oldState := b.state
 	b.state = newState
 	prefs := b.prefs
 	cc := b.cc
+	netMap := b.netMap
 	networkUp := b.prevIfState.AnyInterfaceUp()
 	activeLogin := b.activeLogin
 	authURL := b.authURL
+	if newState == ipn.Running {
+		b.authURL = ""
+		b.authURLSticky = ""
+	} else if oldState == ipn.Running {
+		// Transitioning away from running.
+		b.closePeerAPIListenersLocked()
+	}
 	b.mu.Unlock()
 
-	if state == newState {
+	if oldState == newState {
 		return
 	}
 	b.logf("Switching ipn state %v -> %v (WantRunning=%v)",
-		state, newState, prefs.WantRunning)
+		oldState, newState, prefs.WantRunning)
 	health.SetIPNState(newState.String(), prefs.WantRunning)
 	b.send(ipn.Notify{State: &newState})
 
 	if cc != nil {
-		cc.SetPaused(newState == ipn.Stopped || !networkUp)
+		cc.SetPaused((newState == ipn.Stopped && netMap != nil) || !networkUp)
 	}
 
 	switch newState {
@@ -2008,6 +2148,15 @@ func (b *LocalBackend) enterState(newState ipn.State) {
 
 }
 
+func (b *LocalBackend) hasNodeKey() bool {
+	// we can't use b.Prefs(), because it strips the keys, oops!
+	b.mu.Lock()
+	p := b.prefs
+	b.mu.Unlock()
+
+	return p.Persist != nil && !p.Persist.PrivateNodeKey.IsZero()
+}
+
 // nextState returns the state the backend seems to be in, based on
 // its internal state.
 func (b *LocalBackend) nextState() ipn.State {
@@ -2018,17 +2167,35 @@ func (b *LocalBackend) nextState() ipn.State {
 		netMap      = b.netMap
 		state       = b.state
 		wantRunning = b.prefs.WantRunning
+		loggedOut   = b.prefs.LoggedOut
 	)
 	b.mu.Unlock()
 
 	switch {
+	case !wantRunning && !loggedOut && b.hasNodeKey():
+		return ipn.Stopped
 	case netMap == nil:
-		if cc.AuthCantContinue() {
+		if cc.AuthCantContinue() || loggedOut {
 			// Auth was interrupted or waiting for URL visit,
 			// so it won't proceed without human help.
 			return ipn.NeedsLogin
+		} else if state == ipn.Stopped {
+			// If we were already in the Stopped state, then
+			// we can assume auth is in good shape (or we would
+			// have been in NeedsLogin), so transition to Starting
+			// right away.
+			return ipn.Starting
+		} else if state == ipn.NoState {
+			// Our first time connecting to control, and we
+			// don't know if we'll NeedsLogin or not yet.
+			// UIs should print "Loading..." in this state.
+			return ipn.NoState
+		} else if state == ipn.Starting ||
+			state == ipn.Running ||
+			state == ipn.NeedsLogin {
+			return state
 		} else {
-			// Auth or map request needs to finish
+			b.logf("unexpected no-netmap state transition for %v", state)
 			return state
 		}
 	case !wantRunning:
@@ -2115,16 +2282,13 @@ func (b *LocalBackend) ResetForClientDisconnect() {
 	b.setNetMapLocked(nil)
 	b.prefs = new(ipn.Prefs)
 	b.authURL = ""
+	b.authURLSticky = ""
 	b.activeLogin = ""
 }
 
 // Logout tells the controlclient that we want to log out, and
 // transitions the local engine to the logged-out state without
 // waiting for controlclient to be in that state.
-//
-// NOTE(apenwarr): No easy way to persist logged-out status.
-//  Maybe that's for the better; if someone logs out accidentally,
-//  rebooting will fix it.
 func (b *LocalBackend) Logout() {
 	b.logout(context.Background(), false)
 }
@@ -2141,7 +2305,8 @@ func (b *LocalBackend) logout(ctx context.Context, sync bool) error {
 
 	b.EditPrefs(&ipn.MaskedPrefs{
 		WantRunningSet: true,
-		Prefs:          ipn.Prefs{WantRunning: false},
+		LoggedOutSet:   true,
+		Prefs:          ipn.Prefs{WantRunning: false, LoggedOut: true},
 	})
 
 	if cc == nil {
@@ -2193,6 +2358,17 @@ func (b *LocalBackend) setNetInfo(ni *tailcfg.NetInfo) {
 	cc.SetNetInfo(ni)
 }
 
+func hasCapability(nm *netmap.NetworkMap, cap string) bool {
+	if nm != nil && nm.SelfNode != nil {
+		for _, c := range nm.SelfNode.Capabilities {
+			if c == cap {
+				return true
+			}
+		}
+	}
+	return false
+}
+
 func (b *LocalBackend) setNetMapLocked(nm *netmap.NetworkMap) {
 	var login string
 	if nm != nil {
@@ -2207,6 +2383,13 @@ func (b *LocalBackend) setNetMapLocked(nm *netmap.NetworkMap) {
 		b.activeLogin = login
 	}
 
+	// Determine if file sharing is enabled
+	fs := hasCapability(nm, tailcfg.CapabilityFileSharing)
+	if fs != b.capFileSharing {
+		osshare.SetFileSharingEnabled(fs, b.logf)
+	}
+	b.capFileSharing = fs
+
 	if nm == nil {
 		b.nodeByAddr = nil
 		return
@@ -2315,20 +2498,7 @@ func (b *LocalBackend) OpenFile(name string) (rc io.ReadCloser, size int64, err
 func (b *LocalBackend) hasCapFileSharing() bool {
 	b.mu.Lock()
 	defer b.mu.Unlock()
-	return b.hasCapFileSharingLocked()
-}
-
-func (b *LocalBackend) hasCapFileSharingLocked() bool {
-	nm := b.netMap
-	if nm == nil || nm.SelfNode == nil {
-		return false
-	}
-	for _, c := range nm.SelfNode.Capabilities {
-		if c == tailcfg.CapabilityFileSharing {
-			return true
-		}
-	}
-	return false
+	return b.capFileSharing
 }
 
 // FileTargets lists nodes that the current node can send files to.
@@ -2337,13 +2507,13 @@ func (b *LocalBackend) FileTargets() ([]*apitype.FileTarget, error) {
 
 	b.mu.Lock()
 	defer b.mu.Unlock()
-	if !b.hasCapFileSharingLocked() {
-		return nil, errors.New("file sharing not enabled by Tailscale admin")
-	}
 	nm := b.netMap
 	if b.state != ipn.Running || nm == nil {
 		return nil, errors.New("not connected")
 	}
+	if !b.capFileSharing {
+		return nil, errors.New("file sharing not enabled by Tailscale admin")
+	}
 	for _, p := range nm.Peers {
 		if p.User != nm.User {
 			continue

ipn/ipnlocal/local_test.go

@@ -5,14 +5,20 @@
 package ipnlocal
 
 import (
+	"fmt"
+	"net/http"
 	"reflect"
 	"testing"
+	"time"
 
 	"inet.af/netaddr"
+	"tailscale.com/ipn"
 	"tailscale.com/net/interfaces"
 	"tailscale.com/net/tsaddr"
 	"tailscale.com/tailcfg"
+	"tailscale.com/types/logger"
 	"tailscale.com/types/netmap"
+	"tailscale.com/wgengine"
 	"tailscale.com/wgengine/wgcfg"
 )
 
@@ -419,3 +425,71 @@ func TestPeerAPIBase(t *testing.T) {
 		})
 	}
 }
+
+type panicOnUseTransport struct{}
+
+func (panicOnUseTransport) RoundTrip(*http.Request) (*http.Response, error) {
+	panic("unexpected HTTP request")
+}
+
+// Issue 1573: don't generate a machine key if we don't want to be running.
+func TestLazyMachineKeyGeneration(t *testing.T) {
+	defer func(old bool) { panicOnMachineKeyGeneration = old }(panicOnMachineKeyGeneration)
+	panicOnMachineKeyGeneration = true
+
+	var logf logger.Logf = logger.Discard
+	store := new(ipn.MemoryStore)
+	eng, err := wgengine.NewFakeUserspaceEngine(logf, 0)
+	if err != nil {
+		t.Fatalf("NewFakeUserspaceEngine: %v", err)
+	}
+	lb, err := NewLocalBackend(logf, "logid", store, eng)
+	if err != nil {
+		t.Fatalf("NewLocalBackend: %v", err)
+	}
+
+	lb.SetHTTPTestClient(&http.Client{
+		Transport: panicOnUseTransport{}, // validate we don't send HTTP requests
+	})
+
+	if err := lb.Start(ipn.Options{
+		StateKey: ipn.GlobalDaemonStateKey,
+	}); err != nil {
+		t.Fatalf("Start: %v", err)
+	}
+
+	// Give the controlclient package goroutines (if they're
+	// accidentally started) extra time to schedule and run (and thus
+	// hit panicOnUseTransport).
+	time.Sleep(500 * time.Millisecond)
+}
+
+func TestFileTargets(t *testing.T) {
+	b := new(LocalBackend)
+	_, err := b.FileTargets()
+	if got, want := fmt.Sprint(err), "not connected"; got != want {
+		t.Errorf("before connect: got %q; want %q", got, want)
+	}
+
+	b.netMap = new(netmap.NetworkMap)
+	_, err = b.FileTargets()
+	if got, want := fmt.Sprint(err), "not connected"; got != want {
+		t.Errorf("non-running netmap: got %q; want %q", got, want)
+	}
+
+	b.state = ipn.Running
+	_, err = b.FileTargets()
+	if got, want := fmt.Sprint(err), "file sharing not enabled by Tailscale admin"; got != want {
+		t.Errorf("without cap: got %q; want %q", got, want)
+	}
+
+	b.capFileSharing = true
+	got, err := b.FileTargets()
+	if err != nil {
+		t.Fatal(err)
+	}
+	if len(got) != 0 {
+		t.Fatalf("unexpected %d peers", len(got))
+	}
+	// (other cases handled by TestPeerAPIBase above)
+}

ipn/ipnlocal/loglines_test.go

@@ -15,6 +15,7 @@ import (
 	"tailscale.com/tailcfg"
 	"tailscale.com/tstest"
 	"tailscale.com/types/key"
+	"tailscale.com/types/logger"
 	"tailscale.com/types/persist"
 	"tailscale.com/wgengine"
 )
@@ -30,6 +31,12 @@ func TestLocalLogLines(t *testing.T) {
 	})
 	defer logListen.Close()
 
+	// Put a rate-limiter with a burst of 0 between the components below.
+	// This instructs the rate-limiter to eliminate all logging that
+	// isn't explicitly exempt from rate-limiting.
+	// This lets the logListen tracker verify that the rate-limiter allows these key lines.
+	logf := logger.RateLimitedFnWithClock(logListen.Logf, 5*time.Second, 0, 10, time.Now)
+
 	logid := func(hex byte) logtail.PublicID {
 		var ret logtail.PublicID
 		for i := 0; i < len(ret); i++ {
@@ -41,12 +48,12 @@ func TestLocalLogLines(t *testing.T) {
 
 	// set up a LocalBackend, super bare bones. No functional data.
 	store := &ipn.MemoryStore{}
-	e, err := wgengine.NewFakeUserspaceEngine(logListen.Logf, 0)
+	e, err := wgengine.NewFakeUserspaceEngine(logf, 0)
 	if err != nil {
 		t.Fatal(err)
 	}
 
-	lb, err := NewLocalBackend(logListen.Logf, idA.String(), store, e)
+	lb, err := NewLocalBackend(logf, idA.String(), store, e)
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -61,6 +68,7 @@ func TestLocalLogLines(t *testing.T) {
 	testWantRemain := func(wantRemain ...string) func(t *testing.T) {
 		return func(t *testing.T) {
 			if remain := logListen.Check(); !reflect.DeepEqual(remain, wantRemain) {
+				t.Helper()
 				t.Errorf("remain %q, want %q", remain, wantRemain)
 			}
 		}
@@ -75,17 +83,30 @@ func TestLocalLogLines(t *testing.T) {
 	t.Run("after_prefs", testWantRemain("[v1] peer keys: %s", "[v1] v%v peers: %v"))
 
 	// log peers, peer keys
-	status := &wgengine.Status{
+	lb.mu.Lock()
+	lb.parseWgStatusLocked(&wgengine.Status{
 		Peers: []ipnstate.PeerStatusLite{{
 			TxBytes:       10,
 			RxBytes:       10,
 			LastHandshake: time.Now(),
 			NodeKey:       tailcfg.NodeKey(key.NewPrivate()),
 		}},
-	}
-	lb.mu.Lock()
-	lb.parseWgStatusLocked(status)
+	})
 	lb.mu.Unlock()
 
 	t.Run("after_peers", testWantRemain())
+
+	// Log it again with different stats to ensure it's not dup-suppressed.
+	logListen.Reset()
+	lb.mu.Lock()
+	lb.parseWgStatusLocked(&wgengine.Status{
+		Peers: []ipnstate.PeerStatusLite{{
+			TxBytes:       11,
+			RxBytes:       12,
+			LastHandshake: time.Now(),
+			NodeKey:       tailcfg.NodeKey(key.NewPrivate()),
+		}},
+	})
+	lb.mu.Unlock()
+	t.Run("after_second_peer_status", testWantRemain("SetPrefs: %v"))
 }

ipn/ipnlocal/peerapi.go

@@ -11,6 +11,7 @@ import (
 	"hash/crc32"
 	"html"
 	"io"
+	"io/fs"
 	"net"
 	"net/http"
 	"net/url"
@@ -18,6 +19,7 @@ import (
 	"path"
 	"path/filepath"
 	"runtime"
+	"sort"
 	"strconv"
 	"strings"
 	"sync"
@@ -28,6 +30,7 @@ import (
 	"inet.af/netaddr"
 	"tailscale.com/client/tailscale/apitype"
 	"tailscale.com/ipn"
+	"tailscale.com/logtail/backoff"
 	"tailscale.com/net/interfaces"
 	"tailscale.com/syncs"
 	"tailscale.com/tailcfg"
@@ -47,10 +50,24 @@ type peerAPIServer struct {
 	// download directory (as *.partial files), rather than making
 	// the frontend retrieve it over localapi HTTP and write it
 	// somewhere itself. This is used on GUI macOS version.
+	// In directFileMode, the peerapi doesn't do the final rename
+	// from "foo.jpg.partial" to "foo.jpg".
 	directFileMode bool
 }
 
-const partialSuffix = ".partial"
+const (
+	// partialSuffix is the suffix appened to files while they're
+	// still in the process of being transferred.
+	partialSuffix = ".partial"
+
+	// deletedSuffix is the suffix for a deleted marker file
+	// that's placed next to a file (without the suffix) that we
+	// tried to delete, but Windows wouldn't let us. These are
+	// only written on Windows (and in tests), but they're not
+	// permitted to be uploaded directly on any platform, like
+	// partial files.
+	deletedSuffix = ".deleted"
+)
 
 func validFilenameRune(r rune) bool {
 	switch r {
@@ -83,6 +100,7 @@ func (s *peerAPIServer) diskPath(baseName string) (fullPath string, ok bool) {
 	clean := path.Clean(baseName)
 	if clean != baseName ||
 		clean == "." || clean == ".." ||
+		strings.HasSuffix(clean, deletedSuffix) ||
 		strings.HasSuffix(clean, partialSuffix) {
 		return "", false
 	}
@@ -116,11 +134,28 @@ func (s *peerAPIServer) hasFilesWaiting() bool {
 	for {
 		des, err := f.ReadDir(10)
 		for _, de := range des {
-			if strings.HasSuffix(de.Name(), partialSuffix) {
+			name := de.Name()
+			if strings.HasSuffix(name, partialSuffix) {
+				continue
+			}
+			if strings.HasSuffix(name, deletedSuffix) { // for Windows + tests
+				// After we're done looping over files, then try
+				// to delete this file. Don't do it proactively,
+				// as the OS may return "foo.jpg.deleted" before "foo.jpg"
+				// and we don't want to delete the ".deleted" file before
+				// enumerating to the "foo.jpg" file.
+				defer tryDeleteAgain(filepath.Join(s.rootDir, strings.TrimSuffix(name, deletedSuffix)))
 				continue
 			}
 			if de.Type().IsRegular() {
-				return true
+				_, err := os.Stat(filepath.Join(s.rootDir, name+deletedSuffix))
+				if os.IsNotExist(err) {
+					return true
+				}
+				if err == nil {
+					tryDeleteAgain(filepath.Join(s.rootDir, name))
+					continue
+				}
 			}
 		}
 		if err == io.EOF {
@@ -133,6 +168,12 @@ func (s *peerAPIServer) hasFilesWaiting() bool {
 	return false
 }
 
+// WaitingFiles returns the list of files that have been sent by a
+// peer that are waiting in the buffered "pick up" directory owned by
+// the Tailscale daemon.
+//
+// As a side effect, it also does any lazy deletion of files as
+// required by Windows.
 func (s *peerAPIServer) WaitingFiles() (ret []apitype.WaitingFile, err error) {
 	if s.rootDir == "" {
 		return nil, errors.New("peerapi disabled; no storage configured")
@@ -145,6 +186,7 @@ func (s *peerAPIServer) WaitingFiles() (ret []apitype.WaitingFile, err error) {
 		return nil, err
 	}
 	defer f.Close()
+	var deleted map[string]bool // "foo.jpg" => true (if "foo.jpg.deleted" exists)
 	for {
 		des, err := f.ReadDir(10)
 		for _, de := range des {
@@ -152,6 +194,13 @@ func (s *peerAPIServer) WaitingFiles() (ret []apitype.WaitingFile, err error) {
 			if strings.HasSuffix(name, partialSuffix) {
 				continue
 			}
+			if strings.HasSuffix(name, deletedSuffix) { // for Windows + tests
+				if deleted == nil {
+					deleted = map[string]bool{}
+				}
+				deleted[strings.TrimSuffix(name, deletedSuffix)] = true
+				continue
+			}
 			if de.Type().IsRegular() {
 				fi, err := de.Info()
 				if err != nil {
@@ -170,9 +219,41 @@ func (s *peerAPIServer) WaitingFiles() (ret []apitype.WaitingFile, err error) {
 			return nil, err
 		}
 	}
+	if len(deleted) > 0 {
+		// Filter out any return values "foo.jpg" where a
+		// "foo.jpg.deleted" marker file exists on disk.
+		all := ret
+		ret = ret[:0]
+		for _, wf := range all {
+			if !deleted[wf.Name] {
+				ret = append(ret, wf)
+			}
+		}
+		// And do some opportunistic deleting while we're here.
+		// Maybe Windows is done virus scanning the file we tried
+		// to delete a long time ago and will let us delete it now.
+		for name := range deleted {
+			tryDeleteAgain(filepath.Join(s.rootDir, name))
+		}
+	}
+	sort.Slice(ret, func(i, j int) bool { return ret[i].Name < ret[j].Name })
 	return ret, nil
 }
 
+// tryDeleteAgain tries to delete path (and path+deletedSuffix) after
+// it failed earlier.  This happens on Windows when various anti-virus
+// tools hook into filesystem operations and have the file open still
+// while we're trying to delete it. In that case we instead mark it as
+// deleted (writing a "foo.jpg.deleted" marker file), but then we
+// later try to clean them up.
+//
+// fullPath is the full path to the file without the deleted suffix.
+func tryDeleteAgain(fullPath string) {
+	if err := os.Remove(fullPath); err == nil || os.IsNotExist(err) {
+		os.Remove(fullPath + deletedSuffix)
+	}
+}
+
 func (s *peerAPIServer) DeleteFile(baseName string) error {
 	if s.rootDir == "" {
 		return errors.New("peerapi disabled; no storage configured")
@@ -184,11 +265,59 @@ func (s *peerAPIServer) DeleteFile(baseName string) error {
 	if !ok {
 		return errors.New("bad filename")
 	}
-	err := os.Remove(path)
-	if err != nil && !os.IsNotExist(err) {
-		return err
+	var bo *backoff.Backoff
+	logf := s.b.logf
+	t0 := time.Now()
+	for {
+		err := os.Remove(path)
+		if err != nil && !os.IsNotExist(err) {
+			err = redactErr(err)
+			// Put a retry loop around deletes on Windows. Windows
+			// file descriptor closes are effectively asynchronous,
+			// as a bunch of hooks run on/after close, and we can't
+			// necessarily delete the file for a while after close,
+			// as we need to wait for everybody to be done with
+			// it. (on Windows, unlike Unix, a file can't be deleted
+			// if it's open anywhere)
+			// So try a few times but ultimately just leave a
+			// "foo.jpg.deleted" marker file to note that it's
+			// deleted and we clean it up later.
+			if runtime.GOOS == "windows" {
+				if bo == nil {
+					bo = backoff.NewBackoff("delete-retry", logf, 1*time.Second)
+				}
+				if time.Since(t0) < 5*time.Second {
+					bo.BackOff(context.Background(), err)
+					continue
+				}
+				if err := touchFile(path + deletedSuffix); err != nil {
+					logf("peerapi: failed to leave deleted marker: %v", err)
+				}
+			}
+			logf("peerapi: failed to DeleteFile: %v", err)
+			return err
+		}
+		return nil
 	}
-	return nil
+}
+
+// redacted is a fake path name we use in errors, to avoid
+// accidentally logging actual filenames anywhere.
+const redacted = "redacted"
+
+func redactErr(err error) error {
+	if pe, ok := err.(*os.PathError); ok {
+		pe.Path = redacted
+	}
+	return err
+}
+
+func touchFile(path string) error {
+	f, err := os.OpenFile(path, os.O_RDWR|os.O_CREATE, 0666)
+	if err != nil {
+		return redactErr(err)
+	}
+	return f.Close()
 }
 
 func (s *peerAPIServer) OpenFile(baseName string) (rc io.ReadCloser, size int64, err error) {
@@ -202,14 +331,18 @@ func (s *peerAPIServer) OpenFile(baseName string) (rc io.ReadCloser, size int64,
 	if !ok {
 		return nil, 0, errors.New("bad filename")
 	}
+	if fi, err := os.Stat(path + deletedSuffix); err == nil && fi.Mode().IsRegular() {
+		tryDeleteAgain(path)
+		return nil, 0, &fs.PathError{Op: "open", Path: redacted, Err: fs.ErrNotExist}
+	}
 	f, err := os.Open(path)
 	if err != nil {
-		return nil, 0, err
+		return nil, 0, redactErr(err)
 	}
 	fi, err := f.Stat()
 	if err != nil {
 		f.Close()
-		return nil, 0, err
+		return nil, 0, redactErr(err)
 	}
 	return f, fi.Size(), nil
 }
@@ -367,6 +500,10 @@ func (h *peerAPIHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
 		h.handlePeerPut(w, r)
 		return
 	}
+	if r.URL.Path == "/v0/goroutines" {
+		h.handleServeGoroutines(w, r)
+		return
+	}
 	who := h.peerUser.DisplayName
 	fmt.Fprintf(w, `<html>
 <meta name="viewport" content="width=device-width, initial-scale=1">
@@ -479,19 +616,19 @@ func (h *peerAPIHandler) handlePeerPut(w http.ResponseWriter, r *http.Request) {
 		http.Error(w, "bad filename", 400)
 		return
 	}
-	if h.ps.directFileMode {
-		dstFile += partialSuffix
-	}
-	f, err := os.Create(dstFile)
+	t0 := time.Now()
+	// TODO(bradfitz): prevent same filename being sent by two peers at once
+	partialFile := dstFile + partialSuffix
+	f, err := os.Create(partialFile)
 	if err != nil {
-		h.logf("put Create error: %v", err)
+		h.logf("put Create error: %v", redactErr(err))
 		http.Error(w, err.Error(), http.StatusInternalServerError)
 		return
 	}
 	var success bool
 	defer func() {
 		if !success {
-			os.Remove(dstFile)
+			os.Remove(partialFile)
 		}
 	}()
 	var finalSize int64
@@ -505,12 +642,13 @@ func (h *peerAPIHandler) handlePeerPut(w http.ResponseWriter, r *http.Request) {
 			ph:      h,
 		}
 		if h.ps.directFileMode {
-			inFile.partialPath = dstFile
+			inFile.partialPath = partialFile
 		}
 		h.ps.b.registerIncomingFile(inFile, true)
 		defer h.ps.b.registerIncomingFile(inFile, false)
 		n, err := io.Copy(inFile, r.Body)
 		if err != nil {
+			err = redactErr(err)
 			f.Close()
 			h.logf("put Copy error: %v", err)
 			http.Error(w, err.Error(), http.StatusInternalServerError)
@@ -518,7 +656,7 @@ func (h *peerAPIHandler) handlePeerPut(w http.ResponseWriter, r *http.Request) {
 		}
 		finalSize = n
 	}
-	if err := f.Close(); err != nil {
+	if err := redactErr(f.Close()); err != nil {
 		h.logf("put Close error: %v", err)
 		http.Error(w, err.Error(), http.StatusInternalServerError)
 		return
@@ -527,9 +665,17 @@ func (h *peerAPIHandler) handlePeerPut(w http.ResponseWriter, r *http.Request) {
 		if inFile != nil { // non-zero length; TODO: notify even for zero length
 			inFile.markAndNotifyDone()
 		}
+	} else {
+		if err := os.Rename(partialFile, dstFile); err != nil {
+			err = redactErr(err)
+			h.logf("put final rename: %v", err)
+			http.Error(w, err.Error(), http.StatusInternalServerError)
+			return
+		}
 	}
 
-	h.logf("put of %s from %v/%v", approxSize(finalSize), h.remoteAddr.IP, h.peerNode.ComputedName)
+	d := time.Since(t0).Round(time.Second / 10)
+	h.logf("got put of %s in %v from %v/%v", approxSize(finalSize), d, h.remoteAddr.IP, h.peerNode.ComputedName)
 
 	// TODO: set modtime
 	// TODO: some real response
@@ -546,5 +692,21 @@ func approxSize(n int64) string {
 	if n <= 1<<20 {
 		return "<=1MB"
 	}
-	return fmt.Sprintf("~%dMB", n/1<<20)
+	return fmt.Sprintf("~%dMB", n>>20)
+}
+
+func (h *peerAPIHandler) handleServeGoroutines(w http.ResponseWriter, r *http.Request) {
+	if !h.isSelf {
+		http.Error(w, "not owner", http.StatusForbidden)
+		return
+	}
+	var buf []byte
+	for size := 4 << 10; size <= 2<<20; size *= 2 {
+		buf = make([]byte, size)
+		buf = buf[:runtime.Stack(buf, true)]
+		if len(buf) < size {
+			break
+		}
+	}
+	w.Write(buf)
 }

ipn/ipnlocal/peerapi_test.go

@@ -10,15 +10,16 @@ import (
 	"io"
 	"io/fs"
 	"io/ioutil"
+	"math/rand"
 	"net/http"
 	"net/http/httptest"
 	"os"
 	"path/filepath"
+	"runtime"
 	"strings"
 	"testing"
 
 	"tailscale.com/tailcfg"
-	"tailscale.com/types/netmap"
 )
 
 type peerAPITestEnv struct {
@@ -102,7 +103,7 @@ func hexAll(v string) string {
 	return sb.String()
 }
 
-func TestHandlePeerPut(t *testing.T) {
+func TestHandlePeerAPI(t *testing.T) {
 	tests := []struct {
 		name       string
 		isSelf     bool // the peer sending the request is owned by us
@@ -133,6 +134,21 @@ func TestHandlePeerPut(t *testing.T) {
 				bodyNotContains("You are the owner of this node."),
 			),
 		},
+		{
+			name:   "peer_api_goroutines_deny",
+			isSelf: false,
+			req:    httptest.NewRequest("GET", "/v0/goroutines", nil),
+			checks: checks(httpStatus(403)),
+		},
+		{
+			name:   "peer_api_goroutines",
+			isSelf: true,
+			req:    httptest.NewRequest("GET", "/v0/goroutines", nil),
+			checks: checks(
+				httpStatus(200),
+				bodyContains("ServeHTTP"),
+			),
+		},
 		{
 			name:       "reject_non_owner_put",
 			isSelf:     false,
@@ -220,6 +236,16 @@ func TestHandlePeerPut(t *testing.T) {
 				bodyContains("bad filename"),
 			),
 		},
+		{
+			name:       "bad_filename_deleted",
+			isSelf:     true,
+			capSharing: true,
+			req:        httptest.NewRequest("PUT", "/v0/put/foo.deleted", nil),
+			checks: checks(
+				httpStatus(400),
+				bodyContains("bad filename"),
+			),
+		},
 		{
 			name:       "bad_filename_dot",
 			isSelf:     true,
@@ -375,18 +401,10 @@ func TestHandlePeerPut(t *testing.T) {
 	}
 	for _, tt := range tests {
 		t.Run(tt.name, func(t *testing.T) {
-			var caps []string
-			if tt.capSharing {
-				caps = append(caps, tailcfg.CapabilityFileSharing)
-			}
 			var e peerAPITestEnv
 			lb := &LocalBackend{
-				netMap: &netmap.NetworkMap{
-					SelfNode: &tailcfg.Node{
-						Capabilities: caps,
-					},
-				},
-				logf: e.logf,
+				logf:           e.logf,
+				capFileSharing: tt.capSharing,
 			}
 			e.ph = &peerAPIHandler{
 				isSelf: tt.isSelf,
@@ -422,3 +440,134 @@ func TestHandlePeerPut(t *testing.T) {
 		})
 	}
 }
+
+// Windows likes to hold on to file descriptors for some indeterminate
+// amount of time after you close them and not let you delete them for
+// a bit. So test that we work around that sufficiently.
+func TestFileDeleteRace(t *testing.T) {
+	dir := t.TempDir()
+	ps := &peerAPIServer{
+		b: &LocalBackend{
+			logf:           t.Logf,
+			capFileSharing: true,
+		},
+		rootDir: dir,
+	}
+	ph := &peerAPIHandler{
+		isSelf: true,
+		peerNode: &tailcfg.Node{
+			ComputedName: "some-peer-name",
+		},
+		ps: ps,
+	}
+	buf := make([]byte, 2<<20)
+	for i := 0; i < 30; i++ {
+		rr := httptest.NewRecorder()
+		ph.ServeHTTP(rr, httptest.NewRequest("PUT", "/v0/put/foo.txt", bytes.NewReader(buf[:rand.Intn(len(buf))])))
+		if res := rr.Result(); res.StatusCode != 200 {
+			t.Fatal(res.Status)
+		}
+		wfs, err := ps.WaitingFiles()
+		if err != nil {
+			t.Fatal(err)
+		}
+		if len(wfs) != 1 {
+			t.Fatalf("waiting files = %d; want 1", len(wfs))
+		}
+
+		if err := ps.DeleteFile("foo.txt"); err != nil {
+			t.Fatal(err)
+		}
+		wfs, err = ps.WaitingFiles()
+		if err != nil {
+			t.Fatal(err)
+		}
+		if len(wfs) != 0 {
+			t.Fatalf("waiting files = %d; want 0", len(wfs))
+		}
+	}
+}
+
+// Tests "foo.jpg.deleted" marks (for Windows).
+func TestDeletedMarkers(t *testing.T) {
+	dir := t.TempDir()
+	ps := &peerAPIServer{
+		b: &LocalBackend{
+			logf:           t.Logf,
+			capFileSharing: true,
+		},
+		rootDir: dir,
+	}
+
+	nothingWaiting := func() {
+		t.Helper()
+		ps.knownEmpty.Set(false)
+		if ps.hasFilesWaiting() {
+			t.Fatal("unexpected files waiting")
+		}
+	}
+	touch := func(base string) {
+		t.Helper()
+		if err := touchFile(filepath.Join(dir, base)); err != nil {
+			t.Fatal(err)
+		}
+	}
+	wantEmptyTempDir := func() {
+		t.Helper()
+		if fis, err := ioutil.ReadDir(dir); err != nil {
+			t.Fatal(err)
+		} else if len(fis) > 0 && runtime.GOOS != "windows" {
+			for _, fi := range fis {
+				t.Errorf("unexpected file in tempdir: %q", fi.Name())
+			}
+		}
+	}
+
+	nothingWaiting()
+	wantEmptyTempDir()
+
+	touch("foo.jpg.deleted")
+	nothingWaiting()
+	wantEmptyTempDir()
+
+	touch("foo.jpg.deleted")
+	touch("foo.jpg")
+	nothingWaiting()
+	wantEmptyTempDir()
+
+	touch("foo.jpg.deleted")
+	touch("foo.jpg")
+	wf, err := ps.WaitingFiles()
+	if err != nil {
+		t.Fatal(err)
+	}
+	if len(wf) != 0 {
+		t.Fatalf("WaitingFiles = %d; want 0", len(wf))
+	}
+	wantEmptyTempDir()
+
+	touch("foo.jpg.deleted")
+	touch("foo.jpg")
+	if rc, _, err := ps.OpenFile("foo.jpg"); err == nil {
+		rc.Close()
+		t.Fatal("unexpected foo.jpg open")
+	}
+	wantEmptyTempDir()
+
+	// And verify basics still work in non-deleted cases.
+	touch("foo.jpg")
+	touch("bar.jpg.deleted")
+	if wf, err := ps.WaitingFiles(); err != nil {
+		t.Error(err)
+	} else if len(wf) != 1 {
+		t.Errorf("WaitingFiles = %d; want 1", len(wf))
+	} else if wf[0].Name != "foo.jpg" {
+		t.Errorf("unexpected waiting file %+v", wf[0])
+	}
+	if rc, _, err := ps.OpenFile("foo.jpg"); err != nil {
+		t.Fatal(err)
+	} else {
+		rc.Close()
+	}
+
+}

ipn/ipnlocal/state_test.go

@@ -0,0 +1,812 @@
+// Copyright (c) 2021 Tailscale Inc & AUTHORS All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package ipnlocal
+
+import (
+	"context"
+	"sync"
+	"testing"
+	"time"
+
+	qt "github.com/frankban/quicktest"
+
+	"tailscale.com/control/controlclient"
+	"tailscale.com/ipn"
+	"tailscale.com/tailcfg"
+	"tailscale.com/types/empty"
+	"tailscale.com/types/logger"
+	"tailscale.com/types/netmap"
+	"tailscale.com/types/persist"
+	"tailscale.com/types/wgkey"
+	"tailscale.com/wgengine"
+)
+
+// notifyThrottler receives notifications from an ipn.Backend, blocking
+// (with eventual timeout and t.Fatal) if there are too many and complaining
+// (also with t.Fatal) if they are too few.
+type notifyThrottler struct {
+	t *testing.T
+
+	// ch gets replaced frequently. Lock the mutex before getting or
+	// setting it, but not while waiting on it.
+	mu sync.Mutex
+	ch chan ipn.Notify
+}
+
+// expect tells the throttler to expect count upcoming notifications.
+func (nt *notifyThrottler) expect(count int) {
+	nt.mu.Lock()
+	nt.ch = make(chan ipn.Notify, count)
+	nt.mu.Unlock()
+}
+
+// put adds one notification into the throttler's queue.
+func (nt *notifyThrottler) put(n ipn.Notify) {
+	nt.mu.Lock()
+	ch := nt.ch
+	nt.mu.Unlock()
+
+	select {
+	case ch <- n:
+		return
+	default:
+		nt.t.Fatalf("put: channel full: %v", n)
+	}
+}
+
+// drain pulls the notifications out of the queue, asserting that there are
+// exactly count notifications that have been put so far.
+func (nt *notifyThrottler) drain(count int) []ipn.Notify {
+	nt.mu.Lock()
+	ch := nt.ch
+	nt.mu.Unlock()
+
+	nn := []ipn.Notify{}
+	for i := 0; i < count; i++ {
+		select {
+		case n := <-ch:
+			nn = append(nn, n)
+		case <-time.After(6 * time.Second):
+			nt.t.Fatalf("drain: channel empty after %d/%d", i, count)
+		}
+	}
+
+	// no more notifications expected
+	close(ch)
+
+	return nn
+}
+
+// mockControl is a mock implementation of controlclient.Client.
+// Much of the backend state machine depends on callbacks and state
+// in the controlclient.Client, so by controlling it, we can check that
+// the state machine works as expected.
+type mockControl struct {
+	opts       controlclient.Options
+	logf       logger.Logf
+	statusFunc func(controlclient.Status)
+
+	mu          sync.Mutex
+	calls       []string
+	authBlocked bool
+	persist     persist.Persist
+	machineKey  wgkey.Private
+}
+
+func newMockControl() *mockControl {
+	return &mockControl{
+		calls:       []string{},
+		authBlocked: true,
+	}
+}
+
+func (cc *mockControl) SetStatusFunc(fn func(controlclient.Status)) {
+	cc.statusFunc = fn
+}
+
+func (cc *mockControl) populateKeys() (newKeys bool) {
+	cc.mu.Lock()
+	defer cc.mu.Unlock()
+
+	if cc.machineKey.IsZero() {
+		cc.logf("Copying machineKey.")
+		cc.machineKey, _ = cc.opts.GetMachinePrivateKey()
+		newKeys = true
+	}
+
+	if cc.persist.PrivateNodeKey.IsZero() {
+		cc.logf("Generating a new nodekey.")
+		cc.persist.OldPrivateNodeKey = cc.persist.PrivateNodeKey
+		cc.persist.PrivateNodeKey, _ = wgkey.NewPrivate()
+		newKeys = true
+	}
+
+	return newKeys
+}
+
+// send publishes a controlclient.Status notification upstream.
+// (In our tests here, upstream is the ipnlocal.Local instance.)
+func (cc *mockControl) send(err error, url string, loginFinished bool, nm *netmap.NetworkMap) {
+	if cc.statusFunc != nil {
+		s := controlclient.Status{
+			URL:     url,
+			NetMap:  nm,
+			Persist: &cc.persist,
+		}
+		if err != nil {
+			s.Err = err.Error()
+		}
+		if loginFinished {
+			s.LoginFinished = &empty.Message{}
+		}
+		cc.statusFunc(s)
+	}
+}
+
+// called records that a particular function name was called.
+func (cc *mockControl) called(s string) {
+	cc.mu.Lock()
+	defer cc.mu.Unlock()
+
+	cc.calls = append(cc.calls, s)
+}
+
+// getCalls returns the list of functions that have been called since the
+// last time getCalls was run.
+func (cc *mockControl) getCalls() []string {
+	cc.mu.Lock()
+	defer cc.mu.Unlock()
+
+	r := cc.calls
+	cc.calls = []string{}
+	return r
+}
+
+// setAuthBlocked changes the return value of AuthCantContinue.
+// Auth is blocked if you haven't called Login, the control server hasn't
+// provided an auth URL, or it has provided an auth URL and you haven't
+// visited it yet.
+func (cc *mockControl) setAuthBlocked(blocked bool) {
+	cc.mu.Lock()
+	defer cc.mu.Unlock()
+
+	cc.authBlocked = blocked
+}
+
+// Shutdown disconnects the client.
+//
+// Note that in a normal controlclient, Shutdown would be the last thing you
+// do before discarding the object. In this mock, we don't actually discard
+// the object, but if you see a call to Shutdown, you should always see a
+// call to New right after it, if the object continues to be used.
+// (Note that "New" is the ccGen function here; it means ipn.Backend wanted
+// to create an entirely new controlclient.)
+func (cc *mockControl) Shutdown() {
+	cc.logf("Shutdown")
+	cc.called("Shutdown")
+}
+
+// Login starts a login process.
+// Note that in this mock, we don't automatically generate notifications
+// about the progress of the login operation. You have to call setAuthBlocked()
+// and send() as required by the test.
+func (cc *mockControl) Login(t *tailcfg.Oauth2Token, flags controlclient.LoginFlags) {
+	cc.logf("Login token=%v flags=%v", t, flags)
+	cc.called("Login")
+	newKeys := cc.populateKeys()
+
+	interact := (flags & controlclient.LoginInteractive) != 0
+	cc.logf("Login: interact=%v newKeys=%v", interact, newKeys)
+	cc.setAuthBlocked(interact || newKeys)
+}
+
+func (cc *mockControl) StartLogout() {
+	cc.logf("StartLogout")
+	cc.called("StartLogout")
+}
+
+func (cc *mockControl) Logout(ctx context.Context) error {
+	cc.logf("Logout")
+	cc.called("Logout")
+	return nil
+}
+
+func (cc *mockControl) SetPaused(paused bool) {
+	cc.logf("SetPaused=%v", paused)
+	if paused {
+		cc.called("pause")
+	} else {
+		cc.called("unpause")
+	}
+}
+
+func (cc *mockControl) AuthCantContinue() bool {
+	cc.mu.Lock()
+	defer cc.mu.Unlock()
+
+	return cc.authBlocked
+}
+
+func (cc *mockControl) SetHostinfo(hi *tailcfg.Hostinfo) {
+	cc.logf("SetHostinfo: %v", *hi)
+	cc.called("SetHostinfo")
+}
+
+func (cc *mockControl) SetNetInfo(ni *tailcfg.NetInfo) {
+	cc.called("SetNetinfo")
+	cc.logf("SetNetInfo: %v", *ni)
+	cc.called("SetNetInfo")
+}
+
+func (cc *mockControl) UpdateEndpoints(localPort uint16, endpoints []tailcfg.Endpoint) {
+	// validate endpoint information here?
+	cc.logf("UpdateEndpoints: lp=%v ep=%v", localPort, endpoints)
+	cc.called("UpdateEndpoints")
+}
+
+// A very precise test of the sequence of function calls generated by
+// ipnlocal.Local into its controlclient instance, and the events it
+// produces upstream into the UI.
+//
+// [apenwarr] Normally I'm not a fan of "mock" style tests, but the precise
+// sequence of this state machine is so important for writing our multiple
+// frontends, that it's worth validating it all in one place.
+//
+// Any changes that affect this test will most likely require carefully
+// re-testing all our GUIs (and the CLI) to make sure we didn't break
+// anything.
+//
+// Note also that this test doesn't have any timers, goroutines, or duplicate
+// detection. It expects messages to be produced in exactly the right order,
+// with no duplicates, without doing network activity (other than through
+// controlclient, which we fake, so there's no network activity there either).
+//
+// TODO: A few messages that depend on magicsock (which actually might have
+// network delays) are just ignored for now, which makes the test
+// predictable, but maybe a bit less thorough. This is more of an overall
+// state machine test than a test of the wgengine+magicsock integration.
+func TestStateMachine(t *testing.T) {
+	c := qt.New(t)
+
+	logf := t.Logf
+	store := new(ipn.MemoryStore)
+	e, err := wgengine.NewFakeUserspaceEngine(logf, 0)
+	if err != nil {
+		t.Fatalf("NewFakeUserspaceEngine: %v", err)
+	}
+
+	cc := newMockControl()
+	b, err := NewLocalBackend(logf, "logid", store, e)
+	if err != nil {
+		t.Fatalf("NewLocalBackend: %v", err)
+	}
+	b.SetControlClientGetterForTesting(func(opts controlclient.Options) (controlclient.Client, error) {
+		cc.mu.Lock()
+		cc.opts = opts
+		cc.logf = opts.Logf
+		cc.authBlocked = true
+		cc.persist = cc.opts.Persist
+		cc.mu.Unlock()
+
+		cc.logf("ccGen: new mockControl.")
+		cc.called("New")
+		return cc, nil
+	})
+
+	notifies := &notifyThrottler{t: t}
+	notifies.expect(0)
+
+	b.SetNotifyCallback(func(n ipn.Notify) {
+		if n.State != nil ||
+			n.Prefs != nil ||
+			n.BrowseToURL != nil ||
+			n.LoginFinished != nil {
+			logf("\n%v\n\n", n)
+			notifies.put(n)
+		} else {
+			logf("\n(ignored) %v\n\n", n)
+		}
+	})
+
+	// Check that it hasn't called us right away.
+	// The state machine should be idle until we call Start().
+	c.Assert(cc.getCalls(), qt.HasLen, 0)
+
+	// Start the state machine.
+	// Since !WantRunning by default, it'll create a controlclient,
+	// but not ask it to do anything yet.
+	t.Logf("\n\nStart")
+	notifies.expect(2)
+	c.Assert(b.Start(ipn.Options{StateKey: ipn.GlobalDaemonStateKey}), qt.IsNil)
+	{
+		// BUG: strictly, it should pause, not unpause, here, since !WantRunning.
+		c.Assert([]string{"New", "unpause"}, qt.DeepEquals, cc.getCalls())
+
+		nn := notifies.drain(2)
+		c.Assert(cc.getCalls(), qt.HasLen, 0)
+		c.Assert(nn[0].Prefs, qt.Not(qt.IsNil))
+		c.Assert(nn[1].State, qt.Not(qt.IsNil))
+		prefs := *nn[0].Prefs
+		// Note: a totally fresh system has Prefs.LoggedOut=false by
+		// default. We are logged out, but not because the user asked
+		// for it, so it doesn't count as Prefs.LoggedOut==true.
+		c.Assert(nn[0].Prefs.LoggedOut, qt.IsFalse)
+		c.Assert(prefs.WantRunning, qt.IsFalse)
+		c.Assert(ipn.NeedsLogin, qt.Equals, *nn[1].State)
+		c.Assert(ipn.NeedsLogin, qt.Equals, b.State())
+	}
+
+	// Restart the state machine.
+	// It's designed to handle frontends coming and going sporadically.
+	// Make the sure the restart not only works, but generates the same
+	// events as the first time, so UIs always know what to expect.
+	t.Logf("\n\nStart2")
+	notifies.expect(2)
+	c.Assert(b.Start(ipn.Options{StateKey: ipn.GlobalDaemonStateKey}), qt.IsNil)
+	{
+		// BUG: strictly, it should pause, not unpause, here, since !WantRunning.
+		c.Assert([]string{"Shutdown", "New", "unpause"}, qt.DeepEquals, cc.getCalls())
+
+		nn := notifies.drain(2)
+		c.Assert(cc.getCalls(), qt.HasLen, 0)
+		c.Assert(nn[0].Prefs, qt.Not(qt.IsNil))
+		c.Assert(nn[1].State, qt.Not(qt.IsNil))
+		c.Assert(nn[0].Prefs.LoggedOut, qt.IsFalse)
+		c.Assert(nn[0].Prefs.WantRunning, qt.IsFalse)
+		c.Assert(ipn.NeedsLogin, qt.Equals, *nn[1].State)
+		c.Assert(ipn.NeedsLogin, qt.Equals, b.State())
+	}
+
+	// Start non-interactive login with no token.
+	// This will ask controlclient to start its own Login() process,
+	// then wait for us to respond.
+	t.Logf("\n\nLogin (noninteractive)")
+	notifies.expect(0)
+	b.Login(nil)
+	{
+		c.Assert(cc.getCalls(), qt.DeepEquals, []string{"Login"})
+		notifies.drain(0)
+		// BUG: this should immediately set WantRunning to true.
+		// Users don't log in if they don't want to also connect.
+		// (Generally, we're inconsistent about who is supposed to
+		// update Prefs at what time. But the overall philosophy is:
+		// update it when the user's intent changes. This is clearly
+		// at the time the user *requests* Login, not at the time
+		// the login finishes.)
+	}
+
+	// Attempted non-interactive login with no key; indicate that
+	// the user needs to visit a login URL.
+	t.Logf("\n\nLogin (url response)")
+	notifies.expect(1)
+	url1 := "http://localhost:1/1"
+	cc.send(nil, url1, false, nil)
+	{
+		c.Assert(cc.getCalls(), qt.HasLen, 0)
+
+		// ...but backend eats that notification, because the user
+		// didn't explicitly request interactive login yet, and
+		// we're already in NeedsLogin state.
+		nn := notifies.drain(1)
+
+		// Trying to log in automatically sets WantRunning.
+		// BUG: that should have happened right after Login().
+		c.Assert(nn[0].Prefs, qt.Not(qt.IsNil))
+		c.Assert(nn[0].Prefs.LoggedOut, qt.IsFalse)
+		c.Assert(nn[0].Prefs.WantRunning, qt.IsTrue)
+	}
+
+	// Now we'll try an interactive login.
+	// Since we provided an interactive URL earlier, this shouldn't
+	// ask control to do anything. Instead backend will emit an event
+	// indicating that the UI should browse to the given URL.
+	t.Logf("\n\nLogin (interactive)")
+	notifies.expect(1)
+	b.StartLoginInteractive()
+	{
+		nn := notifies.drain(1)
+		// BUG: UpdateEndpoints shouldn't be called yet.
+		// We're still not logged in so there's nothing we can do
+		// with it. (And empirically, it's providing an empty list
+		// of endpoints.)
+		c.Assert([]string{"UpdateEndpoints"}, qt.DeepEquals, cc.getCalls())
+		c.Assert(nn[0].BrowseToURL, qt.Not(qt.IsNil))
+		c.Assert(url1, qt.Equals, *nn[0].BrowseToURL)
+	}
+
+	// Sometimes users press the Login button again, in the middle of
+	// a login sequence. For example, they might have closed their
+	// browser window without logging in, or they waited too long and
+	// the login URL expired. If they start another interactive login,
+	// we must always get a *new* login URL first.
+	t.Logf("\n\nLogin2 (interactive)")
+	notifies.expect(0)
+	b.StartLoginInteractive()
+	{
+		notifies.drain(0)
+		// backend asks control for another login sequence
+		c.Assert([]string{"Login"}, qt.DeepEquals, cc.getCalls())
+	}
+
+	// Provide a new interactive login URL.
+	t.Logf("\n\nLogin2 (url response)")
+	notifies.expect(1)
+	url2 := "http://localhost:1/2"
+	cc.send(nil, url2, false, nil)
+	{
+		// BUG: UpdateEndpoints again, this is getting silly.
+		c.Assert([]string{"UpdateEndpoints"}, qt.DeepEquals, cc.getCalls())
+
+		// This time, backend should emit it to the UI right away,
+		// because the UI is anxiously awaiting a new URL to visit.
+		nn := notifies.drain(1)
+		c.Assert(nn[0].BrowseToURL, qt.Not(qt.IsNil))
+		c.Assert(url2, qt.Equals, *nn[0].BrowseToURL)
+	}
+
+	// Pretend that the interactive login actually happened.
+	// Controlclient always sends the netmap and LoginFinished at the
+	// same time.
+	// The backend should propagate this upward for the UI.
+	t.Logf("\n\nLoginFinished")
+	notifies.expect(2)
+	cc.setAuthBlocked(false)
+	cc.send(nil, "", true, &netmap.NetworkMap{})
+	{
+		nn := notifies.drain(2)
+		// BUG: still too soon for UpdateEndpoints.
+		//
+		// Arguably it makes sense to unpause now, since the machine
+		// authorization status is part of the netmap.
+		//
+		// BUG: backend unblocks wgengine at this point, even though
+		// our machine key is not authorized. It probably should
+		// wait until it gets into Starting.
+		// TODO: (Currently this test doesn't detect that bug, but
+		// it's visible in the logs)
+		c.Assert([]string{"unpause", "UpdateEndpoints"}, qt.DeepEquals, cc.getCalls())
+		c.Assert(nn[0].LoginFinished, qt.Not(qt.IsNil))
+		c.Assert(nn[1].State, qt.Not(qt.IsNil))
+		c.Assert(ipn.NeedsMachineAuth, qt.Equals, *nn[1].State)
+	}
+
+	// TODO: check that the logged-in username propagates from control
+	// through to the UI notifications. I think it's used as a hint
+	// for future logins, to pre-fill the username box? Not really sure
+	// how it works.
+
+	// Pretend that the administrator has authorized our machine.
+	t.Logf("\n\nMachineAuthorized")
+	notifies.expect(1)
+	// BUG: the real controlclient sends LoginFinished with every
+	// notification while it's in StateAuthenticated, but not StateSynced.
+	// We should send it exactly once, or every time we're authenticated,
+	// but the current code is brittle.
+	// (ie. I suspect it would be better to change false->true in send()
+	// below, and do the same in the real controlclient.)
+	cc.send(nil, "", false, &netmap.NetworkMap{
+		MachineStatus: tailcfg.MachineAuthorized,
+	})
+	{
+		nn := notifies.drain(1)
+		c.Assert([]string{"unpause", "UpdateEndpoints"}, qt.DeepEquals, cc.getCalls())
+		c.Assert(nn[0].State, qt.Not(qt.IsNil))
+		c.Assert(ipn.Starting, qt.Equals, *nn[0].State)
+	}
+
+	// TODO: add a fake DERP server to our fake netmap, so we can
+	// transition to the Running state here.
+
+	// TODO: test what happens when the admin forcibly deletes our key.
+	// (ie. unsolicited logout)
+
+	// TODO: test what happens when our key expires, client side.
+	// (and when it gets close to expiring)
+
+	// The user changes their preference to !WantRunning.
+	t.Logf("\n\nWantRunning -> false")
+	notifies.expect(2)
+	b.EditPrefs(&ipn.MaskedPrefs{
+		WantRunningSet: true,
+		Prefs:          ipn.Prefs{WantRunning: false},
+	})
+	{
+		nn := notifies.drain(2)
+		c.Assert([]string{"pause"}, qt.DeepEquals, cc.getCalls())
+		// BUG: I would expect Prefs to change first, and state after.
+		c.Assert(nn[0].State, qt.Not(qt.IsNil))
+		c.Assert(nn[1].Prefs, qt.Not(qt.IsNil))
+		c.Assert(ipn.Stopped, qt.Equals, *nn[0].State)
+	}
+
+	// The user changes their preference to WantRunning after all.
+	t.Logf("\n\nWantRunning -> true")
+	notifies.expect(2)
+	b.EditPrefs(&ipn.MaskedPrefs{
+		WantRunningSet: true,
+		Prefs:          ipn.Prefs{WantRunning: true},
+	})
+	{
+		nn := notifies.drain(2)
+		// BUG: UpdateEndpoints isn't needed here.
+		// BUG: Login isn't needed here. We never logged out.
+		c.Assert([]string{"Login", "unpause", "UpdateEndpoints"}, qt.DeepEquals, cc.getCalls())
+		// BUG: I would expect Prefs to change first, and state after.
+		c.Assert(nn[0].State, qt.Not(qt.IsNil))
+		c.Assert(nn[1].Prefs, qt.Not(qt.IsNil))
+		c.Assert(ipn.Starting, qt.Equals, *nn[0].State)
+	}
+
+	// Test the fast-path frontend reconnection.
+	// This one is very finicky, so we have to force State==Running.
+	// TODO: actually get to State==Running, rather than cheating.
+	//  That'll require spinning up a fake DERP server and putting it in
+	//  the netmap.
+	t.Logf("\n\nFastpath Start()")
+	notifies.expect(1)
+	b.state = ipn.Running
+	c.Assert(b.Start(ipn.Options{StateKey: ipn.GlobalDaemonStateKey}), qt.IsNil)
+	{
+		nn := notifies.drain(1)
+		c.Assert(cc.getCalls(), qt.HasLen, 0)
+		c.Assert(nn[0].State, qt.Not(qt.IsNil))
+		c.Assert(nn[0].LoginFinished, qt.Not(qt.IsNil))
+		c.Assert(nn[0].NetMap, qt.Not(qt.IsNil))
+		// BUG: Prefs should be sent too, or the UI could end up in
+		// a bad state. (iOS, the only current user of this feature,
+		// probably wouldn't notice because it happens to not display
+		// any prefs. Maybe exit nodes will look weird?)
+	}
+
+	// undo the state hack above.
+	b.state = ipn.Starting
+
+	// User wants to logout.
+	t.Logf("\n\nLogout (async)")
+	notifies.expect(2)
+	b.Logout()
+	{
+		nn := notifies.drain(2)
+		// BUG: now is not the time to unpause.
+		c.Assert([]string{"unpause", "StartLogout"}, qt.DeepEquals, cc.getCalls())
+		c.Assert(nn[0].State, qt.Not(qt.IsNil))
+		c.Assert(nn[1].Prefs, qt.Not(qt.IsNil))
+		c.Assert(ipn.NeedsLogin, qt.Equals, *nn[0].State)
+		c.Assert(nn[1].Prefs.LoggedOut, qt.IsTrue)
+		c.Assert(nn[1].Prefs.WantRunning, qt.IsFalse)
+		c.Assert(ipn.NeedsLogin, qt.Equals, b.State())
+	}
+
+	// Let's make the logout succeed.
+	t.Logf("\n\nLogout (async) - succeed")
+	notifies.expect(1)
+	cc.setAuthBlocked(true)
+	cc.send(nil, "", false, nil)
+	{
+		nn := notifies.drain(1)
+		c.Assert(cc.getCalls(), qt.HasLen, 0)
+		c.Assert(nn[0].Prefs, qt.Not(qt.IsNil))
+		c.Assert(nn[0].Prefs.LoggedOut, qt.IsTrue)
+		// BUG: WantRunning should be false after manual logout.
+		c.Assert(nn[0].Prefs.WantRunning, qt.IsTrue)
+		c.Assert(ipn.NeedsLogin, qt.Equals, b.State())
+	}
+
+	// A second logout should do nothing, since the prefs haven't changed.
+	t.Logf("\n\nLogout2 (async)")
+	notifies.expect(1)
+	b.Logout()
+	{
+		nn := notifies.drain(1)
+		// BUG: the backend has already called StartLogout, and we're
+		// still logged out. So it shouldn't call it again.
+		c.Assert([]string{"StartLogout"}, qt.DeepEquals, cc.getCalls())
+		// BUG: Prefs should not change here. Already logged out.
+		c.Assert(nn[0].Prefs, qt.Not(qt.IsNil))
+		c.Assert(nn[0].Prefs.LoggedOut, qt.IsTrue)
+		c.Assert(nn[0].Prefs.WantRunning, qt.IsFalse)
+		c.Assert(ipn.NeedsLogin, qt.Equals, b.State())
+	}
+
+	// Let's acknowledge the second logout too.
+	t.Logf("\n\nLogout2 (async) - succeed")
+	notifies.expect(1)
+	cc.setAuthBlocked(true)
+	cc.send(nil, "", false, nil)
+	{
+		nn := notifies.drain(1)
+		c.Assert(cc.getCalls(), qt.HasLen, 0)
+		c.Assert(nn[0].Prefs, qt.Not(qt.IsNil))
+		c.Assert(nn[0].Prefs.LoggedOut, qt.IsTrue)
+		// BUG: second logout shouldn't cause WantRunning->true !!
+		c.Assert(nn[0].Prefs.WantRunning, qt.IsTrue)
+		c.Assert(ipn.NeedsLogin, qt.Equals, b.State())
+	}
+
+	// Try the synchronous logout feature.
+	t.Logf("\n\nLogout3 (sync)")
+	notifies.expect(1)
+	b.LogoutSync(context.Background())
+	// NOTE: This returns as soon as cc.Logout() returns, which is okay
+	// I guess, since that's supposed to be synchronous.
+	{
+		nn := notifies.drain(1)
+		c.Assert([]string{"Logout"}, qt.DeepEquals, cc.getCalls())
+		c.Assert(nn[0].Prefs, qt.Not(qt.IsNil))
+		c.Assert(nn[0].Prefs.LoggedOut, qt.IsTrue)
+		c.Assert(nn[0].Prefs.WantRunning, qt.IsFalse)
+		c.Assert(ipn.NeedsLogin, qt.Equals, b.State())
+	}
+
+	// Generate the third logout event.
+	t.Logf("\n\nLogout3 (sync) - succeed")
+	notifies.expect(1)
+	cc.setAuthBlocked(true)
+	cc.send(nil, "", false, nil)
+	{
+		nn := notifies.drain(1)
+		c.Assert(cc.getCalls(), qt.HasLen, 0)
+		c.Assert(nn[0].Prefs, qt.Not(qt.IsNil))
+		c.Assert(nn[0].Prefs.LoggedOut, qt.IsTrue)
+		// BUG: third logout shouldn't cause WantRunning->true !!
+		c.Assert(nn[0].Prefs.WantRunning, qt.IsTrue)
+		c.Assert(ipn.NeedsLogin, qt.Equals, b.State())
+	}
+
+	// Shut down the backend.
+	t.Logf("\n\nShutdown")
+	notifies.expect(0)
+	b.Shutdown()
+	{
+		notifies.drain(0)
+		// BUG: I expect a transition to ipn.NoState here.
+		c.Assert(cc.getCalls(), qt.DeepEquals, []string{"Shutdown"})
+	}
+
+	// Oh, you thought we were done? Ha! Now we have to test what
+	// happens if the user exits and restarts while logged out.
+	// Note that it's explicitly okay to call b.Start() over and over
+	// again, every time the frontend reconnects.
+	//
+	// BUG: WantRunning is true here (because of the bug above).
+	//  We'll have to adjust the following test's expectations if we
+	//  fix that.
+
+	// TODO: test user switching between statekeys.
+
+	// The frontend restarts!
+	t.Logf("\n\nStart3")
+	notifies.expect(2)
+	c.Assert(b.Start(ipn.Options{StateKey: ipn.GlobalDaemonStateKey}), qt.IsNil)
+	{
+		// BUG: We already called Shutdown(), no need to do it again.
+		// BUG: Way too soon for UpdateEndpoints.
+		// BUG: don't unpause because we're not logged in.
+		c.Assert([]string{"Shutdown", "New", "UpdateEndpoints", "unpause"}, qt.DeepEquals, cc.getCalls())
+
+		nn := notifies.drain(2)
+		c.Assert(cc.getCalls(), qt.HasLen, 0)
+		c.Assert(nn[0].Prefs, qt.Not(qt.IsNil))
+		c.Assert(nn[1].State, qt.Not(qt.IsNil))
+		c.Assert(nn[0].Prefs.LoggedOut, qt.IsTrue)
+		c.Assert(nn[0].Prefs.WantRunning, qt.IsTrue)
+		c.Assert(ipn.NeedsLogin, qt.Equals, *nn[1].State)
+		c.Assert(ipn.NeedsLogin, qt.Equals, b.State())
+	}
+
+	// Let's break the rules a little. Our control server accepts
+	// your invalid login attempt, with no need for an interactive login.
+	// (This simulates an admin reviving a key that you previously
+	// disabled.)
+	t.Logf("\n\nLoginFinished3")
+	notifies.expect(3)
+	cc.setAuthBlocked(false)
+	cc.send(nil, "", true, &netmap.NetworkMap{
+		MachineStatus: tailcfg.MachineAuthorized,
+	})
+	{
+		nn := notifies.drain(3)
+		c.Assert([]string{"unpause"}, qt.DeepEquals, cc.getCalls())
+		c.Assert(nn[0].Prefs, qt.Not(qt.IsNil))
+		c.Assert(nn[1].LoginFinished, qt.Not(qt.IsNil))
+		c.Assert(nn[2].State, qt.Not(qt.IsNil))
+		c.Assert(nn[0].Prefs.LoggedOut, qt.IsFalse)
+		c.Assert(ipn.Starting, qt.Equals, *nn[2].State)
+	}
+
+	// Now we've logged in successfully. Let's disconnect.
+	t.Logf("\n\nWantRunning -> false")
+	notifies.expect(2)
+	b.EditPrefs(&ipn.MaskedPrefs{
+		WantRunningSet: true,
+		Prefs:          ipn.Prefs{WantRunning: false},
+	})
+	{
+		nn := notifies.drain(2)
+		c.Assert([]string{"pause"}, qt.DeepEquals, cc.getCalls())
+		// BUG: I would expect Prefs to change first, and state after.
+		c.Assert(nn[0].State, qt.Not(qt.IsNil))
+		c.Assert(nn[1].Prefs, qt.Not(qt.IsNil))
+		c.Assert(ipn.Stopped, qt.Equals, *nn[0].State)
+		c.Assert(nn[1].Prefs.LoggedOut, qt.IsFalse)
+	}
+
+	// One more restart, this time with a valid key, but WantRunning=false.
+	t.Logf("\n\nStart4")
+	notifies.expect(2)
+	c.Assert(b.Start(ipn.Options{StateKey: ipn.GlobalDaemonStateKey}), qt.IsNil)
+	{
+		// NOTE: cc.Shutdown() is correct here, since we didn't call
+		// b.Shutdown() explicitly ourselves.
+		// BUG: UpdateEndpoints should be called here since we're not WantRunning.
+		// Note: unpause happens because ipn needs to get at least one netmap
+		//  on startup, otherwise UIs can't show the node list, login
+		//  name, etc when in state ipn.Stopped.
+		//  Arguably they shouldn't try. But they currently do.
+		c.Assert([]string{"Shutdown", "New", "UpdateEndpoints", "Login", "unpause"}, qt.DeepEquals, cc.getCalls())
+
+		nn := notifies.drain(2)
+		c.Assert(cc.getCalls(), qt.HasLen, 0)
+		c.Assert(nn[0].Prefs, qt.Not(qt.IsNil))
+		c.Assert(nn[1].State, qt.Not(qt.IsNil))
+		c.Assert(nn[0].Prefs.WantRunning, qt.IsFalse)
+		c.Assert(nn[0].Prefs.LoggedOut, qt.IsFalse)
+		c.Assert(ipn.Stopped, qt.Equals, *nn[1].State)
+	}
+
+	// Request connection.
+	// The state machine didn't call Login() earlier, so now it needs to.
+	t.Logf("\n\nWantRunning4 -> true")
+	notifies.expect(2)
+	b.EditPrefs(&ipn.MaskedPrefs{
+		WantRunningSet: true,
+		Prefs:          ipn.Prefs{WantRunning: true},
+	})
+	{
+		nn := notifies.drain(2)
+		c.Assert([]string{"Login", "unpause"}, qt.DeepEquals, cc.getCalls())
+		// BUG: I would expect Prefs to change first, and state after.
+		c.Assert(nn[0].State, qt.Not(qt.IsNil))
+		c.Assert(nn[1].Prefs, qt.Not(qt.IsNil))
+		c.Assert(ipn.Starting, qt.Equals, *nn[0].State)
+	}
+
+	// The last test case is the most common one: restarting when both
+	// logged in and WantRunning.
+	t.Logf("\n\nStart5")
+	notifies.expect(1)
+	c.Assert(b.Start(ipn.Options{StateKey: ipn.GlobalDaemonStateKey}), qt.IsNil)
+	{
+		// NOTE: cc.Shutdown() is correct here, since we didn't call
+		// b.Shutdown() ourselves.
+		c.Assert([]string{"Shutdown", "New", "UpdateEndpoints", "Login"}, qt.DeepEquals, cc.getCalls())
+
+		nn := notifies.drain(1)
+		c.Assert(cc.getCalls(), qt.HasLen, 0)
+		c.Assert(nn[0].Prefs, qt.Not(qt.IsNil))
+		c.Assert(nn[0].Prefs.LoggedOut, qt.IsFalse)
+		c.Assert(nn[0].Prefs.WantRunning, qt.IsTrue)
+		c.Assert(ipn.NoState, qt.Equals, b.State())
+	}
+
+	// Control server accepts our valid key from before.
+	t.Logf("\n\nLoginFinished5")
+	notifies.expect(2)
+	cc.setAuthBlocked(false)
+	cc.send(nil, "", true, &netmap.NetworkMap{
+		MachineStatus: tailcfg.MachineAuthorized,
+	})
+	{
+		nn := notifies.drain(2)
+		c.Assert([]string{"unpause"}, qt.DeepEquals, cc.getCalls())
+		c.Assert(nn[0].LoginFinished, qt.Not(qt.IsNil))
+		c.Assert(nn[1].State, qt.Not(qt.IsNil))
+		c.Assert(ipn.Starting, qt.Equals, *nn[1].State)
+		// NOTE: No prefs change this time. WantRunning stays true.
+		// We were in Starting in the first place, so that doesn't
+		// change either.
+		c.Assert(ipn.Starting, qt.Equals, b.State())
+	}
+}

ipn/message.go

@@ -93,17 +93,27 @@ type BackendServer struct {
 	GotQuit       bool                 // a Quit command was received
 }
 
+// NewBackendServer creates a new BackendServer using b.
+//
+// If sendNotifyMsg is non-nil, it additionally sets the Backend's
+// notification callback to call the func with ipn.Notify messages in
+// JSON form. If nil, it does not change the notification callback.
 func NewBackendServer(logf logger.Logf, b Backend, sendNotifyMsg func(b []byte)) *BackendServer {
 	bs := &BackendServer{
 		logf:          logf,
 		b:             b,
 		sendNotifyMsg: sendNotifyMsg,
 	}
-	b.SetNotifyCallback(bs.send)
+	if sendNotifyMsg != nil {
+		b.SetNotifyCallback(bs.send)
+	}
 	return bs
 }
 
 func (bs *BackendServer) send(n Notify) {
+	if bs.sendNotifyMsg == nil {
+		return
+	}
 	n.Version = version.Long
 	b, err := json.Marshal(n)
 	if err != nil {

ipn/message_test.go

@@ -87,6 +87,8 @@ func TestClientServer(t *testing.T) {
 		t.Logf("c: "+fmt, args...)
 	}
 	bs = NewBackendServer(slogf, b, serverToClient)
+	// Verify that this doesn't break bs's callback:
+	NewBackendServer(slogf, b, nil)
 	bc = NewBackendClient(clogf, clientToServer)
 
 	ch := make(chan Notify, 256)

ipn/prefs.go

@@ -37,6 +37,15 @@ type Prefs struct {
 	// If empty, the default for new installs, DefaultControlURL
 	// is used. It's set non-empty once the daemon has been started
 	// for the first time.
+	//
+	// TODO(apenwarr): Make it safe to update this with SetPrefs().
+	// Right now, you have to pass it in the initial prefs in Start(),
+	// which is the only code that actually uses the ControlURL value.
+	// It would be more consistent to restart controlclient
+	// automatically whenever this variable changes.
+	//
+	// Meanwhile, you have to provide this as part of Options.Prefs or
+	// Options.UpdatePrefs when calling Backend.Start().
 	ControlURL string
 
 	// RouteAll specifies whether to accept subnets advertised by
@@ -87,6 +96,14 @@ type Prefs struct {
 	// this node.
 	WantRunning bool
 
+	// LoggedOut indicates whether the user intends to be logged out.
+	// There are other reasons we may be logged out, including no valid
+	// keys.
+	// We need to remember this state so that, on next startup, we can
+	// generate the "Login" vs "Connect" buttons correctly, without having
+	// to contact the server to confirm our nodekey status first.
+	LoggedOut bool
+
 	// ShieldsUp indicates whether to block all incoming connections,
 	// regardless of the control-provided packet filter. If false, we
 	// use the packet filter as provided. If true, we block incoming
@@ -177,6 +194,7 @@ type MaskedPrefs struct {
 	ExitNodeAllowLANAccessSet bool `json:",omitempty"`
 	CorpDNSSet                bool `json:",omitempty"`
 	WantRunningSet            bool `json:",omitempty"`
+	LoggedOutSet              bool `json:",omitempty"`
 	ShieldsUpSet              bool `json:",omitempty"`
 	AdvertiseTagsSet          bool `json:",omitempty"`
 	HostnameSet               bool `json:",omitempty"`
@@ -246,6 +264,9 @@ func (p *Prefs) pretty(goos string) string {
 		sb.WriteString("mesh=false ")
 	}
 	fmt.Fprintf(&sb, "dns=%v want=%v ", p.CorpDNS, p.WantRunning)
+	if p.LoggedOut {
+		sb.WriteString("loggedout=true ")
+	}
 	if p.ForceDaemon {
 		sb.WriteString("server=true ")
 	}
@@ -315,6 +336,7 @@ func (p *Prefs) Equals(p2 *Prefs) bool {
 		p.ExitNodeAllowLANAccess == p2.ExitNodeAllowLANAccess &&
 		p.CorpDNS == p2.CorpDNS &&
 		p.WantRunning == p2.WantRunning &&
+		p.LoggedOut == p2.LoggedOut &&
 		p.NotepadURLs == p2.NotepadURLs &&
 		p.ShieldsUp == p2.ShieldsUp &&
 		p.NoSNAT == p2.NoSNAT &&

ipn/prefs_clone.go

@@ -41,6 +41,7 @@ var _PrefsNeedsRegeneration = Prefs(struct {
 	ExitNodeAllowLANAccess bool
 	CorpDNS                bool
 	WantRunning            bool
+	LoggedOut              bool
 	ShieldsUp              bool
 	AdvertiseTags          []string
 	Hostname               string

ipn/prefs_test.go

@@ -42,6 +42,7 @@ func TestPrefsEqual(t *testing.T) {
 		"ExitNodeAllowLANAccess",
 		"CorpDNS",
 		"WantRunning",
+		"LoggedOut",
 		"ShieldsUp",
 		"AdvertiseTags",
 		"Hostname",

logtail/filch/filch.go

@@ -15,6 +15,7 @@ import (
 	"sync"
 )
 
+//lint:ignore U1000 work around false positive: https://github.com/dominikh/go-tools/issues/983
 var stderrFD = 2 // a variable for testing
 
 type Options struct {

net/dns/debian_resolvconf.go

@@ -2,6 +2,8 @@
 // Use of this source code is governed by a BSD-style
 // license that can be found in the LICENSE file.
 
+// +build linux freebsd openbsd
+
 package dns
 
 import (

net/dns/direct.go

@@ -2,6 +2,8 @@
 // Use of this source code is governed by a BSD-style
 // license that can be found in the LICENSE file.
 
+// +build linux freebsd openbsd
+
 package dns
 
 import (

net/dns/manager.go

@@ -5,6 +5,7 @@
 package dns
 
 import (
+	"runtime"
 	"strings"
 	"time"
 
@@ -118,7 +119,27 @@ func (m *Manager) compileConfig(cfg Config) (resolver.Config, OSConfig, error) {
 	var rcfg resolver.Config
 	var ocfg OSConfig
 
-	if !cfg.hasHosts() && cfg.singleResolverSet() != nil && m.os.SupportsSplitDNS() {
+	// Workaround for
+	// https://github.com/tailscale/corp/issues/1662. Even though
+	// Windows natively supports split DNS, it only configures linux
+	// containers using whatever the primary is, and doesn't apply
+	// NRPT rules to DNS traffic coming from WSL.
+	//
+	// In order to make WSL work okay when the host Windows is using
+	// Tailscale, we need to set up quad-100 as a "full proxy"
+	// resolver, regardless of whether Windows itself can do split
+	// DNS. We still make Windows do split DNS itself when it can, but
+	// quad-100 will still have the full split configuration as well,
+	// and so can service WSL requests correctly.
+	//
+	// This bool is used in a couple of places below to implement this
+	// workaround.
+	isWindows := runtime.GOOS == "windows"
+
+	// The windows check is for
+	// . See also below
+	// for further routing workarounds there.
+	if !cfg.hasHosts() && cfg.singleResolverSet() != nil && m.os.SupportsSplitDNS() && !isWindows {
 		// Split DNS configuration requested, where all split domains
 		// go to the same resolvers. We can let the OS do it.
 		return resolver.Config{}, OSConfig{
@@ -148,7 +169,8 @@ func (m *Manager) compileConfig(cfg Config) (resolver.Config, OSConfig, error) {
 	// resolver config and blend it into our config.
 	if m.os.SupportsSplitDNS() {
 		ocfg.MatchDomains = cfg.matchDomains()
-	} else {
+	}
+	if !m.os.SupportsSplitDNS() || isWindows {
 		bcfg, err := m.os.GetBaseConfig()
 		if err != nil {
 			// Temporary hack to make OSes where split-DNS isn't fully

net/dns/manager_linux.go

@@ -16,6 +16,7 @@ import (
 
 	"github.com/godbus/dbus/v5"
 	"tailscale.com/types/logger"
+	"tailscale.com/util/cmpver"
 )
 
 type kv struct {
@@ -64,18 +65,54 @@ func NewOSConfigurator(logf logger.Logf, interfaceName string) (ret OSConfigurat
 			return newResolvedManager(logf)
 		}
 		dbg("nm-resolved", "yes")
-		return newNMManager(interfaceName)
+
+		// Version of NetworkManager before 1.26.6 programmed resolved
+		// incorrectly, such that NM's settings would always take
+		// precedence over other settings set by other resolved
+		// clients.
+		//
+		// If we're dealing with such a version, we have to set our
+		// DNS settings through NM to have them take.
+		//
+		// However, versions 1.26.6 later both fixed the resolved
+		// programming issue _and_ started ignoring DNS settings for
+		// "unmanaged" interfaces - meaning NM 1.26.6 and later
+		// actively ignore DNS configuration we give it. So, for those
+		// NM versions, we can and must use resolved directly.
+		old, err := nmVersionOlderThan("1.26.6")
+		if err != nil {
+			// Failed to figure out NM's version, can't make a correct
+			// decision.
+			return nil, fmt.Errorf("checking NetworkManager version: %v", err)
+		}
+		if old {
+			dbg("nm-old", "yes")
+			return newNMManager(interfaceName)
+		}
+		dbg("nm-old", "no")
+		return newResolvedManager(logf)
 	case "resolvconf":
 		dbg("rc", "resolvconf")
 		if err := resolvconfSourceIsNM(bs); err == nil {
 			dbg("src-is-nm", "yes")
 			if err := dbusPing("org.freedesktop.NetworkManager", "/org/freedesktop/NetworkManager/DnsManager"); err == nil {
 				dbg("nm", "yes")
-				return newNMManager(interfaceName)
+				old, err := nmVersionOlderThan("1.26.6")
+				if err != nil {
+					return nil, fmt.Errorf("checking NetworkManager version: %v", err)
+				}
+				if old {
+					dbg("nm-old", "yes")
+					return newNMManager(interfaceName)
+				} else {
+					dbg("nm-old", "no")
+				}
+			} else {
+				dbg("nm", "no")
 			}
-			dbg("nm", "no")
+		} else {
+			dbg("src-is-nm", "no")
 		}
-		dbg("src-is-nm", "no")
 		if _, err := exec.LookPath("resolvconf"); err != nil {
 			dbg("resolvconf", "no")
 			return newDirectManager()
@@ -89,7 +126,16 @@ func NewOSConfigurator(logf logger.Logf, interfaceName string) (ret OSConfigurat
 			return newDirectManager()
 		}
 		dbg("nm", "yes")
-		return newNMManager(interfaceName)
+		old, err := nmVersionOlderThan("1.26.6")
+		if err != nil {
+			return nil, fmt.Errorf("checking NetworkManager version: %v", err)
+		}
+		if old {
+			dbg("nm-old", "yes")
+			return newNMManager(interfaceName)
+		}
+		dbg("nm-old", "no")
+		return newDirectManager()
 	default:
 		dbg("rc", "unknown")
 		return newDirectManager()
@@ -135,6 +181,27 @@ func resolvconfSourceIsNM(resolvDotConf []byte) error {
 	return nil
 }
 
+func nmVersionOlderThan(want string) (bool, error) {
+	conn, err := dbus.SystemBus()
+	if err != nil {
+		// DBus probably not running.
+		return false, err
+	}
+
+	nm := conn.Object("org.freedesktop.NetworkManager", dbus.ObjectPath("/org/freedesktop/NetworkManager"))
+	v, err := nm.GetProperty("org.freedesktop.NetworkManager.Version")
+	if err != nil {
+		return false, err
+	}
+
+	version, ok := v.Value().(string)
+	if !ok {
+		return false, fmt.Errorf("unexpected type %T for NM version", v.Value())
+	}
+
+	return cmpver.Compare(version, want) < 0, nil
+}
+
 func nmIsUsingResolved() error {
 	conn, err := dbus.SystemBus()
 	if err != nil {

net/dns/manager_test.go

@@ -5,6 +5,7 @@
 package dns
 
 import (
+	"runtime"
 	"testing"
 
 	"github.com/google/go-cmp/cmp"
@@ -45,6 +46,10 @@ func (c *fakeOSConfigurator) GetBaseConfig() (OSConfig, error) {
 func (c *fakeOSConfigurator) Close() error { return nil }
 
 func TestManager(t *testing.T) {
+	if runtime.GOOS == "windows" {
+		t.Skipf("test's assumptions break because of https://github.com/tailscale/corp/issues/1662")
+	}
+
 	// Note: these tests assume that it's safe to switch the
 	// OSConfigurator's split-dns support on and off between Set
 	// calls. Empirically this is currently true, because we reprobe

net/dns/manager_windows.go

@@ -8,6 +8,7 @@ import (
 	"errors"
 	"fmt"
 	"os/exec"
+	"sort"
 	"strings"
 	"syscall"
 	"time"
@@ -343,10 +344,11 @@ func (m windowsManager) getBasePrimaryResolver() (resolvers []netaddr.IP, err er
 		return nil, err
 	}
 
-	var (
-		primary winipcfg.LUID
-		best    = ^uint32(0)
-	)
+	type candidate struct {
+		id     winipcfg.LUID
+		metric uint32
+	}
+	var candidates []candidate
 	for _, row := range ifrows {
 		if !row.Connected {
 			continue
@@ -354,29 +356,57 @@ func (m windowsManager) getBasePrimaryResolver() (resolvers []netaddr.IP, err er
 		if row.InterfaceLUID == tsLUID {
 			continue
 		}
-		if row.Metric < best {
-			primary = row.InterfaceLUID
-			best = row.Metric
-		}
+		candidates = append(candidates, candidate{row.InterfaceLUID, row.Metric})
 	}
-	if primary == 0 {
+	if len(candidates) == 0 {
 		// No resolvers set outside of Tailscale.
 		return nil, nil
 	}
 
-	ips, err := primary.DNS()
-	if err != nil {
-		return nil, err
-	}
-	for _, stdip := range ips {
-		if ip, ok := netaddr.FromStdIP(stdip); ok {
+	sort.Slice(candidates, func(i, j int) bool { return candidates[i].metric < candidates[j].metric })
+
+	for _, candidate := range candidates {
+		ips, err := candidate.id.DNS()
+		if err != nil {
+			return nil, err
+		}
+
+	ipLoop:
+		for _, stdip := range ips {
+			ip, ok := netaddr.FromStdIP(stdip)
+			if !ok {
+				continue
+			}
+			// Skip IPv6 site-local resolvers. These are an ancient
+			// and obsolete IPv6 RFC, which Windows still faithfully
+			// implements. The net result is that some low-metric
+			// interfaces can "have" DNS resolvers, but they're just
+			// site-local resolver IPs that don't go anywhere. So, we
+			// skip the site-local resolvers in order to find the
+			// first interface that has real DNS servers configured.
+			for _, sl := range siteLocalResolvers {
+				if ip.WithZone("") == sl {
+					continue ipLoop
+				}
+			}
 			resolvers = append(resolvers, ip)
 		}
+
+		if len(resolvers) > 0 {
+			// Found some resolvers, we're done.
+			break
+		}
 	}
 
 	return resolvers, nil
 }
 
+var siteLocalResolvers = []netaddr.IP{
+	netaddr.MustParseIP("fec0:0:0:ffff::1"),
+	netaddr.MustParseIP("fec0:0:0:ffff::2"),
+	netaddr.MustParseIP("fec0:0:0:ffff::3"),
+}
+
 func isWindows7() bool {
 	key, err := registry.OpenKey(registry.LOCAL_MACHINE, versionKey, registry.READ)
 	if err != nil {

net/dns/nm.go

@@ -16,6 +16,7 @@ import (
 
 	"github.com/godbus/dbus/v5"
 	"inet.af/netaddr"
+	"tailscale.com/net/interfaces"
 	"tailscale.com/util/dnsname"
 	"tailscale.com/util/endian"
 )
@@ -137,6 +138,22 @@ func (m *nmManager) trySet(ctx context.Context, config OSConfig) error {
 		}
 	}
 
+	// NetworkManager wipes out IPv6 address configuration unless we
+	// tell it explicitly to keep it. Read out the current interface
+	// settings and mirror them out to NetworkManager.
+	var addrs6 []map[string]interface{}
+	addrs, _, err := interfaces.Tailscale()
+	if err == nil {
+		for _, a := range addrs {
+			if a.Is6() {
+				addrs6 = append(addrs6, map[string]interface{}{
+					"address": a.String(),
+					"prefix":  uint32(128),
+				})
+			}
+		}
+	}
+
 	seen := map[dnsname.FQDN]bool{}
 	var search []string
 	for _, dom := range config.SearchDomains {
@@ -195,6 +212,9 @@ func (m *nmManager) trySet(ctx context.Context, config OSConfig) error {
 	// (none of its business anyway, we handle our own default
 	// routing).
 	ipv6Map["method"] = dbus.MakeVariant("auto")
+	if len(addrs6) > 0 {
+		ipv6Map["address-data"] = dbus.MakeVariant(addrs6)
+	}
 	ipv6Map["ignore-auto-routes"] = dbus.MakeVariant(true)
 	ipv6Map["ignore-auto-dns"] = dbus.MakeVariant(true)
 	ipv6Map["never-default"] = dbus.MakeVariant(true)

net/dns/openresolv.go

@@ -2,6 +2,8 @@
 // Use of this source code is governed by a BSD-style
 // license that can be found in the LICENSE file.
 
+// +build linux freebsd openbsd
+
 package dns
 
 import (

net/dns/resolvconf.go

@@ -2,6 +2,8 @@
 // Use of this source code is governed by a BSD-style
 // license that can be found in the LICENSE file.
 
+// +build linux freebsd openbsd
+
 package dns
 
 import (

net/dns/resolver/forwarder.go

@@ -13,7 +13,6 @@ import (
 	"hash/crc32"
 	"math/rand"
 	"net"
-	"os"
 	"sync"
 	"time"
 
@@ -39,8 +38,6 @@ const (
 
 var errNoUpstreams = errors.New("upstream nameservers not set")
 
-var aLongTimeAgo = time.Unix(0, 1)
-
 type forwardingRecord struct {
 	src       netaddr.IPPort
 	createdAt time.Time
@@ -303,8 +300,6 @@ type fwdConn struct {
 	// logf allows a fwdConn to log.
 	logf logger.Logf
 
-	// wg tracks the number of outstanding conn.Read and conn.Write calls.
-	wg sync.WaitGroup
 	// change allows calls to read to block until a the network connection has been replaced.
 	change *sync.Cond
 
@@ -352,15 +347,12 @@ func (c *fwdConn) send(packet []byte, dst netaddr.IPPort) {
 		}
 		c.mu.Unlock()
 
-		a := dst.UDPAddr()
-		c.wg.Add(1)
-		_, err := conn.WriteTo(packet, a)
-		c.wg.Done()
+		_, err := conn.WriteTo(packet, dst.UDPAddr())
 		if err == nil {
 			// Success
 			return
 		}
-		if errors.Is(err, os.ErrDeadlineExceeded) {
+		if errors.Is(err, net.ErrClosed) {
 			// We intentionally closed this connection.
 			// It has been replaced by a new connection. Try again.
 			continue
@@ -429,14 +421,12 @@ func (c *fwdConn) read(out []byte) int {
 		}
 		c.mu.Unlock()
 
-		c.wg.Add(1)
 		n, _, err := conn.ReadFrom(out)
-		c.wg.Done()
 		if err == nil {
 			// Success.
 			return n
 		}
-		if errors.Is(err, os.ErrDeadlineExceeded) {
+		if errors.Is(err, net.ErrClosed) {
 			// We intentionally closed this connection.
 			// It has been replaced by a new connection. Try again.
 			continue
@@ -468,10 +458,7 @@ func (c *fwdConn) closeConnLocked() {
 	if c.conn == nil {
 		return
 	}
-	// Unblock all readers/writers, wait for them, close the connection.
-	c.conn.SetDeadline(aLongTimeAgo)
-	c.wg.Wait()
-	c.conn.Close()
+	c.conn.Close() // unblocks all readers/writers, they'll pick up the next connection.
 	c.conn = nil
 }
 

net/interfaces/interfaces.go

@@ -26,7 +26,7 @@ var LoginEndpointForProxyDetermination = "https://login.tailscale.com/"
 // Tailscale returns the current machine's Tailscale interface, if any.
 // If none is found, all zero values are returned.
 // A non-nil error is only returned on a problem listing the system interfaces.
-func Tailscale() (net.IP, *net.Interface, error) {
+func Tailscale() ([]netaddr.IP, *net.Interface, error) {
 	ifs, err := net.Interfaces()
 	if err != nil {
 		return nil, nil, err
@@ -39,14 +39,18 @@ func Tailscale() (net.IP, *net.Interface, error) {
 		if err != nil {
 			continue
 		}
+		var tsIPs []netaddr.IP
 		for _, a := range addrs {
 			if ipnet, ok := a.(*net.IPNet); ok {
 				nip, ok := netaddr.FromStdIP(ipnet.IP)
 				if ok && tsaddr.IsTailscaleIP(nip) {
-					return ipnet.IP, &iface, nil
+					tsIPs = append(tsIPs, nip)
 				}
 			}
 		}
+		if len(tsIPs) > 0 {
+			return tsIPs, &iface, nil
+		}
 	}
 	return nil, nil, nil
 }

net/interfaces/interfaces_darwin.go

@@ -10,6 +10,7 @@ import (
 	"log"
 	"net"
 	"syscall"
+	"time"
 
 	"golang.org/x/net/route"
 	"golang.org/x/sys/unix"
@@ -28,6 +29,34 @@ func DefaultRouteInterface() (string, error) {
 	return iface.Name, nil
 }
 
+// fetchRoutingTable is a retry loop around route.FetchRIB, fetching NET_RT_DUMP2.
+//
+// The retry loop is due to a bug in the BSDs (or Go?). See
+// https://github.com/tailscale/tailscale/issues/1345
+func fetchRoutingTable() (rib []byte, err error) {
+	fails := 0
+	for {
+		rib, err := route.FetchRIB(syscall.AF_UNSPEC, syscall.NET_RT_DUMP2, 0)
+		if err == nil {
+			return rib, nil
+		}
+		fails++
+		if fails < 10 {
+			// Empirically, 1 retry is enough. In a long
+			// stress test while toggling wifi on & off, I
+			// only saw a few occurrences of 2 and one 3.
+			// So 10 should be more plenty.
+			if fails > 5 {
+				time.Sleep(5 * time.Millisecond)
+			}
+			continue
+		}
+		if err != nil {
+			return nil, fmt.Errorf("route.FetchRIB: %w", err)
+		}
+	}
+}
+
 func DefaultRouteInterfaceIndex() (int, error) {
 	// $ netstat -nr
 	// Routing tables
@@ -43,7 +72,7 @@ func DefaultRouteInterfaceIndex() (int, error) {
 	// c       RTF_PRCLONING    Protocol-specified generate new routes on use
 	// I       RTF_IFSCOPE      Route is associated with an interface scope
 
-	rib, err := route.FetchRIB(syscall.AF_UNSPEC, syscall.NET_RT_DUMP2, 0)
+	rib, err := fetchRoutingTable()
 	if err != nil {
 		return 0, fmt.Errorf("route.FetchRIB: %w", err)
 	}
@@ -83,7 +112,7 @@ func init() {
 }
 
 func likelyHomeRouterIPDarwinFetchRIB() (ret netaddr.IP, ok bool) {
-	rib, err := route.FetchRIB(syscall.AF_UNSPEC, syscall.NET_RT_DUMP2, 0)
+	rib, err := fetchRoutingTable()
 	if err != nil {
 		log.Printf("routerIP/FetchRIB: %v", err)
 		return ret, false

net/interfaces/interfaces_darwin_test.go

@@ -83,4 +83,14 @@ func likelyHomeRouterIPDarwinExec() (ret netaddr.IP, ok bool) {
 	return ret, !ret.IsZero()
 }
 
+func TestFetchRoutingTable(t *testing.T) {
+	// Issue 1345: this used to be flaky on darwin.
+	for i := 0; i < 20; i++ {
+		_, err := fetchRoutingTable()
+		if err != nil {
+			t.Fatal(err)
+		}
+	}
+}
+
 var errStopReadingNetstatTable = errors.New("found private gateway")

net/stun/stuntest/stuntest.go

@@ -26,11 +26,11 @@ type stunStats struct {
 	readIPv6 int
 }
 
-func Serve(t *testing.T) (addr *net.UDPAddr, cleanupFn func()) {
+func Serve(t testing.TB) (addr *net.UDPAddr, cleanupFn func()) {
 	return ServeWithPacketListener(t, nettype.Std{})
 }
 
-func ServeWithPacketListener(t *testing.T, ln nettype.PacketListener) (addr *net.UDPAddr, cleanupFn func()) {
+func ServeWithPacketListener(t testing.TB, ln nettype.PacketListener) (addr *net.UDPAddr, cleanupFn func()) {
 	t.Helper()
 
 	// TODO(crawshaw): use stats to test re-STUN logic
@@ -52,7 +52,7 @@ func ServeWithPacketListener(t *testing.T, ln nettype.PacketListener) (addr *net
 	}
 }
 
-func runSTUN(t *testing.T, pc net.PacketConn, stats *stunStats, done chan<- struct{}) {
+func runSTUN(t testing.TB, pc net.PacketConn, stats *stunStats, done chan<- struct{}) {
 	defer close(done)
 
 	var buf [64 << 10]byte

net/tsaddr/tsaddr.go

@@ -138,3 +138,50 @@ type onceIP struct {
 	sync.Once
 	v netaddr.IP
 }
+
+// NewContainsIPFunc returns a func that reports whether ip is in addrs.
+//
+// It's optimized for the cases of addrs being empty and addrs
+// containing 1 or 2 single-IP prefixes (such as one IPv4 address and
+// one IPv6 address).
+//
+// Otherwise the implementation is somewhat slow.
+func NewContainsIPFunc(addrs []netaddr.IPPrefix) func(ip netaddr.IP) bool {
+	// Specialize the three common cases: no address, just IPv4
+	// (or just IPv6), and both IPv4 and IPv6.
+	if len(addrs) == 0 {
+		return func(netaddr.IP) bool { return false }
+	}
+	// If any addr is more than a single IP, then just do the slow
+	// linear thing until
+	// https://github.com/inetaf/netaddr/issues/139 is done.
+	for _, a := range addrs {
+		if a.IsSingleIP() {
+			continue
+		}
+		acopy := append([]netaddr.IPPrefix(nil), addrs...)
+		return func(ip netaddr.IP) bool {
+			for _, a := range acopy {
+				if a.Contains(ip) {
+					return true
+				}
+			}
+			return false
+		}
+	}
+	// Fast paths for 1 and 2 IPs:
+	if len(addrs) == 1 {
+		a := addrs[0]
+		return func(ip netaddr.IP) bool { return ip == a.IP }
+	}
+	if len(addrs) == 2 {
+		a, b := addrs[0], addrs[1]
+		return func(ip netaddr.IP) bool { return ip == a.IP || ip == b.IP }
+	}
+	// General case:
+	m := map[netaddr.IP]bool{}
+	for _, a := range addrs {
+		m[a.IP] = true
+	}
+	return func(ip netaddr.IP) bool { return m[ip] }
+}

net/tsaddr/tsaddr_test.go

@@ -64,3 +64,32 @@ func TestIsUla(t *testing.T) {
 		}
 	}
 }
+
+func TestNewContainsIPFunc(t *testing.T) {
+	f := NewContainsIPFunc([]netaddr.IPPrefix{netaddr.MustParseIPPrefix("10.0.0.0/8")})
+	if f(netaddr.MustParseIP("8.8.8.8")) {
+		t.Fatal("bad")
+	}
+	if !f(netaddr.MustParseIP("10.1.2.3")) {
+		t.Fatal("bad")
+	}
+	f = NewContainsIPFunc([]netaddr.IPPrefix{netaddr.MustParseIPPrefix("10.1.2.3/32")})
+	if !f(netaddr.MustParseIP("10.1.2.3")) {
+		t.Fatal("bad")
+	}
+	f = NewContainsIPFunc([]netaddr.IPPrefix{
+		netaddr.MustParseIPPrefix("10.1.2.3/32"),
+		netaddr.MustParseIPPrefix("::2/128"),
+	})
+	if !f(netaddr.MustParseIP("::2")) {
+		t.Fatal("bad")
+	}
+	f = NewContainsIPFunc([]netaddr.IPPrefix{
+		netaddr.MustParseIPPrefix("10.1.2.3/32"),
+		netaddr.MustParseIPPrefix("10.1.2.4/32"),
+		netaddr.MustParseIPPrefix("::2/128"),
+	})
+	if !f(netaddr.MustParseIP("10.1.2.4")) {
+		t.Fatal("bad")
+	}
+}

net/tstun/ifstatus_windows.go

@@ -93,7 +93,6 @@ func waitInterfaceUp(iface tun.Device, timeout time.Duration, logf logger.Logf)
 			iw.logf("TUN interface is up after %v", time.Since(t0))
 			return nil
 		case <-ticker.C:
-			break
 		}
 
 		if iw.isUp() {

net/tstun/wrap.go

@@ -90,7 +90,12 @@ type Wrapper struct {
 	// to discard an empty packet instead of sending it through t.outbound.
 	outbound chan []byte
 
-	// fitler stores the currently active package filter
+	// eventsUpDown yields up and down tun.Events that arrive on a Wrapper's events channel.
+	eventsUpDown chan tun.Event
+	// eventsOther yields non-up-and-down tun.Events that arrive on a Wrapper's events channel.
+	eventsOther chan tun.Event
+
+	// filter atomically stores the currently active packet filter
 	filter atomic.Value // of *filter.Filter
 	// filterFlags control the verbosity of logging packet drops/accepts.
 	filterFlags filter.RunFlags
@@ -130,11 +135,14 @@ func Wrap(logf logger.Logf, tdev tun.Device) *Wrapper {
 		closed:         make(chan struct{}),
 		errors:         make(chan error),
 		outbound:       make(chan []byte),
+		eventsUpDown:   make(chan tun.Event),
+		eventsOther:    make(chan tun.Event),
 		// TODO(dmytro): (highly rate-limited) hexdumps should happen on unknown packets.
 		filterFlags: filter.LogAccepts | filter.LogDrops,
 	}
 
 	go tun.poll()
+	go tun.pumpEvents()
 	// The buffer starts out consumed.
 	tun.bufferConsumed <- struct{}{}
 
@@ -160,8 +168,50 @@ func (t *Wrapper) Close() error {
 	return err
 }
 
+// pumpEvents copies events from t.tdev to t.eventsUpDown and t.eventsOther.
+// pumpEvents exits when t.tdev.events or t.closed is closed.
+// pumpEvents closes t.eventsUpDown and t.eventsOther when it exits.
+func (t *Wrapper) pumpEvents() {
+	defer close(t.eventsUpDown)
+	defer close(t.eventsOther)
+	src := t.tdev.Events()
+	for {
+		// Retrieve an event from the TUN device.
+		var event tun.Event
+		var ok bool
+		select {
+		case <-t.closed:
+			return
+		case event, ok = <-src:
+			if !ok {
+				return
+			}
+		}
+
+		// Pass along event to the correct recipient.
+		// Though event is a bitmask, in practice there is only ever one bit set at a time.
+		dst := t.eventsOther
+		if event&(tun.EventUp|tun.EventDown) != 0 {
+			dst = t.eventsUpDown
+		}
+		select {
+		case <-t.closed:
+			return
+		case dst <- event:
+		}
+	}
+}
+
+// EventsUpDown returns a TUN event channel that contains all Up and Down events.
+func (t *Wrapper) EventsUpDown() chan tun.Event {
+	return t.eventsUpDown
+}
+
+// Events returns a TUN event channel that contains all non-Up, non-Down events.
+// It is named Events because it is the set of events that we want to expose to wireguard-go,
+// and Events is the name specified by the wireguard-go tun.Device interface.
 func (t *Wrapper) Events() chan tun.Event {
-	return t.tdev.Events()
+	return t.eventsOther
 }
 
 func (t *Wrapper) File() *os.File {
@@ -407,13 +457,22 @@ func (t *Wrapper) filterIn(buf []byte) filter.Response {
 // like wireguard-go/tun.Device.Write.
 func (t *Wrapper) Write(buf []byte, offset int) (int, error) {
 	if !t.disableFilter {
-		res := t.filterIn(buf[offset:])
-		if res == filter.DropSilently {
+		if t.filterIn(buf[offset:]) != filter.Accept {
+			// If we're not accepting the packet, lie to wireguard-go and pretend
+			// that everything is okay with a nil error, so wireguard-go
+			// doesn't log about this Write "failure".
+			//
+			// We return len(buf), but the ill-defined wireguard-go/tun.Device.Write
+			// method doesn't specify how the offset affects the return value.
+			// In fact, the Linux implementation does one of two different things depending
+			// on how the /dev/net/tun was created. But fortunately the wireguard-go
+			// code ignores the int return and only looks at the error:
+			//
+			//     device/receive.go: _, err = device.tun.device.Write(....)
+			//
+			// TODO(bradfitz): fix upstream interface docs, implementation.
 			return len(buf), nil
 		}
-		if res != filter.Accept {
-			return 0, ErrFiltered
-		}
 	}
 
 	t.noteActivity()

net/tstun/wrap_test.go

@@ -329,11 +329,14 @@ func TestFilter(t *testing.T) {
 			var filtered bool
 
 			if tt.dir == in {
+				// Use the side effect of updating the last
+				// activity atomic to determine whether the
+				// data was actually filtered.
+				// If it stays zero, nothing made it through
+				// to the wrapped TUN.
+				atomic.StoreInt64(&tun.lastActivityAtomic, 0)
 				_, err = tun.Write(tt.data, 0)
-				if err == ErrFiltered {
-					filtered = true
-					err = nil
-				}
+				filtered = atomic.LoadInt64(&tun.lastActivityAtomic) == 0
 			} else {
 				chtun.Outbound <- tt.data
 				n, err = tun.Read(buf[:], 0)

paths/paths.go

@@ -26,6 +26,13 @@ func DefaultTailscaledSocket() string {
 	if runtime.GOOS == "darwin" {
 		return "/var/run/tailscaled.socket"
 	}
+	if runtime.GOOS == "linux" {
+		// TODO(crawshaw): does this path change with DSM7?
+		const synologySock = "/volume1/@appstore/Tailscale/var/tailscaled.sock" // SYNOPKG_PKGDEST in scripts/installer
+		if fi, err := os.Stat(filepath.Dir(synologySock)); err == nil && fi.IsDir() {
+			return synologySock
+		}
+	}
 	if fi, err := os.Stat("/var/run"); err == nil && fi.IsDir() {
 		return "/var/run/tailscale/tailscaled.sock"
 	}

portlist/portlist_windows.go

@@ -23,7 +23,7 @@ func listPorts() (List, error) {
 }
 
 func addProcesses(pl []Port) ([]Port, error) {
-	// OpenCurrentProcessToken instead of GetCurrentProcessToken,
+	//lint:ignore SA1019 OpenCurrentProcessToken instead of GetCurrentProcessToken,
 	// as GetCurrentProcessToken only works on Windows 8+.
 	tok, err := windows.OpenCurrentProcessToken()
 	if err != nil {

safesocket/pipe_windows.go

@@ -11,10 +11,6 @@ import (
 	"syscall"
 )
 
-func path(vendor, name string, port uint16) string {
-	return fmt.Sprintf("127.0.0.1:%v", port)
-}
-
 func connect(path string, port uint16) (net.Conn, error) {
 	pipe, err := net.Dial("tcp", fmt.Sprintf("127.0.0.1:%d", port))
 	if err != nil {

safesocket/safesocket.go

@@ -61,8 +61,12 @@ func LocalTCPPortAndToken() (port int, token string, err error) {
 
 // PlatformUsesPeerCreds reports whether the current platform uses peer credentials
 // to authenticate connections.
-func PlatformUsesPeerCreds() bool {
-	switch runtime.GOOS {
+func PlatformUsesPeerCreds() bool { return GOOSUsesPeerCreds(runtime.GOOS) }
+
+// GOOSUsesPeerCreds is like PlatformUsesPeerCreds but takes a
+// runtime.GOOS value instead of using the current one.
+func GOOSUsesPeerCreds(goos string) bool {
+	switch goos {
 	case "linux", "darwin", "freebsd":
 		return true
 	}

syncs/watchdog_test.go

@@ -6,7 +6,6 @@ package syncs
 
 import (
 	"context"
-	"runtime"
 	"sync"
 	"testing"
 	"time"
@@ -49,11 +48,11 @@ func TestWatchContended(t *testing.T) {
 }
 
 func TestWatchMultipleValues(t *testing.T) {
-	if cibuild.On() && runtime.GOOS == "windows" {
+	if cibuild.On() {
 		// On the CI machine, it sometimes takes 500ms to start a new goroutine.
 		// When this happens, we don't get enough events quickly enough.
 		// Nothing's wrong, and it's not worth working around. Just skip the test.
-		t.Skip("flaky on Windows CI")
+		t.Skip("flaky on CI")
 	}
 	mu := new(sync.Mutex)
 	ctx, cancel := context.WithCancel(context.Background())

tailcfg/tailcfg.go

@@ -41,7 +41,8 @@ import (
 //    16: 2021-04-15: client understands Node.Online, MapResponse.OnlineChange
 //    17: 2021-04-18: MapResponse.Domain empty means unchanged
 //    18: 2021-04-19: MapResponse.Node nil means unchanged (all fields now omitempty)
-const CurrentMapRequestVersion = 18
+//    19: 2021-04-21: MapResponse.Debug.SleepSeconds
+const CurrentMapRequestVersion = 19
 
 type StableID string
 
@@ -883,6 +884,24 @@ type PingRequest struct {
 	// Log is whether to log about this ping in the success case.
 	// For failure cases, the client will log regardless.
 	Log bool `json:",omitempty"`
+
+	Initiator        string // admin@email; "system" (for Tailscale)
+	TestIP           netaddr.IP
+	Types            string // empty means all: TSMP+ICMP+disco
+	StopAfterNDirect int    // 1 means stop on 1st direct ping; 4 means 4 direct pings; 0 means do MaxPings and stop
+	MaxPings         int    // MaxPings total, direct or DERPed
+	PayloadSize      int    // default: 0 extra bytes
+}
+
+type StreamedPingResult struct {
+	IP      netaddr.IP
+	SeqNum  int     // somewhat redundant with TxID but for clarity
+	SentTo  NodeID  // for exit/subnet relays
+	TxID    string  // N hex bytes random
+	Dir     string  // "in"/"out"
+	Type    string  // ICMP, disco, TSMP, ...
+	Via     string  // "direct", "derp-nyc", ...
+	Seconds float64 // for Dir "in" only
 }
 
 type MapResponse struct {
@@ -1012,6 +1031,12 @@ type Debug struct {
 	// GoroutineDumpURL, if non-empty, requests that the client do
 	// a one-time dump of its active goroutines to the given URL.
 	GoroutineDumpURL string `json:",omitempty"`
+
+	// SleepSeconds requests that the client sleep for the
+	// provided number of seconds.
+	// The client can (and should) limit the value (such as 5
+	// minutes).
+	SleepSeconds float64 `json:",omitempty"`
 }
 
 func (k MachineKey) String() string                   { return fmt.Sprintf("mkey:%x", k[:]) }

tstest/doc.go

@@ -1,6 +0,0 @@
-// Copyright (c) 2020 Tailscale Inc & AUTHORS All rights reserved.
-// Use of this source code is governed by a BSD-style
-// license that can be found in the LICENSE file.
-
-// Package tstest provides utilities for use in unit tests.
-package tstest

tstest/integration/integration_test.go

@@ -0,0 +1,456 @@
+// Copyright (c) 2021 Tailscale Inc & AUTHORS All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+// Package integration contains Tailscale integration tests.
+package integration
+
+import (
+	"bytes"
+	crand "crypto/rand"
+	"crypto/tls"
+	"encoding/json"
+	"fmt"
+	"io"
+	"io/ioutil"
+	"log"
+	"net"
+	"net/http"
+	"net/http/httptest"
+	"os"
+	"os/exec"
+	"path"
+	"path/filepath"
+	"runtime"
+	"strings"
+	"sync"
+	"sync/atomic"
+	"testing"
+	"time"
+
+	"go4.org/mem"
+	"tailscale.com/derp"
+	"tailscale.com/derp/derphttp"
+	"tailscale.com/ipn/ipnstate"
+	"tailscale.com/net/stun/stuntest"
+	"tailscale.com/safesocket"
+	"tailscale.com/smallzstd"
+	"tailscale.com/tailcfg"
+	"tailscale.com/tstest"
+	"tailscale.com/tstest/integration/testcontrol"
+	"tailscale.com/types/key"
+	"tailscale.com/types/logger"
+	"tailscale.com/types/nettype"
+)
+
+var mainError atomic.Value // of error
+
+func TestMain(m *testing.M) {
+	v := m.Run()
+	if v != 0 {
+		os.Exit(v)
+	}
+	if err, ok := mainError.Load().(error); ok {
+		fmt.Fprintf(os.Stderr, "FAIL: %v\n", err)
+		os.Exit(1)
+	}
+	os.Exit(0)
+}
+
+func TestIntegration(t *testing.T) {
+	if runtime.GOOS == "windows" {
+		t.Skip("not tested/working on Windows yet")
+	}
+
+	bins := buildTestBinaries(t)
+
+	env := newTestEnv(t, bins)
+	defer env.Close()
+
+	n1 := newTestNode(t, env)
+
+	dcmd := n1.StartDaemon(t)
+	defer dcmd.Process.Kill()
+
+	n1.AwaitListening(t)
+
+	st := n1.MustStatus(t)
+	t.Logf("Status: %s", st.BackendState)
+
+	if err := tstest.WaitFor(20*time.Second, func() error {
+		const sub = `Program starting: `
+		if !env.LogCatcher.logsContains(mem.S(sub)) {
+			return fmt.Errorf("log catcher didn't see %#q; got %s", sub, env.LogCatcher.logsString())
+		}
+		return nil
+	}); err != nil {
+		t.Error(err)
+	}
+
+	t.Logf("Running up --login-server=%s ...", env.ControlServer.URL)
+	if err := n1.Tailscale("up", "--login-server="+env.ControlServer.URL).Run(); err != nil {
+		t.Fatalf("up: %v", err)
+	}
+
+	if d, _ := time.ParseDuration(os.Getenv("TS_POST_UP_SLEEP")); d > 0 {
+		t.Logf("Sleeping for %v to give 'up' time to misbehave (https://github.com/tailscale/tailscale/issues/1840) ...", d)
+		time.Sleep(d)
+	}
+
+	var ip string
+	if err := tstest.WaitFor(20*time.Second, func() error {
+		out, err := n1.Tailscale("ip").Output()
+		if err != nil {
+			return err
+		}
+		ip = string(out)
+		return nil
+	}); err != nil {
+		t.Error(err)
+	}
+	t.Logf("Got IP: %v", ip)
+
+	dcmd.Process.Signal(os.Interrupt)
+
+	ps, err := dcmd.Process.Wait()
+	if err != nil {
+		t.Fatalf("tailscaled Wait: %v", err)
+	}
+	if ps.ExitCode() != 0 {
+		t.Errorf("tailscaled ExitCode = %d; want 0", ps.ExitCode())
+	}
+
+	t.Logf("number of HTTP logcatcher requests: %v", env.LogCatcher.numRequests())
+	if err := env.TrafficTrap.Err(); err != nil {
+		t.Errorf("traffic trap: %v", err)
+		t.Logf("logs: %s", env.LogCatcher.logsString())
+	}
+}
+
+// testBinaries are the paths to a tailscaled and tailscale binary.
+// These can be shared by multiple nodes.
+type testBinaries struct {
+	dir    string // temp dir for tailscale & tailscaled
+	daemon string // tailscaled
+	cli    string // tailscale
+}
+
+// buildTestBinaries builds tailscale and tailscaled, failing the test
+// if they fail to compile.
+func buildTestBinaries(t testing.TB) *testBinaries {
+	td := t.TempDir()
+	return &testBinaries{
+		dir:    td,
+		daemon: build(t, td, "tailscale.com/cmd/tailscaled"),
+		cli:    build(t, td, "tailscale.com/cmd/tailscale"),
+	}
+}
+
+// testEnv contains the test environment (set of servers) used by one
+// or more nodes.
+type testEnv struct {
+	Binaries *testBinaries
+
+	LogCatcher       *logCatcher
+	LogCatcherServer *httptest.Server
+
+	Control       *testcontrol.Server
+	ControlServer *httptest.Server
+
+	TrafficTrap       *trafficTrap
+	TrafficTrapServer *httptest.Server
+
+	derpShutdown func()
+}
+
+// newTestEnv starts a bunch of services and returns a new test
+// environment.
+//
+// Call Close to shut everything down.
+func newTestEnv(t testing.TB, bins *testBinaries) *testEnv {
+	derpMap, derpShutdown := runDERPAndStun(t, logger.Discard)
+	logc := new(logCatcher)
+	control := &testcontrol.Server{
+		DERPMap: derpMap,
+	}
+	trafficTrap := new(trafficTrap)
+	e := &testEnv{
+		Binaries:          bins,
+		LogCatcher:        logc,
+		LogCatcherServer:  httptest.NewServer(logc),
+		Control:           control,
+		ControlServer:     httptest.NewServer(control),
+		TrafficTrap:       trafficTrap,
+		TrafficTrapServer: httptest.NewServer(trafficTrap),
+		derpShutdown:      derpShutdown,
+	}
+	return e
+}
+
+func (e *testEnv) Close() error {
+	e.LogCatcherServer.Close()
+	e.TrafficTrapServer.Close()
+	e.ControlServer.Close()
+	e.derpShutdown()
+	return nil
+}
+
+// testNode is a machine with a tailscale & tailscaled.
+// Currently, the test is simplistic and user==node==machine.
+// That may grow complexity later to test more.
+type testNode struct {
+	env *testEnv
+
+	dir       string // temp dir for sock & state
+	sockFile  string
+	stateFile string
+}
+
+// newTestNode allocates a temp directory for a new test node.
+// The node is not started automatically.
+func newTestNode(t *testing.T, env *testEnv) *testNode {
+	dir := t.TempDir()
+	return &testNode{
+		env:       env,
+		dir:       dir,
+		sockFile:  filepath.Join(dir, "tailscale.sock"),
+		stateFile: filepath.Join(dir, "tailscale.state"),
+	}
+}
+
+// StartDaemon starts the node's tailscaled, failing if it fails to
+// start.
+func (n *testNode) StartDaemon(t testing.TB) *exec.Cmd {
+	cmd := exec.Command(n.env.Binaries.daemon,
+		"--tun=userspace-networking",
+		"--state="+n.stateFile,
+		"--socket="+n.sockFile,
+	)
+	cmd.Env = append(os.Environ(),
+		"TS_LOG_TARGET="+n.env.LogCatcherServer.URL,
+		"HTTP_PROXY="+n.env.TrafficTrapServer.URL,
+		"HTTPS_PROXY="+n.env.TrafficTrapServer.URL,
+	)
+	if err := cmd.Start(); err != nil {
+		t.Fatalf("starting tailscaled: %v", err)
+	}
+	return cmd
+}
+
+// AwaitListening waits for the tailscaled to be serving local clients
+// over its localhost IPC mechanism. (Unix socket, etc)
+func (n *testNode) AwaitListening(t testing.TB) {
+	if err := tstest.WaitFor(20*time.Second, func() (err error) {
+		c, err := safesocket.Connect(n.sockFile, 41112)
+		if err != nil {
+			return err
+		}
+		c.Close()
+		return nil
+	}); err != nil {
+		t.Fatal(err)
+	}
+}
+
+// Tailscale returns a command that runs the tailscale CLI with the provided arguments.
+// It does not start the process.
+func (n *testNode) Tailscale(arg ...string) *exec.Cmd {
+	cmd := exec.Command(n.env.Binaries.cli, "--socket="+n.sockFile)
+	cmd.Args = append(cmd.Args, arg...)
+	cmd.Dir = n.dir
+	return cmd
+}
+
+func (n *testNode) MustStatus(tb testing.TB) *ipnstate.Status {
+	tb.Helper()
+	out, err := n.Tailscale("status", "--json").CombinedOutput()
+	if err != nil {
+		tb.Fatalf("getting status: %v, %s", err, out)
+	}
+	st := new(ipnstate.Status)
+	if err := json.Unmarshal(out, st); err != nil {
+		tb.Fatalf("parsing status json: %v, from: %s", err, out)
+	}
+	return st
+}
+
+func exe() string {
+	if runtime.GOOS == "windows" {
+		return ".exe"
+	}
+	return ""
+}
+
+func findGo(t testing.TB) string {
+	goBin := filepath.Join(runtime.GOROOT(), "bin", "go"+exe())
+	if fi, err := os.Stat(goBin); err != nil {
+		if os.IsNotExist(err) {
+			t.Fatalf("failed to find go at %v", goBin)
+		}
+		t.Fatalf("looking for go binary: %v", err)
+	} else if !fi.Mode().IsRegular() {
+		t.Fatalf("%v is unexpected %v", goBin, fi.Mode())
+	}
+	t.Logf("using go binary %v", goBin)
+	return goBin
+}
+
+func build(t testing.TB, outDir, target string) string {
+	exe := ""
+	if runtime.GOOS == "windows" {
+		exe = ".exe"
+	}
+	bin := filepath.Join(outDir, path.Base(target)) + exe
+	errOut, err := exec.Command(findGo(t), "build", "-o", bin, target).CombinedOutput()
+	if err != nil {
+		t.Fatalf("failed to build %v: %v, %s", target, err, errOut)
+	}
+	return bin
+}
+
+// logCatcher is a minimal logcatcher for the logtail upload client.
+type logCatcher struct {
+	mu     sync.Mutex
+	buf    bytes.Buffer
+	gotErr error
+	reqs   int
+}
+
+func (lc *logCatcher) logsContains(sub mem.RO) bool {
+	lc.mu.Lock()
+	defer lc.mu.Unlock()
+	return mem.Contains(mem.B(lc.buf.Bytes()), sub)
+}
+
+func (lc *logCatcher) numRequests() int {
+	lc.mu.Lock()
+	defer lc.mu.Unlock()
+	return lc.reqs
+}
+
+func (lc *logCatcher) logsString() string {
+	lc.mu.Lock()
+	defer lc.mu.Unlock()
+	return lc.buf.String()
+}
+
+func (lc *logCatcher) ServeHTTP(w http.ResponseWriter, r *http.Request) {
+	var body io.Reader = r.Body
+	if r.Header.Get("Content-Encoding") == "zstd" {
+		var err error
+		body, err = smallzstd.NewDecoder(body)
+		if err != nil {
+			log.Printf("bad caught zstd: %v", err)
+			http.Error(w, err.Error(), 400)
+			return
+		}
+	}
+	bodyBytes, _ := ioutil.ReadAll(body)
+
+	type Entry struct {
+		Logtail struct {
+			ClientTime time.Time `json:"client_time"`
+			ServerTime time.Time `json:"server_time"`
+			Error      struct {
+				BadData string `json:"bad_data"`
+			} `json:"error"`
+		} `json:"logtail"`
+		Text string `json:"text"`
+	}
+	var jreq []Entry
+	var err error
+	if len(bodyBytes) > 0 && bodyBytes[0] == '[' {
+		err = json.Unmarshal(bodyBytes, &jreq)
+	} else {
+		var ent Entry
+		err = json.Unmarshal(bodyBytes, &ent)
+		jreq = append(jreq, ent)
+	}
+
+	lc.mu.Lock()
+	defer lc.mu.Unlock()
+	lc.reqs++
+	if lc.gotErr == nil && err != nil {
+		lc.gotErr = err
+	}
+	if err != nil {
+		fmt.Fprintf(&lc.buf, "error from %s of %#q: %v\n", r.Method, bodyBytes, err)
+	} else {
+		for _, ent := range jreq {
+			fmt.Fprintf(&lc.buf, "%s\n", strings.TrimSpace(ent.Text))
+		}
+	}
+	w.WriteHeader(200) // must have no content, but not a 204
+}
+
+// trafficTrap is an HTTP proxy handler to note whether any
+// HTTP traffic tries to leave localhost from tailscaled. We don't
+// expect any, so any request triggers a failure.
+type trafficTrap struct {
+	atomicErr atomic.Value // of error
+}
+
+func (tt *trafficTrap) Err() error {
+	if err, ok := tt.atomicErr.Load().(error); ok {
+		return err
+	}
+	return nil
+}
+
+func (tt *trafficTrap) ServeHTTP(w http.ResponseWriter, r *http.Request) {
+	var got bytes.Buffer
+	r.Write(&got)
+	err := fmt.Errorf("unexpected HTTP proxy via proxy: %s", got.Bytes())
+	mainError.Store(err)
+	if tt.Err() == nil {
+		// Best effort at remembering the first request.
+		tt.atomicErr.Store(err)
+	}
+	log.Printf("Error: %v", err)
+	w.WriteHeader(403)
+}
+
+func runDERPAndStun(t testing.TB, logf logger.Logf) (derpMap *tailcfg.DERPMap, cleanup func()) {
+	var serverPrivateKey key.Private
+	if _, err := crand.Read(serverPrivateKey[:]); err != nil {
+		t.Fatal(err)
+	}
+	d := derp.NewServer(serverPrivateKey, logf)
+
+	httpsrv := httptest.NewUnstartedServer(derphttp.Handler(d))
+	httpsrv.Config.ErrorLog = logger.StdLogger(logf)
+	httpsrv.Config.TLSNextProto = make(map[string]func(*http.Server, *tls.Conn, http.Handler))
+	httpsrv.StartTLS()
+
+	stunAddr, stunCleanup := stuntest.ServeWithPacketListener(t, nettype.Std{})
+
+	m := &tailcfg.DERPMap{
+		Regions: map[int]*tailcfg.DERPRegion{
+			1: {
+				RegionID:   1,
+				RegionCode: "test",
+				Nodes: []*tailcfg.DERPNode{
+					{
+						Name:         "t1",
+						RegionID:     1,
+						HostName:     "127.0.0.1", // to bypass HTTP proxy
+						IPv4:         "127.0.0.1",
+						IPv6:         "none",
+						STUNPort:     stunAddr.Port,
+						DERPTestPort: httpsrv.Listener.Addr().(*net.TCPAddr).Port,
+						STUNTestIP:   stunAddr.IP.String(),
+					},
+				},
+			},
+		},
+	}
+
+	cleanup = func() {
+		httpsrv.CloseClientConnections()
+		httpsrv.Close()
+		d.Close()
+		stunCleanup()
+	}
+
+	return m, cleanup
+}

tstest/integration/testcontrol/testcontrol.go

@@ -0,0 +1,545 @@
+// Copyright (c) 2021 Tailscale Inc & AUTHORS All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+// Package testcontrol contains a minimal control plane server for testing purposes.
+package testcontrol
+
+import (
+	"bytes"
+	crand "crypto/rand"
+	"encoding/binary"
+	"encoding/json"
+	"errors"
+	"fmt"
+	"io"
+	"io/ioutil"
+	"log"
+	"math/rand"
+	"net/http"
+	"strings"
+	"sync"
+	"time"
+
+	"github.com/klauspost/compress/zstd"
+	"golang.org/x/crypto/nacl/box"
+	"inet.af/netaddr"
+	"tailscale.com/derp/derpmap"
+	"tailscale.com/smallzstd"
+	"tailscale.com/tailcfg"
+	"tailscale.com/types/logger"
+	"tailscale.com/types/wgkey"
+)
+
+// Server is a control plane server. Its zero value is ready for use.
+// Everything is stored in-memory in one tailnet.
+type Server struct {
+	Logf    logger.Logf      // nil means to use the log package
+	DERPMap *tailcfg.DERPMap // nil means to use prod DERP map
+
+	initMuxOnce sync.Once
+	mux         *http.ServeMux
+
+	mu      sync.Mutex
+	pubKey  wgkey.Key
+	privKey wgkey.Private
+	nodes   map[tailcfg.NodeKey]*tailcfg.Node
+	users   map[tailcfg.NodeKey]*tailcfg.User
+	logins  map[tailcfg.NodeKey]*tailcfg.Login
+	updates map[tailcfg.NodeID]chan updateType
+}
+
+func (s *Server) logf(format string, a ...interface{}) {
+	if s.Logf != nil {
+		s.Logf(format, a...)
+	} else {
+		log.Printf(format, a...)
+	}
+}
+
+func (s *Server) initMux() {
+	s.mux = http.NewServeMux()
+	s.mux.HandleFunc("/", s.serveUnhandled)
+	s.mux.HandleFunc("/key", s.serveKey)
+	s.mux.HandleFunc("/machine/", s.serveMachine)
+}
+
+func (s *Server) ServeHTTP(w http.ResponseWriter, r *http.Request) {
+	s.initMuxOnce.Do(s.initMux)
+	s.mux.ServeHTTP(w, r)
+}
+
+func (s *Server) serveUnhandled(w http.ResponseWriter, r *http.Request) {
+	var got bytes.Buffer
+	r.Write(&got)
+	go panic(fmt.Sprintf("testcontrol.Server received unhandled request: %s", got.Bytes()))
+}
+
+func (s *Server) publicKey() wgkey.Key {
+	pub, _ := s.keyPair()
+	return pub
+}
+
+func (s *Server) privateKey() wgkey.Private {
+	_, priv := s.keyPair()
+	return priv
+}
+
+func (s *Server) keyPair() (pub wgkey.Key, priv wgkey.Private) {
+	s.mu.Lock()
+	defer s.mu.Unlock()
+	if s.pubKey.IsZero() {
+		var err error
+		s.privKey, err = wgkey.NewPrivate()
+		if err != nil {
+			go panic(err) // bring down test, even if in http.Handler
+		}
+		s.pubKey = s.privKey.Public()
+	}
+	return s.pubKey, s.privKey
+}
+
+func (s *Server) serveKey(w http.ResponseWriter, r *http.Request) {
+	w.Header().Set("Content-Type", "text/plain")
+	w.WriteHeader(200)
+	io.WriteString(w, s.publicKey().HexString())
+}
+
+func (s *Server) serveMachine(w http.ResponseWriter, r *http.Request) {
+	mkeyStr := strings.TrimPrefix(r.URL.Path, "/machine/")
+	rem := ""
+	if i := strings.IndexByte(mkeyStr, '/'); i != -1 {
+		rem = mkeyStr[i:]
+		mkeyStr = mkeyStr[:i]
+	}
+
+	key, err := wgkey.ParseHex(mkeyStr)
+	if err != nil {
+		http.Error(w, "bad machine key hex", 400)
+		return
+	}
+	mkey := tailcfg.MachineKey(key)
+
+	if r.Method != "POST" {
+		http.Error(w, "POST required", 400)
+		return
+	}
+
+	switch rem {
+	case "":
+		s.serveRegister(w, r, mkey)
+	case "/map":
+		s.serveMap(w, r, mkey)
+	default:
+		s.serveUnhandled(w, r)
+	}
+}
+
+// Node returns the node for nodeKey. It's always nil or cloned memory.
+func (s *Server) Node(nodeKey tailcfg.NodeKey) *tailcfg.Node {
+	s.mu.Lock()
+	defer s.mu.Unlock()
+	return s.nodes[nodeKey].Clone()
+}
+
+func (s *Server) getUser(nodeKey tailcfg.NodeKey) (*tailcfg.User, *tailcfg.Login) {
+	s.mu.Lock()
+	defer s.mu.Unlock()
+	if s.users == nil {
+		s.users = map[tailcfg.NodeKey]*tailcfg.User{}
+	}
+	if s.logins == nil {
+		s.logins = map[tailcfg.NodeKey]*tailcfg.Login{}
+	}
+	if u, ok := s.users[nodeKey]; ok {
+		return u, s.logins[nodeKey]
+	}
+	id := tailcfg.UserID(len(s.users) + 1)
+	domain := "fake-control.example.net"
+	loginName := fmt.Sprintf("user-%d@%s", id, domain)
+	displayName := fmt.Sprintf("User %d", id)
+	login := &tailcfg.Login{
+		ID:            tailcfg.LoginID(id),
+		Provider:      "testcontrol",
+		LoginName:     loginName,
+		DisplayName:   displayName,
+		ProfilePicURL: "https://tailscale.com/static/images/marketing/team-carney.jpg",
+		Domain:        domain,
+	}
+	user := &tailcfg.User{
+		ID:          id,
+		LoginName:   loginName,
+		DisplayName: displayName,
+		Domain:      domain,
+		Logins:      []tailcfg.LoginID{login.ID},
+	}
+	s.users[nodeKey] = user
+	s.logins[nodeKey] = login
+	return user, login
+}
+
+func (s *Server) serveRegister(w http.ResponseWriter, r *http.Request, mkey tailcfg.MachineKey) {
+	var req tailcfg.RegisterRequest
+	if err := s.decode(mkey, r.Body, &req); err != nil {
+		panic(fmt.Sprintf("serveRegister: decode: %v", err))
+	}
+	if req.Version != 1 {
+		panic(fmt.Sprintf("serveRegister: unsupported version: %d", req.Version))
+	}
+	if req.NodeKey.IsZero() {
+		panic("serveRegister: request has zero node key")
+	}
+
+	user, login := s.getUser(req.NodeKey)
+	s.mu.Lock()
+	if s.nodes == nil {
+		s.nodes = map[tailcfg.NodeKey]*tailcfg.Node{}
+	}
+	s.nodes[req.NodeKey] = &tailcfg.Node{
+		ID:                tailcfg.NodeID(user.ID),
+		StableID:          tailcfg.StableNodeID(fmt.Sprintf("TESTCTRL%08x", int(user.ID))),
+		User:              user.ID,
+		Machine:           mkey,
+		Key:               req.NodeKey,
+		MachineAuthorized: true,
+	}
+	s.mu.Unlock()
+
+	res, err := s.encode(mkey, false, tailcfg.RegisterResponse{
+		User:              *user,
+		Login:             *login,
+		NodeKeyExpired:    false,
+		MachineAuthorized: true,
+		AuthURL:           "", // all good; TODO(bradfitz): add ways to not start all good.
+	})
+	if err != nil {
+		go panic(fmt.Sprintf("serveRegister: encode: %v", err))
+	}
+	w.WriteHeader(200)
+	w.Write(res)
+}
+
+// updateType indicates why a long-polling map request is being woken
+// up for an update.
+type updateType int
+
+const (
+	// updatePeerChanged is an update that a peer has changed.
+	updatePeerChanged updateType = iota + 1
+
+	// updateSelfChanged is an update that the node changed itself
+	// via a lite endpoint update. These ones are never dup-suppressed,
+	// as the client is expecting an answer regardless.
+	updateSelfChanged
+)
+
+func (s *Server) updateLocked(source string, peers []tailcfg.NodeID) {
+	for _, peer := range peers {
+		sendUpdate(s.updates[peer], updatePeerChanged)
+	}
+}
+
+// sendUpdate sends updateType to dst if dst is non-nil and
+// has capacity.
+func sendUpdate(dst chan<- updateType, updateType updateType) {
+	if dst == nil {
+		return
+	}
+	// The dst channel has a buffer size of 1.
+	// If we fail to insert an update into the buffer that
+	// means there is already an update pending.
+	select {
+	case dst <- updateType:
+	default:
+	}
+}
+
+func (s *Server) serveMap(w http.ResponseWriter, r *http.Request, mkey tailcfg.MachineKey) {
+	ctx := r.Context()
+
+	req := new(tailcfg.MapRequest)
+	if err := s.decode(mkey, r.Body, req); err != nil {
+		go panic(fmt.Sprintf("bad map request: %v", err))
+	}
+
+	jitter := time.Duration(rand.Intn(8000)) * time.Millisecond
+	keepAlive := 50*time.Second + jitter
+
+	node := s.Node(req.NodeKey)
+	if node == nil {
+		http.Error(w, "node not found", 400)
+		return
+	}
+	if node.Machine != mkey {
+		http.Error(w, "node doesn't match machine key", 400)
+		return
+	}
+
+	var peersToUpdate []tailcfg.NodeID
+	if !req.ReadOnly {
+		endpoints := filterInvalidIPv6Endpoints(req.Endpoints)
+		node.Endpoints = endpoints
+		// TODO: more
+		// TODO: register node,
+		//s.UpdateEndpoint(mkey, req.NodeKey,
+		// XXX
+	}
+
+	nodeID := node.ID
+
+	s.mu.Lock()
+	updatesCh := make(chan updateType, 1)
+	oldUpdatesCh := s.updates[nodeID]
+	if breakSameNodeMapResponseStreams(req) {
+		if oldUpdatesCh != nil {
+			close(oldUpdatesCh)
+		}
+		if s.updates == nil {
+			s.updates = map[tailcfg.NodeID]chan updateType{}
+		}
+		s.updates[nodeID] = updatesCh
+	} else {
+		sendUpdate(oldUpdatesCh, updateSelfChanged)
+	}
+	s.updateLocked("serveMap", peersToUpdate)
+	s.mu.Unlock()
+
+	// ReadOnly implies no streaming, as it doesn't
+	// register an updatesCh to get updates.
+	streaming := req.Stream && !req.ReadOnly
+	compress := req.Compress != ""
+
+	w.WriteHeader(200)
+	for {
+		res, err := s.MapResponse(req)
+		if err != nil {
+			// TODO: log
+			return
+		}
+		if res == nil {
+			return // done
+		}
+		// TODO: add minner if/when needed
+		resBytes, err := json.Marshal(res)
+		if err != nil {
+			s.logf("json.Marshal: %v", err)
+			return
+		}
+		if err := s.sendMapMsg(w, mkey, compress, resBytes); err != nil {
+			return
+		}
+		if !streaming {
+			return
+		}
+	keepAliveLoop:
+		for {
+			var keepAliveTimer *time.Timer
+			var keepAliveTimerCh <-chan time.Time
+			if keepAlive > 0 {
+				keepAliveTimer = time.NewTimer(keepAlive)
+				keepAliveTimerCh = keepAliveTimer.C
+			}
+			select {
+			case <-ctx.Done():
+				if keepAliveTimer != nil {
+					keepAliveTimer.Stop()
+				}
+				return
+			case _, ok := <-updatesCh:
+				if !ok {
+					// replaced by new poll request
+					return
+				}
+				break keepAliveLoop
+			case <-keepAliveTimerCh:
+				if err := s.sendMapMsg(w, mkey, compress, keepAliveMsg); err != nil {
+					return
+				}
+			}
+		}
+	}
+}
+
+var keepAliveMsg = &struct {
+	KeepAlive bool
+}{
+	KeepAlive: true,
+}
+
+var prodDERPMap = derpmap.Prod()
+
+// MapResponse generates a MapResponse for a MapRequest.
+//
+// No updates to s are done here.
+func (s *Server) MapResponse(req *tailcfg.MapRequest) (res *tailcfg.MapResponse, err error) {
+	node := s.Node(req.NodeKey)
+	if node == nil {
+		// node key rotated away (once test server supports that)
+		return nil, nil
+	}
+	derpMap := s.DERPMap
+	if derpMap == nil {
+		derpMap = prodDERPMap
+	}
+	user, _ := s.getUser(req.NodeKey)
+	res = &tailcfg.MapResponse{
+		Node:            node,
+		DERPMap:         derpMap,
+		Domain:          string(user.Domain),
+		CollectServices: "true",
+		PacketFilter:    tailcfg.FilterAllowAll,
+	}
+	res.Node.Addresses = []netaddr.IPPrefix{
+		netaddr.MustParseIPPrefix(fmt.Sprintf("100.64.%d.%d/32", uint8(node.ID>>8), uint8(node.ID))),
+	}
+	res.Node.AllowedIPs = res.Node.Addresses
+	return res, nil
+}
+
+func (s *Server) sendMapMsg(w http.ResponseWriter, mkey tailcfg.MachineKey, compress bool, msg interface{}) error {
+	resBytes, err := s.encode(mkey, compress, msg)
+	if err != nil {
+		return err
+	}
+	if len(resBytes) > 16<<20 {
+		return fmt.Errorf("map message too big: %d", len(resBytes))
+	}
+	var siz [4]byte
+	binary.LittleEndian.PutUint32(siz[:], uint32(len(resBytes)))
+	if _, err := w.Write(siz[:]); err != nil {
+		return err
+	}
+	if _, err := w.Write(resBytes); err != nil {
+		return err
+	}
+	if f, ok := w.(http.Flusher); ok {
+		f.Flush()
+	} else {
+		s.logf("[unexpected] ResponseWriter %T is not a Flusher", w)
+	}
+	return nil
+}
+
+func (s *Server) decode(mkey tailcfg.MachineKey, r io.Reader, v interface{}) error {
+	if c, _ := r.(io.Closer); c != nil {
+		defer c.Close()
+	}
+	const msgLimit = 1 << 20
+	msg, err := ioutil.ReadAll(io.LimitReader(r, msgLimit))
+	if err != nil {
+		return err
+	}
+	if len(msg) == msgLimit {
+		return errors.New("encrypted message too long")
+	}
+
+	var nonce [24]byte
+	if len(msg) < len(nonce)+1 {
+		return errors.New("missing nonce")
+	}
+	copy(nonce[:], msg)
+	msg = msg[len(nonce):]
+
+	priv := s.privateKey()
+	pub, pri := (*[32]byte)(&mkey), (*[32]byte)(&priv)
+	decrypted, ok := box.Open(nil, msg, &nonce, pub, pri)
+	if !ok {
+		return errors.New("can't decrypt request")
+	}
+	return json.Unmarshal(decrypted, v)
+}
+
+var zstdEncoderPool = &sync.Pool{
+	New: func() interface{} {
+		encoder, err := smallzstd.NewEncoder(nil, zstd.WithEncoderLevel(zstd.SpeedFastest))
+		if err != nil {
+			panic(err)
+		}
+		return encoder
+	},
+}
+
+func (s *Server) encode(mkey tailcfg.MachineKey, compress bool, v interface{}) (b []byte, err error) {
+	var isBytes bool
+	if b, isBytes = v.([]byte); !isBytes {
+		b, err = json.Marshal(v)
+		if err != nil {
+			return nil, err
+		}
+	}
+	if compress {
+		encoder := zstdEncoderPool.Get().(*zstd.Encoder)
+		b = encoder.EncodeAll(b, nil)
+		encoder.Close()
+		zstdEncoderPool.Put(encoder)
+	}
+	var nonce [24]byte
+	if _, err := io.ReadFull(crand.Reader, nonce[:]); err != nil {
+		panic(err)
+	}
+	priv := s.privateKey()
+	pub, pri := (*[32]byte)(&mkey), (*[32]byte)(&priv)
+	msgData := box.Seal(nonce[:], b, &nonce, pub, pri)
+	return msgData, nil
+}
+
+// filterInvalidIPv6Endpoints removes invalid IPv6 endpoints from eps,
+// modify the slice in place, returning the potentially smaller subset (aliasing
+// the original memory).
+//
+// Two types of IPv6 endpoints are considered invalid: link-local
+// addresses, and anything with a zone.
+func filterInvalidIPv6Endpoints(eps []string) []string {
+	clean := eps[:0]
+	for _, ep := range eps {
+		if keepClientEndpoint(ep) {
+			clean = append(clean, ep)
+		}
+	}
+	return clean
+}
+
+func keepClientEndpoint(ep string) bool {
+	ipp, err := netaddr.ParseIPPort(ep)
+	if err != nil {
+		// Shouldn't have made it this far if we unmarshalled
+		// the incoming JSON response.
+		return false
+	}
+	ip := ipp.IP
+	if ip.Zone() != "" {
+		return false
+	}
+	if ip.Is6() && ip.IsLinkLocalUnicast() {
+		// We let clients send these for now, but
+		// tailscaled doesn't know how to use them yet
+		// so we filter them out for now. A future
+		// MapRequest.Version might signal that
+		// clients know how to use them (e.g. try all
+		// local scopes).
+		return false
+	}
+	return true
+}
+
+// breakSameNodeMapResponseStreams reports whether req should break a
+// prior long-polling MapResponse stream (if active) from the same
+// node ID.
+func breakSameNodeMapResponseStreams(req *tailcfg.MapRequest) bool {
+	if req.ReadOnly {
+		// Don't register our updatesCh for closability
+		// nor close another peer's if we're a read-only request.
+		return false
+	}
+	if !req.Stream && req.OmitPeers {
+		// Likewise, if we're not streaming and not asking for peers,
+		// (but still mutable, without Readonly set), consider this an endpoint
+		// update request only, and don't close any existing map response
+		// for this nodeID. It's likely the same client with a built-up
+		// compression context. We want to let them update their
+		// new endpoints with us without breaking that other long-running
+		// map response.
+		return false
+	}
+	return true
+}

tstest/log.go

@@ -107,6 +107,15 @@ func (lt *LogLineTracker) Check() []string {
 	return notSeen
 }
 
+// Reset forgets everything that it's seen.
+func (lt *LogLineTracker) Reset() {
+	lt.mu.Lock()
+	defer lt.mu.Unlock()
+	for _, line := range lt.listenFor {
+		lt.seen[line] = false
+	}
+}
+
 // Close closes lt. After calling Close, calls to Logf become no-ops.
 func (lt *LogLineTracker) Close() {
 	lt.mu.Lock()

tstest/tstest.go

@@ -0,0 +1,31 @@
+// Copyright (c) 2020 Tailscale Inc & AUTHORS All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+// Package tstest provides utilities for use in unit tests.
+package tstest
+
+import (
+	"context"
+	"time"
+
+	"tailscale.com/logtail/backoff"
+	"tailscale.com/types/logger"
+)
+
+// WaitFor retries try for up to maxWait.
+// It returns nil once try returns nil the first time.
+// If maxWait passes without success, it returns try's last error.
+func WaitFor(maxWait time.Duration, try func() error) error {
+	bo := backoff.NewBackoff("wait-for", logger.Discard, maxWait/4)
+	deadline := time.Now().Add(maxWait)
+	var err error
+	for time.Now().Before(deadline) {
+		err = try()
+		if err == nil {
+			break
+		}
+		bo.BackOff(context.Background(), err)
+	}
+	return err
+}

types/logger/logger.go

@@ -18,8 +18,6 @@ import (
 	"strings"
 	"sync"
 	"time"
-
-	"golang.org/x/time/rate"
 )
 
 // Logf is the basic Tailscale logger type: a printf-like func.
@@ -57,45 +55,53 @@ func Discard(string, ...interface{}) {}
 // limitData is used to keep track of each format string's associated
 // rate-limiting data.
 type limitData struct {
-	lim        *rate.Limiter // the token bucket associated with this string
-	msgBlocked bool          // whether a "duplicate error" message has already been logged
-	ele        *list.Element // list element used to access this string in the cache
+	bucket   *tokenBucket  // the token bucket associated with this string
+	nBlocked int           // number of messages skipped
+	ele      *list.Element // list element used to access this string in the cache
 }
 
 var disableRateLimit = os.Getenv("TS_DEBUG_LOG_RATE") == "all"
 
 // rateFree are format string substrings that are exempt from rate limiting.
-// Things should not be added to this unless they're already limited otherwise.
+// Things should not be added to this unless they're already limited otherwise
+// or are critical for generating important stats from the logs.
 var rateFree = []string{
 	"magicsock: disco: ",
-	"magicsock: CreateEndpoint:",
+	"magicsock: ParseEndpoint:",
+	// grinder stats lines
+	"SetPrefs: %v",
+	"peer keys: %s",
+	"v%v peers: %v",
 }
 
-// RateLimitedFn returns a rate-limiting Logf wrapping the given logf.
-// Messages are allowed through at a maximum of one message every f (where f is a time.Duration), in
-// bursts of up to burst messages at a time. Up to maxCache strings will be held at a time.
+// RateLimitedFn is a wrapper for RateLimitedFnWithClock that includes the
+// current time automatically. This is mainly for backward compatibility.
 func RateLimitedFn(logf Logf, f time.Duration, burst int, maxCache int) Logf {
+	return RateLimitedFnWithClock(logf, f, burst, maxCache, time.Now)
+}
+
+// RateLimitedFnWithClock returns a rate-limiting Logf wrapping the given
+// logf. Messages are allowed through at a maximum of one message every f
+// (where f is a time.Duration), in bursts of up to burst messages at a
+// time. Up to maxCache format strings will be tracked separately.
+// timeNow is a function that returns the current time, used for calculating
+// rate limits.
+func RateLimitedFnWithClock(logf Logf, f time.Duration, burst int, maxCache int, timeNow func() time.Time) Logf {
 	if disableRateLimit {
 		return logf
 	}
-	r := rate.Every(f)
 	var (
 		mu       sync.Mutex
 		msgLim   = make(map[string]*limitData) // keyed by logf format
 		msgCache = list.New()                  // a rudimentary LRU that limits the size of the map
 	)
 
-	type verdict int
-	const (
-		allow verdict = iota
-		warn
-		block
-	)
-
-	judge := func(format string) verdict {
+	return func(format string, args ...interface{}) {
+		// Shortcut for formats with no rate limit
 		for _, sub := range rateFree {
 			if strings.Contains(format, sub) {
-				return allow
+				logf(format, args...)
+				return
 			}
 		}
 
@@ -106,8 +112,8 @@ func RateLimitedFn(logf Logf, f time.Duration, burst int, maxCache int) Logf {
 			msgCache.MoveToFront(rl.ele)
 		} else {
 			rl = &limitData{
-				lim: rate.NewLimiter(r, burst),
-				ele: msgCache.PushFront(format),
+				bucket: newTokenBucket(f, burst, timeNow()),
+				ele:    msgCache.PushFront(format),
 			}
 			msgLim[format] = rl
 			if msgCache.Len() > maxCache {
@@ -115,24 +121,39 @@ func RateLimitedFn(logf Logf, f time.Duration, burst int, maxCache int) Logf {
 				msgCache.Remove(msgCache.Back())
 			}
 		}
-		if rl.lim.Allow() {
-			rl.msgBlocked = false
-			return allow
-		}
-		if !rl.msgBlocked {
-			rl.msgBlocked = true
-			return warn
-		}
-		return block
-	}
 
-	return func(format string, args ...interface{}) {
-		switch judge(format) {
-		case allow:
+		rl.bucket.AdvanceTo(timeNow())
+
+		// Make sure there's enough room for at least a few
+		// more logs before we unblock, so we don't alternate
+		// between blocking and unblocking.
+		if rl.nBlocked > 0 && rl.bucket.remaining >= 2 {
+			// Only print this if we dropped more than 1
+			// message. Otherwise we'd *increase* the total
+			// number of log lines printed.
+			if rl.nBlocked > 1 {
+				logf("[RATELIMIT] format(%q) (%d dropped)",
+					format, rl.nBlocked-1)
+			}
+			rl.nBlocked = 0
+		}
+		if rl.nBlocked == 0 && rl.bucket.Get() {
 			logf(format, args...)
-		case warn:
-			// For the warning, log the specific format string
-			logf("[RATE LIMITED] format string \"%s\" (example: \"%s\")", format, strings.TrimSpace(fmt.Sprintf(format, args...)))
+			if rl.bucket.remaining == 0 {
+				// Enter "blocked" mode immediately after
+				// reaching the burst limit. We want to
+				// always accompany the format() message
+				// with an example of the format, which is
+				// effectively the same as printing the
+				// message anyway. But this way they can
+				// be on two separate lines and we don't
+				// corrupt the original message.
+				logf("[RATELIMIT] format(%q)", format)
+				rl.nBlocked = 1
+			}
+			return
+		} else {
+			rl.nBlocked++
 		}
 	}
 }
@@ -163,7 +184,6 @@ func LogOnChange(logf Logf, maxInterval time.Duration, timeNow func() time.Time)
 		// as it might contain formatting directives.)
 		logf(format, args...)
 	}
-
 }
 
 // ArgWriter is a fmt.Formatter that can be passed to any Logf func to

types/logger/logger_test.go

@@ -44,16 +44,27 @@ func TestRateLimiter(t *testing.T) {
 		"boring string with constant formatting (constant)",
 		"templated format string no. 0",
 		"boring string with constant formatting (constant)",
+		"[RATELIMIT] format(\"boring string with constant formatting %s\")",
 		"templated format string no. 1",
-		"[RATE LIMITED] format string \"boring string with constant formatting %s\" (example: \"boring string with constant formatting (constant)\")",
-		"[RATE LIMITED] format string \"templated format string no. %d\" (example: \"templated format string no. 2\")",
+		"[RATELIMIT] format(\"templated format string no. %d\")",
 		"Make sure this string makes it through the rest (that are blocked) 4",
 		"4 shouldn't get filtered.",
+		"hello 1",
+		"hello 2",
+		"[RATELIMIT] format(\"hello %v\")",
+		"[RATELIMIT] format(\"hello %v\") (2 dropped)",
+		"hello 5",
+		"hello 6",
+		"[RATELIMIT] format(\"hello %v\")",
+		"hello 7",
 	}
 
+	var now time.Time
+	nowf := func() time.Time { return now }
+
 	testsRun := 0
 	lgtest := logTester(want, t, &testsRun)
-	lg := RateLimitedFn(lgtest, 1*time.Minute, 2, 50)
+	lg := RateLimitedFnWithClock(lgtest, 1*time.Minute, 2, 50, nowf)
 	var prefixed Logf
 	for i := 0; i < 10; i++ {
 		lg("boring string with constant formatting %s", "(constant)")
@@ -64,6 +75,19 @@ func TestRateLimiter(t *testing.T) {
 			prefixed(" shouldn't get filtered.")
 		}
 	}
+
+	lg("hello %v", 1)
+	lg("hello %v", 2) // printed, but rate limit starts
+	lg("hello %v", 3) // rate limited (not printed)
+	now = now.Add(1 * time.Minute)
+	lg("hello %v", 4) // still limited (not printed)
+	now = now.Add(1 * time.Minute)
+	lg("hello %v", 5) // restriction lifted; prints drop count + message
+
+	lg("hello %v", 6) // printed, but rate limit starts
+	now = now.Add(2 * time.Minute)
+	lg("hello %v", 7) // restriction lifted; no drop count needed
+
 	if testsRun < len(want) {
 		t.Fatalf("Tests after %s weren't logged.", want[testsRun])
 	}

types/logger/tokenbucket.go

@@ -0,0 +1,63 @@
+// Copyright (c) 2021 Tailscale Inc & AUTHORS All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package logger
+
+import (
+	"time"
+)
+
+// tokenBucket is a simple token bucket style rate limiter.
+
+// It's similar in function to golang.org/x/time/rate.Limiter, which we
+// can't use because:
+// - It doesn't give access to the number of accumulated tokens, which we
+//   need for implementing hysteresis;
+// - It doesn't let us provide our own time function, which we need for
+//   implementing proper unit tests.
+// rate.Limiter is also much more complex than necessary, but that wouldn't
+// be enough to disqualify it on its own.
+//
+// Unlike rate.Limiter, this token bucket does not attempt to
+// do any locking of its own. Don't try to access it re-entrantly.
+// That's fine inside this types/logger package because we already have
+// locking at a higher level.
+type tokenBucket struct {
+	remaining int
+	max       int
+	tick      time.Duration
+	t         time.Time
+}
+
+func newTokenBucket(tick time.Duration, max int, now time.Time) *tokenBucket {
+	return &tokenBucket{max, max, tick, now}
+}
+
+func (tb *tokenBucket) Get() bool {
+	if tb.remaining > 0 {
+		tb.remaining--
+		return true
+	}
+	return false
+}
+
+func (tb *tokenBucket) Refund(n int) {
+	b := tb.remaining + n
+	if b > tb.max {
+		tb.remaining = tb.max
+	} else {
+		tb.remaining = b
+	}
+}
+
+func (tb *tokenBucket) AdvanceTo(t time.Time) {
+	diff := t.Sub(tb.t)
+
+	// only use up whole ticks. The remainder will be used up
+	// next time.
+	ticks := int(diff / tb.tick)
+	tb.t = tb.t.Add(time.Duration(ticks) * tb.tick)
+
+	tb.Refund(ticks)
+}

types/netmap/netmap.go

@@ -31,8 +31,8 @@ type NetworkMap struct {
 	Expiry     time.Time
 	// Name is the DNS name assigned to this node.
 	Name          string
-	Addresses     []netaddr.IPPrefix
-	LocalPort     uint16 // used for debugging
+	Addresses     []netaddr.IPPrefix // same as tailcfg.Node.Addresses (IP addresses of this Node directly)
+	LocalPort     uint16             // used for debugging
 	MachineStatus tailcfg.MachineStatus
 	MachineKey    tailcfg.MachineKey
 	Peers         []*tailcfg.Node // sorted by Node.ID

types/wgkey/key.go

@@ -27,7 +27,7 @@ import (
 const Size = 32
 
 // A Key is a curve25519 key.
-// It is used by WireGuard to represent public keys.
+// It is used by WireGuard to represent public and preshared keys.
 type Key [Size]byte
 
 // NewPreshared generates a new random Key.
@@ -90,14 +90,12 @@ func (k *Key) IsZero() bool {
 	return subtle.ConstantTimeCompare(zeros[:], k[:]) == 1
 }
 
-func (k *Key) MarshalJSON() ([]byte, error) {
-	if k == nil {
-		return []byte("null"), nil
-	}
-	// TODO(josharian): use encoding/hex instead?
-	buf := new(bytes.Buffer)
-	fmt.Fprintf(buf, `"%x"`, k[:])
-	return buf.Bytes(), nil
+func (k Key) MarshalJSON() ([]byte, error) {
+	buf := make([]byte, 2+len(k)*2)
+	buf[0] = '"'
+	hex.Encode(buf[1:], k[:])
+	buf[len(buf)-1] = '"'
+	return buf, nil
 }
 
 func (k *Key) UnmarshalJSON(b []byte) error {

types/wgkey/key_test.go

@@ -6,6 +6,7 @@ package wgkey
 
 import (
 	"bytes"
+	"encoding/json"
 	"testing"
 )
 
@@ -20,7 +21,7 @@ func TestKeyBasics(t *testing.T) {
 		t.Fatal(err)
 	}
 
-	t.Run("JSON round-trip", func(t *testing.T) {
+	t.Run("JSON round-trip (pointer)", func(t *testing.T) {
 		// should preserve the keys
 		k2 := new(Key)
 		if err := k2.UnmarshalJSON(b); err != nil {
@@ -55,6 +56,27 @@ func TestKeyBasics(t *testing.T) {
 			t.Fatalf("base64-encoded keys match: %s, %s", b1, b2)
 		}
 	})
+
+	t.Run("JSON round-trip (value)", func(t *testing.T) {
+		type T struct {
+			K Key
+		}
+		v := T{K: *k1}
+		b, err := json.Marshal(v)
+		if err != nil {
+			t.Fatal(err)
+		}
+		var u T
+		if err := json.Unmarshal(b, &u); err != nil {
+			t.Fatal(err)
+		}
+		if !bytes.Equal(v.K[:], u.K[:]) {
+			t.Fatalf("v.K %v != u.K %v", v.K[:], u.K[:])
+		}
+		if b1, b2 := v.K.String(), u.K.String(); b1 != b2 {
+			t.Fatalf("base64-encoded keys do not match: %s, %s", b1, b2)
+		}
+	})
 }
 func TestPrivateKeyBasics(t *testing.T) {
 	pri, err := NewPrivate()
@@ -109,3 +131,28 @@ func TestPrivateKeyBasics(t *testing.T) {
 		}
 	})
 }
+
+func TestMarshalJSONAllocs(t *testing.T) {
+	var k Key
+	f := testing.AllocsPerRun(100, func() {
+		k.MarshalJSON()
+	})
+	n := int(f)
+	if n != 1 {
+		t.Fatalf("max one alloc per Key.MarshalJSON, got %d", n)
+	}
+}
+
+var sink []byte
+
+func BenchmarkMarshalJSON(b *testing.B) {
+	b.ReportAllocs()
+	var k Key
+	for i := 0; i < b.N; i++ {
+		var err error
+		sink, err = k.MarshalJSON()
+		if err != nil {
+			b.Fatal(err)
+		}
+	}
+}

util/cmpver/version.go

@@ -0,0 +1,96 @@
+// Copyright (c) 2021 Tailscale Inc & AUTHORS All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+// Package cmpver implements a variant of debian version number
+// comparison.
+//
+// A version is a string consisting of alternating non-numeric and
+// numeric fields. When comparing two versions, each one is broken
+// down into its respective fields, and the fields are compared
+// pairwise. The comparison is lexicographic for non-numeric fields,
+// numeric for numeric fields. The first non-equal field pair
+// determines the ordering of the two versions.
+//
+// This comparison scheme is a simplified version of Debian's version
+// number comparisons. Debian differs in a few details of
+// lexicographical field comparison, where certain characters have
+// special meaning and ordering. We don't need that, because Tailscale
+// version numbers don't need it.
+package cmpver
+
+import (
+	"fmt"
+	"strconv"
+	"strings"
+	"unicode"
+)
+
+// Compare returns an integer comparing two strings as version
+// numbers. The result will be 0 if v1==v2, -1 if v1 < v2, and +1 if
+// v1 > v2.
+func Compare(v1, v2 string) int {
+	notNumber := func(r rune) bool { return !unicode.IsNumber(r) }
+
+	var (
+		f1, f2 string
+		n1, n2 uint64
+		err    error
+	)
+	for v1 != "" || v2 != "" {
+		// Compare the non-numeric character run lexicographically.
+		f1, v1 = splitPrefixFunc(v1, notNumber)
+		f2, v2 = splitPrefixFunc(v2, notNumber)
+
+		if res := strings.Compare(f1, f2); res != 0 {
+			return res
+		}
+
+		// Compare the numeric character run numerically.
+		f1, v1 = splitPrefixFunc(v1, unicode.IsNumber)
+		f2, v2 = splitPrefixFunc(v2, unicode.IsNumber)
+
+		// ParseUint refuses to parse empty strings, which would only
+		// happen if we reached end-of-string. We follow the Debian
+		// convention that empty strings mean zero, because
+		// empirically that produces reasonable-feeling comparison
+		// behavior.
+		n1 = 0
+		if f1 != "" {
+			n1, err = strconv.ParseUint(f1, 10, 64)
+			if err != nil {
+				panic(fmt.Sprintf("all-number string %q didn't parse as string: %s", f1, err))
+			}
+		}
+
+		n2 = 0
+		if f2 != "" {
+			n2, err = strconv.ParseUint(f2, 10, 64)
+			if err != nil {
+				panic(fmt.Sprintf("all-number string %q didn't parse as string: %s", f2, err))
+			}
+		}
+
+		switch {
+		case n1 == n2:
+		case n1 < n2:
+			return -1
+		case n1 > n2:
+			return 1
+		}
+	}
+
+	// Only way to reach here is if v1 and v2 run out of fields
+	// simultaneously - i.e. exactly equal versions.
+	return 0
+}
+
+// splitPrefixFunc splits s at the first rune where f(rune) is false.
+func splitPrefixFunc(s string, f func(rune) bool) (string, string) {
+	for i, r := range s {
+		if !f(r) {
+			return s[:i], s[i:]
+		}
+	}
+	return s, s[:0]
+}

util/cmpver/version_test.go

@@ -0,0 +1,106 @@
+// Copyright (c) 2021 Tailscale Inc & AUTHORS All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package cmpver
+
+import "testing"
+
+func TestCompare(t *testing.T) {
+	tests := []struct {
+		name   string
+		v1, v2 string
+		want   int
+	}{
+		{
+			name: "both empty",
+			want: 0,
+		},
+		{
+			name: "v1 empty",
+			v2:   "1.2.3",
+			want: -1,
+		},
+		{
+			name: "v2 empty",
+			v1:   "1.2.3",
+			want: 1,
+		},
+
+		{
+			name: "semver major",
+			v1:   "2.0.0",
+			v2:   "1.9.9",
+			want: 1,
+		},
+		{
+			name: "semver major",
+			v1:   "2.0.0",
+			v2:   "1.9.9",
+			want: 1,
+		},
+		{
+			name: "semver minor",
+			v1:   "1.9.0",
+			v2:   "1.8.9",
+			want: 1,
+		},
+		{
+			name: "semver patch",
+			v1:   "1.9.9",
+			v2:   "1.9.8",
+			want: 1,
+		},
+		{
+			name: "semver equal",
+			v1:   "1.9.8",
+			v2:   "1.9.8",
+			want: 0,
+		},
+
+		{
+			name: "tailscale major",
+			v1:   "1.0-0",
+			v2:   "0.97-105",
+			want: 1,
+		},
+		{
+			name: "tailscale minor",
+			v1:   "0.98-0",
+			v2:   "0.97-105",
+			want: 1,
+		},
+		{
+			name: "tailscale patch",
+			v1:   "0.97-120",
+			v2:   "0.97-105",
+			want: 1,
+		},
+		{
+			name: "tailscale equal",
+			v1:   "0.97-105",
+			v2:   "0.97-105",
+			want: 0,
+		},
+		{
+			name: "tailscale weird extra field",
+			v1:   "0.96.1-0", // more fields == larger
+			v2:   "0.96-105",
+			want: 1,
+		},
+	}
+
+	for _, test := range tests {
+		t.Run(test.name, func(t *testing.T) {
+			got := Compare(test.v1, test.v2)
+			if got != test.want {
+				t.Errorf("Compare(%v, %v) = %v, want %v", test.v1, test.v2, got, test.want)
+			}
+			// Reversing the comparison should reverse the outcome.
+			got2 := Compare(test.v2, test.v1)
+			if got2 != -test.want {
+				t.Errorf("Compare(%v, %v) = %v, want %v", test.v2, test.v1, got2, -test.want)
+			}
+		})
+	}
+}

util/osshare/filesharingstatus_noop.go

@@ -0,0 +1,13 @@
+// Copyright (c) 2021 Tailscale Inc & AUTHORS All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+// +build !windows
+
+package osshare
+
+import (
+	"tailscale.com/types/logger"
+)
+
+func SetFileSharingEnabled(enabled bool, logf logger.Logf) {}

util/osshare/filesharingstatus_windows.go

@@ -0,0 +1,107 @@
+// Copyright (c) 2021 Tailscale Inc & AUTHORS All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+// +build windows
+
+package osshare
+
+import (
+	"fmt"
+	"os"
+	"path/filepath"
+	"sync"
+
+	"golang.org/x/sys/windows/registry"
+	"tailscale.com/types/logger"
+)
+
+const (
+	sendFileShellKey = `*\shell\tailscale`
+)
+
+var ipnExePath struct {
+	sync.Mutex
+	cache string // absolute path of tailscale-ipn.exe, populated lazily on first use
+}
+
+func getIpnExePath(logf logger.Logf) string {
+	ipnExePath.Lock()
+	defer ipnExePath.Unlock()
+
+	if ipnExePath.cache != "" {
+		return ipnExePath.cache
+	}
+
+	// Find the absolute path of tailscale-ipn.exe assuming that it's in the same
+	// directory as this executable (tailscaled.exe).
+	p, err := os.Executable()
+	if err != nil {
+		logf("os.Executable error: %v", err)
+		return ""
+	}
+	if p, err = filepath.EvalSymlinks(p); err != nil {
+		logf("filepath.EvalSymlinks error: %v", err)
+		return ""
+	}
+	p = filepath.Join(filepath.Dir(p), "tailscale-ipn.exe")
+	if p, err = filepath.Abs(p); err != nil {
+		logf("filepath.Abs error: %v", err)
+		return ""
+	}
+	ipnExePath.cache = p
+
+	return p
+}
+
+// SetFileSharingEnabled adds/removes "Send with Tailscale" from the Windows shell menu.
+func SetFileSharingEnabled(enabled bool, logf logger.Logf) {
+	logf = logger.WithPrefix(logf, fmt.Sprintf("SetFileSharingEnabled(%v) error: ", enabled))
+	if enabled {
+		enableFileSharing(logf)
+	} else {
+		disableFileSharing(logf)
+	}
+}
+
+func enableFileSharing(logf logger.Logf) {
+	path := getIpnExePath(logf)
+	if path == "" {
+		return
+	}
+
+	k, _, err := registry.CreateKey(registry.CLASSES_ROOT, sendFileShellKey, registry.WRITE)
+	if err != nil {
+		logf("failed to create HKEY_CLASSES_ROOT\\%s reg key: %v", sendFileShellKey, err)
+		return
+	}
+	defer k.Close()
+	if err := k.SetStringValue("", "Send with Tailscale..."); err != nil {
+		logf("k.SetStringValue error: %v", err)
+		return
+	}
+	if err := k.SetStringValue("Icon", path+",0"); err != nil {
+		logf("k.SetStringValue error: %v", err)
+		return
+	}
+	c, _, err := registry.CreateKey(k, "command", registry.WRITE)
+	if err != nil {
+		logf("failed to create HKEY_CLASSES_ROOT\\%s\\command reg key: %v", sendFileShellKey, err)
+		return
+	}
+	defer c.Close()
+	if err := c.SetStringValue("", "\""+path+"\" /push \"%1\""); err != nil {
+		logf("c.SetStringValue error: %v", err)
+	}
+}
+
+func disableFileSharing(logf logger.Logf) {
+	if err := registry.DeleteKey(registry.CLASSES_ROOT, sendFileShellKey+"\\command"); err != nil &&
+		err != registry.ErrNotExist {
+		logf("registry.DeleteKey error: %v\n", err)
+		return
+	}
+	if err := registry.DeleteKey(registry.CLASSES_ROOT, sendFileShellKey); err != nil && err != registry.ErrNotExist {
+		logf("registry.DeleteKey error: %v\n", err)
+	}
+}

version/prop.go

@@ -4,7 +4,13 @@
 
 package version
 
-import "runtime"
+import (
+	"os"
+	"path/filepath"
+	"runtime"
+	"strings"
+	"sync/atomic"
+)
 
 // IsMobile reports whether this is a mobile client build.
 func IsMobile() bool {
@@ -22,3 +28,37 @@ func OS() string {
 	}
 	return runtime.GOOS
 }
+
+// IsSandboxedMacOS reports whether this process is a sandboxed macOS
+// process. It is true for the Mac App Store and macsys (System
+// Extension) version on macOS, and false for tailscaled-on-macOS.
+func IsSandboxedMacOS() bool {
+	if runtime.GOOS != "darwin" {
+		return false
+	}
+	if IsMacSysExt() {
+		return true
+	}
+	exe, _ := os.Executable()
+	return strings.HasSuffix(exe, "/Contents/MacOS/Tailscale")
+}
+
+var isMacSysExt atomic.Value
+
+// IsMacSysExt whether this binary is from the standalone "System
+// Extension" (a.k.a. "macsys") version of Tailscale for macOS.
+func IsMacSysExt() bool {
+	if runtime.GOOS != "darwin" {
+		return false
+	}
+	if b, ok := isMacSysExt.Load().(bool); ok {
+		return b
+	}
+	exe, err := os.Executable()
+	if err != nil {
+		return false
+	}
+	v := filepath.Base(exe) == "io.tailscale.ipn.macsys.network-extension"
+	isMacSysExt.Store(v)
+	return v
+}

version/version.go

@@ -10,7 +10,7 @@ package version
 // Long is a full version number for this build, of the form
 // "x.y.z-commithash", or "date.yyyymmdd" if no actual version was
 // provided.
-const Long = "date.20210415"
+const Long = "date.20210505"
 
 // Short is a short version number for this build, of the form
 // "x.y.z", or "date.yyyymmdd" if no actual version was provided.

wgengine/bench/bench.go

@@ -0,0 +1,398 @@
+// Copyright (c) 2021 Tailscale Inc & AUTHORS All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+// Create two wgengine instances and pass data through them, measuring
+// throughput, latency, and packet loss.
+package main
+
+import (
+	"bufio"
+	"io"
+	"log"
+	"net"
+	"net/http"
+	"net/http/pprof"
+	"os"
+	"strconv"
+	"time"
+
+	"inet.af/netaddr"
+	"tailscale.com/types/logger"
+)
+
+const PayloadSize = 1000
+const ICMPMinSize = 24
+
+var Addr1 = netaddr.MustParseIPPrefix("100.64.1.1/32")
+var Addr2 = netaddr.MustParseIPPrefix("100.64.1.2/32")
+
+func main() {
+	var logf logger.Logf = log.Printf
+	log.SetFlags(0)
+
+	debugMux := newDebugMux()
+	go runDebugServer(debugMux, "0.0.0.0:8999")
+
+	mode, err := strconv.Atoi(os.Args[1])
+	if err != nil {
+		log.Fatalf("%q: %v", os.Args[1], err)
+	}
+
+	traf := NewTrafficGen(nil)
+
+	// Sample test results below are using GOMAXPROCS=2 (for some
+	// tests, including wireguard-go, higher GOMAXPROCS goes slower)
+	// on apenwarr's old Linux box:
+	//   Intel(R) Core(TM) i7-4785T CPU @ 2.20GHz
+	// My 2019 Mac Mini is about 20% faster on most tests.
+
+	switch mode {
+	// tx=8786325 rx=8786326 (0 = 0.00% loss) (70768.7 Mbits/sec)
+	case 1:
+		setupTrivialNoAllocTest(logf, traf)
+
+	// tx=6476293 rx=6476293 (0 = 0.00% loss) (52249.7 Mbits/sec)
+	case 2:
+		setupTrivialTest(logf, traf)
+
+	// tx=1957974 rx=1958379 (0 = 0.00% loss) (15939.8 Mbits/sec)
+	case 11:
+		setupBlockingChannelTest(logf, traf)
+
+	// tx=728621 rx=701825 (26620 = 3.65% loss) (5525.2 Mbits/sec)
+	// (much faster on macOS??)
+	case 12:
+		setupNonblockingChannelTest(logf, traf)
+
+	// tx=1024260 rx=941098 (83334 = 8.14% loss) (7516.6 Mbits/sec)
+	// (much faster on macOS??)
+	case 13:
+		setupDoubleChannelTest(logf, traf)
+
+	// tx=265468 rx=263189 (2279 = 0.86% loss) (2162.0 Mbits/sec)
+	case 21:
+		setupUDPTest(logf, traf)
+
+	// tx=1493580 rx=1493580 (0 = 0.00% loss) (12210.4 Mbits/sec)
+	case 31:
+		setupBatchTCPTest(logf, traf)
+
+	// tx=134236 rx=133166 (1070 = 0.80% loss) (1088.9 Mbits/sec)
+	case 101:
+		setupWGTest(logf, traf, Addr1, Addr2)
+
+	default:
+		log.Fatalf("provide a valid test number (0..n)")
+	}
+
+	logf("initialized ok.")
+	traf.Start(Addr1.IP, Addr2.IP, PayloadSize+ICMPMinSize, 0)
+
+	var cur, prev Snapshot
+	var pps int64
+	i := 0
+	for {
+		i += 1
+		time.Sleep(10 * time.Millisecond)
+
+		if (i % 100) == 0 {
+			prev = cur
+			cur = traf.Snap()
+			d := cur.Sub(prev)
+
+			if prev.WhenNsec == 0 {
+				logf("tx=%-6d rx=%-6d", d.TxPackets, d.RxPackets)
+			} else {
+				logf("%v @%7d pkt/s", d, pps)
+			}
+		}
+
+		pps = traf.Adjust()
+	}
+}
+
+func newDebugMux() *http.ServeMux {
+	mux := http.NewServeMux()
+	mux.HandleFunc("/debug/pprof/", pprof.Index)
+	mux.HandleFunc("/debug/pprof/cmdline", pprof.Cmdline)
+	mux.HandleFunc("/debug/pprof/profile", pprof.Profile)
+	mux.HandleFunc("/debug/pprof/symbol", pprof.Symbol)
+	mux.HandleFunc("/debug/pprof/trace", pprof.Trace)
+	return mux
+}
+
+func runDebugServer(mux *http.ServeMux, addr string) {
+	srv := &http.Server{
+		Addr:    addr,
+		Handler: mux,
+	}
+	if err := srv.ListenAndServe(); err != nil {
+		log.Fatal(err)
+	}
+}
+
+// The absolute minimal test of the traffic generator: have it fill
+// a packet buffer, then absorb it again. Zero packet loss.
+func setupTrivialNoAllocTest(logf logger.Logf, traf *TrafficGen) {
+	go func() {
+		b := make([]byte, 1600)
+		for {
+			n := traf.Generate(b, 16)
+			if n == 0 {
+				break
+			}
+			traf.GotPacket(b[0:n+16], 16)
+		}
+	}()
+}
+
+// Almost the same, but this time allocate a fresh buffer each time
+// through the loop. Still zero packet loss. Runs about 2/3 as fast for me.
+func setupTrivialTest(logf logger.Logf, traf *TrafficGen) {
+	go func() {
+		for {
+			b := make([]byte, 1600)
+			n := traf.Generate(b, 16)
+			if n == 0 {
+				break
+			}
+			traf.GotPacket(b[0:n+16], 16)
+		}
+	}()
+}
+
+// Pass packets through a blocking channel between sender and receiver.
+// Still zero packet loss since the sender stops when the channel is full.
+// Max speed depends on channel length (I'm not sure why).
+func setupBlockingChannelTest(logf logger.Logf, traf *TrafficGen) {
+	ch := make(chan []byte, 1000)
+
+	go func() {
+		// transmitter
+		for {
+			b := make([]byte, 1600)
+			n := traf.Generate(b, 16)
+			if n == 0 {
+				close(ch)
+				break
+			}
+			ch <- b[0 : n+16]
+		}
+	}()
+
+	go func() {
+		// receiver
+		for b := range ch {
+			traf.GotPacket(b, 16)
+		}
+	}()
+}
+
+// Same as setupBlockingChannelTest, but now we drop packets whenever the
+// channel is full. Max speed is about the same as the above test, but
+// now with nonzero packet loss.
+func setupNonblockingChannelTest(logf logger.Logf, traf *TrafficGen) {
+	ch := make(chan []byte, 1000)
+
+	go func() {
+		// transmitter
+		for {
+			b := make([]byte, 1600)
+			n := traf.Generate(b, 16)
+			if n == 0 {
+				close(ch)
+				break
+			}
+			select {
+			case ch <- b[0 : n+16]:
+			default:
+			}
+		}
+	}()
+
+	go func() {
+		// receiver
+		for b := range ch {
+			traf.GotPacket(b, 16)
+		}
+	}()
+}
+
+// Same as above, but at an intermediate blocking channel and goroutine
+// to make things a little more like wireguard-go. Roughly 20% slower than
+// the single-channel verison.
+func setupDoubleChannelTest(logf logger.Logf, traf *TrafficGen) {
+	ch := make(chan []byte, 1000)
+	ch2 := make(chan []byte, 1000)
+
+	go func() {
+		// transmitter
+		for {
+			b := make([]byte, 1600)
+			n := traf.Generate(b, 16)
+			if n == 0 {
+				close(ch)
+				break
+			}
+			select {
+			case ch <- b[0 : n+16]:
+			default:
+			}
+		}
+	}()
+
+	go func() {
+		// intermediary
+		for b := range ch {
+			ch2 <- b
+		}
+		close(ch2)
+	}()
+
+	go func() {
+		// receiver
+		for b := range ch2 {
+			traf.GotPacket(b, 16)
+		}
+	}()
+}
+
+// Instead of a channel, pass packets through a UDP socket.
+func setupUDPTest(logf logger.Logf, traf *TrafficGen) {
+	la, err := net.ResolveUDPAddr("udp", ":0")
+	if err != nil {
+		log.Fatalf("resolve: %v", err)
+	}
+
+	s1, err := net.ListenUDP("udp", la)
+	if err != nil {
+		log.Fatalf("listen1: %v", err)
+	}
+	s2, err := net.ListenUDP("udp", la)
+	if err != nil {
+		log.Fatalf("listen2: %v", err)
+	}
+
+	a2 := s2.LocalAddr()
+
+	// On macOS (but not Linux), you can't transmit to 0.0.0.0:port,
+	// which is what returns from .LocalAddr() above. We have to
+	// force it to localhost instead.
+	a2.(*net.UDPAddr).IP = net.ParseIP("127.0.0.1")
+
+	s1.SetWriteBuffer(1024 * 1024)
+	s2.SetReadBuffer(1024 * 1024)
+
+	go func() {
+		// transmitter
+		b := make([]byte, 1600)
+		for {
+			n := traf.Generate(b, 16)
+			if n == 0 {
+				break
+			}
+			s1.WriteTo(b[16:n+16], a2)
+		}
+	}()
+
+	go func() {
+		// receiver
+		b := make([]byte, 1600)
+		for traf.Running() {
+			// Use ReadFrom instead of Read, to be more like
+			// how wireguard-go does it, even though we're not
+			// going to actually look at the address.
+			n, _, err := s2.ReadFrom(b)
+			if err != nil {
+				log.Fatalf("s2.Read: %v", err)
+			}
+			traf.GotPacket(b[:n], 0)
+		}
+	}()
+}
+
+// Instead of a channel, pass packets through a TCP socket.
+// TCP is a single stream, so we can amortize one syscall across
+// multiple packets. 10x amortization seems to make it go ~10x faster,
+// as expected, getting us close to the speed of the channel tests above.
+// There's also zero packet loss.
+func setupBatchTCPTest(logf logger.Logf, traf *TrafficGen) {
+	sl, err := net.Listen("tcp", ":0")
+	if err != nil {
+		log.Fatalf("listen: %v", err)
+	}
+
+	s1, err := net.Dial("tcp", sl.Addr().String())
+	if err != nil {
+		log.Fatalf("dial: %v", err)
+	}
+
+	s2, err := sl.Accept()
+	if err != nil {
+		log.Fatalf("accept: %v", err)
+	}
+
+	s1.(*net.TCPConn).SetWriteBuffer(1024 * 1024)
+	s2.(*net.TCPConn).SetReadBuffer(1024 * 1024)
+
+	ch := make(chan int)
+
+	go func() {
+		// transmitter
+
+		bs1 := bufio.NewWriterSize(s1, 1024*1024)
+
+		b := make([]byte, 1600)
+		i := 0
+		for {
+			i += 1
+			n := traf.Generate(b, 16)
+			if n == 0 {
+				break
+			}
+			if i == 1 {
+				ch <- n
+			}
+			bs1.Write(b[16 : n+16])
+
+			// TODO: this is a pretty half-baked batching
+			// function, which we'd never want to employ in
+			// a real-life program.
+			//
+			// In real life, we'd probably want to flush
+			// immediately when there are no more packets to
+			// generate, and queue up only if we fall behind.
+			//
+			// In our case however, we just want to see the
+			// technical benefits of batching 10 syscalls
+			// into 1, so a fixed ratio makes more sense.
+			if (i % 10) == 0 {
+				bs1.Flush()
+			}
+		}
+	}()
+
+	go func() {
+		// receiver
+
+		bs2 := bufio.NewReaderSize(s2, 1024*1024)
+
+		// Find out the packet size (we happen to know they're
+		// all the same size)
+		packetSize := <-ch
+
+		b := make([]byte, packetSize)
+		for traf.Running() {
+			// TODO: can't use ReadFrom() here, which is
+			// unfair compared to UDP. (ReadFrom for UDP
+			// apparently allocates memory per packet, which
+			// this test does not.)
+			n, err := io.ReadFull(bs2, b)
+			if err != nil {
+				log.Fatalf("s2.Read: %v", err)
+			}
+			traf.GotPacket(b[:n], 0)
+		}
+	}()
+}

wgengine/bench/bench_test.go

@@ -0,0 +1,108 @@
+// Copyright (c) 2021 Tailscale Inc & AUTHORS All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+// Create two wgengine instances and pass data through them, measuring
+// throughput, latency, and packet loss.
+package main
+
+import (
+	"fmt"
+	"testing"
+	"time"
+
+	"tailscale.com/types/logger"
+)
+
+func BenchmarkTrivialNoAlloc(b *testing.B) {
+	run(b, setupTrivialNoAllocTest)
+}
+func BenchmarkTrivial(b *testing.B) {
+	run(b, setupTrivialTest)
+}
+
+func BenchmarkBlockingChannel(b *testing.B) {
+	run(b, setupBlockingChannelTest)
+}
+
+func BenchmarkNonblockingChannel(b *testing.B) {
+	run(b, setupNonblockingChannelTest)
+}
+
+func BenchmarkDoubleChannel(b *testing.B) {
+	run(b, setupDoubleChannelTest)
+}
+
+func BenchmarkUDP(b *testing.B) {
+	run(b, setupUDPTest)
+}
+
+func BenchmarkBatchTCP(b *testing.B) {
+	run(b, setupBatchTCPTest)
+}
+
+func BenchmarkWireGuardTest(b *testing.B) {
+	run(b, func(logf logger.Logf, traf *TrafficGen) {
+		setupWGTest(logf, traf, Addr1, Addr2)
+	})
+}
+
+type SetupFunc func(logger.Logf, *TrafficGen)
+
+func run(b *testing.B, setup SetupFunc) {
+	sizes := []int{
+		ICMPMinSize + 8,
+		ICMPMinSize + 100,
+		ICMPMinSize + 1000,
+	}
+
+	for _, size := range sizes {
+		b.Run(fmt.Sprintf("%d", size), func(b *testing.B) {
+			runOnce(b, setup, size)
+		})
+	}
+}
+
+func runOnce(b *testing.B, setup SetupFunc, payload int) {
+	b.StopTimer()
+	b.ReportAllocs()
+
+	var logf logger.Logf = b.Logf
+	if !testing.Verbose() {
+		logf = logger.Discard
+	}
+
+	traf := NewTrafficGen(b.StartTimer)
+	setup(logf, traf)
+
+	logf("initialized. (n=%v)", b.N)
+	b.SetBytes(int64(payload))
+
+	traf.Start(Addr1.IP, Addr2.IP, payload, int64(b.N))
+
+	var cur, prev Snapshot
+	var pps int64
+	i := 0
+	for traf.Running() {
+		i += 1
+		time.Sleep(10 * time.Millisecond)
+
+		if (i % 100) == 0 {
+			prev = cur
+			cur = traf.Snap()
+			d := cur.Sub(prev)
+
+			if prev.WhenNsec != 0 {
+				logf("%v @%7d pkt/sec", d, pps)
+			}
+		}
+
+		pps = traf.Adjust()
+	}
+
+	cur = traf.Snap()
+	d := cur.Sub(prev)
+	loss := float64(d.LostPackets) / float64(d.RxPackets)
+
+	b.ReportMetric(loss*100, "%lost")
+}

wgengine/bench/trafficgen.go

@@ -0,0 +1,262 @@
+// Copyright (c) 2021 Tailscale Inc & AUTHORS All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package main
+
+import (
+	"encoding/binary"
+	"fmt"
+	"log"
+	"sync"
+	"time"
+
+	"inet.af/netaddr"
+	"tailscale.com/net/packet"
+	"tailscale.com/types/ipproto"
+)
+
+type Snapshot struct {
+	WhenNsec int64 // current time
+	timeAcc  int64 // accumulated time (+NSecPerTx per transmit)
+
+	LastSeqTx    int64 // last sequence number sent
+	LastSeqRx    int64 // last sequence number received
+	TotalLost    int64 // packets out-of-order or lost so far
+	TotalOOO     int64 // packets out-of-order so far
+	TotalBytesRx int64 // total bytes received so far
+}
+
+type Delta struct {
+	DurationNsec int64
+	TxPackets    int64
+	RxPackets    int64
+	LostPackets  int64
+	OOOPackets   int64
+	Bytes        int64
+}
+
+func (b Snapshot) Sub(a Snapshot) Delta {
+	return Delta{
+		DurationNsec: b.WhenNsec - a.WhenNsec,
+		TxPackets:    b.LastSeqTx - a.LastSeqTx,
+		RxPackets: (b.LastSeqRx - a.LastSeqRx) -
+			(b.TotalLost - a.TotalLost) +
+			(b.TotalOOO - a.TotalOOO),
+		LostPackets: b.TotalLost - a.TotalLost,
+		OOOPackets:  b.TotalOOO - a.TotalOOO,
+		Bytes:       b.TotalBytesRx - a.TotalBytesRx,
+	}
+}
+
+func (d Delta) String() string {
+	return fmt.Sprintf("tx=%-6d rx=%-4d (%6d = %.1f%% loss) (%d OOO) (%4.1f Mbit/s)",
+		d.TxPackets, d.RxPackets, d.LostPackets,
+		float64(d.LostPackets)*100/float64(d.TxPackets),
+		d.OOOPackets,
+		float64(d.Bytes)*8*1e9/float64(d.DurationNsec)/1e6)
+}
+
+type TrafficGen struct {
+	mu        sync.Mutex
+	cur, prev Snapshot // snapshots used for rate control
+	buf       []byte   // pre-generated packet buffer
+	done      bool     // true if the test has completed
+
+	onFirstPacket func() // function to call on first received packet
+
+	// maxPackets is the max packets to receive (not send) before
+	// ending the test. If it's zero, the test runs forever.
+	maxPackets int64
+
+	// nsPerPacket is the target average nanoseconds between packets.
+	// It's initially zero, which means transmit as fast as the
+	// caller wants to go.
+	nsPerPacket int64
+
+	// ppsHistory is the observed packets-per-second from recent
+	// samples.
+	ppsHistory [5]int64
+}
+
+// NewTrafficGen creates a new, initially locked, TrafficGen.
+// Until Start() is called, Generate() will block forever.
+func NewTrafficGen(onFirstPacket func()) *TrafficGen {
+	t := TrafficGen{
+		onFirstPacket: onFirstPacket,
+	}
+
+	// initially locked, until first Start()
+	t.mu.Lock()
+
+	return &t
+}
+
+// Start starts the traffic generator. It assumes mu is already locked,
+// and unlocks it.
+func (t *TrafficGen) Start(src, dst netaddr.IP, bytesPerPacket int, maxPackets int64) {
+	h12 := packet.ICMP4Header{
+		IP4Header: packet.IP4Header{
+			IPProto: ipproto.ICMPv4,
+			IPID:    0,
+			Src:     src,
+			Dst:     dst,
+		},
+		Type: packet.ICMP4EchoRequest,
+		Code: packet.ICMP4NoCode,
+	}
+
+	// ensure there's room for ICMP header plus sequence number
+	if bytesPerPacket < ICMPMinSize+8 {
+		log.Fatalf("bytesPerPacket must be > 24+8")
+	}
+
+	t.maxPackets = maxPackets
+
+	payload := make([]byte, bytesPerPacket-ICMPMinSize)
+	t.buf = packet.Generate(h12, payload)
+
+	t.mu.Unlock()
+}
+
+func (t *TrafficGen) Snap() Snapshot {
+	t.mu.Lock()
+	defer t.mu.Unlock()
+
+	t.cur.WhenNsec = time.Now().UnixNano()
+	return t.cur
+}
+
+func (t *TrafficGen) Running() bool {
+	t.mu.Lock()
+	defer t.mu.Unlock()
+
+	return !t.done
+}
+
+// Generate produces the next packet in the sequence. It sleeps if
+// it's too soon for the next packet to be sent.
+//
+// The generated packet is placed into buf at offset ofs, for compatibility
+// with the wireguard-go conventions.
+//
+// The return value is the number of bytes generated in the packet, or 0
+// if the test has finished running.
+func (t *TrafficGen) Generate(b []byte, ofs int) int {
+	t.mu.Lock()
+
+	now := time.Now().UnixNano()
+	if t.nsPerPacket == 0 || t.cur.timeAcc == 0 {
+		t.cur.timeAcc = now - 1
+	}
+	if t.cur.timeAcc >= now {
+		// too soon
+		t.mu.Unlock()
+		time.Sleep(time.Duration(t.cur.timeAcc-now) * time.Nanosecond)
+		t.mu.Lock()
+
+		now = t.cur.timeAcc
+	}
+	if t.done {
+		t.mu.Unlock()
+		return 0
+	}
+
+	t.cur.timeAcc += t.nsPerPacket
+	t.cur.LastSeqTx += 1
+	t.cur.WhenNsec = now
+	seq := t.cur.LastSeqTx
+
+	t.mu.Unlock()
+
+	copy(b[ofs:], t.buf)
+	binary.BigEndian.PutUint64(
+		b[ofs+ICMPMinSize:ofs+ICMPMinSize+8],
+		uint64(seq))
+
+	return len(t.buf)
+}
+
+// GotPacket processes a packet that came back on the receive side.
+func (t *TrafficGen) GotPacket(b []byte, ofs int) {
+	t.mu.Lock()
+
+	s := &t.cur
+	seq := int64(binary.BigEndian.Uint64(
+		b[ofs+ICMPMinSize : ofs+ICMPMinSize+8]))
+	if seq > s.LastSeqRx {
+		if s.LastSeqRx > 0 {
+			// only count lost packets after the very first
+			// successful one.
+			s.TotalLost += seq - s.LastSeqRx - 1
+		}
+		s.LastSeqRx = seq
+	} else {
+		s.TotalOOO += 1
+	}
+
+	// +1 packet since we only start counting after the first one
+	if t.maxPackets > 0 && s.LastSeqRx >= t.maxPackets+1 {
+		t.done = true
+	}
+	s.TotalBytesRx += int64(len(b) - ofs)
+
+	f := t.onFirstPacket
+	t.onFirstPacket = nil
+
+	t.mu.Unlock()
+
+	if f != nil {
+		f()
+	}
+}
+
+// Adjust tunes the transmit rate based on the received packets.
+// The goal is to converge on the fastest transmit rate that still has
+// minimal packet loss. Returns the new target rate in packets/sec.
+//
+// We need to play this guessing game in order to balance out tx and rx
+// rates when there's a lossy network between them. Otherwise we can end
+// up using 99% of the CPU to blast out transmitted packets and leaving only
+// 1% to receive them, leading to a misleading throughput calculation.
+//
+// Call this function multiple times per second.
+func (t *TrafficGen) Adjust() (pps int64) {
+	t.mu.Lock()
+	defer t.mu.Unlock()
+
+	d := t.cur.Sub(t.prev)
+
+	// don't adjust rate until the first full period *after* receiving
+	// the first packet. This skips any handshake time in the underlying
+	// transport.
+	if t.prev.LastSeqRx == 0 || d.DurationNsec == 0 {
+		t.prev = t.cur
+		return 0 // no estimate yet, continue at max speed
+	}
+
+	pps = int64(d.RxPackets) * 1e9 / int64(d.DurationNsec)
+
+	// We use a rate selection algorithm based loosely on TCP BBR.
+	// Basically, we set the transmit rate to be a bit higher than
+	// the best observed transmit rate in the last several time
+	// periods. This guarantees some packet loss, but should converge
+	// quickly on a rate near the sustainable maximum.
+	bestPPS := pps
+	for _, p := range t.ppsHistory {
+		if p > bestPPS {
+			bestPPS = p
+		}
+	}
+	if pps > 0 && t.prev.WhenNsec > 0 {
+		copy(t.ppsHistory[1:], t.ppsHistory[0:len(t.ppsHistory)-1])
+		t.ppsHistory[0] = pps
+	}
+	if bestPPS > 0 {
+		pps = bestPPS * 103 / 100
+		t.nsPerPacket = int64(1e9 / pps)
+	}
+	t.prev = t.cur
+
+	return pps
+}

wgengine/bench/wg.go

@@ -0,0 +1,205 @@
+// Copyright (c) 2021 Tailscale Inc & AUTHORS All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package main
+
+import (
+	"io"
+	"log"
+	"os"
+	"strings"
+	"sync"
+
+	"github.com/tailscale/wireguard-go/tun"
+	"inet.af/netaddr"
+
+	"tailscale.com/net/dns"
+	"tailscale.com/tailcfg"
+	"tailscale.com/types/logger"
+	"tailscale.com/types/netmap"
+	"tailscale.com/types/wgkey"
+	"tailscale.com/wgengine"
+	"tailscale.com/wgengine/filter"
+	"tailscale.com/wgengine/router"
+	"tailscale.com/wgengine/wgcfg"
+)
+
+func setupWGTest(logf logger.Logf, traf *TrafficGen, a1, a2 netaddr.IPPrefix) {
+	l1 := logger.WithPrefix(logf, "e1: ")
+	k1, err := wgkey.NewPrivate()
+	if err != nil {
+		log.Fatalf("e1 NewPrivateKey: %v", err)
+	}
+	c1 := wgcfg.Config{
+		Name:       "e1",
+		PrivateKey: k1,
+		Addresses:  []netaddr.IPPrefix{a1},
+	}
+	t1 := &sourceTun{
+		logf: logger.WithPrefix(logf, "tun1: "),
+		traf: traf,
+	}
+	e1, err := wgengine.NewUserspaceEngine(l1, wgengine.Config{
+		Router:      router.NewFake(l1),
+		LinkMonitor: nil,
+		ListenPort:  0,
+		Tun:         t1,
+	})
+	if err != nil {
+		log.Fatalf("e1 init: %v", err)
+	}
+
+	l2 := logger.WithPrefix(logf, "e2: ")
+	k2, err := wgkey.NewPrivate()
+	if err != nil {
+		log.Fatalf("e2 NewPrivateKey: %v", err)
+	}
+	c2 := wgcfg.Config{
+		Name:       "e2",
+		PrivateKey: k2,
+		Addresses:  []netaddr.IPPrefix{a2},
+	}
+	t2 := &sinkTun{
+		logf: logger.WithPrefix(logf, "tun2: "),
+		traf: traf,
+	}
+	e2, err := wgengine.NewUserspaceEngine(l2, wgengine.Config{
+		Router:      router.NewFake(l2),
+		LinkMonitor: nil,
+		ListenPort:  0,
+		Tun:         t2,
+	})
+	if err != nil {
+		log.Fatalf("e2 init: %v", err)
+	}
+
+	e1.SetFilter(filter.NewAllowAllForTest(l1))
+	e2.SetFilter(filter.NewAllowAllForTest(l2))
+
+	var wait sync.WaitGroup
+	wait.Add(2)
+
+	e1.SetStatusCallback(func(st *wgengine.Status, err error) {
+		if err != nil {
+			log.Fatalf("e1 status err: %v", err)
+		}
+		logf("e1 status: %v", *st)
+
+		var eps []string
+		for _, ep := range st.LocalAddrs {
+			eps = append(eps, ep.Addr.String())
+		}
+
+		n := tailcfg.Node{
+			ID:         tailcfg.NodeID(0),
+			Name:       "n1",
+			Addresses:  []netaddr.IPPrefix{a1},
+			AllowedIPs: []netaddr.IPPrefix{a1},
+			Endpoints:  eps,
+		}
+		e2.SetNetworkMap(&netmap.NetworkMap{
+			NodeKey:    tailcfg.NodeKey(k2),
+			PrivateKey: wgkey.Private(k2),
+			Peers:      []*tailcfg.Node{&n},
+		})
+
+		p := wgcfg.Peer{
+			PublicKey:  c1.PrivateKey.Public(),
+			AllowedIPs: []netaddr.IPPrefix{a1},
+			Endpoints:  strings.Join(eps, ","),
+		}
+		c2.Peers = []wgcfg.Peer{p}
+		e2.Reconfig(&c2, &router.Config{}, new(dns.Config))
+		wait.Done()
+	})
+
+	e2.SetStatusCallback(func(st *wgengine.Status, err error) {
+		if err != nil {
+			log.Fatalf("e2 status err: %v", err)
+		}
+		logf("e2 status: %v", *st)
+
+		var eps []string
+		for _, ep := range st.LocalAddrs {
+			eps = append(eps, ep.Addr.String())
+		}
+
+		n := tailcfg.Node{
+			ID:         tailcfg.NodeID(0),
+			Name:       "n2",
+			Addresses:  []netaddr.IPPrefix{a2},
+			AllowedIPs: []netaddr.IPPrefix{a2},
+			Endpoints:  eps,
+		}
+		e1.SetNetworkMap(&netmap.NetworkMap{
+			NodeKey:    tailcfg.NodeKey(k1),
+			PrivateKey: wgkey.Private(k1),
+			Peers:      []*tailcfg.Node{&n},
+		})
+
+		p := wgcfg.Peer{
+			PublicKey:  c2.PrivateKey.Public(),
+			AllowedIPs: []netaddr.IPPrefix{a2},
+			Endpoints:  strings.Join(eps, ","),
+		}
+		c1.Peers = []wgcfg.Peer{p}
+		e1.Reconfig(&c1, &router.Config{}, new(dns.Config))
+		wait.Done()
+	})
+
+	// Not using DERP in this test (for now?).
+	e1.SetDERPMap(&tailcfg.DERPMap{})
+	e2.SetDERPMap(&tailcfg.DERPMap{})
+
+	wait.Wait()
+}
+
+type sourceTun struct {
+	logf logger.Logf
+	traf *TrafficGen
+}
+
+func (t *sourceTun) Close() error           { return nil }
+func (t *sourceTun) Events() chan tun.Event { return nil }
+func (t *sourceTun) File() *os.File         { return nil }
+func (t *sourceTun) Flush() error           { return nil }
+func (t *sourceTun) MTU() (int, error)      { return 1500, nil }
+func (t *sourceTun) Name() (string, error)  { return "source", nil }
+
+func (t *sourceTun) Write(b []byte, ofs int) (int, error) {
+	// Discard all writes
+	return len(b) - ofs, nil
+}
+
+func (t *sourceTun) Read(b []byte, ofs int) (int, error) {
+	// Continually generate "input" packets
+	n := t.traf.Generate(b, ofs)
+	if n == 0 {
+		return 0, io.EOF
+	}
+	return n, nil
+}
+
+type sinkTun struct {
+	logf logger.Logf
+	traf *TrafficGen
+}
+
+func (t *sinkTun) Close() error           { return nil }
+func (t *sinkTun) Events() chan tun.Event { return nil }
+func (t *sinkTun) File() *os.File         { return nil }
+func (t *sinkTun) Flush() error           { return nil }
+func (t *sinkTun) MTU() (int, error)      { return 1500, nil }
+func (t *sinkTun) Name() (string, error)  { return "sink", nil }
+
+func (t *sinkTun) Read(b []byte, ofs int) (int, error) {
+	// Never returns
+	select {}
+}
+
+func (t *sinkTun) Write(b []byte, ofs int) (int, error) {
+	// Count packets, but discard them
+	t.traf.GotPacket(b, ofs)
+	return len(b) - ofs, nil
+}

wgengine/magicsock/legacy.go

@@ -27,7 +27,6 @@ import (
 	"tailscale.com/types/key"
 	"tailscale.com/types/logger"
 	"tailscale.com/types/wgkey"
-	"tailscale.com/wgengine/wgcfg"
 )
 
 var (
@@ -591,8 +590,8 @@ func init() {
 type messageInitiation struct {
 	Type      uint32
 	Sender    uint32
-	Ephemeral wgcfg.Key
-	Static    [wgcfg.KeySize + poly1305.TagSize]byte
+	Ephemeral wgkey.Key
+	Static    [wgkey.Size + poly1305.TagSize]byte
 	Timestamp [tai64n.TimestampSize + poly1305.TagSize]byte
 	MAC1      [blake2s.Size128]byte
 	MAC2      [blake2s.Size128]byte

wgengine/magicsock/magicsock.go

@@ -52,6 +52,7 @@ import (
 	"tailscale.com/types/netmap"
 	"tailscale.com/types/nettype"
 	"tailscale.com/types/wgkey"
+	"tailscale.com/util/uniq"
 	"tailscale.com/version"
 	"tailscale.com/wgengine/monitor"
 	"tailscale.com/wgengine/wgcfg"
@@ -365,10 +366,6 @@ type activeDerp struct {
 	createTime time.Time
 }
 
-// DefaultPort is the default port to listen on.
-// The current default (zero) means to auto-select a random free port.
-const DefaultPort = 0
-
 // Options contains options for Listen.
 type Options struct {
 	// Logf optionally provides a log function to use.
@@ -1581,6 +1578,8 @@ func (c *Conn) noteRecvActivityFromEndpoint(e conn.Endpoint) {
 
 // receiveIPv6 receives a UDP IPv6 packet. It is called by wireguard-go.
 func (c *Conn) receiveIPv6(b []byte) (int, conn.Endpoint, error) {
+	health.ReceiveIPv6.Enter()
+	defer health.ReceiveIPv6.Exit()
 	for {
 		n, ipp, err := c.pconn6.ReadFromNetaddr(b)
 		if err != nil {
@@ -1594,6 +1593,8 @@ func (c *Conn) receiveIPv6(b []byte) (int, conn.Endpoint, error) {
 
 // receiveIPv4 receives a UDP IPv4 packet. It is called by wireguard-go.
 func (c *Conn) receiveIPv4(b []byte) (n int, ep conn.Endpoint, err error) {
+	health.ReceiveIPv4.Enter()
+	defer health.ReceiveIPv4.Exit()
 	for {
 		n, ipp, err := c.pconn4.ReadFromNetaddr(b)
 		if err != nil {
@@ -1646,6 +1647,8 @@ func (c *Conn) receiveIP(b []byte, ipp netaddr.IPPort, cache *ippEndpointCache)
 // If the packet was a disco message or the peer endpoint wasn't
 // found, the returned error is errLoopAgain.
 func (c *connBind) receiveDERP(b []byte) (n int, ep conn.Endpoint, err error) {
+	health.ReceiveDERP.Enter()
+	defer health.ReceiveDERP.Exit()
 	for dm := range c.derpRecvCh {
 		if c.Closed() {
 			break
@@ -2410,10 +2413,7 @@ func (c *connBind) Open(ignoredPort uint16) ([]conn.ReceiveFunc, uint16, error)
 		return nil, 0, errors.New("magicsock: connBind already open")
 	}
 	c.closed = false
-	fns := []conn.ReceiveFunc{c.receiveIPv4, c.receiveDERP}
-	if c.pconn6 != nil {
-		fns = append(fns, c.receiveIPv6)
-	}
+	fns := []conn.ReceiveFunc{c.receiveIPv4, c.receiveIPv6, c.receiveDERP}
 	// TODO: Combine receiveIPv4 and receiveIPv6 and receiveIP into a single
 	// closure that closes over a *RebindingUDPConn?
 	return fns, c.LocalPort(), nil
@@ -2435,9 +2435,7 @@ func (c *connBind) Close() error {
 	c.closed = true
 	// Unblock all outstanding receives.
 	c.pconn4.Close()
-	if c.pconn6 != nil {
-		c.pconn6.Close()
-	}
+	c.pconn6.Close()
 	// Send an empty read result to unblock receiveDERP,
 	// which will then check connBind.Closed.
 	c.derpRecvCh <- derpReadResult{}
@@ -2585,96 +2583,119 @@ func (c *Conn) ReSTUN(why string) {
 }
 
 func (c *Conn) initialBind() error {
-	if err := c.bind1(&c.pconn4, "udp4"); err != nil {
-		return err
+	if err := c.bindSocket(&c.pconn4, "udp4"); err != nil {
+		return fmt.Errorf("magicsock: initialBind IPv4 failed: %w", err)
 	}
 	c.portMapper.SetLocalPort(c.LocalPort())
-	if err := c.bind1(&c.pconn6, "udp6"); err != nil {
+	if err := c.bindSocket(&c.pconn6, "udp6"); err != nil {
 		c.logf("magicsock: ignoring IPv6 bind failure: %v", err)
 	}
 	return nil
 }
 
-func (c *Conn) listenPacket(ctx context.Context, network, host string, port uint16) (net.PacketConn, error) {
-	addr := net.JoinHostPort(host, fmt.Sprint(port))
+// listenPacket opens a packet listener.
+// The network must be "udp4" or "udp6".
+// Host is the (local) IP address to listen on; use the zero IP to leave unspecified.
+func (c *Conn) listenPacket(network string, host netaddr.IP, port uint16) (net.PacketConn, error) {
+	ctx := context.Background() // unused without DNS name to resolve
+	// Translate host to package net: "" for the zero value, the IP address string otherwise.
+	var s string
+	if !host.IsZero() {
+		s = host.String()
+	}
+	addr := net.JoinHostPort(s, fmt.Sprint(port))
 	if c.packetListener != nil {
 		return c.packetListener.ListenPacket(ctx, network, addr)
 	}
 	return netns.Listener().ListenPacket(ctx, network, addr)
 }
 
-func (c *Conn) bind1(ruc **RebindingUDPConn, which string) error {
-	host := ""
+// bindSocket initializes rucPtr if necessary and binds a UDP socket to it.
+// Network indicates the UDP socket type; it must be "udp4" or "udp6".
+// If rucPtr had an existing UDP socket bound, it closes that socket.
+// The caller is responsible for informing the portMapper of any changes.
+func (c *Conn) bindSocket(rucPtr **RebindingUDPConn, network string) error {
+	var host netaddr.IP
 	if inTest() && !c.simulatedNetwork {
-		host = "127.0.0.1"
-		if which == "udp6" {
-			host = "::1"
+		switch network {
+		case "udp4":
+			host = netaddr.MustParseIP("127.0.0.1")
+		case "udp6":
+			host = netaddr.MustParseIP("::1")
+		default:
+			panic("unrecognized network in bindSocket: " + network)
 		}
 	}
-	var pc net.PacketConn
-	var err error
-	listenCtx := context.Background() // unused without DNS name to resolve
-	if c.port == 0 && DefaultPort != 0 {
-		pc, err = c.listenPacket(listenCtx, which, host, DefaultPort)
+
+	if *rucPtr == nil {
+		*rucPtr = new(RebindingUDPConn)
+	}
+	ruc := *rucPtr
+
+	// Hold the ruc lock the entire time, so that the close+bind is atomic
+	// from the perspective of ruc receive functions.
+	ruc.mu.Lock()
+	defer ruc.mu.Unlock()
+
+	// Build a list of preferred ports.
+	// Best is the port that the user requested.
+	// Second best is the port that is currently in use.
+	// If those fail, fall back to 0.
+	var ports []uint16
+	if c.port != 0 {
+		ports = append(ports, c.port)
+	}
+	if ruc.pconn != nil {
+		curPort := uint16(ruc.localAddrLocked().Port)
+		ports = append(ports, curPort)
+	}
+	ports = append(ports, 0)
+	// Remove duplicates. (All duplicates are consecutive.)
+	uniq.ModifySlice(&ports, func(i, j int) bool { return ports[i] == ports[j] })
+
+	var pconn net.PacketConn
+	for _, port := range ports {
+		// Close the existing conn, in case it is sitting on the port we want.
+		err := ruc.closeLocked()
+		if err != nil && !errors.Is(err, net.ErrClosed) && !errors.Is(err, errNilPConn) {
+			c.logf("magicsock: bindSocket %v close failed: %v", network, err)
+		}
+		// Open a new one with the desired port.
+		pconn, err = c.listenPacket(network, host, port)
 		if err != nil {
-			c.logf("magicsock: bind: default port %s/%v unavailable; picking random", which, DefaultPort)
+			c.logf("magicsock: unable to bind %v port %d: %v", network, port, err)
+			continue
 		}
+		// Success.
+		ruc.pconn = pconn
+		if network == "udp4" {
+			health.SetUDP4Unbound(false)
+		}
+		return nil
 	}
-	if pc == nil {
-		pc, err = c.listenPacket(listenCtx, which, host, c.port)
+
+	// Failed to bind, including on port 0 (!).
+	// Set pconn to a dummy conn whose reads block until closed.
+	// This keeps the receive funcs alive for a future in which
+	// we get a link change and we can try binding again.
+	ruc.pconn = newBlockForeverConn()
+	if network == "udp4" {
+		health.SetUDP4Unbound(true)
 	}
-	if err != nil {
-		c.logf("magicsock: bind(%s/%v): %v", which, c.port, err)
-		return fmt.Errorf("magicsock: bind: %s/%d: %v", which, c.port, err)
-	}
-	if *ruc == nil {
-		*ruc = new(RebindingUDPConn)
-	}
-	(*ruc).Reset(pc)
-	return nil
+	return fmt.Errorf("failed to bind any ports (tried %v)", ports)
 }
 
 // Rebind closes and re-binds the UDP sockets.
 // It should be followed by a call to ReSTUN.
 func (c *Conn) Rebind() {
-	host := ""
-	if inTest() && !c.simulatedNetwork {
-		host = "127.0.0.1"
-	}
-	listenCtx := context.Background() // unused without DNS name to resolve
-
-	if c.port != 0 {
-		c.pconn4.mu.Lock()
-		oldPort := c.pconn4.localAddrLocked().Port
-		if err := c.pconn4.pconn.Close(); err != nil {
-			c.logf("magicsock: link change close failed: %v", err)
-		}
-		packetConn, err := c.listenPacket(listenCtx, "udp4", host, c.port)
-		if err != nil {
-			c.logf("magicsock: link change unable to bind fixed port %d: %v, falling back to random port", c.port, err)
-			packetConn, err = c.listenPacket(listenCtx, "udp4", host, 0)
-			if err != nil {
-				c.logf("magicsock: link change failed to bind random port: %v", err)
-				c.pconn4.mu.Unlock()
-				return
-			}
-			newPort := packetConn.LocalAddr().(*net.UDPAddr).Port
-			c.logf("magicsock: link change rebound port: from %v to %v (failed to get %v)", oldPort, newPort, c.port)
-		} else {
-			c.logf("magicsock: link change rebound port from %d to %d", oldPort, c.port)
-		}
-		c.pconn4.pconn = packetConn
-		c.pconn4.mu.Unlock()
-	} else {
-		c.logf("magicsock: link change, binding new port")
-		packetConn, err := c.listenPacket(listenCtx, "udp4", host, 0)
-		if err != nil {
-			c.logf("magicsock: link change failed to bind new port: %v", err)
-			return
-		}
-		c.pconn4.Reset(packetConn)
+	if err := c.bindSocket(&c.pconn4, "udp4"); err != nil {
+		c.logf("magicsock: Rebind IPv4 failed: %w", err)
+		return
 	}
 	c.portMapper.SetLocalPort(c.LocalPort())
+	if err := c.bindSocket(&c.pconn6, "udp6"); err != nil {
+		c.logf("magicsock: Rebind ignoring IPv6 bind failure: %v", err)
+	}
 
 	c.mu.Lock()
 	c.closeAllDerpLocked("rebind")
@@ -2774,17 +2795,6 @@ func (c *RebindingUDPConn) currentConn() net.PacketConn {
 	return c.pconn
 }
 
-func (c *RebindingUDPConn) Reset(pconn net.PacketConn) {
-	c.mu.Lock()
-	old := c.pconn
-	c.pconn = pconn
-	c.mu.Unlock()
-
-	if old != nil {
-		old.Close()
-	}
-}
-
 // ReadFrom reads a packet from c into b.
 // It returns the number of bytes copied and the source address.
 func (c *RebindingUDPConn) ReadFrom(b []byte) (int, net.Addr, error) {
@@ -2854,9 +2864,20 @@ func (c *RebindingUDPConn) localAddrLocked() *net.UDPAddr {
 	return c.pconn.LocalAddr().(*net.UDPAddr)
 }
 
+// errNilPConn is returned by RebindingUDPConn.Close when there is no current pconn.
+// It is for internal use only and should not be returned to users.
+var errNilPConn = errors.New("nil pconn")
+
 func (c *RebindingUDPConn) Close() error {
 	c.mu.Lock()
 	defer c.mu.Unlock()
+	return c.closeLocked()
+}
+
+func (c *RebindingUDPConn) closeLocked() error {
+	if c.pconn == nil {
+		return errNilPConn
+	}
 	return c.pconn.Close()
 }
 
@@ -2900,6 +2921,52 @@ func (c *RebindingUDPConn) WriteTo(b []byte, addr net.Addr) (int, error) {
 	}
 }
 
+func newBlockForeverConn() *blockForeverConn {
+	c := new(blockForeverConn)
+	c.cond = sync.NewCond(&c.mu)
+	return c
+}
+
+// blockForeverConn is a net.PacketConn whose reads block until it is closed.
+type blockForeverConn struct {
+	mu     sync.Mutex
+	cond   *sync.Cond
+	closed bool
+}
+
+func (c *blockForeverConn) ReadFrom(p []byte) (n int, addr net.Addr, err error) {
+	c.mu.Lock()
+	for !c.closed {
+		c.cond.Wait()
+	}
+	c.mu.Unlock()
+	return 0, nil, net.ErrClosed
+}
+
+func (c *blockForeverConn) WriteTo(p []byte, addr net.Addr) (n int, err error) {
+	// Silently drop writes.
+	return len(p), nil
+}
+
+func (c *blockForeverConn) LocalAddr() net.Addr {
+	// Return a *net.UDPAddr because lots of code assumes that it will.
+	return new(net.UDPAddr)
+}
+
+func (c *blockForeverConn) Close() error {
+	c.mu.Lock()
+	defer c.mu.Unlock()
+	if c.closed {
+		return net.ErrClosed
+	}
+	c.closed = true
+	return nil
+}
+
+func (c *blockForeverConn) SetDeadline(t time.Time) error      { return errors.New("unimplemented") }
+func (c *blockForeverConn) SetReadDeadline(t time.Time) error  { return errors.New("unimplemented") }
+func (c *blockForeverConn) SetWriteDeadline(t time.Time) error { return errors.New("unimplemented") }
+
 // simpleDur rounds d such that it stringifies to something short.
 func simpleDur(d time.Duration) time.Duration {
 	if d < time.Second {

wgengine/magicsock/magicsock_test.go

@@ -10,7 +10,6 @@ import (
 	crand "crypto/rand"
 	"crypto/tls"
 	"encoding/binary"
-	"encoding/json"
 	"errors"
 	"fmt"
 	"io/ioutil"
@@ -26,7 +25,6 @@ import (
 	"time"
 	"unsafe"
 
-	"github.com/google/go-cmp/cmp"
 	"github.com/tailscale/wireguard-go/device"
 	"github.com/tailscale/wireguard-go/tun/tuntest"
 	"golang.org/x/crypto/nacl/box"
@@ -444,7 +442,7 @@ func TestPickDERPFallback(t *testing.T) {
 func makeConfigs(t *testing.T, addrs []netaddr.IPPort) []wgcfg.Config {
 	t.Helper()
 
-	var privKeys []wgcfg.PrivateKey
+	var privKeys []wgkey.Private
 	var addresses [][]netaddr.IPPrefix
 
 	for i := range addrs {
@@ -452,7 +450,7 @@ func makeConfigs(t *testing.T, addrs []netaddr.IPPort) []wgcfg.Config {
 		if err != nil {
 			t.Fatal(err)
 		}
-		privKeys = append(privKeys, wgcfg.PrivateKey(privKey))
+		privKeys = append(privKeys, wgkey.Private(privKey))
 
 		addresses = append(addresses, []netaddr.IPPrefix{
 			parseCIDR(t, fmt.Sprintf("1.0.0.%d/32", i+1)),
@@ -996,148 +994,6 @@ func testTwoDevicePing(t *testing.T, d *devices) {
 		ping1(t)
 		ping2(t)
 	})
-
-	// TODO: Remove this once the following tests are reliable.
-	if run, _ := strconv.ParseBool(os.Getenv("RUN_CURSED_TESTS")); !run {
-		t.Skip("skipping following tests because RUN_CURSED_TESTS is not set.")
-	}
-
-	pingSeq := func(t *testing.T, count int, totalTime time.Duration, strict bool) {
-		msg := func(i int) []byte {
-			b := tuntest.Ping(net.ParseIP("1.0.0.2"), net.ParseIP("1.0.0.1"))
-			b[len(b)-1] = byte(i) // set seq num
-			return b
-		}
-
-		// Space out ping transmissions so that the overall
-		// transmission happens in totalTime.
-		//
-		// We do this because the packet spray logic in magicsock is
-		// time-based to allow for reliable NAT traversal. However,
-		// for the packet spraying test further down, there needs to
-		// be at least 1 sprayed packet that is not the handshake, in
-		// case the handshake gets eaten by the race resolution logic.
-		//
-		// This is an inherent "race by design" in our current
-		// magicsock+wireguard-go codebase: sometimes, racing
-		// handshakes will result in a sub-optimal path for a few
-		// hundred milliseconds, until a subsequent spray corrects the
-		// issue. In order for the test to reflect that magicsock
-		// works as designed, we have to space out packet transmission
-		// here.
-		interPacketGap := totalTime / time.Duration(count)
-		if interPacketGap < 1*time.Millisecond {
-			interPacketGap = 0
-		}
-
-		for i := 0; i < count; i++ {
-			b := msg(i)
-			m1.tun.Outbound <- b
-			time.Sleep(interPacketGap)
-		}
-
-		for i := 0; i < count; i++ {
-			b := msg(i)
-			select {
-			case msgRecv := <-m2.tun.Inbound:
-				if !bytes.Equal(b, msgRecv) {
-					if strict {
-						t.Errorf("return ping %d did not transit correctly: %s", i, cmp.Diff(b, msgRecv))
-					}
-				}
-			case <-time.After(pingTimeout):
-				if strict {
-					t.Errorf("return ping %d did not transit", i)
-				}
-			}
-		}
-	}
-
-	t.Run("ping 1.0.0.1 x50", func(t *testing.T) {
-		setT(t)
-		defer setT(outerT)
-		pingSeq(t, 50, 0, true)
-	})
-
-	// Add DERP relay.
-	derpEp := "127.3.3.40:1"
-	ep0 := cfgs[0].Peers[0].Endpoints
-	ep0 = derpEp + "," + ep0
-	cfgs[0].Peers[0].Endpoints = ep0
-	ep1 := cfgs[1].Peers[0].Endpoints
-	ep1 = derpEp + "," + ep1
-	cfgs[1].Peers[0].Endpoints = ep1
-	if err := m1.Reconfig(&cfgs[0]); err != nil {
-		t.Fatal(err)
-	}
-	if err := m2.Reconfig(&cfgs[1]); err != nil {
-		t.Fatal(err)
-	}
-
-	t.Run("add DERP", func(t *testing.T) {
-		setT(t)
-		defer setT(outerT)
-		pingSeq(t, 20, 0, true)
-	})
-
-	// Disable real route.
-	cfgs[0].Peers[0].Endpoints = derpEp
-	cfgs[1].Peers[0].Endpoints = derpEp
-	if err := m1.Reconfig(&cfgs[0]); err != nil {
-		t.Fatal(err)
-	}
-	if err := m2.Reconfig(&cfgs[1]); err != nil {
-		t.Fatal(err)
-	}
-	time.Sleep(250 * time.Millisecond) // TODO remove
-
-	t.Run("all traffic over DERP", func(t *testing.T) {
-		setT(t)
-		defer setT(outerT)
-		defer func() {
-			if t.Failed() || true {
-				logf("cfg0: %v", stringifyConfig(cfgs[0]))
-				logf("cfg1: %v", stringifyConfig(cfgs[1]))
-			}
-		}()
-		pingSeq(t, 20, 0, true)
-	})
-
-	m1.dev.RemoveAllPeers()
-	m2.dev.RemoveAllPeers()
-
-	// Give one peer a non-DERP endpoint. We expect the other to
-	// accept it via roamAddr.
-	cfgs[0].Peers[0].Endpoints = ep0
-	if ep2 := cfgs[1].Peers[0].Endpoints; len(ep2) != 1 {
-		t.Errorf("unexpected peer endpoints in dev2: %v", ep2)
-	}
-	if err := m2.Reconfig(&cfgs[1]); err != nil {
-		t.Fatal(err)
-	}
-	if err := m1.Reconfig(&cfgs[0]); err != nil {
-		t.Fatal(err)
-	}
-	// Dear future human debugging a test failure here: this test is
-	// flaky, and very infrequently will drop 1-2 of the 50 ping
-	// packets. This does not affect normal operation of tailscaled,
-	// but makes this test fail.
-	//
-	// TODO(danderson): finish root-causing and de-flake this test.
-	t.Run("one real route is enough thanks to spray", func(t *testing.T) {
-		setT(t)
-		defer setT(outerT)
-		pingSeq(t, 50, 700*time.Millisecond, false)
-
-		cfg, err := wgcfg.DeviceConfig(m2.dev)
-		if err != nil {
-			t.Fatal(err)
-		}
-		ep2 := cfg.Peers[0].Endpoints
-		if len(ep2) != 2 {
-			t.Error("handshake spray failed to find real route")
-		}
-	})
 }
 
 // TestAddrSet tests addrSet appendDests and updateDst.
@@ -1362,14 +1218,6 @@ func TestDiscoStringLogRace(t *testing.T) {
 	wg.Wait()
 }
 
-func stringifyConfig(cfg wgcfg.Config) string {
-	j, err := json.Marshal(cfg)
-	if err != nil {
-		panic(err)
-	}
-	return string(j)
-}
-
 func Test32bitAlignment(t *testing.T) {
 	var de discoEndpoint
 

wgengine/netstack/netstack.go

@@ -15,6 +15,7 @@ import (
 	"strconv"
 	"strings"
 	"sync"
+	"sync/atomic"
 	"time"
 
 	"inet.af/netaddr"
@@ -55,6 +56,12 @@ type Impl struct {
 	logf        logger.Logf
 	onlySubnets bool // whether we only want to handle subnet relaying
 
+	// atomicIsLocalIPFunc holds a func that reports whether an IP
+	// is a local (non-subnet) Tailscale IP address of this
+	// machine. It's always a non-nil func. It's changed on netmap
+	// updates.
+	atomicIsLocalIPFunc atomic.Value // of func(netaddr.IP) bool
+
 	mu  sync.Mutex
 	dns DNSMap
 	// connsOpenBySubnetIP keeps track of number of connections open
@@ -119,6 +126,7 @@ func Create(logf logger.Logf, tundev *tstun.Wrapper, e wgengine.Engine, mc *magi
 		connsOpenBySubnetIP: make(map[netaddr.IP]int),
 		onlySubnets:         onlySubnets,
 	}
+	ns.atomicIsLocalIPFunc.Store(tsaddr.NewContainsIPFunc(nil))
 	return ns, nil
 }
 
@@ -145,7 +153,7 @@ func (ns *Impl) Start() error {
 			ns.logf("netstack: could not parse local address %s for incoming TCP connection", ip)
 			return false
 		}
-		if !tsaddr.IsTailscaleIP(ip) {
+		if !ns.isLocalIP(ip) {
 			ns.addSubnetAddress(pn, ip)
 		}
 		return tcpFwd.HandlePacket(tei, pb)
@@ -219,6 +227,7 @@ func ipPrefixToAddressWithPrefix(ipp netaddr.IPPrefix) tcpip.AddressWithPrefix {
 }
 
 func (ns *Impl) updateIPs(nm *netmap.NetworkMap) {
+	ns.atomicIsLocalIPFunc.Store(tsaddr.NewContainsIPFunc(nm.Addresses))
 	ns.updateDNS(nm)
 
 	oldIPs := make(map[tcpip.AddressWithPrefix]bool)
@@ -373,7 +382,19 @@ func (ns *Impl) injectOutbound() {
 	}
 }
 
+// isLocalIP reports whether ip is a Tailscale IP assigned to this
+// node directly (but not a subnet-routed IP).
+func (ns *Impl) isLocalIP(ip netaddr.IP) bool {
+	return ns.atomicIsLocalIPFunc.Load().(func(netaddr.IP) bool)(ip)
+}
+
 func (ns *Impl) injectInbound(p *packet.Parsed, t *tstun.Wrapper) filter.Response {
+	if ns.onlySubnets && ns.isLocalIP(p.Dst.IP) {
+		// In hybrid ("only subnets") mode, bail out early if
+		// the traffic is destined for an actual Tailscale
+		// address. The real host OS interface will handle it.
+		return filter.Accept
+	}
 	var pn tcpip.NetworkProtocolNumber
 	switch p.IPVersion {
 	case 4:
@@ -389,16 +410,21 @@ func (ns *Impl) injectInbound(p *packet.Parsed, t *tstun.Wrapper) filter.Respons
 		Data: vv,
 	})
 	ns.linkEP.InjectInbound(pn, packetBuf)
-	return filter.Accept
+
+	// We've now delivered this to netstack, so we're done.
+	// Instead of returning a filter.Accept here (which would also
+	// potentially deliver it to the host OS), and instead of
+	// filter.Drop (which would log about rejected traffic),
+	// instead return filter.DropSilently which just quietly stops
+	// processing it in the tstun TUN wrapper.
+	return filter.DropSilently
 }
 
 func (ns *Impl) acceptTCP(r *tcp.ForwarderRequest) {
-	if debugNetstack {
-		// Kinda ugly:
-		// ForwarderRequest: &{{{{0 0}}} 0xc0001c30b0 0xc0004c3d40 {1240 6 true 826109390 0 true}
-		ns.logf("[v2] ForwarderRequest: %v", r)
-	}
 	reqDetails := r.ID()
+	if debugNetstack {
+		ns.logf("[v2] TCP ForwarderRequest: %s", stringifyTEI(reqDetails))
+	}
 	dialAddr := reqDetails.LocalAddress
 	dialNetAddr, _ := netaddr.FromStdIP(net.IP(dialAddr))
 	isTailscaleIP := tsaddr.IsTailscaleIP(dialNetAddr)
@@ -472,11 +498,14 @@ func (ns *Impl) forwardTCP(client *gonet.TCPConn, wq *waiter.Queue, dialAddr tcp
 }
 
 func (ns *Impl) acceptUDP(r *udp.ForwarderRequest) {
-	ns.logf("[v2] UDP ForwarderRequest: %v", r)
+	reqDetails := r.ID()
+	if debugNetstack {
+		ns.logf("[v2] UDP ForwarderRequest: %v", stringifyTEI(reqDetails))
+	}
 	var wq waiter.Queue
 	ep, err := r.CreateEndpoint(&wq)
 	if err != nil {
-		ns.logf("Could not create endpoint, exiting")
+		ns.logf("acceptUDP: could not create endpoint: %v", err)
 		return
 	}
 	localAddr, err := ep.GetLocalAddress()
@@ -563,3 +592,9 @@ func startPacketCopy(ctx context.Context, cancel context.CancelFunc, dst net.Pac
 		}
 	}()
 }
+
+func stringifyTEI(tei stack.TransportEndpointID) string {
+	localHostPort := net.JoinHostPort(tei.LocalAddress.String(), strconv.Itoa(int(tei.LocalPort)))
+	remoteHostPort := net.JoinHostPort(tei.RemoteAddress.String(), strconv.Itoa(int(tei.RemotePort)))
+	return fmt.Sprintf("%s -> %s", remoteHostPort, localHostPort)
+}

wgengine/pendopen.go

@@ -104,6 +104,7 @@ func (e *userspaceEngine) trackOpenPostFilterOut(pp *packet.Parsed, t *tstun.Wra
 
 	if pp.IPVersion == 0 ||
 		pp.IPProto != ipproto.TCP ||
+		pp.TCPFlags&packet.TCPAck != 0 ||
 		pp.TCPFlags&packet.TCPSyn == 0 {
 		return
 	}
@@ -190,6 +191,25 @@ func (e *userspaceEngine) onOpenTimeout(flow flowtrack.Tuple) {
 		return
 	}
 	if ps == nil {
+		onlyZeroRoute := true // whether peerForIP returned n only because its /0 route matched
+		for _, r := range n.AllowedIPs {
+			if r.Bits != 0 && r.Contains(flow.Dst.IP) {
+				onlyZeroRoute = false
+				break
+			}
+		}
+		if onlyZeroRoute {
+			// This node was returned by peerForIP because
+			// its exit node /0 route(s) matched, but this
+			// might not be the exit node that's currently
+			// selected.  Rather than log misleading
+			// errors, just don't log at all for now.
+			// TODO(bradfitz): update this code to be
+			// exit-node-aware and make peerForIP return
+			// the node of the currently selected exit
+			// node.
+			return
+		}
 		e.logf("open-conn-track: timeout opening %v; target node %v in netmap but unknown to wireguard", flow, n.Key.ShortString())
 		return
 	}
@@ -200,14 +220,32 @@ func (e *userspaceEngine) onOpenTimeout(flow flowtrack.Tuple) {
 	// handshake completed, which is what I want.
 	_ = ps.LastHandshake
 
-	e.logf("open-conn-track: timeout opening %v to node %v; lastSeen=%v, lastRecv=%v",
+	online := "?"
+	if n.Online != nil {
+		if *n.Online {
+			online = "yes"
+		} else {
+			online = "no"
+		}
+	}
+	e.logf("open-conn-track: timeout opening %v to node %v; lastSeen=%v, online=%v, lastRecv=%v",
 		flow, n.Key.ShortString(),
-		agoOrNever(lastSeen), agoOrNever(e.magicConn.LastRecvActivityOfDisco(n.DiscoKey)))
+		durFmt(lastSeen),
+		online,
+		durFmt(e.magicConn.LastRecvActivityOfDisco(n.DiscoKey)))
 }
 
-func agoOrNever(t time.Time) string {
+func durFmt(t time.Time) string {
 	if t.IsZero() {
 		return "never"
 	}
-	return time.Since(t).Round(time.Second).String()
+	d := time.Since(t).Round(time.Second)
+	if d < 10*time.Minute {
+		// node.LastSeen times are rounded very coarsely, and
+		// we compare times from different clocks (server vs
+		// local), so negative is common when close. Format as
+		// "recent" if negative or actually recent.
+		return "recent"
+	}
+	return d.String()
 }

wgengine/router/ifconfig_windows.go

@@ -359,7 +359,7 @@ func configureInterface(cfg *Config, tun *tun.NativeTun) (retErr error) {
 			firstGateway6 = &ipnet.IP
 		} else if route.IP.Is4() && firstGateway4 == nil {
 			// TODO: do same dummy behavior as v6?
-			return errors.New("Due to a Windows limitation, one cannot have interface routes without an interface address")
+			return errors.New("due to a Windows limitation, one cannot have interface routes without an interface address")
 		}
 
 		ipn := route.IPNet()
@@ -377,7 +377,7 @@ func configureInterface(cfg *Config, tun *tun.NativeTun) (retErr error) {
 			NextHop: gateway,
 			Metric:  0,
 		}
-		if bytes.Compare(r.Destination.IP, gateway) == 0 {
+		if net.IP.Equal(r.Destination.IP, gateway) {
 			// no need to add a route for the interface's
 			// own IP. The kernel does that for us.
 			// If we try to replace it, we'll fail to
@@ -411,7 +411,7 @@ func configureInterface(cfg *Config, tun *tun.NativeTun) (retErr error) {
 		// There's only one way to get to a given IP+Mask, so delete
 		// all matches after the first.
 		if i > 0 &&
-			bytes.Equal(routes[i].Destination.IP, routes[i-1].Destination.IP) &&
+			net.IP.Equal(routes[i].Destination.IP, routes[i-1].Destination.IP) &&
 			bytes.Equal(routes[i].Destination.Mask, routes[i-1].Destination.Mask) {
 			continue
 		}
@@ -713,22 +713,68 @@ func getInterfaceRoutes(ifc *winipcfg.IPAdapterAddresses, family winipcfg.Addres
 	return
 }
 
-// isSingleRouteInPrefixes reports whether r is a single-address
-// prefix that appears in pfxs.
-func isSingleRouteInPrefixes(r net.IPNet, pfxs []netaddr.IPPrefix) bool {
-	rr, ok := netaddr.FromStdIPNet(&r)
-	if !ok {
-		return false
+func getAllInterfaceRoutes(ifc *winipcfg.IPAdapterAddresses) ([]*winipcfg.RouteData, error) {
+	routes4, err := getInterfaceRoutes(ifc, windows.AF_INET)
+	if err != nil {
+		return nil, err
 	}
-	if !rr.IsSingleIP() {
-		return false
+
+	routes6, err := getInterfaceRoutes(ifc, windows.AF_INET6)
+	if err != nil {
+		// TODO: what if v6 unavailable?
+		return nil, err
 	}
-	for _, pfx := range pfxs {
-		if pfx == rr {
-			return true
+	rd := make([]*winipcfg.RouteData, 0, len(routes4)+len(routes6))
+	for _, r := range routes4 {
+		rd = append(rd, &winipcfg.RouteData{
+			Destination: r.DestinationPrefix.IPNet(),
+			NextHop:     r.NextHop.IP(),
+			Metric:      r.Metric,
+		})
+	}
+	for _, r := range routes6 {
+		rd = append(rd, &winipcfg.RouteData{
+			Destination: r.DestinationPrefix.IPNet(),
+			NextHop:     r.NextHop.IP(),
+			Metric:      r.Metric,
+		})
+	}
+	return rd, nil
+}
+
+// filterRoutes removes routes that have been added by Windows and should not
+// be managed by us.
+func filterRoutes(routes []*winipcfg.RouteData, dontDelete []netaddr.IPPrefix) []*winipcfg.RouteData {
+	ddm := make(map[netaddr.IPPrefix]bool)
+	for _, dd := range dontDelete {
+		// See issue 1448: we don't want to touch the routes added
+		// by Windows for our interface addresses.
+		ddm[dd] = true
+	}
+	for _, r := range routes {
+		// We don't want to touch broadcast routes that Windows adds.
+		nr, ok := netaddr.FromStdIPNet(&r.Destination)
+		if !ok {
+			continue
 		}
+		if nr.IsSingleIP() {
+			continue
+		}
+		lastIP := nr.Range().To
+		ddm[netaddr.IPPrefix{
+			IP:   lastIP,
+			Bits: lastIP.BitLen(),
+		}] = true
 	}
-	return false
+	filtered := make([]*winipcfg.RouteData, 0, len(routes))
+	for _, r := range routes {
+		rr, ok := netaddr.FromStdIPNet(&r.Destination)
+		if ok && ddm[rr] {
+			continue
+		}
+		filtered = append(filtered, r)
+	}
+	return filtered
 }
 
 // syncRoutes incrementally sets multiples routes on an interface.
@@ -736,42 +782,11 @@ func isSingleRouteInPrefixes(r net.IPNet, pfxs []netaddr.IPPrefix) bool {
 // dontDelete is a list of interface address routes that the
 // synchronization logic should never delete.
 func syncRoutes(ifc *winipcfg.IPAdapterAddresses, want []*winipcfg.RouteData, dontDelete []netaddr.IPPrefix) error {
-	routes4, err := getInterfaceRoutes(ifc, windows.AF_INET)
+	existingRoutes, err := getAllInterfaceRoutes(ifc)
 	if err != nil {
 		return err
 	}
-
-	routes6, err := getInterfaceRoutes(ifc, windows.AF_INET6)
-	if err != nil {
-		// TODO: what if v6 unavailable?
-		return err
-	}
-
-	got := make([]*winipcfg.RouteData, 0, len(routes4))
-	for _, r := range routes4 {
-		if isSingleRouteInPrefixes(r.DestinationPrefix.IPNet(), dontDelete) {
-			// See issue 1448: we don't want to touch the routes added
-			// by Windows for our interface addresses.
-			continue
-		}
-		got = append(got, &winipcfg.RouteData{
-			Destination: r.DestinationPrefix.IPNet(),
-			NextHop:     r.NextHop.IP(),
-			Metric:      r.Metric,
-		})
-	}
-	for _, r := range routes6 {
-		if isSingleRouteInPrefixes(r.DestinationPrefix.IPNet(), dontDelete) {
-			// See issue 1448: we don't want to touch the routes added
-			// by Windows for our interface addresses.
-			continue
-		}
-		got = append(got, &winipcfg.RouteData{
-			Destination: r.DestinationPrefix.IPNet(),
-			NextHop:     r.NextHop.IP(),
-			Metric:      r.Metric,
-		})
-	}
+	got := filterRoutes(existingRoutes, dontDelete)
 
 	add, del := deltaRouteData(got, want)
 
@@ -783,6 +798,7 @@ func syncRoutes(ifc *winipcfg.IPAdapterAddresses, want []*winipcfg.RouteData, do
 			if dstStr == "169.254.255.255/32" {
 				// Issue 785. Ignore these routes
 				// failing to delete. Harmless.
+				// TODO(maisem): do we still need this?
 				continue
 			}
 			errs = append(errs, fmt.Errorf("deleting route %v: %w", dstStr, err))

wgengine/router/ifconfig_windows_test.go

@@ -193,6 +193,14 @@ func TestDeltaNets(t *testing.T) {
 	}
 }
 
+func formatRouteData(rds []*winipcfg.RouteData) string {
+	var b strings.Builder
+	for _, rd := range rds {
+		b.WriteString(fmt.Sprintf("%+v", rd))
+	}
+	return b.String()
+}
+
 func equalRouteDatas(a, b []*winipcfg.RouteData) bool {
 	if len(a) != len(b) {
 		return false
@@ -205,6 +213,43 @@ func equalRouteDatas(a, b []*winipcfg.RouteData) bool {
 	return true
 }
 
+func TestFilterRoutes(t *testing.T) {
+	var h0 net.IP
+	in := []*winipcfg.RouteData{
+		// LinkLocal and Loopback routes.
+		{*ipnet4("169.254.0.0", 16), h0, 1},
+		{*ipnet4("169.254.255.255", 32), h0, 1},
+		{*ipnet4("127.0.0.0", 8), h0, 1},
+		{*ipnet4("127.255.255.255", 32), h0, 1},
+		// Local LAN routes.
+		{*ipnet4("192.168.0.0", 24), h0, 1},
+		{*ipnet4("192.168.0.255", 32), h0, 1},
+		{*ipnet4("192.168.1.0", 25), h0, 1},
+		{*ipnet4("192.168.1.127", 32), h0, 1},
+		// Some random other route.
+		{*ipnet4("192.168.2.23", 32), h0, 1},
+		// Our own tailscale address.
+		{*ipnet4("100.100.100.100", 32), h0, 1},
+		// Other tailscale addresses.
+		{*ipnet4("100.100.100.101", 32), h0, 1},
+		{*ipnet4("100.100.100.102", 32), h0, 1},
+	}
+	want := []*winipcfg.RouteData{
+		{*ipnet4("169.254.0.0", 16), h0, 1},
+		{*ipnet4("127.0.0.0", 8), h0, 1},
+		{*ipnet4("192.168.0.0", 24), h0, 1},
+		{*ipnet4("192.168.1.0", 25), h0, 1},
+		{*ipnet4("192.168.2.23", 32), h0, 1},
+		{*ipnet4("100.100.100.101", 32), h0, 1},
+		{*ipnet4("100.100.100.102", 32), h0, 1},
+	}
+
+	got := filterRoutes(in, mustCIDRs("100.100.100.100/32"))
+	if !equalRouteDatas(got, want) {
+		t.Errorf("\ngot:  %v\n  want: %v\n", formatRouteData(got), formatRouteData(want))
+	}
+}
+
 func TestDeltaRouteData(t *testing.T) {
 	var h0 net.IP
 	h1 := net.ParseIP("99.99.99.99")
@@ -232,9 +277,9 @@ func TestDeltaRouteData(t *testing.T) {
 	}
 
 	if !equalRouteDatas(add, wantAdd) {
-		t.Errorf("add:\n   got: %v\n  want: %v\n", add, wantAdd)
+		t.Errorf("add:\n   got: %v\n  want: %v\n", formatRouteData(add), formatRouteData(wantAdd))
 	}
 	if !equalRouteDatas(del, wantDel) {
-		t.Errorf("del:\n   got: %v\n  want: %v\n", del, wantDel)
+		t.Errorf("del:\n   got: %v\n  want: %v\n", formatRouteData(del), formatRouteData(wantDel))
 	}
 }

wgengine/router/router_linux_test.go

@@ -20,22 +20,6 @@ import (
 	"inet.af/netaddr"
 )
 
-func mustCIDR(s string) netaddr.IPPrefix {
-	pfx, err := netaddr.ParseIPPrefix(s)
-	if err != nil {
-		panic(err)
-	}
-	return pfx
-}
-
-func mustCIDRs(ss ...string) []netaddr.IPPrefix {
-	var ret []netaddr.IPPrefix
-	for _, s := range ss {
-		ret = append(ret, mustCIDR(s))
-	}
-	return ret
-}
-
 func TestRouterStates(t *testing.T) {
 	basic := `
 ip rule add -4 pref 5210 fwmark 0x80000 table main

wgengine/router/router_test.go

@@ -0,0 +1,17 @@
+// Copyright (c) 2021 Tailscale Inc & AUTHORS All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+// +build linux windows
+
+package router
+
+import "inet.af/netaddr"
+
+func mustCIDRs(ss ...string) []netaddr.IPPrefix {
+	var ret []netaddr.IPPrefix
+	for _, s := range ss {
+		ret = append(ret, netaddr.MustParseIPPrefix(s))
+	}
+	return ret
+}

wgengine/router/router_windows.go

@@ -30,15 +30,6 @@ type winRouter struct {
 	nativeTun           *tun.NativeTun
 	routeChangeCallback *winipcfg.RouteChangeCallback
 	firewall            *firewallTweaker
-
-	// firewallSubproc is a subprocess that runs a tweaked version of
-	// wireguard-windows's "default route killswitch" code. We run it
-	// as a subprocess because it does unsafe callouts to the WFP API,
-	// and we want to defend against memory corruption in our main
-	// process. Owned and mutated only by Set, and doesn't need a lock
-	// because Set is only called with wgengine's lock held,
-	// preventing concurrent reconfigs.
-	firewallSubproc *exec.Cmd
 }
 
 func newUserspaceRouter(logf logger.Logf, tundev tun.Device) (Router, error) {

wgengine/userspace.go

@@ -76,6 +76,10 @@ const (
 	packetSendRecheckWireguardThreshold = 1 * time.Minute
 )
 
+// statusPollInterval is how often we ask wireguard-go for its engine
+// status (as long as there's activity). See docs on its use below.
+const statusPollInterval = 1 * time.Minute
+
 type userspaceEngine struct {
 	logf              logger.Logf
 	wgLogger          *wglog.Logger //a wireguard-go logging wrapper
@@ -108,6 +112,7 @@ type userspaceEngine struct {
 	sentActivityAt      map[netaddr.IP]*int64     // value is atomic int64 of unixtime
 	destIPActivityFuncs map[netaddr.IP]func()
 	statusBufioReader   *bufio.Reader // reusable for UAPI
+	lastStatusPollTime  time.Time     // last time we polled the engine status
 
 	mu                  sync.Mutex         // guards following; see lock order comment below
 	netMap              *netmap.NetworkMap // or nil
@@ -238,7 +243,7 @@ func NewUserspaceEngine(logf logger.Logf, conf Config) (_ Engine, reterr error)
 		router:  conf.Router,
 		pingers: make(map[wgkey.Key]*pinger),
 	}
-	e.isLocalAddr.Store(genLocalAddrFunc(nil))
+	e.isLocalAddr.Store(tsaddr.NewContainsIPFunc(nil))
 
 	if conf.LinkMonitor != nil {
 		e.linkMon = conf.LinkMonitor
@@ -373,11 +378,7 @@ func NewUserspaceEngine(logf logger.Logf, conf Config) (_ Engine, reterr error)
 
 	go func() {
 		up := false
-		for event := range e.tundev.Events() {
-			if event&tun.EventMTUUpdate != 0 {
-				mtu, err := e.tundev.MTU()
-				e.logf("external route MTU: %d (%v)", mtu, err)
-			}
+		for event := range e.tundev.EventsUpDown() {
 			if event&tun.EventUp != 0 && !up {
 				e.logf("external route: up")
 				e.RequestStatus()
@@ -712,6 +713,18 @@ func (e *userspaceEngine) noteReceiveActivity(dk tailcfg.DiscoKey) {
 	now := e.timeNow()
 	e.recvActivityAt[dk] = now
 
+	// As long as there's activity, periodically poll the engine to get
+	// stats for the far away side effect of
+	// ipn/ipnlocal.LocalBackend.parseWgStatusLocked to log activity, for
+	// use in various admin dashboards.
+	// This particularly matters on platforms without a connected GUI, as
+	// the GUIs generally poll this enough to cause that logging. But
+	// tailscaled alone did not, hence this.
+	if e.lastStatusPollTime.IsZero() || now.Sub(e.lastStatusPollTime) >= statusPollInterval {
+		e.lastStatusPollTime = now
+		go e.RequestStatus()
+	}
+
 	// If the last activity time jumped a bunch (say, at least
 	// half the idle timeout) then see if we need to reprogram
 	// Wireguard. This could probably be just
@@ -923,28 +936,6 @@ func (e *userspaceEngine) updateActivityMapsLocked(trackDisco []tailcfg.DiscoKey
 	e.tundev.SetDestIPActivityFuncs(e.destIPActivityFuncs)
 }
 
-// genLocalAddrFunc returns a func that reports whether an IP is in addrs.
-// addrs is assumed to be all /32 or /128 entries.
-func genLocalAddrFunc(addrs []netaddr.IPPrefix) func(netaddr.IP) bool {
-	// Specialize the three common cases: no address, just IPv4
-	// (or just IPv6), and both IPv4 and IPv6.
-	if len(addrs) == 0 {
-		return func(netaddr.IP) bool { return false }
-	}
-	if len(addrs) == 1 {
-		return func(t netaddr.IP) bool { return t == addrs[0].IP }
-	}
-	if len(addrs) == 2 {
-		return func(t netaddr.IP) bool { return t == addrs[0].IP || t == addrs[1].IP }
-	}
-	// Otherwise, the general implementation: a map lookup.
-	m := map[netaddr.IP]bool{}
-	for _, a := range addrs {
-		m[a.IP] = true
-	}
-	return func(t netaddr.IP) bool { return m[t] }
-}
-
 func (e *userspaceEngine) Reconfig(cfg *wgcfg.Config, routerCfg *router.Config, dnsCfg *dns.Config) error {
 	if routerCfg == nil {
 		panic("routerCfg must not be nil")
@@ -953,7 +944,7 @@ func (e *userspaceEngine) Reconfig(cfg *wgcfg.Config, routerCfg *router.Config,
 		panic("dnsCfg must not be nil")
 	}
 
-	e.isLocalAddr.Store(genLocalAddrFunc(routerCfg.LocalAddrs))
+	e.isLocalAddr.Store(tsaddr.NewContainsIPFunc(routerCfg.LocalAddrs))
 
 	e.wgLock.Lock()
 	defer e.wgLock.Unlock()
@@ -1211,7 +1202,7 @@ func (e *userspaceEngine) RequestStatus() {
 	case <-e.reqCh:
 		s, err := e.getStatus()
 		if s == nil && err == nil {
-			e.logf("RequestStatus: weird: both s and err are nil")
+			e.logf("[unexpected] RequestStatus: both s and err are nil")
 			return
 		}
 		if cb := e.getStatusCallback(); cb != nil {