wgengine/netstack: add netstack port rewriting mechanism

Signed-off-by: Brad Fitzpatrick <bradfitz@tailscale.com>

.github/ISSUE_TEMPLATE/bug_report.md

@@ -2,36 +2,7 @@
 name: Bug report
 about: Create a bug report
 title: ''
-labels: ''
+labels: 'needs-triage'
 assignees: ''
 
 ---
-
-<!-- Please note, this template is for definite bugs, not requests for
-support. If you need help with Tailscale, please email
-support@tailscale.com. We don't provide support via Github issues. -->
-
-**Describe the bug**
-A clear and concise description of what the bug is.
-
-**To Reproduce**
-Steps to reproduce the behavior:
-1. Go to '...'
-2. Click on '....'
-3. Scroll down to '....'
-4. See error
-
-**Expected behavior**
-A clear and concise description of what you expected to happen.
-
-**Screenshots**
-If applicable, add screenshots to help explain your problem.
-
-**Version information:**
- - Device: [e.g. iPhone X, laptop]
- - OS: [e.g. Windows, MacOS]
- - OS version: [e.g. Windows 10, Ubuntu 18.04]
- - Tailscale version: [e.g. 0.95-0]
-
-**Additional context**
-Add any other context about the problem here.

.github/ISSUE_TEMPLATE/feature_request.md

@@ -2,25 +2,6 @@
 name: Feature request
 about: Suggest an idea for this project
 title: ''
-labels: ''
+labels: 'needs-triage'
 assignees: ''
-
 ---
-
-**Is your feature request related to a problem? Please describe.**
-
-A clear and concise description of what the problem is. Ex. I'm always
-frustrated when [...]
-
-**Describe the solution you'd like**
-
-A clear and concise description of what you want to happen.
-
-**Describe alternatives you've considered**
-
-A clear and concise description of any alternative solutions or
-features you've considered.
-
-**Additional context**
-
-Add any other context or screenshots about the feature request here.

.github/workflows/linux-race.yml

@@ -28,8 +28,8 @@ jobs:
     - name: Basic build
       run: go build ./cmd/...
 
-    - name: Run tests with -race flag on linux
-      run: go test -race ./...
+    - name: Run tests and benchmarks with -race flag on linux
+      run: go test -race -bench=. -benchtime=1x ./...
 
     - uses: k0kubun/action-slack@v2.0.0
       with:

.github/workflows/linux.yml

@@ -29,7 +29,7 @@ jobs:
       run: go build ./cmd/...
 
     - name: Run tests on linux
-      run: go test ./...
+      run: go test -bench=. -benchtime=1x ./...
 
     - uses: k0kubun/action-slack@v2.0.0
       with:

.github/workflows/linux32.yml

@@ -29,7 +29,7 @@ jobs:
       run: GOARCH=386 go build ./cmd/...
 
     - name: Run tests on linux
-      run: GOARCH=386 go test ./...
+      run: GOARCH=386 go test -bench=. -benchtime=1x ./...
 
     - uses: k0kubun/action-slack@v2.0.0
       with:

.github/workflows/staticcheck.yml

@@ -24,11 +24,23 @@ jobs:
     - name: Run go vet
       run: go vet ./...
 
-    - name: Print staticcheck version
-      run: go run honnef.co/go/tools/cmd/staticcheck -version
+    - name: Install staticcheck
+      run: "GOBIN=~/.local/bin go install honnef.co/go/tools/cmd/staticcheck"
 
-    - name: Run staticcheck
-      run: "go run honnef.co/go/tools/cmd/staticcheck -- $(go list ./... | grep -v tempfork)"
+    - name: Print staticcheck version
+      run: "staticcheck -version"
+
+    - name: Run staticcheck (linux/amd64)
+      run: "GOOS=linux GOARCH=amd64 staticcheck -- $(go list ./... | grep -v tempfork)"
+
+    - name: Run staticcheck (darwin/amd64)
+      run: "GOOS=darwin GOARCH=amd64 staticcheck -- $(go list ./... | grep -v tempfork)"
+
+    - name: Run staticcheck (windows/amd64)
+      run: "GOOS=windows GOARCH=amd64 staticcheck -- $(go list ./... | grep -v tempfork)"
+
+    - name: Run staticcheck (windows/386)
+      run: "GOOS=windows GOARCH=386 staticcheck -- $(go list ./... | grep -v tempfork)"
 
     - uses: k0kubun/action-slack@v2.0.0
       with:

.github/workflows/windows-race.yml

@@ -33,7 +33,10 @@ jobs:
           ${{ runner.os }}-go-
 
     - name: Test with -race flag
-      run: go test -race ./...
+      # Don't use -bench=. -benchtime=1x.
+      # Somewhere in the layers (powershell?)
+      # the equals signs cause great confusion.
+      run: go test -race -bench . -benchtime 1x ./...
 
     - uses: k0kubun/action-slack@v2.0.0
       with:

.github/workflows/windows.yml

@@ -33,7 +33,10 @@ jobs:
           ${{ runner.os }}-go-
 
     - name: Test
-      run: go test ./...
+      # Don't use -bench=. -benchtime=1x.
+      # Somewhere in the layers (powershell?)
+      # the equals signs cause great confusion.
+      run: go test -bench . -benchtime 1x ./...
 
     - uses: k0kubun/action-slack@v2.0.0
       with:

VERSION.txt

@@ -1 +1 @@
-1.7.0
+1.9.0

cmd/tailscale/cli/cli.go

@@ -8,8 +8,10 @@ package cli
 
 import (
 	"context"
+	"errors"
 	"flag"
 	"fmt"
+	"io"
 	"log"
 	"net"
 	"os"
@@ -49,6 +51,14 @@ func ActLikeCLI() bool {
 		return false
 	}
 
+	// Xcode adds the -NSDocumentRevisionsDebugMode flag on execution.
+	// If present, we are almost certainly being run as a GUI.
+	for _, arg := range os.Args {
+		if arg == "-NSDocumentRevisionsDebugMode" {
+			return false
+		}
+	}
+
 	// Looking at the environment of the GUI Tailscale app (ps eww
 	// $PID), empirically none of these environment variables are
 	// present. But all or some of these should be present with
@@ -169,21 +179,22 @@ func connect(ctx context.Context) (net.Conn, *ipn.BackendClient, context.Context
 }
 
 // pump receives backend messages on conn and pushes them into bc.
-func pump(ctx context.Context, bc *ipn.BackendClient, conn net.Conn) {
+func pump(ctx context.Context, bc *ipn.BackendClient, conn net.Conn) error {
 	defer conn.Close()
 	for ctx.Err() == nil {
 		msg, err := ipn.ReadMsg(conn)
 		if err != nil {
 			if ctx.Err() != nil {
-				return
+				return ctx.Err()
 			}
-			if !gotSignal.Get() {
-				log.Printf("ReadMsg: %v\n", err)
+			if errors.Is(err, io.EOF) || errors.Is(err, net.ErrClosed) {
+				return fmt.Errorf("%w (tailscaled stopped running?)", err)
 			}
-			break
+			return err
 		}
 		bc.GotNotifyMsg(msg)
 	}
+	return ctx.Err()
 }
 
 func strSliceContains(ss []string, s string) bool {

cmd/tailscale/cli/cli_test.go

@@ -9,131 +9,98 @@ import (
 	"encoding/json"
 	"flag"
 	"fmt"
+	"reflect"
 	"strings"
 	"testing"
 
 	"inet.af/netaddr"
 	"tailscale.com/ipn"
 	"tailscale.com/ipn/ipnstate"
-	"tailscale.com/types/logger"
 	"tailscale.com/types/preftype"
 )
 
+// geese is a collection of gooses. It need not be complete.
+// But it should include anything handled specially (e.g. linux, windows)
+// and at least one thing that's not (darwin, freebsd).
+var geese = []string{"linux", "darwin", "windows", "freebsd"}
+
 // Test that checkForAccidentalSettingReverts's updateMaskedPrefsFromUpFlag can handle
 // all flags. This will panic if a new flag creeps in that's unhandled.
+//
+// Also, issue 1880: advertise-exit-node was being ignored. Verify that all flags cause an edit.
 func TestUpdateMaskedPrefsFromUpFlag(t *testing.T) {
-	mp := new(ipn.MaskedPrefs)
-	upFlagSet.VisitAll(func(f *flag.Flag) {
-		updateMaskedPrefsFromUpFlag(mp, f.Name)
-	})
+	for _, goos := range geese {
+		var upArgs upArgsT
+		fs := newUpFlagSet(goos, &upArgs)
+		fs.VisitAll(func(f *flag.Flag) {
+			mp := new(ipn.MaskedPrefs)
+			updateMaskedPrefsFromUpFlag(mp, f.Name)
+			got := mp.Pretty()
+			wantEmpty := preflessFlag(f.Name)
+			isEmpty := got == "MaskedPrefs{}"
+			if isEmpty != wantEmpty {
+				t.Errorf("flag %q created MaskedPrefs %s; want empty=%v", f.Name, got, wantEmpty)
+			}
+		})
+	}
 }
 
 func TestCheckForAccidentalSettingReverts(t *testing.T) {
-	f := func(flags ...string) map[string]bool {
-		m := make(map[string]bool)
-		for _, f := range flags {
-			m[f] = true
-		}
-		return m
-	}
 	tests := []struct {
 		name     string
-		flagSet  map[string]bool
+		flags    []string // argv to be parsed by FlagSet
 		curPrefs *ipn.Prefs
-		curUser  string // os.Getenv("USER") on the client side
-		mp       *ipn.MaskedPrefs
-		want     string
+
+		curExitNodeIP netaddr.IP
+		curUser       string // os.Getenv("USER") on the client side
+		goos          string // empty means "linux"
+
+		want string
 	}{
 		{
-			name:    "bare_up_means_up",
-			flagSet: f(),
+			name:  "bare_up_means_up",
+			flags: []string{},
 			curPrefs: &ipn.Prefs{
 				ControlURL:  ipn.DefaultControlURL,
 				WantRunning: false,
 				Hostname:    "foo",
 			},
-			mp: &ipn.MaskedPrefs{
-				Prefs: ipn.Prefs{
-					WantRunning: true,
-				},
-				WantRunningSet: true,
-			},
 			want: "",
 		},
 		{
-			name:    "losing_hostname",
-			flagSet: f("accept-dns"),
+			name:  "losing_hostname",
+			flags: []string{"--accept-dns"},
 			curPrefs: &ipn.Prefs{
-				ControlURL:  ipn.DefaultControlURL,
-				WantRunning: false,
-				Hostname:    "foo",
-				CorpDNS:     true,
-			},
-			mp: &ipn.MaskedPrefs{
-				Prefs: ipn.Prefs{
-					ControlURL:  ipn.DefaultControlURL,
-					WantRunning: true,
-					CorpDNS:     true,
-				},
-				ControlURLSet:  true,
-				WantRunningSet: true,
-				CorpDNSSet:     true,
+				ControlURL:       ipn.DefaultControlURL,
+				WantRunning:      false,
+				Hostname:         "foo",
+				CorpDNS:          true,
+				NetfilterMode:    preftype.NetfilterOn,
+				AllowSingleHosts: true,
 			},
 			want: accidentalUpPrefix + " --accept-dns --hostname=foo",
 		},
 		{
-			name:    "hostname_changing_explicitly",
-			flagSet: f("hostname"),
+			name:  "hostname_changing_explicitly",
+			flags: []string{"--hostname=bar"},
 			curPrefs: &ipn.Prefs{
-				ControlURL:  ipn.DefaultControlURL,
-				WantRunning: false,
-				Hostname:    "foo",
-			},
-			mp: &ipn.MaskedPrefs{
-				Prefs: ipn.Prefs{
-					ControlURL:  ipn.DefaultControlURL,
-					WantRunning: true,
-					Hostname:    "bar",
-				},
-				ControlURLSet:  true,
-				WantRunningSet: true,
-				HostnameSet:    true,
+				ControlURL:       ipn.DefaultControlURL,
+				CorpDNS:          true,
+				NetfilterMode:    preftype.NetfilterOn,
+				AllowSingleHosts: true,
+				Hostname:         "foo",
 			},
 			want: "",
 		},
 		{
-			name:    "hostname_changing_empty_explicitly",
-			flagSet: f("hostname"),
+			name:  "hostname_changing_empty_explicitly",
+			flags: []string{"--hostname="},
 			curPrefs: &ipn.Prefs{
-				ControlURL:  ipn.DefaultControlURL,
-				WantRunning: false,
-				Hostname:    "foo",
-			},
-			mp: &ipn.MaskedPrefs{
-				Prefs: ipn.Prefs{
-					ControlURL:  ipn.DefaultControlURL,
-					WantRunning: true,
-					Hostname:    "",
-				},
-				ControlURLSet:  true,
-				WantRunningSet: true,
-				HostnameSet:    true,
-			},
-			want: "",
-		},
-		{
-			name:    "empty_slice_equals_nil_slice",
-			flagSet: f("hostname"),
-			curPrefs: &ipn.Prefs{
-				ControlURL:      ipn.DefaultControlURL,
-				AdvertiseRoutes: []netaddr.IPPrefix{},
-			},
-			mp: &ipn.MaskedPrefs{
-				Prefs: ipn.Prefs{
-					ControlURL:      ipn.DefaultControlURL,
-					AdvertiseRoutes: nil,
-				},
-				ControlURLSet: true,
+				ControlURL:       ipn.DefaultControlURL,
+				CorpDNS:          true,
+				NetfilterMode:    preftype.NetfilterOn,
+				AllowSingleHosts: true,
+				Hostname:         "foo",
 			},
 			want: "",
 		},
@@ -141,192 +108,174 @@ func TestCheckForAccidentalSettingReverts(t *testing.T) {
 			// Issue 1725: "tailscale up --authkey=..." (or other non-empty flags) works from
 			// a fresh server's initial prefs.
 			name:     "up_with_default_prefs",
-			flagSet:  f("authkey"),
+			flags:    []string{"--authkey=foosdlkfjskdljf"},
 			curPrefs: ipn.NewPrefs(),
-			mp: &ipn.MaskedPrefs{
-				Prefs:          *defaultPrefsFromUpArgs(t),
-				WantRunningSet: true,
-			},
-			want: "",
+			want:     "",
 		},
 		{
-			name:    "implicit_operator_change",
-			flagSet: f("hostname"),
+			name:  "implicit_operator_change",
+			flags: []string{"--hostname=foo"},
 			curPrefs: &ipn.Prefs{
-				ControlURL:   ipn.DefaultControlURL,
-				OperatorUser: "alice",
+				ControlURL:       ipn.DefaultControlURL,
+				OperatorUser:     "alice",
+				AllowSingleHosts: true,
+				CorpDNS:          true,
+				NetfilterMode:    preftype.NetfilterOn,
 			},
 			curUser: "eve",
-			mp: &ipn.MaskedPrefs{
-				Prefs: ipn.Prefs{
-					ControlURL: ipn.DefaultControlURL,
-				},
-				ControlURLSet: true,
-			},
-			want: accidentalUpPrefix + " --hostname= --operator=alice",
+			want:    accidentalUpPrefix + " --hostname=foo --operator=alice",
 		},
 		{
-			name:    "implicit_operator_matches_shell_user",
-			flagSet: f("hostname"),
+			name:  "implicit_operator_matches_shell_user",
+			flags: []string{"--hostname=foo"},
 			curPrefs: &ipn.Prefs{
-				ControlURL:   ipn.DefaultControlURL,
-				OperatorUser: "alice",
+				ControlURL:       ipn.DefaultControlURL,
+				AllowSingleHosts: true,
+				CorpDNS:          true,
+				NetfilterMode:    preftype.NetfilterOn,
+				OperatorUser:     "alice",
 			},
 			curUser: "alice",
-			mp: &ipn.MaskedPrefs{
-				Prefs: ipn.Prefs{
-					ControlURL: ipn.DefaultControlURL,
-				},
-				ControlURLSet: true,
-			},
-			want: "",
+			want:    "",
 		},
 		{
-			name:    "error_advertised_routes_exit_node_removed",
-			flagSet: f("advertise-routes"),
+			name:  "error_advertised_routes_exit_node_removed",
+			flags: []string{"--advertise-routes=10.0.42.0/24"},
 			curPrefs: &ipn.Prefs{
-				ControlURL: ipn.DefaultControlURL,
+				ControlURL:       ipn.DefaultControlURL,
+				AllowSingleHosts: true,
+				CorpDNS:          true,
+				NetfilterMode:    preftype.NetfilterOn,
 				AdvertiseRoutes: []netaddr.IPPrefix{
 					netaddr.MustParseIPPrefix("10.0.42.0/24"),
 					netaddr.MustParseIPPrefix("0.0.0.0/0"),
 					netaddr.MustParseIPPrefix("::/0"),
 				},
 			},
-			mp: &ipn.MaskedPrefs{
-				Prefs: ipn.Prefs{
-					ControlURL: ipn.DefaultControlURL,
-					AdvertiseRoutes: []netaddr.IPPrefix{
-						netaddr.MustParseIPPrefix("10.0.42.0/24"),
-					},
-				},
-				AdvertiseRoutesSet: true,
-			},
 			want: accidentalUpPrefix + " --advertise-routes=10.0.42.0/24 --advertise-exit-node",
 		},
 		{
-			name:    "advertised_routes_exit_node_removed",
-			flagSet: f("advertise-routes", "advertise-exit-node"),
+			name:  "advertised_routes_exit_node_removed_explicit",
+			flags: []string{"--advertise-routes=10.0.42.0/24", "--advertise-exit-node=false"},
 			curPrefs: &ipn.Prefs{
-				ControlURL: ipn.DefaultControlURL,
+				ControlURL:       ipn.DefaultControlURL,
+				AllowSingleHosts: true,
+				CorpDNS:          true,
+				NetfilterMode:    preftype.NetfilterOn,
 				AdvertiseRoutes: []netaddr.IPPrefix{
 					netaddr.MustParseIPPrefix("10.0.42.0/24"),
 					netaddr.MustParseIPPrefix("0.0.0.0/0"),
 					netaddr.MustParseIPPrefix("::/0"),
 				},
 			},
-			mp: &ipn.MaskedPrefs{
-				Prefs: ipn.Prefs{
-					ControlURL: ipn.DefaultControlURL,
-					AdvertiseRoutes: []netaddr.IPPrefix{
-						netaddr.MustParseIPPrefix("10.0.42.0/24"),
-					},
-				},
-				AdvertiseRoutesSet: true,
-			},
 			want: "",
 		},
 		{
-			name:    "advertised_routes_includes_the_0_routes", // but no --advertise-exit-node
-			flagSet: f("advertise-routes"),
+			name:  "advertised_routes_includes_the_0_routes", // but no --advertise-exit-node
+			flags: []string{"--advertise-routes=11.1.43.0/24,0.0.0.0/0,::/0"},
 			curPrefs: &ipn.Prefs{
-				ControlURL: ipn.DefaultControlURL,
+				ControlURL:       ipn.DefaultControlURL,
+				AllowSingleHosts: true,
+				CorpDNS:          true,
+				NetfilterMode:    preftype.NetfilterOn,
 				AdvertiseRoutes: []netaddr.IPPrefix{
 					netaddr.MustParseIPPrefix("10.0.42.0/24"),
 					netaddr.MustParseIPPrefix("0.0.0.0/0"),
 					netaddr.MustParseIPPrefix("::/0"),
 				},
 			},
-			mp: &ipn.MaskedPrefs{
-				Prefs: ipn.Prefs{
-					ControlURL: ipn.DefaultControlURL,
-					AdvertiseRoutes: []netaddr.IPPrefix{
-						netaddr.MustParseIPPrefix("11.1.43.0/24"),
-						netaddr.MustParseIPPrefix("0.0.0.0/0"),
-						netaddr.MustParseIPPrefix("::/0"),
-					},
-				},
-				AdvertiseRoutesSet: true,
-			},
 			want: "",
 		},
 		{
-			name:    "advertised_routes_includes_only_one_0_route", // and no --advertise-exit-node
-			flagSet: f("advertise-routes"),
+			name:  "advertise_exit_node", // Issue 1859
+			flags: []string{"--advertise-exit-node"},
 			curPrefs: &ipn.Prefs{
-				ControlURL: ipn.DefaultControlURL,
-				AdvertiseRoutes: []netaddr.IPPrefix{
-					netaddr.MustParseIPPrefix("10.0.42.0/24"),
-					netaddr.MustParseIPPrefix("0.0.0.0/0"),
-					netaddr.MustParseIPPrefix("::/0"),
-				},
+				ControlURL:       ipn.DefaultControlURL,
+				AllowSingleHosts: true,
+				CorpDNS:          true,
+				NetfilterMode:    preftype.NetfilterOn,
 			},
-			mp: &ipn.MaskedPrefs{
-				Prefs: ipn.Prefs{
-					ControlURL: ipn.DefaultControlURL,
-					AdvertiseRoutes: []netaddr.IPPrefix{
-						netaddr.MustParseIPPrefix("11.1.43.0/24"),
-						netaddr.MustParseIPPrefix("0.0.0.0/0"),
-					},
-				},
-				AdvertiseRoutesSet: true,
-			},
-			want: accidentalUpPrefix + " --advertise-routes=11.1.43.0/24,0.0.0.0/0 --advertise-exit-node",
+			want: "",
 		},
 		{
-			name:    "exit_node_clearing", // Issue 1777
-			flagSet: f("exit-node"),
+			name:  "advertise_exit_node_over_existing_routes",
+			flags: []string{"--advertise-exit-node"},
 			curPrefs: &ipn.Prefs{
-				ControlURL: ipn.DefaultControlURL,
+				ControlURL:       ipn.DefaultControlURL,
+				AllowSingleHosts: true,
+				CorpDNS:          true,
+				NetfilterMode:    preftype.NetfilterOn,
+
+				AdvertiseRoutes: []netaddr.IPPrefix{
+					netaddr.MustParseIPPrefix("1.2.0.0/16"),
+				},
+			},
+			want: accidentalUpPrefix + " --advertise-exit-node --advertise-routes=1.2.0.0/16",
+		},
+		{
+			name:  "advertise_exit_node_over_existing_routes_and_exit_node",
+			flags: []string{"--advertise-exit-node"},
+			curPrefs: &ipn.Prefs{
+				ControlURL:       ipn.DefaultControlURL,
+				AllowSingleHosts: true,
+				CorpDNS:          true,
+				NetfilterMode:    preftype.NetfilterOn,
+				AdvertiseRoutes: []netaddr.IPPrefix{
+					netaddr.MustParseIPPrefix("0.0.0.0/0"),
+					netaddr.MustParseIPPrefix("::/0"),
+					netaddr.MustParseIPPrefix("1.2.0.0/16"),
+				},
+			},
+			want: accidentalUpPrefix + " --advertise-exit-node --advertise-routes=1.2.0.0/16",
+		},
+		{
+			name:  "exit_node_clearing", // Issue 1777
+			flags: []string{"--exit-node="},
+			curPrefs: &ipn.Prefs{
+				ControlURL:       ipn.DefaultControlURL,
+				AllowSingleHosts: true,
+				CorpDNS:          true,
+				NetfilterMode:    preftype.NetfilterOn,
+
 				ExitNodeID: "fooID",
 			},
-			mp: &ipn.MaskedPrefs{
-				Prefs: ipn.Prefs{
-					ControlURL: ipn.DefaultControlURL,
-					ExitNodeIP: netaddr.IP{},
-				},
-				ExitNodeIPSet: true,
-			},
 			want: "",
 		},
 		{
-			name:    "remove_all_implicit",
-			flagSet: f("force-reauth"),
+			name:  "remove_all_implicit",
+			flags: []string{"--force-reauth"},
 			curPrefs: &ipn.Prefs{
 				WantRunning:      true,
 				ControlURL:       ipn.DefaultControlURL,
 				RouteAll:         true,
 				AllowSingleHosts: false,
 				ExitNodeIP:       netaddr.MustParseIP("100.64.5.6"),
-				CorpDNS:          true,
+				CorpDNS:          false,
 				ShieldsUp:        true,
 				AdvertiseTags:    []string{"tag:foo", "tag:bar"},
 				Hostname:         "myhostname",
 				ForceDaemon:      true,
 				AdvertiseRoutes: []netaddr.IPPrefix{
 					netaddr.MustParseIPPrefix("10.0.0.0/16"),
+					netaddr.MustParseIPPrefix("0.0.0.0/0"),
+					netaddr.MustParseIPPrefix("::/0"),
 				},
 				NetfilterMode: preftype.NetfilterNoDivert,
 				OperatorUser:  "alice",
 			},
 			curUser: "eve",
-			mp: &ipn.MaskedPrefs{
-				Prefs: ipn.Prefs{
-					ControlURL:  ipn.DefaultControlURL,
-					WantRunning: true,
-				},
-			},
-			want: accidentalUpPrefix + " --accept-routes --exit-node=100.64.5.6 --accept-dns --shields-up --advertise-tags=tag:foo,tag:bar --hostname=myhostname --unattended --advertise-routes=10.0.0.0/16 --netfilter-mode=nodivert --operator=alice",
+			want:    accidentalUpPrefix + " --force-reauth --accept-dns=false --accept-routes --advertise-exit-node --advertise-routes=10.0.0.0/16 --advertise-tags=tag:foo,tag:bar --exit-node=100.64.5.6 --host-routes=false --hostname=myhostname --netfilter-mode=nodivert --operator=alice --shields-up",
 		},
 		{
-			name:    "remove_all_implicit_except_hostname",
-			flagSet: f("hostname"),
+			name:  "remove_all_implicit_except_hostname",
+			flags: []string{"--hostname=newhostname"},
 			curPrefs: &ipn.Prefs{
 				WantRunning:      true,
 				ControlURL:       ipn.DefaultControlURL,
 				RouteAll:         true,
 				AllowSingleHosts: false,
 				ExitNodeIP:       netaddr.MustParseIP("100.64.5.6"),
-				CorpDNS:          true,
+				CorpDNS:          false,
 				ShieldsUp:        true,
 				AdvertiseTags:    []string{"tag:foo", "tag:bar"},
 				Hostname:         "myhostname",
@@ -338,21 +287,141 @@ func TestCheckForAccidentalSettingReverts(t *testing.T) {
 				OperatorUser:  "alice",
 			},
 			curUser: "eve",
-			mp: &ipn.MaskedPrefs{
-				Prefs: ipn.Prefs{
-					ControlURL:  ipn.DefaultControlURL,
-					WantRunning: true,
-					Hostname:    "newhostname",
-				},
-				HostnameSet: true,
+			want:    accidentalUpPrefix + " --hostname=newhostname --accept-dns=false --accept-routes --advertise-routes=10.0.0.0/16 --advertise-tags=tag:foo,tag:bar --exit-node=100.64.5.6 --host-routes=false --netfilter-mode=nodivert --operator=alice --shields-up",
+		},
+		{
+			name:  "loggedout_is_implicit",
+			flags: []string{"--hostname=foo"},
+			curPrefs: &ipn.Prefs{
+				ControlURL:       ipn.DefaultControlURL,
+				LoggedOut:        true,
+				AllowSingleHosts: true,
+				CorpDNS:          true,
+				NetfilterMode:    preftype.NetfilterOn,
 			},
-			want: accidentalUpPrefix + " --hostname=newhostname --accept-routes --exit-node=100.64.5.6 --accept-dns --shields-up --advertise-tags=tag:foo,tag:bar --unattended --advertise-routes=10.0.0.0/16 --netfilter-mode=nodivert --operator=alice",
+			want: "", // not an error. LoggedOut is implicit.
+		},
+		{
+			// Test that a pre-1.8 version of Tailscale with bogus NoSNAT pref
+			// values is able to enable exit nodes without warnings.
+			name:  "make_windows_exit_node",
+			flags: []string{"--advertise-exit-node"},
+			curPrefs: &ipn.Prefs{
+				ControlURL:       ipn.DefaultControlURL,
+				AllowSingleHosts: true,
+				CorpDNS:          true,
+
+				// And assume this no-op accidental pre-1.8 value:
+				NoSNAT: true,
+			},
+			goos: "windows",
+			want: "", // not an error
+		},
+		{
+			name:  "ignore_netfilter_change_non_linux",
+			flags: []string{"--accept-dns"},
+			curPrefs: &ipn.Prefs{
+				ControlURL:       ipn.DefaultControlURL,
+				AllowSingleHosts: true,
+
+				NetfilterMode: preftype.NetfilterNoDivert, // we never had this bug, but pretend it got set non-zero on Windows somehow
+			},
+			goos: "windows",
+			want: "", // not an error
+		},
+		{
+			name:  "operator_losing_routes_step1", // https://twitter.com/EXPbits/status/1390418145047887877
+			flags: []string{"--operator=expbits"},
+			curPrefs: &ipn.Prefs{
+				ControlURL:       ipn.DefaultControlURL,
+				AllowSingleHosts: true,
+				CorpDNS:          true,
+				NetfilterMode:    preftype.NetfilterOn,
+				AdvertiseRoutes: []netaddr.IPPrefix{
+					netaddr.MustParseIPPrefix("0.0.0.0/0"),
+					netaddr.MustParseIPPrefix("::/0"),
+					netaddr.MustParseIPPrefix("1.2.0.0/16"),
+				},
+			},
+			want: accidentalUpPrefix + " --operator=expbits --advertise-exit-node --advertise-routes=1.2.0.0/16",
+		},
+		{
+			name:  "operator_losing_routes_step2", // https://twitter.com/EXPbits/status/1390418145047887877
+			flags: []string{"--operator=expbits", "--advertise-routes=1.2.0.0/16"},
+			curPrefs: &ipn.Prefs{
+				ControlURL:       ipn.DefaultControlURL,
+				AllowSingleHosts: true,
+				CorpDNS:          true,
+				NetfilterMode:    preftype.NetfilterOn,
+				AdvertiseRoutes: []netaddr.IPPrefix{
+					netaddr.MustParseIPPrefix("0.0.0.0/0"),
+					netaddr.MustParseIPPrefix("::/0"),
+					netaddr.MustParseIPPrefix("1.2.0.0/16"),
+				},
+			},
+			want: accidentalUpPrefix + " --advertise-routes=1.2.0.0/16 --operator=expbits --advertise-exit-node",
+		},
+		{
+			name:  "errors_preserve_explicit_flags",
+			flags: []string{"--reset", "--force-reauth=false", "--authkey=secretrand"},
+			curPrefs: &ipn.Prefs{
+				ControlURL:       ipn.DefaultControlURL,
+				WantRunning:      false,
+				CorpDNS:          true,
+				NetfilterMode:    preftype.NetfilterOn,
+				AllowSingleHosts: true,
+
+				Hostname: "foo",
+			},
+			want: accidentalUpPrefix + " --authkey=secretrand --force-reauth=false --reset --hostname=foo",
+		},
+		{
+			name:  "error_exit_node_omit_with_ip_pref",
+			flags: []string{"--hostname=foo"},
+			curPrefs: &ipn.Prefs{
+				ControlURL:       ipn.DefaultControlURL,
+				AllowSingleHosts: true,
+				CorpDNS:          true,
+				NetfilterMode:    preftype.NetfilterOn,
+
+				ExitNodeIP: netaddr.MustParseIP("100.64.5.4"),
+			},
+			want: accidentalUpPrefix + " --hostname=foo --exit-node=100.64.5.4",
+		},
+		{
+			name:          "error_exit_node_omit_with_id_pref",
+			flags:         []string{"--hostname=foo"},
+			curExitNodeIP: netaddr.MustParseIP("100.64.5.7"),
+			curPrefs: &ipn.Prefs{
+				ControlURL:       ipn.DefaultControlURL,
+				AllowSingleHosts: true,
+				CorpDNS:          true,
+				NetfilterMode:    preftype.NetfilterOn,
+
+				ExitNodeID: "some_stable_id",
+			},
+			want: accidentalUpPrefix + " --hostname=foo --exit-node=100.64.5.7",
 		},
 	}
 	for _, tt := range tests {
 		t.Run(tt.name, func(t *testing.T) {
+			goos := "linux"
+			if tt.goos != "" {
+				goos = tt.goos
+			}
+			var upArgs upArgsT
+			flagSet := newUpFlagSet(goos, &upArgs)
+			flagSet.Parse(tt.flags)
+			newPrefs, err := prefsFromUpArgs(upArgs, t.Logf, new(ipnstate.Status), goos)
+			if err != nil {
+				t.Fatal(err)
+			}
+			applyImplicitPrefs(newPrefs, tt.curPrefs, tt.curUser)
 			var got string
-			if err := checkForAccidentalSettingReverts(tt.flagSet, tt.curPrefs, tt.mp, tt.curUser); err != nil {
+			if err := checkForAccidentalSettingReverts(flagSet, tt.curPrefs, newPrefs, upCheckEnv{
+				goos:          goos,
+				curExitNodeIP: tt.curExitNodeIP,
+			}); err != nil {
 				got = err.Error()
 			}
 			if strings.TrimSpace(got) != tt.want {
@@ -362,19 +431,10 @@ func TestCheckForAccidentalSettingReverts(t *testing.T) {
 	}
 }
 
-func defaultPrefsFromUpArgs(t testing.TB) *ipn.Prefs {
-	upFlagSet.Parse(nil) // populates upArgs
-	if upFlagSet.Lookup("netfilter-mode") == nil && upArgs.netfilterMode == "" {
-		// This flag is not compiled on on-Linux platforms,
-		// but prefsFromUpArgs requires it be populated.
-		upArgs.netfilterMode = defaultNetfilterMode()
-	}
-	prefs, err := prefsFromUpArgs(upArgs, logger.Discard, new(ipnstate.Status), "linux")
-	if err != nil {
-		t.Fatalf("defaultPrefsFromUpArgs: %v", err)
-	}
-	prefs.WantRunning = true
-	return prefs
+func upArgsFromOSArgs(goos string, flagArgs ...string) (args upArgsT) {
+	fs := newUpFlagSet(goos, &args)
+	fs.Parse(flagArgs) // populates args
+	return
 }
 
 func TestPrefsFromUpArgs(t *testing.T) {
@@ -388,13 +448,43 @@ func TestPrefsFromUpArgs(t *testing.T) {
 		wantWarn string
 	}{
 		{
-			name: "zero",
-			goos: "windows",
-			args: upArgsT{},
+			name: "default_linux",
+			goos: "linux",
+			args: upArgsFromOSArgs("linux"),
 			want: &ipn.Prefs{
-				WantRunning:   true,
-				NoSNAT:        true,
-				NetfilterMode: preftype.NetfilterOn, // silly, but default from ipn.NewPref currently
+				ControlURL:       ipn.DefaultControlURL,
+				WantRunning:      true,
+				NoSNAT:           false,
+				NetfilterMode:    preftype.NetfilterOn,
+				CorpDNS:          true,
+				AllowSingleHosts: true,
+			},
+		},
+		{
+			name: "default_windows",
+			goos: "windows",
+			args: upArgsFromOSArgs("windows"),
+			want: &ipn.Prefs{
+				ControlURL:       ipn.DefaultControlURL,
+				WantRunning:      true,
+				CorpDNS:          true,
+				AllowSingleHosts: true,
+				NetfilterMode:    preftype.NetfilterOn,
+			},
+		},
+		{
+			name: "advertise_default_route",
+			args: upArgsFromOSArgs("linux", "--advertise-exit-node"),
+			want: &ipn.Prefs{
+				ControlURL:       ipn.DefaultControlURL,
+				WantRunning:      true,
+				AllowSingleHosts: true,
+				CorpDNS:          true,
+				AdvertiseRoutes: []netaddr.IPPrefix{
+					netaddr.MustParseIPPrefix("0.0.0.0/0"),
+					netaddr.MustParseIPPrefix("::/0"),
+				},
+				NetfilterMode: preftype.NetfilterOn,
 			},
 		},
 		{
@@ -416,7 +506,7 @@ func TestPrefsFromUpArgs(t *testing.T) {
 			args: upArgsT{
 				exitNodeIP: "foo",
 			},
-			wantErr: `invalid IP address "foo" for --exit-node: unable to parse IP`,
+			wantErr: `invalid IP address "foo" for --exit-node: ParseIP("foo"): unable to parse IP`,
 		},
 		{
 			name: "error_exit_node_allow_lan_without_exit_node",
@@ -534,3 +624,45 @@ func TestPrefsFromUpArgs(t *testing.T) {
 	}
 
 }
+
+func TestPrefFlagMapping(t *testing.T) {
+	prefHasFlag := map[string]bool{}
+	for _, pv := range prefsOfFlag {
+		for _, pref := range pv {
+			prefHasFlag[pref] = true
+		}
+	}
+
+	prefType := reflect.TypeOf(ipn.Prefs{})
+	for i := 0; i < prefType.NumField(); i++ {
+		prefName := prefType.Field(i).Name
+		if prefHasFlag[prefName] {
+			continue
+		}
+		switch prefName {
+		case "WantRunning", "Persist", "LoggedOut":
+			// All explicitly handled (ignored) by checkForAccidentalSettingReverts.
+			continue
+		case "OSVersion", "DeviceModel":
+			// Only used by Android, which doesn't have a CLI mode anyway, so
+			// fine to not map.
+			continue
+		case "NotepadURLs":
+			// TODO(bradfitz): https://github.com/tailscale/tailscale/issues/1830
+			continue
+		}
+		t.Errorf("unexpected new ipn.Pref field %q is not handled by up.go (see addPrefFlagMapping and checkForAccidentalSettingReverts)", prefName)
+	}
+}
+
+func TestFlagAppliesToOS(t *testing.T) {
+	for _, goos := range geese {
+		var upArgs upArgsT
+		fs := newUpFlagSet(goos, &upArgs)
+		fs.VisitAll(func(f *flag.Flag) {
+			if !flagAppliesToOS(f.Name, goos) {
+				t.Errorf("flagAppliesToOS(%q, %q) = false but found in %s set", f.Name, goos, goos)
+			}
+		})
+	}
+}

cmd/tailscale/cli/file.go

@@ -29,6 +29,7 @@ import (
 	"tailscale.com/client/tailscale"
 	"tailscale.com/client/tailscale/apitype"
 	"tailscale.com/ipn"
+	"tailscale.com/net/tsaddr"
 	"tailscale.com/version"
 )
 
@@ -98,7 +99,7 @@ func runCp(ctx context.Context, args []string) error {
 
 	peerAPIBase, lastSeen, isOffline, err := discoverPeerAPIBase(ctx, ip)
 	if err != nil {
-		return err
+		return fmt.Errorf("can't send to %s: %v", target, err)
 	}
 	if isOffline {
 		fmt.Fprintf(os.Stderr, "# warning: %s is offline\n", target)
@@ -203,7 +204,32 @@ func discoverPeerAPIBase(ctx context.Context, ipStr string) (base string, lastSe
 			return ft.PeerAPIURL, lastSeen, isOffline, nil
 		}
 	}
-	return "", time.Time{}, false, errors.New("target seems to be running an old Tailscale version")
+	return "", time.Time{}, false, fileTargetErrorDetail(ctx, ip)
+}
+
+// fileTargetErrorDetail returns a non-nil error saying why ip is an
+// invalid file sharing target.
+func fileTargetErrorDetail(ctx context.Context, ip netaddr.IP) error {
+	found := false
+	if st, err := tailscale.Status(ctx); err == nil && st.Self != nil {
+		for _, peer := range st.Peer {
+			for _, pip := range peer.TailscaleIPs {
+				if pip == ip {
+					found = true
+					if peer.UserID != st.Self.UserID {
+						return errors.New("owned by different user; can only send files to your own devices")
+					}
+				}
+			}
+		}
+	}
+	if found {
+		return errors.New("target seems to be running an old Tailscale version")
+	}
+	if !tsaddr.IsTailscaleIP(ip) {
+		return fmt.Errorf("unknown target; %v is not a Tailscale IP address", ip)
+	}
+	return errors.New("unknown target; not in your Tailnet")
 }
 
 const maxSniff = 4 << 20

cmd/tailscale/cli/ping.go

@@ -80,7 +80,8 @@ func runPing(ctx context.Context, args []string) error {
 			prc <- pr
 		}
 	})
-	go pump(ctx, bc, c)
+	pumpErr := make(chan error, 1)
+	go func() { pumpErr <- pump(ctx, bc, c) }()
 
 	hostOrIP := args[0]
 	ip, err := tailscaleIPFromArg(ctx, hostOrIP)
@@ -101,6 +102,8 @@ func runPing(ctx context.Context, args []string) error {
 		select {
 		case <-timer.C:
 			fmt.Printf("timeout waiting for ping reply\n")
+		case err := <-pumpErr:
+			return err
 		case pr := <-prc:
 			timer.Stop()
 			if pr.Err != "" {
@@ -136,6 +139,9 @@ func runPing(ctx context.Context, args []string) error {
 			if !anyPong {
 				return errors.New("no reply")
 			}
+			if pingArgs.untilDirect {
+				return errors.New("direct connection not established")
+			}
 			return nil
 		}
 	}

cmd/tailscale/cli/up.go

@@ -13,7 +13,6 @@ import (
 	"reflect"
 	"runtime"
 	"sort"
-	"strconv"
 	"strings"
 	"sync"
 
@@ -52,7 +51,9 @@ flag is also used.
 	Exec:    runUp,
 }
 
-var upFlagSet = (func() *flag.FlagSet {
+var upFlagSet = newUpFlagSet(runtime.GOOS, &upArgs)
+
+func newUpFlagSet(goos string, upArgs *upArgsT) *flag.FlagSet {
 	upf := flag.NewFlagSet("up", flag.ExitOnError)
 
 	upf.BoolVar(&upArgs.forceReauth, "force-reauth", false, "force reauthentication")
@@ -70,18 +71,18 @@ var upFlagSet = (func() *flag.FlagSet {
 	upf.StringVar(&upArgs.hostname, "hostname", "", "hostname to use instead of the one provided by the OS")
 	upf.StringVar(&upArgs.advertiseRoutes, "advertise-routes", "", "routes to advertise to other nodes (comma-separated, e.g. \"10.0.0.0/8,192.168.0.0/24\")")
 	upf.BoolVar(&upArgs.advertiseDefaultRoute, "advertise-exit-node", false, "offer to be an exit node for internet traffic for the tailnet")
-	if safesocket.PlatformUsesPeerCreds() {
+	if safesocket.GOOSUsesPeerCreds(goos) {
 		upf.StringVar(&upArgs.opUser, "operator", "", "Unix username to allow to operate on tailscaled without sudo")
 	}
-	if runtime.GOOS == "linux" {
+	switch goos {
+	case "linux":
 		upf.BoolVar(&upArgs.snat, "snat-subnet-routes", true, "source NAT traffic to local routes advertised with --advertise-routes")
 		upf.StringVar(&upArgs.netfilterMode, "netfilter-mode", defaultNetfilterMode(), "netfilter mode (one of on, nodivert, off)")
-	}
-	if runtime.GOOS == "windows" {
+	case "windows":
 		upf.BoolVar(&upArgs.forceDaemon, "unattended", false, "run in \"Unattended Mode\" where Tailscale keeps running even after the current GUI user logs out (Windows-only)")
 	}
 	return upf
-})()
+}
 
 func defaultNetfilterMode() string {
 	if distro.Get() == distro.Synology {
@@ -214,12 +215,13 @@ func prefsFromUpArgs(upArgs upArgsT, warnf logger.Logf, st *ipnstate.Status, goo
 	prefs.ShieldsUp = upArgs.shieldsUp
 	prefs.AdvertiseRoutes = routes
 	prefs.AdvertiseTags = tags
-	prefs.NoSNAT = !upArgs.snat
 	prefs.Hostname = upArgs.hostname
 	prefs.ForceDaemon = upArgs.forceDaemon
 	prefs.OperatorUser = upArgs.opUser
 
 	if goos == "linux" {
+		prefs.NoSNAT = !upArgs.snat
+
 		switch upArgs.netfilterMode {
 		case "on":
 			prefs.NetfilterMode = preftype.NetfilterOn
@@ -292,17 +294,13 @@ func runUp(ctx context.Context, args []string) error {
 		return err
 	}
 
-	flagSet := map[string]bool{}
-	mp := new(ipn.MaskedPrefs)
-	mp.WantRunningSet = true
-	mp.Prefs = *prefs
-	upFlagSet.Visit(func(f *flag.Flag) {
-		updateMaskedPrefsFromUpFlag(mp, f.Name)
-		flagSet[f.Name] = true
-	})
-
 	if !upArgs.reset {
-		if err := checkForAccidentalSettingReverts(flagSet, curPrefs, mp, os.Getenv("USER")); err != nil {
+		applyImplicitPrefs(prefs, curPrefs, os.Getenv("USER"))
+
+		if err := checkForAccidentalSettingReverts(upFlagSet, curPrefs, prefs, upCheckEnv{
+			goos:          runtime.GOOS,
+			curExitNodeIP: exitNodeIP(prefs, st),
+		}); err != nil {
 			fatalf("%s", err)
 		}
 	}
@@ -314,7 +312,7 @@ func runUp(ctx context.Context, args []string) error {
 
 	// If we're already running and none of the flags require a
 	// restart, we can just do an EditPrefs call and change the
-	// prefs at runtime (e.g. changing hostname, changinged
+	// prefs at runtime (e.g. changing hostname, changing
 	// advertised tags, routes, etc)
 	justEdit := st.BackendState == ipn.Running.String() &&
 		!upArgs.forceReauth &&
@@ -322,6 +320,13 @@ func runUp(ctx context.Context, args []string) error {
 		upArgs.authKey == "" &&
 		!controlURLChanged
 	if justEdit {
+		mp := new(ipn.MaskedPrefs)
+		mp.WantRunningSet = true
+		mp.Prefs = *prefs
+		upFlagSet.Visit(func(f *flag.Flag) {
+			updateMaskedPrefsFromUpFlag(mp, f.Name)
+		})
+
 		_, err := tailscale.EditPrefs(ctx, mp)
 		return err
 	}
@@ -329,7 +334,7 @@ func runUp(ctx context.Context, args []string) error {
 	// simpleUp is whether we're running a simple "tailscale up"
 	// to transition to running from a previously-logged-in but
 	// down state, without changing any settings.
-	simpleUp := len(flagSet) == 0 &&
+	simpleUp := upFlagSet.NFlag() == 0 &&
 		curPrefs.Persist != nil &&
 		curPrefs.Persist.LoginName != "" &&
 		st.BackendState != ipn.NeedsLogin.String()
@@ -341,7 +346,8 @@ func runUp(ctx context.Context, args []string) error {
 
 	startingOrRunning := make(chan bool, 1) // gets value once starting or running
 	gotEngineUpdate := make(chan bool, 1)   // gets value upon an engine update
-	go pump(pumpCtx, bc, c)
+	pumpErr := make(chan error, 1)
+	go func() { pumpErr <- pump(pumpCtx, bc, c) }()
 
 	printed := !simpleUp
 	var loginOnce sync.Once
@@ -401,6 +407,8 @@ func runUp(ctx context.Context, args []string) error {
 	case <-gotEngineUpdate:
 	case <-pumpCtx.Done():
 		return pumpCtx.Err()
+	case err := <-pumpErr:
+		return err
 	}
 
 	// Special case: bare "tailscale up" means to just start
@@ -416,11 +424,10 @@ func runUp(ctx context.Context, args []string) error {
 			return err
 		}
 	} else {
-		bc.SetPrefs(prefs)
-
 		opts := ipn.Options{
-			StateKey: ipn.GlobalDaemonStateKey,
-			AuthKey:  upArgs.authKey,
+			StateKey:    ipn.GlobalDaemonStateKey,
+			AuthKey:     upArgs.authKey,
+			UpdatePrefs: prefs,
 		}
 		// On Windows, we still run in mostly the "legacy" way that
 		// predated the server's StateStore. That is, we send an empty
@@ -453,18 +460,26 @@ func runUp(ctx context.Context, args []string) error {
 		default:
 		}
 		return pumpCtx.Err()
+	case err := <-pumpErr:
+		return err
 	}
 }
 
 var (
-	flagForPref = map[string]string{} // "ExitNodeIP" => "exit-node"
-	prefsOfFlag = map[string][]string{}
+	prefsOfFlag = map[string][]string{} // "exit-node" => ExitNodeIP, ExitNodeID
 )
 
 func init() {
+	// Both these have the same ipn.Pref:
+	addPrefFlagMapping("advertise-exit-node", "AdvertiseRoutes")
+	addPrefFlagMapping("advertise-routes", "AdvertiseRoutes")
+
+	// And this flag has two ipn.Prefs:
+	addPrefFlagMapping("exit-node", "ExitNodeIP", "ExitNodeID")
+
+	// The rest are 1:1:
 	addPrefFlagMapping("accept-dns", "CorpDNS")
 	addPrefFlagMapping("accept-routes", "RouteAll")
-	addPrefFlagMapping("advertise-routes", "AdvertiseRoutes")
 	addPrefFlagMapping("advertise-tags", "AdvertiseTags")
 	addPrefFlagMapping("host-routes", "AllowSingleHosts")
 	addPrefFlagMapping("hostname", "Hostname")
@@ -472,7 +487,6 @@ func init() {
 	addPrefFlagMapping("netfilter-mode", "NetfilterMode")
 	addPrefFlagMapping("shields-up", "ShieldsUp")
 	addPrefFlagMapping("snat-subnet-routes", "NoSNAT")
-	addPrefFlagMapping("exit-node", "ExitNodeIP", "ExitNodeID")
 	addPrefFlagMapping("exit-node-allow-lan-access", "ExitNodeAllowLANAccess")
 	addPrefFlagMapping("unattended", "ForceDaemon")
 	addPrefFlagMapping("operator", "OperatorUser")
@@ -482,8 +496,6 @@ func addPrefFlagMapping(flagName string, prefNames ...string) {
 	prefsOfFlag[flagName] = prefNames
 	prefType := reflect.TypeOf(ipn.Prefs{})
 	for _, pref := range prefNames {
-		flagForPref[pref] = flagName
-
 		// Crash at runtime if there's a typo in the prefName.
 		if _, ok := prefType.FieldByName(pref); !ok {
 			panic(fmt.Sprintf("invalid ipn.Prefs field %q", pref))
@@ -491,21 +503,27 @@ func addPrefFlagMapping(flagName string, prefNames ...string) {
 	}
 }
 
+// preflessFlag reports whether flagName is a flag that doesn't
+// correspond to an ipn.Pref.
+func preflessFlag(flagName string) bool {
+	switch flagName {
+	case "authkey", "force-reauth", "reset":
+		return true
+	}
+	return false
+}
+
 func updateMaskedPrefsFromUpFlag(mp *ipn.MaskedPrefs, flagName string) {
+	if preflessFlag(flagName) {
+		return
+	}
 	if prefs, ok := prefsOfFlag[flagName]; ok {
 		for _, pref := range prefs {
 			reflect.ValueOf(mp).Elem().FieldByName(pref + "Set").SetBool(true)
 		}
 		return
 	}
-	switch flagName {
-	case "authkey", "force-reauth", "reset":
-		// Not pref-related flags.
-	case "advertise-exit-node":
-		// This pref is a shorthand for advertise-routes.
-	default:
-		panic(fmt.Sprintf("internal error: unhandled flag %q", flagName))
-	}
+	panic(fmt.Sprintf("internal error: unhandled flag %q", flagName))
 }
 
 const accidentalUpPrefix = "Error: changing settings via 'tailscale up' requires mentioning all\n" +
@@ -514,9 +532,16 @@ const accidentalUpPrefix = "Error: changing settings via 'tailscale up' requires
 	"all non-default settings:\n\n" +
 	"\ttailscale up"
 
-// checkForAccidentalSettingReverts checks for people running
-// "tailscale up" with a subset of the flags they originally ran it
-// with.
+// upCheckEnv are extra parameters describing the environment as
+// needed by checkForAccidentalSettingReverts and friends.
+type upCheckEnv struct {
+	goos          string
+	curExitNodeIP netaddr.IP
+}
+
+// checkForAccidentalSettingReverts (the "up checker") checks for
+// people running "tailscale up" with a subset of the flags they
+// originally ran it with.
 //
 // For example, in Tailscale 1.6 and prior, a user might've advertised
 // a tag, but later tried to change just one other setting and forgot
@@ -528,135 +553,171 @@ const accidentalUpPrefix = "Error: changing settings via 'tailscale up' requires
 //
 // mp is the mask of settings actually set, where mp.Prefs is the new
 // preferences to set, including any values set from implicit flags.
-func checkForAccidentalSettingReverts(flagSet map[string]bool, curPrefs *ipn.Prefs, mp *ipn.MaskedPrefs, curUser string) error {
-	if len(flagSet) == 0 {
-		// A bare "tailscale up" is a special case to just
-		// mean bringing the network up without any changes.
-		return nil
-	}
+func checkForAccidentalSettingReverts(flagSet *flag.FlagSet, curPrefs, newPrefs *ipn.Prefs, env upCheckEnv) error {
 	if curPrefs.ControlURL == "" {
 		// Don't validate things on initial "up" before a control URL has been set.
 		return nil
 	}
-	curWithExplicitEdits := curPrefs.Clone()
-	curWithExplicitEdits.ApplyEdits(mp)
 
-	prefType := reflect.TypeOf(ipn.Prefs{})
+	flagIsSet := map[string]bool{}
+	flagSet.Visit(func(f *flag.Flag) {
+		flagIsSet[f.Name] = true
+	})
 
-	// Explicit values (current + explicit edit):
-	ev := reflect.ValueOf(curWithExplicitEdits).Elem()
-	// Implicit values (what we'd get if we replaced everything with flag defaults):
-	iv := reflect.ValueOf(&mp.Prefs).Elem()
+	if len(flagIsSet) == 0 {
+		// A bare "tailscale up" is a special case to just
+		// mean bringing the network up without any changes.
+		return nil
+	}
+
+	// flagsCur is what flags we'd need to use to keep the exact
+	// settings as-is.
+	flagsCur := prefsToFlags(env, curPrefs)
+	flagsNew := prefsToFlags(env, newPrefs)
 
 	var missing []string
-	flagExplicitValue := map[string]interface{}{} // e.g. "accept-dns" => true (from flagSet)
-	for i := 0; i < prefType.NumField(); i++ {
-		prefName := prefType.Field(i).Name
-		if prefName == "Persist" {
+	for flagName := range flagsCur {
+		valCur, valNew := flagsCur[flagName], flagsNew[flagName]
+		if flagIsSet[flagName] {
 			continue
 		}
-		flagName, hasFlag := flagForPref[prefName]
-
-		// Special case for advertise-exit-node; which is a
-		// flag but doesn't have a corresponding pref.  The
-		// flag augments advertise-routes, so we have to infer
-		// the imaginary pref's current value from the routes.
-		if prefName == "AdvertiseRoutes" &&
-			hasExitNodeRoutes(curPrefs.AdvertiseRoutes) &&
-			!hasExitNodeRoutes(curWithExplicitEdits.AdvertiseRoutes) &&
-			!flagSet["advertise-exit-node"] {
-			missing = append(missing, "--advertise-exit-node")
-		}
-
-		if hasFlag && flagSet[flagName] {
-			flagExplicitValue[flagName] = ev.Field(i).Interface()
+		if reflect.DeepEqual(valCur, valNew) {
 			continue
 		}
-		// Get explicit value and implicit value
-		ex, im := ev.Field(i), iv.Field(i)
-		switch ex.Kind() {
-		case reflect.String, reflect.Slice:
-			if ex.Kind() == reflect.Slice && ex.Len() == 0 && im.Len() == 0 {
-				// Treat nil and non-nil empty slices as equivalent.
-				continue
-			}
-		}
-		exi, imi := ex.Interface(), im.Interface()
-
-		if reflect.DeepEqual(exi, imi) {
-			continue
-		}
-		if flagName == "operator" && imi == "" && exi == curUser {
-			// Don't require setting operator if the current user matches
-			// the configured operator.
-			continue
-		}
-		switch flagName {
-		case "":
-			return fmt.Errorf("'tailscale up' without --reset requires all preferences with changing values to be explicitly mentioned; this command would change the value of flagless pref %q", prefName)
-		case "exit-node":
-			if prefName == "ExitNodeIP" {
-				missing = append(missing, fmtFlagValueArg("exit-node", fmtSettingVal(exi)))
-			}
-		default:
-			missing = append(missing, fmtFlagValueArg(flagName, fmtSettingVal(exi)))
-		}
+		missing = append(missing, fmtFlagValueArg(flagName, valCur))
 	}
 	if len(missing) == 0 {
 		return nil
 	}
+	sort.Strings(missing)
+
+	// Compute the stringification of the explicitly provided args in flagSet
+	// to prepend to the command to run.
+	var explicit []string
+	flagSet.Visit(func(f *flag.Flag) {
+		type isBool interface {
+			IsBoolFlag() bool
+		}
+		if ib, ok := f.Value.(isBool); ok && ib.IsBoolFlag() {
+			if f.Value.String() == "false" {
+				explicit = append(explicit, "--"+f.Name+"=false")
+			} else {
+				explicit = append(explicit, "--"+f.Name)
+			}
+		} else {
+			explicit = append(explicit, fmtFlagValueArg(f.Name, f.Value.String()))
+		}
+	})
+
 	var sb strings.Builder
 	sb.WriteString(accidentalUpPrefix)
 
-	var flagSetSorted []string
-	for f := range flagSet {
-		flagSetSorted = append(flagSetSorted, f)
-	}
-	sort.Strings(flagSetSorted)
-	for _, flagName := range flagSetSorted {
-		if ev, ok := flagExplicitValue[flagName]; ok {
-			fmt.Fprintf(&sb, " %s", fmtFlagValueArg(flagName, fmtSettingVal(ev)))
-		}
-	}
-	for _, a := range missing {
+	for _, a := range append(explicit, missing...) {
 		fmt.Fprintf(&sb, " %s", a)
 	}
 	sb.WriteString("\n\n")
 	return errors.New(sb.String())
 }
 
-func fmtFlagValueArg(flagName, val string) string {
-	if val == "true" {
-		// TODO: check flagName's type to see if its Pref is of type bool
+// applyImplicitPrefs mutates prefs to add implicit preferences. Currently
+// this is just the operator user, which only needs to be set if it doesn't
+// match the current user.
+//
+// curUser is os.Getenv("USER"). It's pulled out for testability.
+func applyImplicitPrefs(prefs, oldPrefs *ipn.Prefs, curUser string) {
+	if prefs.OperatorUser == "" && oldPrefs.OperatorUser == curUser {
+		prefs.OperatorUser = oldPrefs.OperatorUser
+	}
+}
+
+func flagAppliesToOS(flag, goos string) bool {
+	switch flag {
+	case "netfilter-mode", "snat-subnet-routes":
+		return goos == "linux"
+	case "unattended":
+		return goos == "windows"
+	}
+	return true
+}
+
+func prefsToFlags(env upCheckEnv, prefs *ipn.Prefs) (flagVal map[string]interface{}) {
+	ret := make(map[string]interface{})
+
+	exitNodeIPStr := func() string {
+		if !prefs.ExitNodeIP.IsZero() {
+			return prefs.ExitNodeIP.String()
+		}
+		if prefs.ExitNodeID.IsZero() || env.curExitNodeIP.IsZero() {
+			return ""
+		}
+		return env.curExitNodeIP.String()
+	}
+
+	fs := newUpFlagSet(env.goos, new(upArgsT) /* dummy */)
+	fs.VisitAll(func(f *flag.Flag) {
+		if preflessFlag(f.Name) {
+			return
+		}
+		set := func(v interface{}) {
+			if flagAppliesToOS(f.Name, env.goos) {
+				ret[f.Name] = v
+			} else {
+				ret[f.Name] = nil
+			}
+		}
+		switch f.Name {
+		default:
+			panic(fmt.Sprintf("unhandled flag %q", f.Name))
+		case "login-server":
+			set(prefs.ControlURL)
+		case "accept-routes":
+			set(prefs.RouteAll)
+		case "host-routes":
+			set(prefs.AllowSingleHosts)
+		case "accept-dns":
+			set(prefs.CorpDNS)
+		case "shields-up":
+			set(prefs.ShieldsUp)
+		case "exit-node":
+			set(exitNodeIPStr())
+		case "exit-node-allow-lan-access":
+			set(prefs.ExitNodeAllowLANAccess)
+		case "advertise-tags":
+			set(strings.Join(prefs.AdvertiseTags, ","))
+		case "hostname":
+			set(prefs.Hostname)
+		case "operator":
+			set(prefs.OperatorUser)
+		case "advertise-routes":
+			var sb strings.Builder
+			for i, r := range withoutExitNodes(prefs.AdvertiseRoutes) {
+				if i > 0 {
+					sb.WriteByte(',')
+				}
+				sb.WriteString(r.String())
+			}
+			set(sb.String())
+		case "advertise-exit-node":
+			set(hasExitNodeRoutes(prefs.AdvertiseRoutes))
+		case "snat-subnet-routes":
+			set(!prefs.NoSNAT)
+		case "netfilter-mode":
+			set(prefs.NetfilterMode.String())
+		case "unattended":
+			set(prefs.ForceDaemon)
+		}
+	})
+	return ret
+}
+
+func fmtFlagValueArg(flagName string, val interface{}) string {
+	if val == true {
 		return "--" + flagName
 	}
 	if val == "" {
 		return "--" + flagName + "="
 	}
-	return fmt.Sprintf("--%s=%v", flagName, shellquote.Join(val))
-}
-
-func fmtSettingVal(v interface{}) string {
-	switch v := v.(type) {
-	case bool:
-		return strconv.FormatBool(v)
-	case string:
-		return v
-	case preftype.NetfilterMode:
-		return v.String()
-	case []string:
-		return strings.Join(v, ",")
-	case []netaddr.IPPrefix:
-		var sb strings.Builder
-		for i, r := range v {
-			if i > 0 {
-				sb.WriteByte(',')
-			}
-			sb.WriteString(r.String())
-		}
-		return sb.String()
-	}
-	return fmt.Sprint(v)
+	return fmt.Sprintf("--%s=%v", flagName, shellquote.Join(fmt.Sprint(val)))
 }
 
 func hasExitNodeRoutes(rr []netaddr.IPPrefix) bool {
@@ -672,3 +733,43 @@ func hasExitNodeRoutes(rr []netaddr.IPPrefix) bool {
 	}
 	return v4 && v6
 }
+
+// withoutExitNodes returns rr unchanged if it has only 1 or 0 /0
+// routes. If it has both IPv4 and IPv6 /0 routes, then it returns
+// a copy with all /0 routes removed.
+func withoutExitNodes(rr []netaddr.IPPrefix) []netaddr.IPPrefix {
+	if !hasExitNodeRoutes(rr) {
+		return rr
+	}
+	var out []netaddr.IPPrefix
+	for _, r := range rr {
+		if r.Bits > 0 {
+			out = append(out, r)
+		}
+	}
+	return out
+}
+
+// exitNodeIP returns the exit node IP from p, using st to map
+// it from its ID form to an IP address if needed.
+func exitNodeIP(p *ipn.Prefs, st *ipnstate.Status) (ip netaddr.IP) {
+	if p == nil {
+		return
+	}
+	if !p.ExitNodeIP.IsZero() {
+		return p.ExitNodeIP
+	}
+	id := p.ExitNodeID
+	if id.IsZero() {
+		return
+	}
+	for _, p := range st.Peer {
+		if p.ID == id {
+			if len(p.TailscaleIPs) > 0 {
+				return p.TailscaleIPs[0]
+			}
+			break
+		}
+	}
+	return
+}

cmd/tailscale/cli/web.go

@@ -73,7 +73,11 @@ func runWeb(ctx context.Context, args []string) error {
 	}
 
 	if webArgs.cgi {
-		return cgi.Serve(http.HandlerFunc(webHandler))
+		if err := cgi.Serve(http.HandlerFunc(webHandler)); err != nil {
+			log.Printf("tailscale.cgi: %v", err)
+			return err
+		}
+		return nil
 	}
 	return http.ListenAndServe(webArgs.listen, http.HandlerFunc(webHandler))
 }
@@ -208,7 +212,7 @@ func webHandler(w http.ResponseWriter, r *http.Request) {
 	if r.Method == "POST" {
 		type mi map[string]interface{}
 		w.Header().Set("Content-Type", "application/json")
-		url, err := tailscaleUp(r.Context())
+		url, err := tailscaleUpForceReauth(r.Context())
 		if err != nil {
 			json.NewEncoder(w).Encode(mi{"error": err})
 			return
@@ -244,7 +248,7 @@ func webHandler(w http.ResponseWriter, r *http.Request) {
 }
 
 // TODO(crawshaw): some of this is very similar to the code in 'tailscale up', can we share anything?
-func tailscaleUp(ctx context.Context) (authURL string, retErr error) {
+func tailscaleUpForceReauth(ctx context.Context) (authURL string, retErr error) {
 	prefs := ipn.NewPrefs()
 	prefs.ControlURL = ipn.DefaultControlURL
 	prefs.WantRunning = true
@@ -256,10 +260,31 @@ func tailscaleUp(ctx context.Context) (authURL string, retErr error) {
 		prefs.NetfilterMode = preftype.NetfilterOff
 	}
 
-	c, bc, ctx, cancel := connect(ctx)
+	st, err := tailscale.Status(ctx)
+	if err != nil {
+		return "", fmt.Errorf("can't fetch status: %v", err)
+	}
+	origAuthURL := st.AuthURL
+
+	// printAuthURL reports whether we should print out the
+	// provided auth URL from an IPN notify.
+	printAuthURL := func(url string) bool {
+		return url != origAuthURL
+	}
+
+	c, bc, pumpCtx, cancel := connect(ctx)
 	defer cancel()
 
+	gotEngineUpdate := make(chan bool, 1) // gets value upon an engine update
+	go pump(pumpCtx, bc, c)
+
 	bc.SetNotifyCallback(func(n ipn.Notify) {
+		if n.Engine != nil {
+			select {
+			case gotEngineUpdate <- true:
+			default:
+			}
+		}
 		if n.ErrMessage != nil {
 			msg := *n.ErrMessage
 			if msg == ipn.ErrMsgPermissionDenied {
@@ -272,11 +297,21 @@ func tailscaleUp(ctx context.Context) (authURL string, retErr error) {
 			}
 			retErr = fmt.Errorf("backend error: %v", msg)
 			cancel()
-		} else if url := n.BrowseToURL; url != nil {
+		} else if url := n.BrowseToURL; url != nil && printAuthURL(*url) {
 			authURL = *url
 			cancel()
 		}
 	})
+	// Wait for backend client to be connected so we know
+	// we're subscribed to updates. Otherwise we can miss
+	// an update upon its transition to running. Do so by causing some traffic
+	// back to the bus that we then wait on.
+	bc.RequestEngineStatus()
+	select {
+	case <-gotEngineUpdate:
+	case <-pumpCtx.Done():
+		return authURL, pumpCtx.Err()
+	}
 
 	bc.SetPrefs(prefs)
 
@@ -284,7 +319,6 @@ func tailscaleUp(ctx context.Context) (authURL string, retErr error) {
 		StateKey: ipn.GlobalDaemonStateKey,
 	})
 	bc.StartLoginInteractive()
-	pump(ctx, bc, c)
 
 	if authURL == "" && retErr == nil {
 		return "", fmt.Errorf("login failed with no backend error message")

cmd/tailscale/cli/web.html

@@ -11,140 +11,133 @@
 </head>
 
 <body class="py-14">
-	<main class="container max-w-lg mx-auto py-6 px-8 bg-white rounded-md shadow-2xl" style="width: 95%">
-		<header class="flex justify-between items-center min-width-0 py-2 mb-8">
-			<svg width="26" height="26" viewBox="0 0 23 23" title="Tailscale" fill="none" xmlns="http://www.w3.org/2000/svg"
-				class="flex-shrink-0 mr-4">
-				<circle opacity="0.2" cx="3.4" cy="3.25" r="2.7" fill="currentColor"></circle>
-				<circle cx="3.4" cy="11.3" r="2.7" fill="currentColor"></circle>
-				<circle opacity="0.2" cx="3.4" cy="19.5" r="2.7" fill="currentColor"></circle>
-				<circle cx="11.5" cy="11.3" r="2.7" fill="currentColor"></circle>
-				<circle cx="11.5" cy="19.5" r="2.7" fill="currentColor"></circle>
-				<circle opacity="0.2" cx="11.5" cy="3.25" r="2.7" fill="currentColor"></circle>
-				<circle opacity="0.2" cx="19.5" cy="3.25" r="2.7" fill="currentColor"></circle>
-				<circle cx="19.5" cy="11.3" r="2.7" fill="currentColor"></circle>
-				<circle opacity="0.2" cx="19.5" cy="19.5" r="2.7" fill="currentColor"></circle>
-			</svg>
-			<div class="flex items-center justify-end space-x-2 w-2/3">
-				{{ with .Profile.LoginName }}
-				<div class="text-right truncate leading-4">
-					<h4 class="truncate">{{.}}</h4>
-					<a href="#" class="text-xs text-gray-500 hover:text-gray-700 js-loginButton">Switch account</a>
-				</div>
+<main class="container max-w-lg mx-auto py-6 px-8 bg-white rounded-md shadow-2xl" style="width: 95%">
+	<header class="flex justify-between items-center min-width-0 py-2 mb-8">
+		<svg width="26" height="26" viewBox="0 0 23 23" title="Tailscale" fill="none" xmlns="http://www.w3.org/2000/svg"
+			class="flex-shrink-0 mr-4">
+			<circle opacity="0.2" cx="3.4" cy="3.25" r="2.7" fill="currentColor"></circle>
+			<circle cx="3.4" cy="11.3" r="2.7" fill="currentColor"></circle>
+			<circle opacity="0.2" cx="3.4" cy="19.5" r="2.7" fill="currentColor"></circle>
+			<circle cx="11.5" cy="11.3" r="2.7" fill="currentColor"></circle>
+			<circle cx="11.5" cy="19.5" r="2.7" fill="currentColor"></circle>
+			<circle opacity="0.2" cx="11.5" cy="3.25" r="2.7" fill="currentColor"></circle>
+			<circle opacity="0.2" cx="19.5" cy="3.25" r="2.7" fill="currentColor"></circle>
+			<circle cx="19.5" cy="11.3" r="2.7" fill="currentColor"></circle>
+			<circle opacity="0.2" cx="19.5" cy="19.5" r="2.7" fill="currentColor"></circle>
+		</svg>
+		<div class="flex items-center justify-end space-x-2 w-2/3">
+			{{ with .Profile.LoginName }}
+			<div class="text-right truncate leading-4">
+				<h4 class="truncate">{{.}}</h4>
+				<a href="#" class="text-xs text-gray-500 hover:text-gray-700 js-loginButton">Switch account</a>
+			</div>
+			{{ end }}
+			<div class="relative flex-shrink-0 w-8 h-8 rounded-full overflow-hidden">
+				{{ with .Profile.ProfilePicURL }}
+				<div class="w-8 h-8 flex pointer-events-none rounded-full bg-gray-200"
+					style="background-image: url('{{.}}'); background-size: cover;"></div>
+				{{ else }}
+				<div class="w-8 h-8 flex pointer-events-none rounded-full border border-gray-400 border-dashed"></div>
 				{{ end }}
-				<div class="relative flex-shrink-0 w-8 h-8 rounded-full overflow-hidden">
-					{{ with .Profile.ProfilePicURL }}
-					<div class="w-8 h-8 flex pointer-events-none rounded-full bg-gray-200"
-						style="background-image: url('{{.}}'); background-size: cover;"></div>
-					{{ else }}
-					<div class="w-8 h-8 flex pointer-events-none rounded-full border border-gray-400 border-dashed"></div>
-					{{ end }}
-				</div>
 			</div>
-		</header>
-		{{ if .IP }}
-		<div
-			class="border border-gray-200 bg-gray-0 rounded-lg p-2 pl-3 pr-3 mb-8 width-full flex items-center justify-between">
-			<div class="flex items-center min-width-0">
-				<svg class="flex-shrink-0 text-gray-600 mr-3 ml-1" xmlns="http://www.w3.org/2000/svg" width="20" height="20"
-					viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round"
-					stroke-linejoin="round">
-					<rect x="2" y="2" width="20" height="8" rx="2" ry="2"></rect>
-					<rect x="2" y="14" width="20" height="8" rx="2" ry="2"></rect>
-					<line x1="6" y1="6" x2="6.01" y2="6"></line>
-					<line x1="6" y1="18" x2="6.01" y2="18"></line>
-				</svg>
-				<h4 class="font-semibold truncate mr-2">{{.DeviceName}}</h4>
-			</div>
-			<h5>{{.IP}}</h5>
 		</div>
-		{{ end }}
-		{{ if or (eq .Status "NeedsLogin") (eq .Status "NoState") }}
-		{{ if .IP }}
-		<div class="mb-6">
-			<p class="text-gray-700">Your device's key has expired. Reauthenticate this device by logging in again, or <a
-					href="https://tailscale.com/kb/1028/key-expiry" class="link" target="_blank">learn more</a>.</p>
+	</header>
+	{{ if .IP }}
+	<div
+		class="border border-gray-200 bg-gray-0 rounded-lg p-2 pl-3 pr-3 mb-8 width-full flex items-center justify-between">
+		<div class="flex items-center min-width-0">
+			<svg class="flex-shrink-0 text-gray-600 mr-3 ml-1" xmlns="http://www.w3.org/2000/svg" width="20" height="20"
+				viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round"
+				stroke-linejoin="round">
+				<rect x="2" y="2" width="20" height="8" rx="2" ry="2"></rect>
+				<rect x="2" y="14" width="20" height="8" rx="2" ry="2"></rect>
+				<line x1="6" y1="6" x2="6.01" y2="6"></line>
+				<line x1="6" y1="18" x2="6.01" y2="18"></line>
+			</svg>
+			<h4 class="font-semibold truncate mr-2">{{.DeviceName}}</h4>
 		</div>
-		<a href="#" class="mb-4 js-loginButton" target="_blank">
-			<button class="button button-blue w-full">Reauthenticate</button>
-		</a>
-		{{ else }}
-		<div class="mb-6">
-			<h3 class="text-3xl font-semibold mb-3">Log in</h3>
-			<p class="text-gray-700">Get started by logging in to your Tailscale network. Or,&nbsp;learn&nbsp;more at <a
-					href="https://tailscale.com/" class="link" target="_blank">tailscale.com</a>.</p>
-		</div>
-		<a href="#" class="mb-4 js-loginButton" target="_blank">
-			<button class="button button-blue w-full">Log In</button>
-		</a>
-		{{ end }}
-		{{ else if eq .Status "NeedsMachineAuth" }}
-		<div class="mb-4">
-			This device is authorized, but needs approval from a network admin before it can connect to the network.
-		</div>
-		{{ else }}
-		<div class="mb-4">
-			<p>You are connected! Access this device over Tailscale using the device name or IP address above.</p>
-		</div>
-		<a href="#" class="mb-4 link font-medium js-loginButton" target="_blank">Reauthenticate</a>
-		{{ end }}
-	</main>
-	<script>
-		(function () {
-			let loginButtons = document.querySelectorAll(".js-loginButton");
-			let fetchingUrl = false;
+		<h5>{{.IP}}</h5>
+	</div>
+	{{ end }}
+	{{ if or (eq .Status "NeedsLogin") (eq .Status "NoState") }}
+	{{ if .IP }}
+	<div class="mb-6">
+		<p class="text-gray-700">Your device's key has expired. Reauthenticate this device by logging in again, or <a
+				href="https://tailscale.com/kb/1028/key-expiry" class="link" target="_blank">learn more</a>.</p>
+	</div>
+	<a href="#" class="mb-4 js-loginButton" target="_blank">
+		<button class="button button-blue w-full">Reauthenticate</button>
+	</a>
+	{{ else }}
+	<div class="mb-6">
+		<h3 class="text-3xl font-semibold mb-3">Log in</h3>
+		<p class="text-gray-700">Get started by logging in to your Tailscale network. Or,&nbsp;learn&nbsp;more at <a
+				href="https://tailscale.com/" class="link" target="_blank">tailscale.com</a>.</p>
+	</div>
+	<a href="#" class="mb-4 js-loginButton" target="_blank">
+		<button class="button button-blue w-full">Log In</button>
+	</a>
+	{{ end }}
+	{{ else if eq .Status "NeedsMachineAuth" }}
+	<div class="mb-4">
+		This device is authorized, but needs approval from a network admin before it can connect to the network.
+	</div>
+	{{ else }}
+	<div class="mb-4">
+		<p>You are connected! Access this device over Tailscale using the device name or IP address above.</p>
+	</div>
+	<a href="#" class="mb-4 link font-medium js-loginButton" target="_blank">Reauthenticate</a>
+	{{ end }}
+</main>
+<script>(function () {
+let loginButtons = document.querySelectorAll(".js-loginButton");
+let fetchingUrl = false;
 
-			function handleClick(e) {
-				e.preventDefault();
+function handleClick(e) {
+	e.preventDefault();
 
-				if (fetchingUrl) {
-					return;
-				}
+	if (fetchingUrl) {
+		return;
+	}
 
-				fetchingUrl = true;
-				const urlParams = new URLSearchParams(window.location.search);
-				const token = urlParams.get("SynoToken");
-				const nextParams = new URLSearchParams({ up: true });
-				if (token) {
-					nextParams.set("SynoToken", token)
-				}
-				const nextUrl = new URL(window.location);
-				nextUrl.search = nextParams.toString()
-				const url = nextUrl.toString();
+	fetchingUrl = true;
+	const urlParams = new URLSearchParams(window.location.search);
+	const token = urlParams.get("SynoToken");
+	const nextParams = new URLSearchParams({ up: true });
+	if (token) {
+		nextParams.set("SynoToken", token)
+	}
+	const nextUrl = new URL(window.location);
+	nextUrl.search = nextParams.toString()
+	const url = nextUrl.toString();
 
-				const tab = window.open("/redirect", "_blank");
+	fetch(url, {
+		method: "POST",
+		headers: {
+			"Accept": "application/json",
+			"Content-Type": "application/json",
+		}
+	}).then(res => res.json()).then(res => {
+		fetchingUrl = false;
+		const err = res["error"];
+		if (err) {
+			throw new Error(err);
+		}
+		const url = res["url"];
+		if (url) {
+			document.location.href = url;
+		} else {
+			location.reload();
+		}
+	}).catch(err => {
+		alert("Failed to log in: " + err.message);
+	});
+}
 
-				fetch(url, {
-					method: "POST",
-					headers: {
-						"Accept": "application/json",
-						"Content-Type": "application/json",
-					}
-				}).then(res => res.json()).then(res => {
-					fetchingUrl = false;
-					const err = res["error"];
-					if (err) {
-						throw new Error(err);
-					}
-					const url = res["url"];
-					if (url) {
-						authUrl = url;
-						tab.location = url;
-						tab.focus();
-					} else {
-						location.reload();
-					}
-				}).catch(err => {
-					tab.close();
-					alert("Failed to log in: " + err.message);
-				});
-			}
-
-			Array.from(loginButtons).forEach(el => {
-				el.addEventListener("click", handleClick);
-			})
-		})();
-	</script>
+Array.from(loginButtons).forEach(el => {
+	el.addEventListener("click", handleClick);
+})
+})();</script>
 </body>
 
 </html>

cmd/tailscale/depaware.txt

@@ -33,7 +33,7 @@ tailscale.com/cmd/tailscale dependencies: (generated by github.com/tailscale/dep
         tailscale.com/net/portmapper                                 from tailscale.com/net/netcheck+
         tailscale.com/net/stun                                       from tailscale.com/net/netcheck
         tailscale.com/net/tlsdial                                    from tailscale.com/derp/derphttp
-        tailscale.com/net/tsaddr                                     from tailscale.com/net/interfaces
+        tailscale.com/net/tsaddr                                     from tailscale.com/net/interfaces+
      💣 tailscale.com/net/tshttpproxy                                from tailscale.com/derp/derphttp+
         tailscale.com/paths                                          from tailscale.com/cmd/tailscale/cli+
         tailscale.com/safesocket                                     from tailscale.com/cmd/tailscale/cli+

cmd/tailscaled/depaware.txt

@@ -78,7 +78,7 @@ tailscale.com/cmd/tailscaled dependencies: (generated by github.com/tailscale/de
         tailscale.com/derp/derpmap                                   from tailscale.com/cmd/tailscaled+
         tailscale.com/disco                                          from tailscale.com/derp+
         tailscale.com/health                                         from tailscale.com/control/controlclient+
-        tailscale.com/internal/deepprint                             from tailscale.com/ipn/ipnlocal+
+        tailscale.com/internal/deephash                              from tailscale.com/ipn/ipnlocal+
         tailscale.com/ipn                                            from tailscale.com/ipn/ipnserver+
         tailscale.com/ipn/ipnlocal                                   from tailscale.com/ipn/ipnserver+
         tailscale.com/ipn/ipnserver                                  from tailscale.com/cmd/tailscaled

cmd/tailscaled/tailscaled_windows.go

@@ -168,7 +168,7 @@ func startIPNServer(ctx context.Context, logid string) error {
 		r, err := router.New(logf, dev)
 		if err != nil {
 			dev.Close()
-			return nil, fmt.Errorf("Router: %w", err)
+			return nil, fmt.Errorf("router: %w", err)
 		}
 		if wrapNetstack {
 			r = netstack.NewSubnetRouterWrapper(r)
@@ -188,7 +188,7 @@ func startIPNServer(ctx context.Context, logid string) error {
 		if err != nil {
 			r.Close()
 			dev.Close()
-			return nil, fmt.Errorf("Engine: %w", err)
+			return nil, fmt.Errorf("engine: %w", err)
 		}
 		onlySubnets := true
 		if wrapNetstack {

control/controlclient/direct.go

@@ -460,10 +460,10 @@ func (c *Direct) doLogin(ctx context.Context, opt loginOpt) (mustRegen bool, new
 			request.NodeKey.ShortString())
 		return true, "", nil
 	}
-	if persist.Provider == "" {
+	if resp.Login.Provider != "" {
 		persist.Provider = resp.Login.Provider
 	}
-	if persist.LoginName == "" {
+	if resp.Login.LoginName != "" {
 		persist.LoginName = resp.Login.LoginName
 	}
 

go.mod

@@ -7,12 +7,13 @@ require (
 	github.com/anmitsu/go-shlex v0.0.0-20161002113705-648efa622239 // indirect
 	github.com/coreos/go-iptables v0.4.5
 	github.com/flynn/go-shlex v0.0.0-20150515145356-3f9db97f8568 // indirect
+	github.com/frankban/quicktest v1.12.1
 	github.com/github/certstore v0.1.0
 	github.com/gliderlabs/ssh v0.2.2
 	github.com/go-multierror/multierror v1.0.2
 	github.com/go-ole/go-ole v1.2.4
 	github.com/godbus/dbus/v5 v5.0.3
-	github.com/google/go-cmp v0.5.4
+	github.com/google/go-cmp v0.5.5
 	github.com/goreleaser/nfpm v1.1.10
 	github.com/jsimonetti/rtnetlink v0.0.0-20210212075122-66c871082f2b
 	github.com/kballard/go-shellquote v0.0.0-20180428030007-95032a82bc51
@@ -24,25 +25,25 @@ require (
 	github.com/pborman/getopt v0.0.0-20190409184431-ee0cd42419d3
 	github.com/peterbourgon/ff/v2 v2.0.0
 	github.com/pkg/errors v0.9.1 // indirect
-	github.com/stretchr/testify v1.4.0
 	github.com/tailscale/depaware v0.0.0-20201214215404-77d1e9757027
-	github.com/tailscale/wireguard-go v0.0.0-20210429195722-6cd106ab1339
+	github.com/tailscale/wireguard-go v0.0.0-20210510192616-d1aa5623121d
 	github.com/tcnksm/go-httpstat v0.2.0
 	github.com/toqueteos/webbrowser v1.2.0
 	go4.org/mem v0.0.0-20201119185036-c04c5a6ff174
 	golang.org/x/crypto v0.0.0-20210317152858-513c2a44f670
-	golang.org/x/net v0.0.0-20210226172049-e18ecbb05110
+	golang.org/x/net v0.0.0-20210510120150-4163338589ed
 	golang.org/x/sync v0.0.0-20210220032951-036812b2e83c
-	golang.org/x/sys v0.0.0-20210403161142-5e06dd20ab57
+	golang.org/x/sys v0.0.0-20210510120138-977fb7262007
 	golang.org/x/term v0.0.0-20210317153231-de623e64d2a6
 	golang.org/x/time v0.0.0-20210220033141-f8bda1e9f3ba
-	golang.org/x/tools v0.0.0-20201211185031-d93e913c1a58
+	golang.org/x/tools v0.1.0
 	golang.zx2c4.com/wireguard/windows v0.1.2-0.20201113162609-9b85be97fdf8
 	gopkg.in/yaml.v2 v2.2.8 // indirect
 	honnef.co/go/tools v0.1.0
-	inet.af/netaddr v0.0.0-20210222205655-a1ec2b7b8c44
+	inet.af/netaddr v0.0.0-20210511181906-37180328850c
 	inet.af/netstack v0.0.0-20210317161235-a1bf4e56ef22
 	inet.af/peercred v0.0.0-20210302202138-56e694897155
+	inet.af/wf v0.0.0-20210424212123-eaa011a774a4
 	rsc.io/goversion v1.2.0
 )
 

go.sum

@@ -23,8 +23,11 @@ github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSs
 github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
 github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/dvyukov/go-fuzz v0.0.0-20201127111758-49e582c6c23d/go.mod h1:11Gm+ccJnvAhCNLlf5+cS9KjtbaD5I5zaZpFMsTHWTw=
+github.com/dvyukov/go-fuzz v0.0.0-20210103155950-6a8e9d1f2415/go.mod h1:11Gm+ccJnvAhCNLlf5+cS9KjtbaD5I5zaZpFMsTHWTw=
 github.com/flynn/go-shlex v0.0.0-20150515145356-3f9db97f8568 h1:BHsljHzVlRcyQhjrss6TZTdY2VfCqZPbv5k3iBFa2ZQ=
 github.com/flynn/go-shlex v0.0.0-20150515145356-3f9db97f8568/go.mod h1:xEzjJPgXI435gkrCt3MPfRiAkVrwSbHsst4LCFVfpJc=
+github.com/frankban/quicktest v1.12.1 h1:P6vQcHwZYgVGIpUzKB5DXzkEeYJppJOStPLuh9aB89c=
+github.com/frankban/quicktest v1.12.1/go.mod h1:qLE0fzW0VuyUAJgPU19zByoIr0HtCHN/r/VLSOOIySU=
 github.com/github/fakeca v0.1.0 h1:Km/MVOFvclqxPM9dZBC4+QE564nU4gz4iZ0D9pMw28I=
 github.com/github/fakeca v0.1.0/go.mod h1:+bormgoGMMuamOscx7N91aOuUST7wdaJ2rNjeohylyo=
 github.com/gliderlabs/ssh v0.2.2 h1:6zsha5zo/TWhRhwqCD3+EarCAgZ2yN28ipRnGPnwkI0=
@@ -42,8 +45,9 @@ github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMyw
 github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
 github.com/google/go-cmp v0.5.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
 github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
-github.com/google/go-cmp v0.5.4 h1:L8R9j+yAqZuZjsqh/z+F1NCffTKKLShY6zXTItVIZ8M=
 github.com/google/go-cmp v0.5.4/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
+github.com/google/go-cmp v0.5.5 h1:Khx7svrCpmxxtHBq5j2mp/xVjsi8hQMfNLvJFAlrGgU=
+github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
 github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI=
 github.com/google/rpmpack v0.0.0-20191226140753-aa36bfddb3a0 h1:BW6OvS3kpT5UEPbCZ+KyX/OB4Ks9/MNMhWjqPPkZxsE=
 github.com/google/rpmpack v0.0.0-20191226140753-aa36bfddb3a0/go.mod h1:RaTPr0KUf2K7fnZYLNDrr8rxAamWs3iNywJLtQ2AzBg=
@@ -66,8 +70,9 @@ github.com/kballard/go-shellquote v0.0.0-20180428030007-95032a82bc51/go.mod h1:C
 github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck=
 github.com/klauspost/compress v1.10.10 h1:a/y8CglcM7gLGYmlbP/stPE5sR3hbhFRUjCBfd/0B3I=
 github.com/klauspost/compress v1.10.10/go.mod h1:aoV0uJVorq1K+umq18yTdKaF57EivdYsUV+/s2qKfXs=
-github.com/kr/pretty v0.1.0 h1:L/CwN0zerZDmRFUapSPitk6f+Q3+0za1rQkzVuMiMFI=
 github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo=
+github.com/kr/pretty v0.2.1 h1:Fmg33tUaq4/8ym9TJN1x7sLJnHVwhP33CNkpYV/7rwI=
+github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI=
 github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
 github.com/kr/pty v1.1.8 h1:AkaSdXYQOWeaO3neb8EM634ahkXXe3jYbVh/F9lq+GI=
 github.com/kr/pty v1.1.8/go.mod h1:O1sed60cT9XZ5uDucP5qwvh+TE3NnUj51EiZO/lmSfw=
@@ -102,6 +107,7 @@ github.com/pborman/getopt v0.0.0-20190409184431-ee0cd42419d3/go.mod h1:85jBQOZwp
 github.com/pelletier/go-toml v1.6.0/go.mod h1:5N711Q9dKgbdkxHL+MEfF31hpT7l0S0s/t2kKREewys=
 github.com/peterbourgon/ff/v2 v2.0.0 h1:lx0oYI5qr/FU1xnpNhQ+EZM04gKgn46jyYvGEEqBBbY=
 github.com/peterbourgon/ff/v2 v2.0.0/go.mod h1:xjwr+t+SjWm4L46fcj/D+Ap+6ME7+HqFzaP22pP5Ggk=
+github.com/peterbourgon/ff/v3 v3.0.0/go.mod h1:UILIFjRH5a/ar8TjXYLTkIvSvekZqPm5Eb/qbGk6CT0=
 github.com/pkg/diff v0.0.0-20200914180035-5b29258ca4f7 h1:+/+DxvQaYifJ+grD4klzrS5y+KJXldn/2YTl5JG+vZ8=
 github.com/pkg/diff v0.0.0-20200914180035-5b29258ca4f7/go.mod h1:zO8QMzTeZd5cpnIkz/Gn6iK0jDfGicM1nynOkkPIl28=
 github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
@@ -121,6 +127,10 @@ github.com/tailscale/depaware v0.0.0-20201214215404-77d1e9757027 h1:lK99QQdH3yBW
 github.com/tailscale/depaware v0.0.0-20201214215404-77d1e9757027/go.mod h1:p9lPsd+cx33L3H9nNoecRRxPssFKUwwI50I3pZ0yT+8=
 github.com/tailscale/wireguard-go v0.0.0-20210429195722-6cd106ab1339 h1:OjLaZ57xeWJUUBAJN5KmsgjsaUABTZhcvgO/lKtZ8sQ=
 github.com/tailscale/wireguard-go v0.0.0-20210429195722-6cd106ab1339/go.mod h1:ys4yUmhKncXy1jWP34qUHKipRjl322VVhxoh1Rkfo7c=
+github.com/tailscale/wireguard-go v0.0.0-20210510175647-030c638da3df h1:ekBw6cxmDhXf9YxTmMZh7SPwUh9rnRRnaoX7HFiGobc=
+github.com/tailscale/wireguard-go v0.0.0-20210510175647-030c638da3df/go.mod h1:ys4yUmhKncXy1jWP34qUHKipRjl322VVhxoh1Rkfo7c=
+github.com/tailscale/wireguard-go v0.0.0-20210510192616-d1aa5623121d h1:qJSz1zlpuPLmfACtnj+tAH4g3iasJMBW8dpeFm5f4wg=
+github.com/tailscale/wireguard-go v0.0.0-20210510192616-d1aa5623121d/go.mod h1:ys4yUmhKncXy1jWP34qUHKipRjl322VVhxoh1Rkfo7c=
 github.com/tcnksm/go-httpstat v0.2.0 h1:rP7T5e5U2HfmOBmZzGgGZjBQ5/GluWUylujl0tJ04I0=
 github.com/tcnksm/go-httpstat v0.2.0/go.mod h1:s3JVJFtQxtBEBC9dwcdTTXS9xFnM3SXAZwPG41aurT8=
 github.com/toqueteos/webbrowser v1.2.0 h1:tVP/gpK69Fx+qMJKsLE7TD8LuGWPnEV71wBN9rrstGQ=
@@ -167,8 +177,9 @@ golang.org/x/net v0.0.0-20201110031124-69a78807bb2b/go.mod h1:sp8m0HH+o8qH0wwXwY
 golang.org/x/net v0.0.0-20201216054612-986b41b23924/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg=
 golang.org/x/net v0.0.0-20201224014010-6772e930b67b/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg=
 golang.org/x/net v0.0.0-20210119194325-5f4716e94777/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg=
-golang.org/x/net v0.0.0-20210226172049-e18ecbb05110 h1:qWPm9rbaAMKs8Bq/9LRpbMqxWRVUAQwMI9fVrssnTfw=
 golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg=
+golang.org/x/net v0.0.0-20210510120150-4163338589ed h1:p9UgmWI9wKpfYmgaV/IZKGdXc5qEK45tDwwwDyjS26I=
+golang.org/x/net v0.0.0-20210510120150-4163338589ed/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
 golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
@@ -192,29 +203,35 @@ golang.org/x/sys v0.0.0-20201118182958-a01c418693c7/go.mod h1:h1NjWce9XRLGQEsW7w
 golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20201218084310-7d0127a74742/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20210110051926-789bb1bd4061/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20210119212857-b64e53b001e4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20210123111255-9b0068b26619/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20210124154548-22da62e12c0c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20210216163648-f7da38b97c65/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20210301091718-77cc2087c03b/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20210305230114-8fe3ee5dd75b/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20210316164454-77fc1eacc6aa/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20210403161142-5e06dd20ab57 h1:F5Gozwx4I1xtr/sr/8CFbb57iKi3297KFs0QDbGN60A=
 golang.org/x/sys v0.0.0-20210403161142-5e06dd20ab57/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20210510120138-977fb7262007 h1:gG67DSER+11cZvqIMb8S8bt0vZtiN6xWYARwirrOSfE=
+golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw=
 golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
 golang.org/x/term v0.0.0-20210317153231-de623e64d2a6 h1:EC6+IGYTjPpRfv9a2b/6Puw0W+hLtAhkV1tPsXhutqs=
 golang.org/x/term v0.0.0-20210317153231-de623e64d2a6/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
 golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
 golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
-golang.org/x/text v0.3.4 h1:0YWbFKbhXG/wIiuHDSKpS0Iy7FSA+u45VtBMfQcFTTc=
 golang.org/x/text v0.3.4/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
+golang.org/x/text v0.3.6 h1:aRYxNxv6iGQlyVaZmk6ZgYEDa+Jg18DxebPSrd6bg1M=
+golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
 golang.org/x/time v0.0.0-20210220033141-f8bda1e9f3ba h1:O8mE0/t419eoIwhTFpKVkHiTs/Igowgfkj25AcZrtiE=
 golang.org/x/time v0.0.0-20210220033141-f8bda1e9f3ba/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
 golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
 golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
 golang.org/x/tools v0.0.0-20191216052735-49a3e744a425/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=
 golang.org/x/tools v0.0.0-20200609164405-eb789aa7ce50/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE=
-golang.org/x/tools v0.0.0-20201211185031-d93e913c1a58 h1:1Bs6RVeBFtLZ8Yi1Hk07DiOqzvwLD/4hln4iahvFlag=
 golang.org/x/tools v0.0.0-20201211185031-d93e913c1a58/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA=
+golang.org/x/tools v0.1.0 h1:po9/4sTYwZU9lPhi1tOrb4hCv3qrhiQ77LZfGa2OjwY=
+golang.org/x/tools v0.1.0/go.mod h1:xkSsbof2nBLbhDlRMhhhyNLN/zl3eTqcnHD5viDpcZ0=
 golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
 golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
 golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
@@ -236,11 +253,16 @@ gopkg.in/yaml.v2 v2.2.8 h1:obN1ZagJSUGI0Ek/LBmuj4SNLPfIny3KsKFopxRdj10=
 gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
 honnef.co/go/tools v0.1.0 h1:AWNL1W1i7f0wNZ8VwOKNJ0sliKvOF/adn0EHenfUh+c=
 honnef.co/go/tools v0.1.0/go.mod h1:XtegFAyX/PfluP4921rXU5IkjkqBCDnUq4W8VCIoKvM=
-inet.af/netaddr v0.0.0-20210222205655-a1ec2b7b8c44 h1:p7fX77zWzZMuNdJUhniBsmN1OvFOrW9SOtvgnzqUZX4=
 inet.af/netaddr v0.0.0-20210222205655-a1ec2b7b8c44/go.mod h1:I2i9ONCXRZDnG1+7O8fSuYzjcPxHQXrIfzD/IkR87x4=
+inet.af/netaddr v0.0.0-20210508014949-da1c2a70a83d h1:9tuJMxDV7THGfXWirKBD/v9rbsBC21bHd2eEYsYuIek=
+inet.af/netaddr v0.0.0-20210508014949-da1c2a70a83d/go.mod h1:z0nx+Dh+7N7CC8V5ayHtHGpZpxLQZZxkIaaz6HN65Ls=
+inet.af/netaddr v0.0.0-20210511181906-37180328850c h1:rzDy/tC8LjEdN94+i0Bu22tTo/qE9cvhKyfD0HMU0NU=
+inet.af/netaddr v0.0.0-20210511181906-37180328850c/go.mod h1:z0nx+Dh+7N7CC8V5ayHtHGpZpxLQZZxkIaaz6HN65Ls=
 inet.af/netstack v0.0.0-20210317161235-a1bf4e56ef22 h1:DNtszwGa6w76qlIr+PbPEnlBJdiRV8SaxeigOy0q1gg=
 inet.af/netstack v0.0.0-20210317161235-a1bf4e56ef22/go.mod h1:GVx+5OZtbG4TVOW5ilmyRZAZXr1cNwfqUEkTOtWK0PM=
 inet.af/peercred v0.0.0-20210302202138-56e694897155 h1:KojYNEYqDkZ2O3LdyTstR1l13L3ePKTIEM2h7ONkfkE=
 inet.af/peercred v0.0.0-20210302202138-56e694897155/go.mod h1:FjawnflS/udxX+SvpsMgZfdqx2aykOlkISeAsADi5IU=
+inet.af/wf v0.0.0-20210424212123-eaa011a774a4 h1:g1VVXY1xRKoO17aKY3g9KeJxDW0lGx1n2Y+WPSWkOL8=
+inet.af/wf v0.0.0-20210424212123-eaa011a774a4/go.mod h1:56/0QVlZ4NmPRh1QuU2OfrKqjSgt5P39R534gD2JMpQ=
 rsc.io/goversion v1.2.0 h1:SPn+NLTiAG7w30IRK/DKp1BjvpWabYgxlLp/+kx5J8w=
 rsc.io/goversion v1.2.0/go.mod h1:Eih9y/uIBS3ulggl7KNJ09xGSLcuNaLgmvvqa07sgfo=

internal/deephash/deephash.go

@@ -0,0 +1,174 @@
+// Copyright (c) 2020 Tailscale Inc & AUTHORS All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+// Package deephash hashes a Go value recursively, in a predictable
+// order, without looping.
+package deephash
+
+import (
+	"bufio"
+	"crypto/sha256"
+	"fmt"
+	"reflect"
+
+	"inet.af/netaddr"
+	"tailscale.com/tailcfg"
+	"tailscale.com/types/wgkey"
+)
+
+func Hash(v ...interface{}) string {
+	h := sha256.New()
+	// 64 matches the chunk size in crypto/sha256/sha256.go
+	b := bufio.NewWriterSize(h, 64)
+	Print(b, v)
+	b.Flush()
+	return fmt.Sprintf("%x", h.Sum(nil))
+}
+
+// UpdateHash sets last to the hash of v and reports whether its value changed.
+func UpdateHash(last *string, v ...interface{}) (changed bool) {
+	sig := Hash(v)
+	if *last != sig {
+		*last = sig
+		return true
+	}
+	return false
+}
+
+func Print(w *bufio.Writer, v ...interface{}) {
+	print(w, reflect.ValueOf(v), make(map[uintptr]bool))
+}
+
+var (
+	netaddrIPType       = reflect.TypeOf(netaddr.IP{})
+	netaddrIPPrefix     = reflect.TypeOf(netaddr.IPPrefix{})
+	wgkeyKeyType        = reflect.TypeOf(wgkey.Key{})
+	wgkeyPrivateType    = reflect.TypeOf(wgkey.Private{})
+	tailcfgDiscoKeyType = reflect.TypeOf(tailcfg.DiscoKey{})
+)
+
+func print(w *bufio.Writer, v reflect.Value, visited map[uintptr]bool) {
+	if !v.IsValid() {
+		return
+	}
+
+	// Special case some common types.
+	if v.CanInterface() {
+		switch v.Type() {
+		case netaddrIPType:
+			var b []byte
+			var err error
+			if v.CanAddr() {
+				x := v.Addr().Interface().(*netaddr.IP)
+				b, err = x.MarshalText()
+			} else {
+				x := v.Interface().(netaddr.IP)
+				b, err = x.MarshalText()
+			}
+			if err == nil {
+				w.Write(b)
+				return
+			}
+		case netaddrIPPrefix:
+			var b []byte
+			var err error
+			if v.CanAddr() {
+				x := v.Addr().Interface().(*netaddr.IPPrefix)
+				b, err = x.MarshalText()
+			} else {
+				x := v.Interface().(netaddr.IPPrefix)
+				b, err = x.MarshalText()
+			}
+			if err == nil {
+				w.Write(b)
+				return
+			}
+		case wgkeyKeyType:
+			if v.CanAddr() {
+				x := v.Addr().Interface().(*wgkey.Key)
+				w.Write(x[:])
+			} else {
+				x := v.Interface().(wgkey.Key)
+				w.Write(x[:])
+			}
+			return
+		case wgkeyPrivateType:
+			if v.CanAddr() {
+				x := v.Addr().Interface().(*wgkey.Private)
+				w.Write(x[:])
+			} else {
+				x := v.Interface().(wgkey.Private)
+				w.Write(x[:])
+			}
+			return
+		case tailcfgDiscoKeyType:
+			if v.CanAddr() {
+				x := v.Addr().Interface().(*tailcfg.DiscoKey)
+				w.Write(x[:])
+			} else {
+				x := v.Interface().(tailcfg.DiscoKey)
+				w.Write(x[:])
+			}
+			return
+		}
+	}
+
+	// Generic handling.
+	switch v.Kind() {
+	default:
+		panic(fmt.Sprintf("unhandled kind %v for type %v", v.Kind(), v.Type()))
+	case reflect.Ptr:
+		ptr := v.Pointer()
+		if visited[ptr] {
+			return
+		}
+		visited[ptr] = true
+		print(w, v.Elem(), visited)
+		return
+	case reflect.Struct:
+		w.WriteString("struct{\n")
+		for i, n := 0, v.NumField(); i < n; i++ {
+			fmt.Fprintf(w, " [%d]: ", i)
+			print(w, v.Field(i), visited)
+			w.WriteString("\n")
+		}
+		w.WriteString("}\n")
+	case reflect.Slice, reflect.Array:
+		if v.Type().Elem().Kind() == reflect.Uint8 && v.CanInterface() {
+			fmt.Fprintf(w, "%q", v.Interface())
+			return
+		}
+		fmt.Fprintf(w, "[%d]{\n", v.Len())
+		for i, ln := 0, v.Len(); i < ln; i++ {
+			fmt.Fprintf(w, " [%d]: ", i)
+			print(w, v.Index(i), visited)
+			w.WriteString("\n")
+		}
+		w.WriteString("}\n")
+	case reflect.Interface:
+		print(w, v.Elem(), visited)
+	case reflect.Map:
+		sm := newSortedMap(v)
+		fmt.Fprintf(w, "map[%d]{\n", len(sm.Key))
+		for i, k := range sm.Key {
+			print(w, k, visited)
+			w.WriteString(": ")
+			print(w, sm.Value[i], visited)
+			w.WriteString("\n")
+		}
+		w.WriteString("}\n")
+	case reflect.String:
+		w.WriteString(v.String())
+	case reflect.Bool:
+		fmt.Fprintf(w, "%v", v.Bool())
+	case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64:
+		fmt.Fprintf(w, "%v", v.Int())
+	case reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64, reflect.Uintptr:
+		fmt.Fprintf(w, "%v", v.Uint())
+	case reflect.Float32, reflect.Float64:
+		fmt.Fprintf(w, "%v", v.Float())
+	case reflect.Complex64, reflect.Complex128:
+		fmt.Fprintf(w, "%v", v.Complex())
+	}
+}

internal/deephash/deephash_test.go

@@ -2,13 +2,14 @@
 // Use of this source code is governed by a BSD-style
 // license that can be found in the LICENSE file.
 
-package deepprint
+package deephash
 
 import (
-	"bytes"
 	"testing"
 
 	"inet.af/netaddr"
+	"tailscale.com/tailcfg"
+	"tailscale.com/util/dnsname"
 	"tailscale.com/wgengine/router"
 	"tailscale.com/wgengine/wgcfg"
 )
@@ -18,10 +19,6 @@ func TestDeepPrint(t *testing.T) {
 	// Mostly we're just testing that we don't panic on handled types.
 	v := getVal()
 
-	var buf bytes.Buffer
-	Print(&buf, v)
-	t.Logf("Got: %s", buf.Bytes())
-
 	hash1 := Hash(v)
 	t.Logf("hash: %v", hash1)
 	for i := 0; i < 20; i++ {
@@ -39,7 +36,9 @@ func getVal() []interface{} {
 			Addresses: []netaddr.IPPrefix{{Bits: 5, IP: netaddr.IPFrom16([16]byte{3: 3})}},
 			Peers: []wgcfg.Peer{
 				{
-					Endpoints: "foo:5",
+					Endpoints: wgcfg.Endpoints{
+						IPPorts: wgcfg.NewIPPortSet(netaddr.MustParseIPPort("42.42.42.42:5")),
+					},
 				},
 			},
 		},
@@ -49,16 +48,25 @@ func getVal() []interface{} {
 				netaddr.MustParseIPPrefix("1234::/64"),
 			},
 		},
-		map[string]string{
-			"key1": "val1",
-			"key2": "val2",
-			"key3": "val3",
-			"key4": "val4",
-			"key5": "val5",
-			"key6": "val6",
-			"key7": "val7",
-			"key8": "val8",
-			"key9": "val9",
+		map[dnsname.FQDN][]netaddr.IP{
+			dnsname.FQDN("a."): {netaddr.MustParseIP("1.2.3.4"), netaddr.MustParseIP("4.3.2.1")},
+			dnsname.FQDN("b."): {netaddr.MustParseIP("8.8.8.8"), netaddr.MustParseIP("9.9.9.9")},
+		},
+		map[dnsname.FQDN][]netaddr.IPPort{
+			dnsname.FQDN("a."): {netaddr.MustParseIPPort("1.2.3.4:11"), netaddr.MustParseIPPort("4.3.2.1:22")},
+			dnsname.FQDN("b."): {netaddr.MustParseIPPort("8.8.8.8:11"), netaddr.MustParseIPPort("9.9.9.9:22")},
+		},
+		map[tailcfg.DiscoKey]bool{
+			{1: 1}: true,
+			{1: 2}: false,
 		},
 	}
 }
+
+func BenchmarkHash(b *testing.B) {
+	b.ReportAllocs()
+	v := getVal()
+	for i := 0; i < b.N; i++ {
+		Hash(v)
+	}
+}

internal/deephash/fmtsort.go

@@ -10,7 +10,7 @@
 
 // This is a slightly modified fork of Go's src/internal/fmtsort/sort.go
 
-package deepprint
+package deephash
 
 import (
 	"reflect"

internal/deepprint/deepprint.go

@@ -1,103 +0,0 @@
-// Copyright (c) 2020 Tailscale Inc & AUTHORS All rights reserved.
-// Use of this source code is governed by a BSD-style
-// license that can be found in the LICENSE file.
-
-// Package deepprint walks a Go value recursively, in a predictable
-// order, without looping, and prints each value out to a given
-// Writer, which is assumed to be a hash.Hash, as this package doesn't
-// format things nicely.
-//
-// This is intended as a lighter version of go-spew, etc. We don't need its
-// features when our writer is just a hash.
-package deepprint
-
-import (
-	"crypto/sha256"
-	"fmt"
-	"io"
-	"reflect"
-)
-
-func Hash(v ...interface{}) string {
-	h := sha256.New()
-	Print(h, v)
-	return fmt.Sprintf("%x", h.Sum(nil))
-}
-
-// UpdateHash sets last to the hash of v and reports whether its value changed.
-func UpdateHash(last *string, v ...interface{}) (changed bool) {
-	sig := Hash(v)
-	if *last != sig {
-		*last = sig
-		return true
-	}
-	return false
-}
-
-func Print(w io.Writer, v ...interface{}) {
-	print(w, reflect.ValueOf(v), make(map[uintptr]bool))
-}
-
-func print(w io.Writer, v reflect.Value, visited map[uintptr]bool) {
-	if !v.IsValid() {
-		return
-	}
-	switch v.Kind() {
-	default:
-		panic(fmt.Sprintf("unhandled kind %v for type %v", v.Kind(), v.Type()))
-	case reflect.Ptr:
-		ptr := v.Pointer()
-		if visited[ptr] {
-			return
-		}
-		visited[ptr] = true
-		print(w, v.Elem(), visited)
-		return
-	case reflect.Struct:
-		fmt.Fprintf(w, "struct{\n")
-		t := v.Type()
-		for i, n := 0, v.NumField(); i < n; i++ {
-			sf := t.Field(i)
-			fmt.Fprintf(w, "%s: ", sf.Name)
-			print(w, v.Field(i), visited)
-			fmt.Fprintf(w, "\n")
-		}
-	case reflect.Slice, reflect.Array:
-		if v.Type().Elem().Kind() == reflect.Uint8 && v.CanInterface() {
-			fmt.Fprintf(w, "%q", v.Interface())
-			return
-		}
-		fmt.Fprintf(w, "[%d]{\n", v.Len())
-		for i, ln := 0, v.Len(); i < ln; i++ {
-			fmt.Fprintf(w, " [%d]: ", i)
-			print(w, v.Index(i), visited)
-			fmt.Fprintf(w, "\n")
-		}
-		fmt.Fprintf(w, "}\n")
-	case reflect.Interface:
-		print(w, v.Elem(), visited)
-	case reflect.Map:
-		sm := newSortedMap(v)
-		fmt.Fprintf(w, "map[%d]{\n", len(sm.Key))
-		for i, k := range sm.Key {
-			print(w, k, visited)
-			fmt.Fprintf(w, ": ")
-			print(w, sm.Value[i], visited)
-			fmt.Fprintf(w, "\n")
-		}
-		fmt.Fprintf(w, "}\n")
-
-	case reflect.String:
-		fmt.Fprintf(w, "%s", v.String())
-	case reflect.Bool:
-		fmt.Fprintf(w, "%v", v.Bool())
-	case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64:
-		fmt.Fprintf(w, "%v", v.Int())
-	case reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64, reflect.Uintptr:
-		fmt.Fprintf(w, "%v", v.Uint())
-	case reflect.Float32, reflect.Float64:
-		fmt.Fprintf(w, "%v", v.Float())
-	case reflect.Complex64, reflect.Complex128:
-		fmt.Fprintf(w, "%v", v.Complex())
-	}
-}

ipn/backend.go

@@ -185,8 +185,30 @@ type Options struct {
 	//    state and use/update that.
 	//  - StateKey!="" && Prefs!=nil: like the previous case, but do
 	//    an initial overwrite of backend state with Prefs.
+	//
+	// NOTE(apenwarr): The above means that this Prefs field does not do
+	// what you probably think it does. It will overwrite your encryption
+	// keys. Do not use unless you know what you're doing.
 	StateKey StateKey
 	Prefs    *Prefs
+	// UpdatePrefs, if provided, overrides Options.Prefs *and* the Prefs
+	// already stored in the backend state, *except* for the Persist
+	// Persist member. If you just want to provide prefs, this is
+	// probably what you want.
+	//
+	// UpdatePrefs.Persist is always ignored. Prefs.Persist will still
+	// be used even if UpdatePrefs is provided. Other than Persist,
+	// UpdatePrefs takes precedence over Prefs.
+	//
+	// This is intended as a purely temporary workaround for the
+	// currently unexpected behaviour of Options.Prefs.
+	//
+	// TODO(apenwarr): Remove this, or rename Prefs to something else
+	//   and rename this to Prefs. Or, move Prefs.Persist elsewhere
+	//   entirely (as it always should have been), and then we wouldn't
+	//   need two separate fields at all. Or, move the fancy state
+	//   migration stuff out of Start().
+	UpdatePrefs *Prefs
 	// AuthKey is an optional node auth key used to authorize a
 	// new node key without user interaction.
 	AuthKey string

ipn/ipnlocal/local.go

@@ -29,7 +29,7 @@ import (
 	"tailscale.com/client/tailscale/apitype"
 	"tailscale.com/control/controlclient"
 	"tailscale.com/health"
-	"tailscale.com/internal/deepprint"
+	"tailscale.com/internal/deephash"
 	"tailscale.com/ipn"
 	"tailscale.com/ipn/ipnstate"
 	"tailscale.com/ipn/policy"
@@ -83,7 +83,6 @@ type LocalBackend struct {
 	keyLogf               logger.Logf        // for printing list of peers on change
 	statsLogf             logger.Logf        // for printing peers stats on change
 	e                     wgengine.Engine
-	ccGen                 clientGen // function for producing controlclient
 	store                 ipn.StateStore
 	backendLogID          string
 	unregisterLinkMon     func()
@@ -99,6 +98,7 @@ type LocalBackend struct {
 	// The mutex protects the following elements.
 	mu             sync.Mutex
 	httpTestClient *http.Client // for controlclient. nil by default, used by tests.
+	ccGen          clientGen    // function for producing controlclient; lazily populated
 	notify         func(ipn.Notify)
 	cc             controlclient.Client
 	stateKey       ipn.StateKey // computed in part from user-provided value
@@ -141,23 +141,13 @@ type LocalBackend struct {
 	statusChanged *sync.Cond
 }
 
+// clientGen is a func that creates a control plane client.
+// It's the type used by LocalBackend.SetControlClientGetterForTesting.
 type clientGen func(controlclient.Options) (controlclient.Client, error)
 
 // NewLocalBackend returns a new LocalBackend that is ready to run,
 // but is not actually running.
 func NewLocalBackend(logf logger.Logf, logid string, store ipn.StateStore, e wgengine.Engine) (*LocalBackend, error) {
-	// TODO(apenwarr): change controlclient.New to return controlclient.Client?
-	// Then we could avoid this wrapper, at the expense of external tests
-	// having to typecast in the interface.
-	ccWrap := func(opts controlclient.Options) (controlclient.Client, error) {
-		return controlclient.New(opts)
-	}
-	return NewLocalBackendWithClientGen(logf, logid, store, e, ccWrap)
-}
-
-// NewLocalBackend returns a new LocalBackend that is ready to run,
-// but is not actually running.
-func NewLocalBackendWithClientGen(logf logger.Logf, logid string, store ipn.StateStore, e wgengine.Engine, ccGen clientGen) (*LocalBackend, error) {
 	if e == nil {
 		panic("ipn.NewLocalBackend: wgengine must not be nil")
 	}
@@ -180,7 +170,6 @@ func NewLocalBackendWithClientGen(logf logger.Logf, logid string, store ipn.Stat
 		keyLogf:        logger.LogOnChange(logf, 5*time.Minute, time.Now),
 		statsLogf:      logger.LogOnChange(logf, 5*time.Minute, time.Now),
 		e:              e,
-		ccGen:          ccGen,
 		store:          store,
 		backendLogID:   logid,
 		state:          ipn.NoState,
@@ -337,6 +326,9 @@ func (b *LocalBackend) updateStatus(sb *ipnstate.StatusBuilder, extraLocked func
 		}
 	})
 	sb.MutateSelfStatus(func(ss *ipnstate.PeerStatus) {
+		if b.netMap != nil && b.netMap.SelfNode != nil {
+			ss.ID = b.netMap.SelfNode.StableID
+		}
 		for _, pln := range b.peerAPIListeners {
 			ss.PeerAPIURL = append(ss.PeerAPIURL, pln.urlStr)
 		}
@@ -376,6 +368,7 @@ func (b *LocalBackend) populatePeerStatusLocked(sb *ipnstate.StatusBuilder) {
 		}
 		sb.AddPeer(key.Public(p.Key), &ipnstate.PeerStatus{
 			InNetworkMap:       true,
+			ID:                 p.StableID,
 			UserID:             p.User,
 			TailAddrDeprecated: tailAddr4,
 			TailscaleIPs:       tailscaleIPs,
@@ -441,14 +434,15 @@ func (b *LocalBackend) setClientStatus(st controlclient.Status) {
 		}
 		return
 	}
-	if st.LoginFinished != nil {
+
+	b.mu.Lock()
+	wasBlocked := b.blocked
+	b.mu.Unlock()
+
+	if st.LoginFinished != nil && wasBlocked {
 		// Auth completed, unblock the engine
 		b.blockEngineUpdates(false)
 		b.authReconfig()
-		b.EditPrefs(&ipn.MaskedPrefs{
-			LoggedOutSet: true,
-			Prefs:        ipn.Prefs{LoggedOut: false},
-		})
 		b.send(ipn.Notify{LoginFinished: &empty.Message{}})
 	}
 
@@ -487,11 +481,15 @@ func (b *LocalBackend) setClientStatus(st controlclient.Status) {
 		b.authURL = st.URL
 		b.authURLSticky = st.URL
 	}
-	if b.state == ipn.NeedsLogin {
-		if !b.prefs.WantRunning {
+	if wasBlocked && st.LoginFinished != nil {
+		// Interactive login finished successfully (URL visited).
+		// After an interactive login, the user always wants
+		// WantRunning.
+		if !b.prefs.WantRunning || b.prefs.LoggedOut {
 			prefsChanged = true
 		}
 		b.prefs.WantRunning = true
+		b.prefs.LoggedOut = false
 	}
 	// Prefs will be written out; this is not safe unless locked or cloned.
 	if prefsChanged {
@@ -573,10 +571,18 @@ func (b *LocalBackend) findExitNodeIDLocked(nm *netmap.NetworkMap) (prefsChanged
 func (b *LocalBackend) setWgengineStatus(s *wgengine.Status, err error) {
 	if err != nil {
 		b.logf("wgengine status error: %v", err)
+
+		b.statusLock.Lock()
+		b.statusChanged.Broadcast()
+		b.statusLock.Unlock()
 		return
 	}
 	if s == nil {
 		b.logf("[unexpected] non-error wgengine update with status=nil: %v", s)
+
+		b.statusLock.Lock()
+		b.statusChanged.Broadcast()
+		b.statusLock.Unlock()
 		return
 	}
 
@@ -614,21 +620,48 @@ func (b *LocalBackend) SetHTTPTestClient(c *http.Client) {
 	b.httpTestClient = c
 }
 
+// SetControlClientGetterForTesting sets the func that creates a
+// control plane client. It can be called at most once, before Start.
+func (b *LocalBackend) SetControlClientGetterForTesting(newControlClient func(controlclient.Options) (controlclient.Client, error)) {
+	b.mu.Lock()
+	defer b.mu.Unlock()
+	if b.ccGen != nil {
+		panic("invalid use of SetControlClientGetterForTesting after Start")
+	}
+	b.ccGen = newControlClient
+}
+
+func (b *LocalBackend) getNewControlClientFunc() clientGen {
+	b.mu.Lock()
+	defer b.mu.Unlock()
+	if b.ccGen == nil {
+		// Initialize it rather than just returning the
+		// default to make any future call to
+		// SetControlClientGetterForTesting panic.
+		b.ccGen = func(opts controlclient.Options) (controlclient.Client, error) {
+			return controlclient.New(opts)
+		}
+	}
+	return b.ccGen
+}
+
 // startIsNoopLocked reports whether a Start call on this LocalBackend
 // with the provided Start Options would be a useless no-op.
 //
 // b.mu must be held.
 func (b *LocalBackend) startIsNoopLocked(opts ipn.Options) bool {
-	// Options has 4 fields; check all of them:
+	// Options has 5 fields; check all of them:
 	//   * FrontendLogID
 	//   * StateKey
 	//   * Prefs
+	//   * UpdatePrefs
 	//   * AuthKey
 	return b.state == ipn.Running &&
 		b.hostinfo != nil &&
 		b.hostinfo.FrontendLogID == opts.FrontendLogID &&
 		b.stateKey == opts.StateKey &&
 		opts.Prefs == nil &&
+		opts.UpdatePrefs == nil &&
 		opts.AuthKey == ""
 }
 
@@ -703,6 +736,12 @@ func (b *LocalBackend) Start(opts ipn.Options) error {
 		return fmt.Errorf("loading requested state: %v", err)
 	}
 
+	if opts.UpdatePrefs != nil {
+		newPrefs := opts.UpdatePrefs
+		newPrefs.Persist = b.prefs.Persist
+		b.prefs = newPrefs
+	}
+
 	wantRunning := b.prefs.WantRunning
 	if wantRunning {
 		if err := b.initMachineKeyLocked(); err != nil {
@@ -765,7 +804,11 @@ func (b *LocalBackend) Start(opts ipn.Options) error {
 		debugFlags = append([]string{"netstack"}, debugFlags...)
 	}
 
-	cc, err := b.ccGen(controlclient.Options{
+	// TODO(apenwarr): The only way to change the ServerURL is to
+	// re-run b.Start(), because this is the only place we create a
+	// new controlclient. SetPrefs() allows you to overwrite ServerURL,
+	// but it won't take effect until the next Start().
+	cc, err := b.getNewControlClientFunc()(controlclient.Options{
 		GetMachinePrivateKey: b.createGetMachinePrivateKeyFunc(),
 		Logf:                 logger.WithPrefix(b.logf, "control: "),
 		Persist:              *persistv,
@@ -873,7 +916,7 @@ func (b *LocalBackend) updateFilter(netMap *netmap.NetworkMap, prefs *ipn.Prefs)
 	localNets := localNetsB.IPSet()
 	logNets := logNetsB.IPSet()
 
-	changed := deepprint.UpdateHash(&b.filterHash, haveNetmap, addrs, packetFilter, localNets.Ranges(), logNets.Ranges(), shieldsUp)
+	changed := deephash.UpdateHash(&b.filterHash, haveNetmap, addrs, packetFilter, localNets.Ranges(), logNets.Ranges(), shieldsUp)
 	if !changed {
 		return
 	}
@@ -2083,8 +2126,8 @@ func (b *LocalBackend) enterState(newState ipn.State) {
 	if oldState == newState {
 		return
 	}
-	b.logf("Switching ipn state %v -> %v (WantRunning=%v)",
-		oldState, newState, prefs.WantRunning)
+	b.logf("Switching ipn state %v -> %v (WantRunning=%v, nm=%v)",
+		oldState, newState, prefs.WantRunning, netMap != nil)
 	health.SetIPNState(newState.String(), prefs.WantRunning)
 	b.send(ipn.Notify{State: &newState})
 
@@ -2140,13 +2183,14 @@ func (b *LocalBackend) nextState() ipn.State {
 		cc          = b.cc
 		netMap      = b.netMap
 		state       = b.state
+		blocked     = b.blocked
 		wantRunning = b.prefs.WantRunning
 		loggedOut   = b.prefs.LoggedOut
 	)
 	b.mu.Unlock()
 
 	switch {
-	case !wantRunning && !loggedOut && b.hasNodeKey():
+	case !wantRunning && !loggedOut && !blocked && b.hasNodeKey():
 		return ipn.Stopped
 	case netMap == nil:
 		if cc.AuthCantContinue() || loggedOut {

ipn/ipnlocal/loglines_test.go

@@ -15,6 +15,7 @@ import (
 	"tailscale.com/tailcfg"
 	"tailscale.com/tstest"
 	"tailscale.com/types/key"
+	"tailscale.com/types/logger"
 	"tailscale.com/types/persist"
 	"tailscale.com/wgengine"
 )
@@ -30,6 +31,12 @@ func TestLocalLogLines(t *testing.T) {
 	})
 	defer logListen.Close()
 
+	// Put a rate-limiter with a burst of 0 between the components below.
+	// This instructs the rate-limiter to eliminate all logging that
+	// isn't explicitly exempt from rate-limiting.
+	// This lets the logListen tracker verify that the rate-limiter allows these key lines.
+	logf := logger.RateLimitedFnWithClock(logListen.Logf, 5*time.Second, 0, 10, time.Now)
+
 	logid := func(hex byte) logtail.PublicID {
 		var ret logtail.PublicID
 		for i := 0; i < len(ret); i++ {
@@ -41,12 +48,12 @@ func TestLocalLogLines(t *testing.T) {
 
 	// set up a LocalBackend, super bare bones. No functional data.
 	store := &ipn.MemoryStore{}
-	e, err := wgengine.NewFakeUserspaceEngine(logListen.Logf, 0)
+	e, err := wgengine.NewFakeUserspaceEngine(logf, 0)
 	if err != nil {
 		t.Fatal(err)
 	}
 
-	lb, err := NewLocalBackend(logListen.Logf, idA.String(), store, e)
+	lb, err := NewLocalBackend(logf, idA.String(), store, e)
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -61,6 +68,7 @@ func TestLocalLogLines(t *testing.T) {
 	testWantRemain := func(wantRemain ...string) func(t *testing.T) {
 		return func(t *testing.T) {
 			if remain := logListen.Check(); !reflect.DeepEqual(remain, wantRemain) {
+				t.Helper()
 				t.Errorf("remain %q, want %q", remain, wantRemain)
 			}
 		}
@@ -75,17 +83,30 @@ func TestLocalLogLines(t *testing.T) {
 	t.Run("after_prefs", testWantRemain("[v1] peer keys: %s", "[v1] v%v peers: %v"))
 
 	// log peers, peer keys
-	status := &wgengine.Status{
+	lb.mu.Lock()
+	lb.parseWgStatusLocked(&wgengine.Status{
 		Peers: []ipnstate.PeerStatusLite{{
 			TxBytes:       10,
 			RxBytes:       10,
 			LastHandshake: time.Now(),
 			NodeKey:       tailcfg.NodeKey(key.NewPrivate()),
 		}},
-	}
-	lb.mu.Lock()
-	lb.parseWgStatusLocked(status)
+	})
 	lb.mu.Unlock()
 
 	t.Run("after_peers", testWantRemain())
+
+	// Log it again with different stats to ensure it's not dup-suppressed.
+	logListen.Reset()
+	lb.mu.Lock()
+	lb.parseWgStatusLocked(&wgengine.Status{
+		Peers: []ipnstate.PeerStatusLite{{
+			TxBytes:       11,
+			RxBytes:       12,
+			LastHandshake: time.Now(),
+			NodeKey:       tailcfg.NodeKey(key.NewPrivate()),
+		}},
+	})
+	lb.mu.Unlock()
+	t.Run("after_second_peer_status", testWantRemain("SetPrefs: %v"))
 }

ipn/ipnlocal/peerapi.go

@@ -290,7 +290,7 @@ func (s *peerAPIServer) DeleteFile(baseName string) error {
 					bo.BackOff(context.Background(), err)
 					continue
 				}
-				if err := redactErr(touchFile(path + deletedSuffix)); err != nil {
+				if err := touchFile(path + deletedSuffix); err != nil {
 					logf("peerapi: failed to leave deleted marker: %v", err)
 				}
 			}
@@ -301,9 +301,13 @@ func (s *peerAPIServer) DeleteFile(baseName string) error {
 	}
 }
 
+// redacted is a fake path name we use in errors, to avoid
+// accidentally logging actual filenames anywhere.
+const redacted = "redacted"
+
 func redactErr(err error) error {
 	if pe, ok := err.(*os.PathError); ok {
-		pe.Path = "redacted"
+		pe.Path = redacted
 	}
 	return err
 }
@@ -311,7 +315,7 @@ func redactErr(err error) error {
 func touchFile(path string) error {
 	f, err := os.OpenFile(path, os.O_RDWR|os.O_CREATE, 0666)
 	if err != nil {
-		return err
+		return redactErr(err)
 	}
 	return f.Close()
 }
@@ -329,16 +333,16 @@ func (s *peerAPIServer) OpenFile(baseName string) (rc io.ReadCloser, size int64,
 	}
 	if fi, err := os.Stat(path + deletedSuffix); err == nil && fi.Mode().IsRegular() {
 		tryDeleteAgain(path)
-		return nil, 0, &fs.PathError{Op: "open", Path: path, Err: fs.ErrNotExist}
+		return nil, 0, &fs.PathError{Op: "open", Path: redacted, Err: fs.ErrNotExist}
 	}
 	f, err := os.Open(path)
 	if err != nil {
-		return nil, 0, err
+		return nil, 0, redactErr(err)
 	}
 	fi, err := f.Stat()
 	if err != nil {
 		f.Close()
-		return nil, 0, err
+		return nil, 0, redactErr(err)
 	}
 	return f, fi.Size(), nil
 }
@@ -612,6 +616,7 @@ func (h *peerAPIHandler) handlePeerPut(w http.ResponseWriter, r *http.Request) {
 		http.Error(w, "bad filename", 400)
 		return
 	}
+	t0 := time.Now()
 	// TODO(bradfitz): prevent same filename being sent by two peers at once
 	partialFile := dstFile + partialSuffix
 	f, err := os.Create(partialFile)
@@ -643,6 +648,7 @@ func (h *peerAPIHandler) handlePeerPut(w http.ResponseWriter, r *http.Request) {
 		defer h.ps.b.registerIncomingFile(inFile, false)
 		n, err := io.Copy(inFile, r.Body)
 		if err != nil {
+			err = redactErr(err)
 			f.Close()
 			h.logf("put Copy error: %v", err)
 			http.Error(w, err.Error(), http.StatusInternalServerError)
@@ -650,7 +656,7 @@ func (h *peerAPIHandler) handlePeerPut(w http.ResponseWriter, r *http.Request) {
 		}
 		finalSize = n
 	}
-	if err := f.Close(); err != nil {
+	if err := redactErr(f.Close()); err != nil {
 		h.logf("put Close error: %v", err)
 		http.Error(w, err.Error(), http.StatusInternalServerError)
 		return
@@ -668,7 +674,8 @@ func (h *peerAPIHandler) handlePeerPut(w http.ResponseWriter, r *http.Request) {
 		}
 	}
 
-	h.logf("put of %s from %v/%v", approxSize(finalSize), h.remoteAddr.IP, h.peerNode.ComputedName)
+	d := time.Since(t0).Round(time.Second / 10)
+	h.logf("got put of %s in %v from %v/%v", approxSize(finalSize), d, h.remoteAddr.IP, h.peerNode.ComputedName)
 
 	// TODO: set modtime
 	// TODO: some real response

ipn/ipnlocal/state_test.go

@@ -10,7 +10,7 @@ import (
 	"testing"
 	"time"
 
-	"github.com/stretchr/testify/assert"
+	qt "github.com/frankban/quicktest"
 
 	"tailscale.com/control/controlclient"
 	"tailscale.com/ipn"
@@ -268,7 +268,7 @@ func (cc *mockControl) UpdateEndpoints(localPort uint16, endpoints []tailcfg.End
 // predictable, but maybe a bit less thorough. This is more of an overall
 // state machine test than a test of the wgengine+magicsock integration.
 func TestStateMachine(t *testing.T) {
-	assert := assert.New(t)
+	c := qt.New(t)
 
 	logf := t.Logf
 	store := new(ipn.MemoryStore)
@@ -278,7 +278,11 @@ func TestStateMachine(t *testing.T) {
 	}
 
 	cc := newMockControl()
-	ccGen := func(opts controlclient.Options) (controlclient.Client, error) {
+	b, err := NewLocalBackend(logf, "logid", store, e)
+	if err != nil {
+		t.Fatalf("NewLocalBackend: %v", err)
+	}
+	b.SetControlClientGetterForTesting(func(opts controlclient.Options) (controlclient.Client, error) {
 		cc.mu.Lock()
 		cc.opts = opts
 		cc.logf = opts.Logf
@@ -289,11 +293,7 @@ func TestStateMachine(t *testing.T) {
 		cc.logf("ccGen: new mockControl.")
 		cc.called("New")
 		return cc, nil
-	}
-	b, err := NewLocalBackendWithClientGen(logf, "logid", store, e, ccGen)
-	if err != nil {
-		t.Fatalf("NewLocalBackend: %v", err)
-	}
+	})
 
 	notifies := &notifyThrottler{t: t}
 	notifies.expect(0)
@@ -312,32 +312,30 @@ func TestStateMachine(t *testing.T) {
 
 	// Check that it hasn't called us right away.
 	// The state machine should be idle until we call Start().
-	assert.Equal(cc.getCalls(), []string{})
+	c.Assert(cc.getCalls(), qt.HasLen, 0)
 
 	// Start the state machine.
 	// Since !WantRunning by default, it'll create a controlclient,
 	// but not ask it to do anything yet.
 	t.Logf("\n\nStart")
 	notifies.expect(2)
-	assert.Nil(b.Start(ipn.Options{
-		StateKey: ipn.GlobalDaemonStateKey,
-	}))
+	c.Assert(b.Start(ipn.Options{StateKey: ipn.GlobalDaemonStateKey}), qt.IsNil)
 	{
 		// BUG: strictly, it should pause, not unpause, here, since !WantRunning.
-		assert.Equal([]string{"New", "unpause"}, cc.getCalls())
+		c.Assert([]string{"New", "unpause"}, qt.DeepEquals, cc.getCalls())
 
 		nn := notifies.drain(2)
-		assert.Equal([]string{}, cc.getCalls())
-		assert.NotNil(nn[0].Prefs)
-		assert.NotNil(nn[1].State)
+		c.Assert(cc.getCalls(), qt.HasLen, 0)
+		c.Assert(nn[0].Prefs, qt.Not(qt.IsNil))
+		c.Assert(nn[1].State, qt.Not(qt.IsNil))
 		prefs := *nn[0].Prefs
 		// Note: a totally fresh system has Prefs.LoggedOut=false by
 		// default. We are logged out, but not because the user asked
 		// for it, so it doesn't count as Prefs.LoggedOut==true.
-		assert.Equal(false, nn[0].Prefs.LoggedOut)
-		assert.Equal(false, prefs.WantRunning)
-		assert.Equal(ipn.NeedsLogin, *nn[1].State)
-		assert.Equal(ipn.NeedsLogin, b.State())
+		c.Assert(nn[0].Prefs.LoggedOut, qt.IsFalse)
+		c.Assert(prefs.WantRunning, qt.IsFalse)
+		c.Assert(ipn.NeedsLogin, qt.Equals, *nn[1].State)
+		c.Assert(ipn.NeedsLogin, qt.Equals, b.State())
 	}
 
 	// Restart the state machine.
@@ -346,21 +344,19 @@ func TestStateMachine(t *testing.T) {
 	// events as the first time, so UIs always know what to expect.
 	t.Logf("\n\nStart2")
 	notifies.expect(2)
-	assert.Nil(b.Start(ipn.Options{
-		StateKey: ipn.GlobalDaemonStateKey,
-	}))
+	c.Assert(b.Start(ipn.Options{StateKey: ipn.GlobalDaemonStateKey}), qt.IsNil)
 	{
 		// BUG: strictly, it should pause, not unpause, here, since !WantRunning.
-		assert.Equal([]string{"Shutdown", "New", "unpause"}, cc.getCalls())
+		c.Assert([]string{"Shutdown", "New", "unpause"}, qt.DeepEquals, cc.getCalls())
 
 		nn := notifies.drain(2)
-		assert.Equal([]string{}, cc.getCalls())
-		assert.NotNil(nn[0].Prefs)
-		assert.NotNil(nn[1].State)
-		assert.Equal(false, nn[0].Prefs.LoggedOut)
-		assert.Equal(false, nn[0].Prefs.WantRunning)
-		assert.Equal(ipn.NeedsLogin, *nn[1].State)
-		assert.Equal(ipn.NeedsLogin, b.State())
+		c.Assert(cc.getCalls(), qt.HasLen, 0)
+		c.Assert(nn[0].Prefs, qt.Not(qt.IsNil))
+		c.Assert(nn[1].State, qt.Not(qt.IsNil))
+		c.Assert(nn[0].Prefs.LoggedOut, qt.IsFalse)
+		c.Assert(nn[0].Prefs.WantRunning, qt.IsFalse)
+		c.Assert(ipn.NeedsLogin, qt.Equals, *nn[1].State)
+		c.Assert(ipn.NeedsLogin, qt.Equals, b.State())
 	}
 
 	// Start non-interactive login with no token.
@@ -370,15 +366,13 @@ func TestStateMachine(t *testing.T) {
 	notifies.expect(0)
 	b.Login(nil)
 	{
-		assert.Equal(cc.getCalls(), []string{"Login"})
+		c.Assert(cc.getCalls(), qt.DeepEquals, []string{"Login"})
 		notifies.drain(0)
-		// BUG: this should immediately set WantRunning to true.
-		// Users don't log in if they don't want to also connect.
-		// (Generally, we're inconsistent about who is supposed to
-		// update Prefs at what time. But the overall philosophy is:
-		// update it when the user's intent changes. This is clearly
-		// at the time the user *requests* Login, not at the time
-		// the login finishes.)
+		// Note: WantRunning isn't true yet. It'll switch to true
+		// after a successful login finishes.
+		// (This behaviour is needed so that b.Login() won't
+		// start connecting to an old account right away, if one
+		// exists when you launch another login.)
 	}
 
 	// Attempted non-interactive login with no key; indicate that
@@ -388,18 +382,16 @@ func TestStateMachine(t *testing.T) {
 	url1 := "http://localhost:1/1"
 	cc.send(nil, url1, false, nil)
 	{
-		assert.Equal([]string{}, cc.getCalls())
+		c.Assert(cc.getCalls(), qt.DeepEquals, []string{})
 
 		// ...but backend eats that notification, because the user
 		// didn't explicitly request interactive login yet, and
 		// we're already in NeedsLogin state.
 		nn := notifies.drain(1)
 
-		// Trying to log in automatically sets WantRunning.
-		// BUG: that should have happened right after Login().
-		assert.NotNil(nn[0].Prefs)
-		assert.Equal(false, nn[0].Prefs.LoggedOut)
-		assert.Equal(true, nn[0].Prefs.WantRunning)
+		c.Assert(nn[0].Prefs, qt.Not(qt.IsNil))
+		c.Assert(nn[0].Prefs.LoggedOut, qt.IsFalse)
+		c.Assert(nn[0].Prefs.WantRunning, qt.IsFalse)
 	}
 
 	// Now we'll try an interactive login.
@@ -415,9 +407,9 @@ func TestStateMachine(t *testing.T) {
 		// We're still not logged in so there's nothing we can do
 		// with it. (And empirically, it's providing an empty list
 		// of endpoints.)
-		assert.Equal([]string{"UpdateEndpoints"}, cc.getCalls())
-		assert.NotNil(nn[0].BrowseToURL)
-		assert.Equal(url1, *nn[0].BrowseToURL)
+		c.Assert([]string{"UpdateEndpoints"}, qt.DeepEquals, cc.getCalls())
+		c.Assert(nn[0].BrowseToURL, qt.Not(qt.IsNil))
+		c.Assert(url1, qt.Equals, *nn[0].BrowseToURL)
 	}
 
 	// Sometimes users press the Login button again, in the middle of
@@ -431,7 +423,7 @@ func TestStateMachine(t *testing.T) {
 	{
 		notifies.drain(0)
 		// backend asks control for another login sequence
-		assert.Equal([]string{"Login"}, cc.getCalls())
+		c.Assert([]string{"Login"}, qt.DeepEquals, cc.getCalls())
 	}
 
 	// Provide a new interactive login URL.
@@ -441,13 +433,13 @@ func TestStateMachine(t *testing.T) {
 	cc.send(nil, url2, false, nil)
 	{
 		// BUG: UpdateEndpoints again, this is getting silly.
-		assert.Equal([]string{"UpdateEndpoints"}, cc.getCalls())
+		c.Assert([]string{"UpdateEndpoints"}, qt.DeepEquals, cc.getCalls())
 
 		// This time, backend should emit it to the UI right away,
 		// because the UI is anxiously awaiting a new URL to visit.
 		nn := notifies.drain(1)
-		assert.NotNil(nn[0].BrowseToURL)
-		assert.Equal(url2, *nn[0].BrowseToURL)
+		c.Assert(nn[0].BrowseToURL, qt.Not(qt.IsNil))
+		c.Assert(url2, qt.Equals, *nn[0].BrowseToURL)
 	}
 
 	// Pretend that the interactive login actually happened.
@@ -455,11 +447,12 @@ func TestStateMachine(t *testing.T) {
 	// same time.
 	// The backend should propagate this upward for the UI.
 	t.Logf("\n\nLoginFinished")
-	notifies.expect(2)
+	notifies.expect(3)
 	cc.setAuthBlocked(false)
+	cc.persist.LoginName = "user1"
 	cc.send(nil, "", true, &netmap.NetworkMap{})
 	{
-		nn := notifies.drain(2)
+		nn := notifies.drain(3)
 		// BUG: still too soon for UpdateEndpoints.
 		//
 		// Arguably it makes sense to unpause now, since the machine
@@ -470,17 +463,14 @@ func TestStateMachine(t *testing.T) {
 		// wait until it gets into Starting.
 		// TODO: (Currently this test doesn't detect that bug, but
 		// it's visible in the logs)
-		assert.Equal([]string{"unpause", "UpdateEndpoints"}, cc.getCalls())
-		assert.NotNil(nn[0].LoginFinished)
-		assert.NotNil(nn[1].State)
-		assert.Equal(ipn.NeedsMachineAuth, *nn[1].State)
+		c.Assert([]string{"unpause", "UpdateEndpoints"}, qt.DeepEquals, cc.getCalls())
+		c.Assert(nn[0].LoginFinished, qt.Not(qt.IsNil))
+		c.Assert(nn[1].Prefs, qt.Not(qt.IsNil))
+		c.Assert(nn[2].State, qt.Not(qt.IsNil))
+		c.Assert(nn[1].Prefs.Persist.LoginName, qt.Equals, "user1")
+		c.Assert(ipn.NeedsMachineAuth, qt.Equals, *nn[2].State)
 	}
 
-	// TODO: check that the logged-in username propagates from control
-	// through to the UI notifications. I think it's used as a hint
-	// for future logins, to pre-fill the username box? Not really sure
-	// how it works.
-
 	// Pretend that the administrator has authorized our machine.
 	t.Logf("\n\nMachineAuthorized")
 	notifies.expect(1)
@@ -495,9 +485,9 @@ func TestStateMachine(t *testing.T) {
 	})
 	{
 		nn := notifies.drain(1)
-		assert.Equal([]string{"unpause", "UpdateEndpoints"}, cc.getCalls())
-		assert.NotNil(nn[0].State)
-		assert.Equal(ipn.Starting, *nn[0].State)
+		c.Assert([]string{"unpause", "UpdateEndpoints"}, qt.DeepEquals, cc.getCalls())
+		c.Assert(nn[0].State, qt.Not(qt.IsNil))
+		c.Assert(ipn.Starting, qt.Equals, *nn[0].State)
 	}
 
 	// TODO: add a fake DERP server to our fake netmap, so we can
@@ -518,11 +508,11 @@ func TestStateMachine(t *testing.T) {
 	})
 	{
 		nn := notifies.drain(2)
-		assert.Equal([]string{"pause"}, cc.getCalls())
+		c.Assert([]string{"pause"}, qt.DeepEquals, cc.getCalls())
 		// BUG: I would expect Prefs to change first, and state after.
-		assert.NotNil(nn[0].State)
-		assert.NotNil(nn[1].Prefs)
-		assert.Equal(ipn.Stopped, *nn[0].State)
+		c.Assert(nn[0].State, qt.Not(qt.IsNil))
+		c.Assert(nn[1].Prefs, qt.Not(qt.IsNil))
+		c.Assert(ipn.Stopped, qt.Equals, *nn[0].State)
 	}
 
 	// The user changes their preference to WantRunning after all.
@@ -536,11 +526,11 @@ func TestStateMachine(t *testing.T) {
 		nn := notifies.drain(2)
 		// BUG: UpdateEndpoints isn't needed here.
 		// BUG: Login isn't needed here. We never logged out.
-		assert.Equal([]string{"Login", "unpause", "UpdateEndpoints"}, cc.getCalls())
+		c.Assert([]string{"Login", "unpause", "UpdateEndpoints"}, qt.DeepEquals, cc.getCalls())
 		// BUG: I would expect Prefs to change first, and state after.
-		assert.NotNil(nn[0].State)
-		assert.NotNil(nn[1].Prefs)
-		assert.Equal(ipn.Starting, *nn[0].State)
+		c.Assert(nn[0].State, qt.Not(qt.IsNil))
+		c.Assert(nn[1].Prefs, qt.Not(qt.IsNil))
+		c.Assert(ipn.Starting, qt.Equals, *nn[0].State)
 	}
 
 	// Test the fast-path frontend reconnection.
@@ -551,15 +541,13 @@ func TestStateMachine(t *testing.T) {
 	t.Logf("\n\nFastpath Start()")
 	notifies.expect(1)
 	b.state = ipn.Running
-	assert.Nil(b.Start(ipn.Options{
-		StateKey: ipn.GlobalDaemonStateKey,
-	}))
+	c.Assert(b.Start(ipn.Options{StateKey: ipn.GlobalDaemonStateKey}), qt.IsNil)
 	{
 		nn := notifies.drain(1)
-		assert.Equal([]string{}, cc.getCalls())
-		assert.NotNil(nn[0].State)
-		assert.NotNil(nn[0].LoginFinished)
-		assert.NotNil(nn[0].NetMap)
+		c.Assert(cc.getCalls(), qt.HasLen, 0)
+		c.Assert(nn[0].State, qt.Not(qt.IsNil))
+		c.Assert(nn[0].LoginFinished, qt.Not(qt.IsNil))
+		c.Assert(nn[0].NetMap, qt.Not(qt.IsNil))
 		// BUG: Prefs should be sent too, or the UI could end up in
 		// a bad state. (iOS, the only current user of this feature,
 		// probably wouldn't notice because it happens to not display
@@ -576,89 +564,84 @@ func TestStateMachine(t *testing.T) {
 	{
 		nn := notifies.drain(2)
 		// BUG: now is not the time to unpause.
-		assert.Equal([]string{"unpause", "StartLogout"}, cc.getCalls())
-		assert.NotNil(nn[0].State)
-		assert.NotNil(nn[1].Prefs)
-		assert.Equal(ipn.NeedsLogin, *nn[0].State)
-		assert.Equal(true, nn[1].Prefs.LoggedOut)
-		assert.Equal(false, nn[1].Prefs.WantRunning)
-		assert.Equal(ipn.NeedsLogin, b.State())
+		c.Assert([]string{"unpause", "StartLogout"}, qt.DeepEquals, cc.getCalls())
+		c.Assert(nn[0].State, qt.Not(qt.IsNil))
+		c.Assert(nn[1].Prefs, qt.Not(qt.IsNil))
+		c.Assert(ipn.NeedsLogin, qt.Equals, *nn[0].State)
+		c.Assert(nn[1].Prefs.LoggedOut, qt.IsTrue)
+		c.Assert(nn[1].Prefs.WantRunning, qt.IsFalse)
+		c.Assert(ipn.NeedsLogin, qt.Equals, b.State())
 	}
 
 	// Let's make the logout succeed.
 	t.Logf("\n\nLogout (async) - succeed")
-	notifies.expect(1)
+	notifies.expect(0)
 	cc.setAuthBlocked(true)
 	cc.send(nil, "", false, nil)
 	{
-		nn := notifies.drain(1)
-		assert.Equal([]string{}, cc.getCalls())
-		assert.NotNil(nn[0].Prefs)
-		assert.Equal(true, nn[0].Prefs.LoggedOut)
-		// BUG: WantRunning should be false after manual logout.
-		assert.Equal(true, nn[0].Prefs.WantRunning)
-		assert.Equal(ipn.NeedsLogin, b.State())
+		notifies.drain(0)
+		c.Assert(cc.getCalls(), qt.HasLen, 0)
+		c.Assert(b.Prefs().LoggedOut, qt.IsTrue)
+		c.Assert(b.Prefs().WantRunning, qt.IsFalse)
+		c.Assert(ipn.NeedsLogin, qt.Equals, b.State())
 	}
 
 	// A second logout should do nothing, since the prefs haven't changed.
 	t.Logf("\n\nLogout2 (async)")
-	notifies.expect(1)
+	notifies.expect(0)
 	b.Logout()
 	{
-		nn := notifies.drain(1)
+		notifies.drain(0)
 		// BUG: the backend has already called StartLogout, and we're
 		// still logged out. So it shouldn't call it again.
-		assert.Equal([]string{"StartLogout"}, cc.getCalls())
-		// BUG: Prefs should not change here. Already logged out.
-		assert.NotNil(nn[0].Prefs)
-		assert.Equal(true, nn[0].Prefs.LoggedOut)
-		assert.Equal(false, nn[0].Prefs.WantRunning)
-		assert.Equal(ipn.NeedsLogin, b.State())
+		c.Assert([]string{"StartLogout"}, qt.DeepEquals, cc.getCalls())
+		c.Assert(cc.getCalls(), qt.HasLen, 0)
+		c.Assert(b.Prefs().LoggedOut, qt.IsTrue)
+		c.Assert(b.Prefs().WantRunning, qt.IsFalse)
+		c.Assert(ipn.NeedsLogin, qt.Equals, b.State())
 	}
 
 	// Let's acknowledge the second logout too.
 	t.Logf("\n\nLogout2 (async) - succeed")
-	notifies.expect(1)
+	notifies.expect(0)
 	cc.setAuthBlocked(true)
 	cc.send(nil, "", false, nil)
 	{
-		nn := notifies.drain(1)
-		assert.Equal([]string{}, cc.getCalls())
-		assert.NotNil(nn[0].Prefs)
-		assert.Equal(true, nn[0].Prefs.LoggedOut)
-		// BUG: second logout shouldn't cause WantRunning->true !!
-		assert.Equal(true, nn[0].Prefs.WantRunning)
-		assert.Equal(ipn.NeedsLogin, b.State())
+		notifies.drain(0)
+		c.Assert(cc.getCalls(), qt.HasLen, 0)
+		c.Assert(cc.getCalls(), qt.HasLen, 0)
+		c.Assert(b.Prefs().LoggedOut, qt.IsTrue)
+		c.Assert(b.Prefs().WantRunning, qt.IsFalse)
+		c.Assert(ipn.NeedsLogin, qt.Equals, b.State())
 	}
 
 	// Try the synchronous logout feature.
 	t.Logf("\n\nLogout3 (sync)")
-	notifies.expect(1)
+	notifies.expect(0)
 	b.LogoutSync(context.Background())
 	// NOTE: This returns as soon as cc.Logout() returns, which is okay
 	// I guess, since that's supposed to be synchronous.
 	{
-		nn := notifies.drain(1)
-		assert.Equal([]string{"Logout"}, cc.getCalls())
-		assert.NotNil(nn[0].Prefs)
-		assert.Equal(true, nn[0].Prefs.LoggedOut)
-		assert.Equal(false, nn[0].Prefs.WantRunning)
-		assert.Equal(ipn.NeedsLogin, b.State())
+		notifies.drain(0)
+		c.Assert([]string{"Logout"}, qt.DeepEquals, cc.getCalls())
+		c.Assert(cc.getCalls(), qt.HasLen, 0)
+		c.Assert(b.Prefs().LoggedOut, qt.IsTrue)
+		c.Assert(b.Prefs().WantRunning, qt.IsFalse)
+		c.Assert(ipn.NeedsLogin, qt.Equals, b.State())
 	}
 
 	// Generate the third logout event.
 	t.Logf("\n\nLogout3 (sync) - succeed")
-	notifies.expect(1)
+	notifies.expect(0)
 	cc.setAuthBlocked(true)
 	cc.send(nil, "", false, nil)
 	{
-		nn := notifies.drain(1)
-		assert.Equal([]string{}, cc.getCalls())
-		assert.NotNil(nn[0].Prefs)
-		assert.Equal(true, nn[0].Prefs.LoggedOut)
-		// BUG: third logout shouldn't cause WantRunning->true !!
-		assert.Equal(true, nn[0].Prefs.WantRunning)
-		assert.Equal(ipn.NeedsLogin, b.State())
+		notifies.drain(0)
+		c.Assert(cc.getCalls(), qt.HasLen, 0)
+		c.Assert(cc.getCalls(), qt.HasLen, 0)
+		c.Assert(b.Prefs().LoggedOut, qt.IsTrue)
+		c.Assert(b.Prefs().WantRunning, qt.IsFalse)
+		c.Assert(ipn.NeedsLogin, qt.Equals, b.State())
 	}
 
 	// Shut down the backend.
@@ -668,40 +651,34 @@ func TestStateMachine(t *testing.T) {
 	{
 		notifies.drain(0)
 		// BUG: I expect a transition to ipn.NoState here.
-		assert.Equal(cc.getCalls(), []string{"Shutdown"})
+		c.Assert(cc.getCalls(), qt.DeepEquals, []string{"Shutdown"})
 	}
 
 	// Oh, you thought we were done? Ha! Now we have to test what
 	// happens if the user exits and restarts while logged out.
 	// Note that it's explicitly okay to call b.Start() over and over
 	// again, every time the frontend reconnects.
-	//
-	// BUG: WantRunning is true here (because of the bug above).
-	//  We'll have to adjust the following test's expectations if we
-	//  fix that.
 
 	// TODO: test user switching between statekeys.
 
 	// The frontend restarts!
 	t.Logf("\n\nStart3")
 	notifies.expect(2)
-	assert.Nil(b.Start(ipn.Options{
-		StateKey: ipn.GlobalDaemonStateKey,
-	}))
+	c.Assert(b.Start(ipn.Options{StateKey: ipn.GlobalDaemonStateKey}), qt.IsNil)
 	{
 		// BUG: We already called Shutdown(), no need to do it again.
 		// BUG: Way too soon for UpdateEndpoints.
 		// BUG: don't unpause because we're not logged in.
-		assert.Equal([]string{"Shutdown", "New", "UpdateEndpoints", "unpause"}, cc.getCalls())
+		c.Assert([]string{"Shutdown", "New", "UpdateEndpoints", "unpause"}, qt.DeepEquals, cc.getCalls())
 
 		nn := notifies.drain(2)
-		assert.Equal([]string{}, cc.getCalls())
-		assert.NotNil(nn[0].Prefs)
-		assert.NotNil(nn[1].State)
-		assert.Equal(true, nn[0].Prefs.LoggedOut)
-		assert.Equal(true, nn[0].Prefs.WantRunning)
-		assert.Equal(ipn.NeedsLogin, *nn[1].State)
-		assert.Equal(ipn.NeedsLogin, b.State())
+		c.Assert(cc.getCalls(), qt.HasLen, 0)
+		c.Assert(nn[0].Prefs, qt.Not(qt.IsNil))
+		c.Assert(nn[1].State, qt.Not(qt.IsNil))
+		c.Assert(nn[0].Prefs.LoggedOut, qt.IsTrue)
+		c.Assert(nn[0].Prefs.WantRunning, qt.IsFalse)
+		c.Assert(ipn.NeedsLogin, qt.Equals, *nn[1].State)
+		c.Assert(ipn.NeedsLogin, qt.Equals, b.State())
 	}
 
 	// Let's break the rules a little. Our control server accepts
@@ -711,17 +688,21 @@ func TestStateMachine(t *testing.T) {
 	t.Logf("\n\nLoginFinished3")
 	notifies.expect(3)
 	cc.setAuthBlocked(false)
+	cc.persist.LoginName = "user2"
 	cc.send(nil, "", true, &netmap.NetworkMap{
 		MachineStatus: tailcfg.MachineAuthorized,
 	})
 	{
 		nn := notifies.drain(3)
-		assert.Equal([]string{"unpause"}, cc.getCalls())
-		assert.NotNil(nn[0].Prefs)
-		assert.NotNil(nn[1].LoginFinished)
-		assert.NotNil(nn[2].State)
-		assert.Equal(false, nn[0].Prefs.LoggedOut)
-		assert.Equal(ipn.Starting, *nn[2].State)
+		c.Assert([]string{"unpause"}, qt.DeepEquals, cc.getCalls())
+		c.Assert(nn[0].LoginFinished, qt.Not(qt.IsNil))
+		c.Assert(nn[1].Prefs, qt.Not(qt.IsNil))
+		c.Assert(nn[2].State, qt.Not(qt.IsNil))
+		// Prefs after finishing the login, so LoginName updated.
+		c.Assert(nn[1].Prefs.Persist.LoginName, qt.Equals, "user2")
+		c.Assert(nn[1].Prefs.LoggedOut, qt.IsFalse)
+		c.Assert(nn[1].Prefs.WantRunning, qt.IsTrue)
+		c.Assert(ipn.Starting, qt.Equals, *nn[2].State)
 	}
 
 	// Now we've logged in successfully. Let's disconnect.
@@ -733,20 +714,18 @@ func TestStateMachine(t *testing.T) {
 	})
 	{
 		nn := notifies.drain(2)
-		assert.Equal([]string{"pause"}, cc.getCalls())
+		c.Assert([]string{"pause"}, qt.DeepEquals, cc.getCalls())
 		// BUG: I would expect Prefs to change first, and state after.
-		assert.NotNil(nn[0].State)
-		assert.NotNil(nn[1].Prefs)
-		assert.Equal(ipn.Stopped, *nn[0].State)
-		assert.Equal(false, nn[1].Prefs.LoggedOut)
+		c.Assert(nn[0].State, qt.Not(qt.IsNil))
+		c.Assert(nn[1].Prefs, qt.Not(qt.IsNil))
+		c.Assert(ipn.Stopped, qt.Equals, *nn[0].State)
+		c.Assert(nn[1].Prefs.LoggedOut, qt.IsFalse)
 	}
 
 	// One more restart, this time with a valid key, but WantRunning=false.
 	t.Logf("\n\nStart4")
 	notifies.expect(2)
-	assert.Nil(b.Start(ipn.Options{
-		StateKey: ipn.GlobalDaemonStateKey,
-	}))
+	c.Assert(b.Start(ipn.Options{StateKey: ipn.GlobalDaemonStateKey}), qt.IsNil)
 	{
 		// NOTE: cc.Shutdown() is correct here, since we didn't call
 		// b.Shutdown() explicitly ourselves.
@@ -755,15 +734,15 @@ func TestStateMachine(t *testing.T) {
 		//  on startup, otherwise UIs can't show the node list, login
 		//  name, etc when in state ipn.Stopped.
 		//  Arguably they shouldn't try. But they currently do.
-		assert.Equal([]string{"Shutdown", "New", "UpdateEndpoints", "Login", "unpause"}, cc.getCalls())
+		c.Assert([]string{"Shutdown", "New", "UpdateEndpoints", "Login", "unpause"}, qt.DeepEquals, cc.getCalls())
 
 		nn := notifies.drain(2)
-		assert.Equal([]string{}, cc.getCalls())
-		assert.NotNil(nn[0].Prefs)
-		assert.NotNil(nn[1].State)
-		assert.Equal(false, nn[0].Prefs.WantRunning)
-		assert.Equal(false, nn[0].Prefs.LoggedOut)
-		assert.Equal(ipn.Stopped, *nn[1].State)
+		c.Assert(cc.getCalls(), qt.HasLen, 0)
+		c.Assert(nn[0].Prefs, qt.Not(qt.IsNil))
+		c.Assert(nn[1].State, qt.Not(qt.IsNil))
+		c.Assert(nn[0].Prefs.WantRunning, qt.IsFalse)
+		c.Assert(nn[0].Prefs.LoggedOut, qt.IsFalse)
+		c.Assert(ipn.Stopped, qt.Equals, *nn[1].State)
 	}
 
 	// Request connection.
@@ -776,49 +755,105 @@ func TestStateMachine(t *testing.T) {
 	})
 	{
 		nn := notifies.drain(2)
-		assert.Equal([]string{"Login", "unpause"}, cc.getCalls())
+		c.Assert([]string{"Login", "unpause"}, qt.DeepEquals, cc.getCalls())
 		// BUG: I would expect Prefs to change first, and state after.
-		assert.NotNil(nn[0].State)
-		assert.NotNil(nn[1].Prefs)
-		assert.Equal(ipn.Starting, *nn[0].State)
+		c.Assert(nn[0].State, qt.Not(qt.IsNil))
+		c.Assert(nn[1].Prefs, qt.Not(qt.IsNil))
+		c.Assert(ipn.Starting, qt.Equals, *nn[0].State)
+	}
+
+	// Disconnect.
+	t.Logf("\n\nStop")
+	notifies.expect(2)
+	b.EditPrefs(&ipn.MaskedPrefs{
+		WantRunningSet: true,
+		Prefs:          ipn.Prefs{WantRunning: false},
+	})
+	{
+		nn := notifies.drain(2)
+		c.Assert([]string{"unpause"}, qt.DeepEquals, cc.getCalls())
+		// BUG: I would expect Prefs to change first, and state after.
+		c.Assert(nn[0].State, qt.Not(qt.IsNil))
+		c.Assert(nn[1].Prefs, qt.Not(qt.IsNil))
+		c.Assert(ipn.Stopped, qt.Equals, *nn[0].State)
+	}
+
+	// We want to try logging in as a different user, while Stopped.
+	// First, start the login process (without logging out first).
+	t.Logf("\n\nLoginDifferent")
+	notifies.expect(2)
+	b.StartLoginInteractive()
+	url3 := "http://localhost:1/3"
+	cc.send(nil, url3, false, nil)
+	{
+		nn := notifies.drain(2)
+		// It might seem like WantRunning should switch to true here,
+		// but that would be risky since we already have a valid
+		// user account. It might try to reconnect to the old account
+		// before the new one is ready. So no change yet.
+		c.Assert([]string{"Login", "unpause"}, qt.DeepEquals, cc.getCalls())
+		c.Assert(nn[0].BrowseToURL, qt.Not(qt.IsNil))
+		c.Assert(nn[1].State, qt.Not(qt.IsNil))
+		c.Assert(*nn[0].BrowseToURL, qt.Equals, url3)
+		c.Assert(ipn.NeedsLogin, qt.Equals, *nn[1].State)
+	}
+
+	// Now, let's say the interactive login completed, using a different
+	// user account than before.
+	t.Logf("\n\nLoginDifferent URL visited")
+	notifies.expect(3)
+	cc.persist.LoginName = "user3"
+	cc.send(nil, "", true, &netmap.NetworkMap{
+		MachineStatus: tailcfg.MachineAuthorized,
+	})
+	{
+		nn := notifies.drain(3)
+		c.Assert([]string{"unpause"}, qt.DeepEquals, cc.getCalls())
+		c.Assert(nn[0].LoginFinished, qt.Not(qt.IsNil))
+		c.Assert(nn[1].Prefs, qt.Not(qt.IsNil))
+		c.Assert(nn[2].State, qt.Not(qt.IsNil))
+		// Prefs after finishing the login, so LoginName updated.
+		c.Assert(nn[1].Prefs.Persist.LoginName, qt.Equals, "user3")
+		c.Assert(nn[1].Prefs.LoggedOut, qt.IsFalse)
+		c.Assert(nn[1].Prefs.WantRunning, qt.IsTrue)
+		c.Assert(ipn.Starting, qt.Equals, *nn[2].State)
 	}
 
 	// The last test case is the most common one: restarting when both
 	// logged in and WantRunning.
 	t.Logf("\n\nStart5")
 	notifies.expect(1)
-	assert.Nil(b.Start(ipn.Options{
-		StateKey: ipn.GlobalDaemonStateKey,
-	}))
+	c.Assert(b.Start(ipn.Options{StateKey: ipn.GlobalDaemonStateKey}), qt.IsNil)
 	{
 		// NOTE: cc.Shutdown() is correct here, since we didn't call
 		// b.Shutdown() ourselves.
-		assert.Equal([]string{"Shutdown", "New", "UpdateEndpoints", "Login"}, cc.getCalls())
+		c.Assert([]string{"Shutdown", "New", "UpdateEndpoints", "Login"}, qt.DeepEquals, cc.getCalls())
 
 		nn := notifies.drain(1)
-		assert.Equal([]string{}, cc.getCalls())
-		assert.NotNil(nn[0].Prefs)
-		assert.Equal(false, nn[0].Prefs.LoggedOut)
-		assert.Equal(true, nn[0].Prefs.WantRunning)
-		assert.Equal(ipn.NoState, b.State())
+		c.Assert(cc.getCalls(), qt.HasLen, 0)
+		c.Assert(nn[0].Prefs, qt.Not(qt.IsNil))
+		c.Assert(nn[0].Prefs.LoggedOut, qt.IsFalse)
+		c.Assert(nn[0].Prefs.WantRunning, qt.IsTrue)
+		c.Assert(ipn.NoState, qt.Equals, b.State())
 	}
 
 	// Control server accepts our valid key from before.
 	t.Logf("\n\nLoginFinished5")
-	notifies.expect(2)
+	notifies.expect(1)
 	cc.setAuthBlocked(false)
 	cc.send(nil, "", true, &netmap.NetworkMap{
 		MachineStatus: tailcfg.MachineAuthorized,
 	})
 	{
-		nn := notifies.drain(2)
-		assert.Equal([]string{"unpause"}, cc.getCalls())
-		assert.NotNil(nn[0].LoginFinished)
-		assert.NotNil(nn[1].State)
-		assert.Equal(ipn.Starting, *nn[1].State)
+		nn := notifies.drain(1)
+		c.Assert([]string{"unpause"}, qt.DeepEquals, cc.getCalls())
+		// NOTE: No LoginFinished message since no interactive
+		// login was needed.
+		c.Assert(nn[0].State, qt.Not(qt.IsNil))
+		c.Assert(ipn.Starting, qt.Equals, *nn[0].State)
 		// NOTE: No prefs change this time. WantRunning stays true.
 		// We were in Starting in the first place, so that doesn't
 		// change either.
-		assert.Equal(ipn.Starting, b.State())
+		c.Assert(ipn.Starting, qt.Equals, b.State())
 	}
 }

ipn/ipnserver/server.go

@@ -6,7 +6,9 @@ package ipnserver
 
 import (
 	"bufio"
+	"bytes"
 	"context"
+	"encoding/json"
 	"errors"
 	"fmt"
 	"io"
@@ -251,8 +253,7 @@ func (s *server) serveConn(ctx context.Context, c net.Conn, logf logger.Logf) {
 			return
 		}
 		defer c.Close()
-		serverToClient := func(b []byte) { ipn.WriteMsg(c, b) }
-		bs := ipn.NewBackendServer(logf, nil, serverToClient)
+		bs := ipn.NewBackendServer(logf, nil, jsonNotifier(c, s.logf))
 		_, occupied := err.(inUseOtherUserError)
 		if occupied {
 			bs.SendInUseOtherUserErrorMessage(err.Error())
@@ -567,7 +568,9 @@ func (s *server) setServerModeUserLocked() {
 	}
 }
 
-func (s *server) writeToClients(b []byte) {
+var jsonEscapedZero = []byte(`\u0000`)
+
+func (s *server) writeToClients(n ipn.Notify) {
 	inServerMode := s.b.InServerMode()
 
 	s.mu.Lock()
@@ -584,8 +587,17 @@ func (s *server) writeToClients(b []byte) {
 		}
 	}
 
-	for c := range s.clients {
-		ipn.WriteMsg(c, b)
+	if len(s.clients) == 0 {
+		// Common case (at least on busy servers): nobody
+		// connected (no GUI, etc), so return before
+		// serializing JSON.
+		return
+	}
+
+	if b, ok := marshalNotify(n, s.logf); ok {
+		for c := range s.clients {
+			ipn.WriteMsg(c, b)
+		}
 	}
 }
 
@@ -671,8 +683,7 @@ func Run(ctx context.Context, logf logger.Logf, logid string, getEngine func() (
 			errMsg := err.Error()
 			go func() {
 				defer c.Close()
-				serverToClient := func(b []byte) { ipn.WriteMsg(c, b) }
-				bs := ipn.NewBackendServer(logf, nil, serverToClient)
+				bs := ipn.NewBackendServer(logf, nil, jsonNotifier(c, logf))
 				bs.SendErrorMessage(errMsg)
 				time.Sleep(time.Second)
 			}()
@@ -962,3 +973,25 @@ func peerPid(entries []netstat.Entry, la, ra netaddr.IPPort) int {
 	}
 	return 0
 }
+
+// jsonNotifier returns a notify-writer func that writes ipn.Notify
+// messages to w.
+func jsonNotifier(w io.Writer, logf logger.Logf) func(ipn.Notify) {
+	return func(n ipn.Notify) {
+		if b, ok := marshalNotify(n, logf); ok {
+			ipn.WriteMsg(w, b)
+		}
+	}
+}
+
+func marshalNotify(n ipn.Notify, logf logger.Logf) (b []byte, ok bool) {
+	b, err := json.Marshal(n)
+	if err != nil {
+		logf("ipnserver: [unexpected] error serializing JSON: %v", err)
+		return nil, false
+	}
+	if bytes.Contains(b, jsonEscapedZero) {
+		logf("[unexpected] zero byte in BackendServer.send notify message: %q", b)
+	}
+	return b, true
+}

ipn/ipnstate/ipnstate.go

@@ -65,6 +65,7 @@ type PeerStatusLite struct {
 }
 
 type PeerStatus struct {
+	ID        tailcfg.StableNodeID
 	PublicKey key.Public
 	HostName  string // HostInfo's Hostname (not a DNS name or necessarily unique)
 	DNSName   string
@@ -203,6 +204,9 @@ func (sb *StatusBuilder) AddPeer(peer key.Public, st *PeerStatus) {
 		return
 	}
 
+	if v := st.ID; v != "" {
+		e.ID = v
+	}
 	if v := st.HostName; v != "" {
 		e.HostName = v
 	}

ipn/message.go

@@ -88,9 +88,9 @@ type Command struct {
 
 type BackendServer struct {
 	logf          logger.Logf
-	b             Backend              // the Backend we are serving up
-	sendNotifyMsg func(jsonMsg []byte) // send a notification message
-	GotQuit       bool                 // a Quit command was received
+	b             Backend      // the Backend we are serving up
+	sendNotifyMsg func(Notify) // send a notification message
+	GotQuit       bool         // a Quit command was received
 }
 
 // NewBackendServer creates a new BackendServer using b.
@@ -98,7 +98,7 @@ type BackendServer struct {
 // If sendNotifyMsg is non-nil, it additionally sets the Backend's
 // notification callback to call the func with ipn.Notify messages in
 // JSON form. If nil, it does not change the notification callback.
-func NewBackendServer(logf logger.Logf, b Backend, sendNotifyMsg func(b []byte)) *BackendServer {
+func NewBackendServer(logf logger.Logf, b Backend, sendNotifyMsg func(Notify)) *BackendServer {
 	bs := &BackendServer{
 		logf:          logf,
 		b:             b,
@@ -115,14 +115,7 @@ func (bs *BackendServer) send(n Notify) {
 		return
 	}
 	n.Version = version.Long
-	b, err := json.Marshal(n)
-	if err != nil {
-		log.Fatalf("Failed json.Marshal(notify): %v\n%#v", err, n)
-	}
-	if bytes.Contains(b, jsonEscapedZero) {
-		log.Printf("[unexpected] zero byte in BackendServer.send notify message: %q", b)
-	}
-	bs.sendNotifyMsg(b)
+	bs.sendNotifyMsg(n)
 }
 
 func (bs *BackendServer) SendErrorMessage(msg string) {

ipn/message_test.go

@@ -7,6 +7,7 @@ package ipn
 import (
 	"bytes"
 	"context"
+	"encoding/json"
 	"testing"
 	"time"
 
@@ -74,7 +75,11 @@ func TestClientServer(t *testing.T) {
 			bc.GotNotifyMsg(b)
 		}
 	}()
-	serverToClient := func(b []byte) {
+	serverToClient := func(n Notify) {
+		b, err := json.Marshal(n)
+		if err != nil {
+			panic(err.Error())
+		}
 		serverToClientCh <- append([]byte{}, b...)
 	}
 	clientToServer := func(b []byte) {

ipn/prefs.go

@@ -37,6 +37,15 @@ type Prefs struct {
 	// If empty, the default for new installs, DefaultControlURL
 	// is used. It's set non-empty once the daemon has been started
 	// for the first time.
+	//
+	// TODO(apenwarr): Make it safe to update this with SetPrefs().
+	// Right now, you have to pass it in the initial prefs in Start(),
+	// which is the only code that actually uses the ControlURL value.
+	// It would be more consistent to restart controlclient
+	// automatically whenever this variable changes.
+	//
+	// Meanwhile, you have to provide this as part of Options.Prefs or
+	// Options.UpdatePrefs when calling Backend.Start().
 	ControlURL string
 
 	// RouteAll specifies whether to accept subnets advertised by

logtail/filch/filch.go

@@ -15,6 +15,7 @@ import (
 	"sync"
 )
 
+//lint:ignore U1000 work around false positive: https://github.com/dominikh/go-tools/issues/983
 var stderrFD = 2 // a variable for testing
 
 type Options struct {

net/dns/manager_linux.go

@@ -57,12 +57,12 @@ func NewOSConfigurator(logf logger.Logf, interfaceName string) (ret OSConfigurat
 		}
 		if err := dbusPing("org.freedesktop.NetworkManager", "/org/freedesktop/NetworkManager/DnsManager"); err != nil {
 			dbg("nm", "no")
-			return newResolvedManager(logf)
+			return newResolvedManager(logf, interfaceName)
 		}
 		dbg("nm", "yes")
 		if err := nmIsUsingResolved(); err != nil {
 			dbg("nm-resolved", "no")
-			return newResolvedManager(logf)
+			return newResolvedManager(logf, interfaceName)
 		}
 		dbg("nm-resolved", "yes")
 
@@ -90,7 +90,7 @@ func NewOSConfigurator(logf logger.Logf, interfaceName string) (ret OSConfigurat
 			return newNMManager(interfaceName)
 		}
 		dbg("nm-old", "no")
-		return newResolvedManager(logf)
+		return newResolvedManager(logf, interfaceName)
 	case "resolvconf":
 		dbg("rc", "resolvconf")
 		if err := resolvconfSourceIsNM(bs); err == nil {

net/dns/nm.go

@@ -175,9 +175,12 @@ func (m *nmManager) trySet(ctx context.Context, config OSConfig) error {
 		search = append(search, "~.")
 	}
 
-	general := settings["connection"]
-	general["llmnr"] = dbus.MakeVariant(0)
-	general["mdns"] = dbus.MakeVariant(0)
+	// Ideally we would like to disable LLMNR and mdns on the
+	// interface here, but older NetworkManagers don't understand
+	// those settings and choke on them, so we don't. Both LLMNR and
+	// mdns will fail since tailscale0 doesn't do multicast, so it's
+	// effectively fine. We used to try and enforce LLMNR and mdns
+	// settings here, but that led to #1870.
 
 	ipv4Map := settings["ipv4"]
 	ipv4Map["dns"] = dbus.MakeVariant(dnsv4)
@@ -247,7 +250,7 @@ func (m *nmManager) trySet(ctx context.Context, config OSConfig) error {
 	}
 
 	if call := device.CallWithContext(ctx, "org.freedesktop.NetworkManager.Device.Reapply", 0, settings, version, uint32(0)); call.Err != nil {
-		return fmt.Errorf("reapply: %w", err)
+		return fmt.Errorf("reapply: %w", call.Err)
 	}
 
 	return nil

net/dns/resolved.go

@@ -12,11 +12,11 @@ import (
 	"context"
 	"errors"
 	"fmt"
+	"net"
 
 	"github.com/godbus/dbus/v5"
 	"golang.org/x/sys/unix"
 	"inet.af/netaddr"
-	"tailscale.com/net/interfaces"
 	"tailscale.com/types/logger"
 	"tailscale.com/util/dnsname"
 )
@@ -85,17 +85,24 @@ func isResolvedActive() bool {
 // resolvedManager uses the systemd-resolved DBus API.
 type resolvedManager struct {
 	logf     logger.Logf
+	ifidx    int
 	resolved dbus.BusObject
 }
 
-func newResolvedManager(logf logger.Logf) (*resolvedManager, error) {
+func newResolvedManager(logf logger.Logf, interfaceName string) (*resolvedManager, error) {
 	conn, err := dbus.SystemBus()
 	if err != nil {
 		return nil, err
 	}
 
+	iface, err := net.InterfaceByName(interfaceName)
+	if err != nil {
+		return nil, err
+	}
+
 	return &resolvedManager{
 		logf:     logf,
+		ifidx:    iface.Index,
 		resolved: conn.Object("org.freedesktop.resolve1", dbus.ObjectPath("/org/freedesktop/resolve1")),
 	}, nil
 }
@@ -105,16 +112,6 @@ func (m *resolvedManager) SetDNS(config OSConfig) error {
 	ctx, cancel := context.WithTimeout(context.Background(), reconfigTimeout)
 	defer cancel()
 
-	// In principle, we could persist this in the manager struct
-	// if we knew that interface indices are persistent. This does not seem to be the case.
-	_, iface, err := interfaces.Tailscale()
-	if err != nil {
-		return fmt.Errorf("getting interface index: %w", err)
-	}
-	if iface == nil {
-		return errNotReady
-	}
-
 	var linkNameservers = make([]resolvedLinkNameserver, len(config.Nameservers))
 	for i, server := range config.Nameservers {
 		ip := server.As16()
@@ -131,9 +128,9 @@ func (m *resolvedManager) SetDNS(config OSConfig) error {
 		}
 	}
 
-	err = m.resolved.CallWithContext(
+	err := m.resolved.CallWithContext(
 		ctx, "org.freedesktop.resolve1.Manager.SetLinkDNS", 0,
-		iface.Index, linkNameservers,
+		m.ifidx, linkNameservers,
 	).Store()
 	if err != nil {
 		return fmt.Errorf("setLinkDNS: %w", err)
@@ -174,13 +171,13 @@ func (m *resolvedManager) SetDNS(config OSConfig) error {
 
 	err = m.resolved.CallWithContext(
 		ctx, "org.freedesktop.resolve1.Manager.SetLinkDomains", 0,
-		iface.Index, linkDomains,
+		m.ifidx, linkDomains,
 	).Store()
 	if err != nil {
 		return fmt.Errorf("setLinkDomains: %w", err)
 	}
 
-	if call := m.resolved.CallWithContext(ctx, "org.freedesktop.resolve1.Manager.SetLinkDefaultRoute", 0, iface.Index, len(config.MatchDomains) == 0); call.Err != nil {
+	if call := m.resolved.CallWithContext(ctx, "org.freedesktop.resolve1.Manager.SetLinkDefaultRoute", 0, m.ifidx, len(config.MatchDomains) == 0); call.Err != nil {
 		return fmt.Errorf("setLinkDefaultRoute: %w", err)
 	}
 
@@ -189,22 +186,22 @@ func (m *resolvedManager) SetDNS(config OSConfig) error {
 	// or something).
 
 	// Disable LLMNR, we don't do multicast.
-	if call := m.resolved.CallWithContext(ctx, "org.freedesktop.resolve1.Manager.SetLinkLLMNR", 0, iface.Index, "no"); call.Err != nil {
+	if call := m.resolved.CallWithContext(ctx, "org.freedesktop.resolve1.Manager.SetLinkLLMNR", 0, m.ifidx, "no"); call.Err != nil {
 		m.logf("[v1] failed to disable LLMNR: %v", call.Err)
 	}
 
 	// Disable mdns.
-	if call := m.resolved.CallWithContext(ctx, "org.freedesktop.resolve1.Manager.SetLinkMulticastDNS", 0, iface.Index, "no"); call.Err != nil {
+	if call := m.resolved.CallWithContext(ctx, "org.freedesktop.resolve1.Manager.SetLinkMulticastDNS", 0, m.ifidx, "no"); call.Err != nil {
 		m.logf("[v1] failed to disable mdns: %v", call.Err)
 	}
 
 	// We don't support dnssec consistently right now, force it off to
 	// avoid partial failures when we split DNS internally.
-	if call := m.resolved.CallWithContext(ctx, "org.freedesktop.resolve1.Manager.SetLinkDNSSEC", 0, iface.Index, "no"); call.Err != nil {
+	if call := m.resolved.CallWithContext(ctx, "org.freedesktop.resolve1.Manager.SetLinkDNSSEC", 0, m.ifidx, "no"); call.Err != nil {
 		m.logf("[v1] failed to disable DNSSEC: %v", call.Err)
 	}
 
-	if call := m.resolved.CallWithContext(ctx, "org.freedesktop.resolve1.Manager.SetLinkDNSOverTLS", 0, iface.Index, "no"); call.Err != nil {
+	if call := m.resolved.CallWithContext(ctx, "org.freedesktop.resolve1.Manager.SetLinkDNSOverTLS", 0, m.ifidx, "no"); call.Err != nil {
 		m.logf("[v1] failed to disable DoT: %v", call.Err)
 	}
 
@@ -227,15 +224,7 @@ func (m *resolvedManager) Close() error {
 	ctx, cancel := context.WithTimeout(context.Background(), reconfigTimeout)
 	defer cancel()
 
-	_, iface, err := interfaces.Tailscale()
-	if err != nil {
-		return fmt.Errorf("getting interface index: %w", err)
-	}
-	if iface == nil {
-		return errNotReady
-	}
-
-	if call := m.resolved.CallWithContext(ctx, "org.freedesktop.resolve1.Manager.RevertLink", 0, iface.Index); call.Err != nil {
+	if call := m.resolved.CallWithContext(ctx, "org.freedesktop.resolve1.Manager.RevertLink", 0, m.ifidx); call.Err != nil {
 		return fmt.Errorf("RevertLink: %w", call.Err)
 	}
 

net/interfaces/interfaces_darwin.go

@@ -10,7 +10,6 @@ import (
 	"log"
 	"net"
 	"syscall"
-	"time"
 
 	"golang.org/x/net/route"
 	"golang.org/x/sys/unix"
@@ -29,32 +28,9 @@ func DefaultRouteInterface() (string, error) {
 	return iface.Name, nil
 }
 
-// fetchRoutingTable is a retry loop around route.FetchRIB, fetching NET_RT_DUMP2.
-//
-// The retry loop is due to a bug in the BSDs (or Go?). See
-// https://github.com/tailscale/tailscale/issues/1345
+// fetchRoutingTable calls route.FetchRIB, fetching NET_RT_DUMP2.
 func fetchRoutingTable() (rib []byte, err error) {
-	fails := 0
-	for {
-		rib, err := route.FetchRIB(syscall.AF_UNSPEC, syscall.NET_RT_DUMP2, 0)
-		if err == nil {
-			return rib, nil
-		}
-		fails++
-		if fails < 10 {
-			// Empirically, 1 retry is enough. In a long
-			// stress test while toggling wifi on & off, I
-			// only saw a few occurrences of 2 and one 3.
-			// So 10 should be more plenty.
-			if fails > 5 {
-				time.Sleep(5 * time.Millisecond)
-			}
-			continue
-		}
-		if err != nil {
-			return nil, fmt.Errorf("route.FetchRIB: %w", err)
-		}
-	}
+	return route.FetchRIB(syscall.AF_UNSPEC, syscall.NET_RT_DUMP2, 0)
 }
 
 func DefaultRouteInterfaceIndex() (int, error) {

net/stun/stuntest/stuntest.go

@@ -26,11 +26,11 @@ type stunStats struct {
 	readIPv6 int
 }
 
-func Serve(t *testing.T) (addr *net.UDPAddr, cleanupFn func()) {
+func Serve(t testing.TB) (addr *net.UDPAddr, cleanupFn func()) {
 	return ServeWithPacketListener(t, nettype.Std{})
 }
 
-func ServeWithPacketListener(t *testing.T, ln nettype.PacketListener) (addr *net.UDPAddr, cleanupFn func()) {
+func ServeWithPacketListener(t testing.TB, ln nettype.PacketListener) (addr *net.UDPAddr, cleanupFn func()) {
 	t.Helper()
 
 	// TODO(crawshaw): use stats to test re-STUN logic
@@ -52,7 +52,7 @@ func ServeWithPacketListener(t *testing.T, ln nettype.PacketListener) (addr *net
 	}
 }
 
-func runSTUN(t *testing.T, pc net.PacketConn, stats *stunStats, done chan<- struct{}) {
+func runSTUN(t testing.TB, pc net.PacketConn, stats *stunStats, done chan<- struct{}) {
 	defer close(done)
 
 	var buf [64 << 10]byte

net/tsaddr/tsaddr.go

@@ -138,3 +138,50 @@ type onceIP struct {
 	sync.Once
 	v netaddr.IP
 }
+
+// NewContainsIPFunc returns a func that reports whether ip is in addrs.
+//
+// It's optimized for the cases of addrs being empty and addrs
+// containing 1 or 2 single-IP prefixes (such as one IPv4 address and
+// one IPv6 address).
+//
+// Otherwise the implementation is somewhat slow.
+func NewContainsIPFunc(addrs []netaddr.IPPrefix) func(ip netaddr.IP) bool {
+	// Specialize the three common cases: no address, just IPv4
+	// (or just IPv6), and both IPv4 and IPv6.
+	if len(addrs) == 0 {
+		return func(netaddr.IP) bool { return false }
+	}
+	// If any addr is more than a single IP, then just do the slow
+	// linear thing until
+	// https://github.com/inetaf/netaddr/issues/139 is done.
+	for _, a := range addrs {
+		if a.IsSingleIP() {
+			continue
+		}
+		acopy := append([]netaddr.IPPrefix(nil), addrs...)
+		return func(ip netaddr.IP) bool {
+			for _, a := range acopy {
+				if a.Contains(ip) {
+					return true
+				}
+			}
+			return false
+		}
+	}
+	// Fast paths for 1 and 2 IPs:
+	if len(addrs) == 1 {
+		a := addrs[0]
+		return func(ip netaddr.IP) bool { return ip == a.IP }
+	}
+	if len(addrs) == 2 {
+		a, b := addrs[0], addrs[1]
+		return func(ip netaddr.IP) bool { return ip == a.IP || ip == b.IP }
+	}
+	// General case:
+	m := map[netaddr.IP]bool{}
+	for _, a := range addrs {
+		m[a.IP] = true
+	}
+	return func(ip netaddr.IP) bool { return m[ip] }
+}

net/tsaddr/tsaddr_test.go

@@ -64,3 +64,32 @@ func TestIsUla(t *testing.T) {
 		}
 	}
 }
+
+func TestNewContainsIPFunc(t *testing.T) {
+	f := NewContainsIPFunc([]netaddr.IPPrefix{netaddr.MustParseIPPrefix("10.0.0.0/8")})
+	if f(netaddr.MustParseIP("8.8.8.8")) {
+		t.Fatal("bad")
+	}
+	if !f(netaddr.MustParseIP("10.1.2.3")) {
+		t.Fatal("bad")
+	}
+	f = NewContainsIPFunc([]netaddr.IPPrefix{netaddr.MustParseIPPrefix("10.1.2.3/32")})
+	if !f(netaddr.MustParseIP("10.1.2.3")) {
+		t.Fatal("bad")
+	}
+	f = NewContainsIPFunc([]netaddr.IPPrefix{
+		netaddr.MustParseIPPrefix("10.1.2.3/32"),
+		netaddr.MustParseIPPrefix("::2/128"),
+	})
+	if !f(netaddr.MustParseIP("::2")) {
+		t.Fatal("bad")
+	}
+	f = NewContainsIPFunc([]netaddr.IPPrefix{
+		netaddr.MustParseIPPrefix("10.1.2.3/32"),
+		netaddr.MustParseIPPrefix("10.1.2.4/32"),
+		netaddr.MustParseIPPrefix("::2/128"),
+	})
+	if !f(netaddr.MustParseIP("10.1.2.4")) {
+		t.Fatal("bad")
+	}
+}

net/tstun/ifstatus_windows.go

@@ -93,7 +93,6 @@ func waitInterfaceUp(iface tun.Device, timeout time.Duration, logf logger.Logf)
 			iw.logf("TUN interface is up after %v", time.Since(t0))
 			return nil
 		case <-ticker.C:
-			break
 		}
 
 		if iw.isUp() {

net/tstun/wrap.go

@@ -457,13 +457,22 @@ func (t *Wrapper) filterIn(buf []byte) filter.Response {
 // like wireguard-go/tun.Device.Write.
 func (t *Wrapper) Write(buf []byte, offset int) (int, error) {
 	if !t.disableFilter {
-		res := t.filterIn(buf[offset:])
-		if res == filter.DropSilently {
+		if t.filterIn(buf[offset:]) != filter.Accept {
+			// If we're not accepting the packet, lie to wireguard-go and pretend
+			// that everything is okay with a nil error, so wireguard-go
+			// doesn't log about this Write "failure".
+			//
+			// We return len(buf), but the ill-defined wireguard-go/tun.Device.Write
+			// method doesn't specify how the offset affects the return value.
+			// In fact, the Linux implementation does one of two different things depending
+			// on how the /dev/net/tun was created. But fortunately the wireguard-go
+			// code ignores the int return and only looks at the error:
+			//
+			//     device/receive.go: _, err = device.tun.device.Write(....)
+			//
+			// TODO(bradfitz): fix upstream interface docs, implementation.
 			return len(buf), nil
 		}
-		if res != filter.Accept {
-			return 0, ErrFiltered
-		}
 	}
 
 	t.noteActivity()

net/tstun/wrap_test.go

@@ -329,11 +329,14 @@ func TestFilter(t *testing.T) {
 			var filtered bool
 
 			if tt.dir == in {
+				// Use the side effect of updating the last
+				// activity atomic to determine whether the
+				// data was actually filtered.
+				// If it stays zero, nothing made it through
+				// to the wrapped TUN.
+				atomic.StoreInt64(&tun.lastActivityAtomic, 0)
 				_, err = tun.Write(tt.data, 0)
-				if err == ErrFiltered {
-					filtered = true
-					err = nil
-				}
+				filtered = atomic.LoadInt64(&tun.lastActivityAtomic) == 0
 			} else {
 				chtun.Outbound <- tt.data
 				n, err = tun.Read(buf[:], 0)

paths/paths.go

@@ -26,6 +26,13 @@ func DefaultTailscaledSocket() string {
 	if runtime.GOOS == "darwin" {
 		return "/var/run/tailscaled.socket"
 	}
+	if runtime.GOOS == "linux" {
+		// TODO(crawshaw): does this path change with DSM7?
+		const synologySock = "/volume1/@appstore/Tailscale/var/tailscaled.sock" // SYNOPKG_PKGDEST in scripts/installer
+		if fi, err := os.Stat(filepath.Dir(synologySock)); err == nil && fi.IsDir() {
+			return synologySock
+		}
+	}
 	if fi, err := os.Stat("/var/run"); err == nil && fi.IsDir() {
 		return "/var/run/tailscale/tailscaled.sock"
 	}

portlist/portlist_windows.go

@@ -23,7 +23,7 @@ func listPorts() (List, error) {
 }
 
 func addProcesses(pl []Port) ([]Port, error) {
-	// OpenCurrentProcessToken instead of GetCurrentProcessToken,
+	//lint:ignore SA1019 OpenCurrentProcessToken instead of GetCurrentProcessToken,
 	// as GetCurrentProcessToken only works on Windows 8+.
 	tok, err := windows.OpenCurrentProcessToken()
 	if err != nil {

safesocket/pipe_windows.go

@@ -11,10 +11,6 @@ import (
 	"syscall"
 )
 
-func path(vendor, name string, port uint16) string {
-	return fmt.Sprintf("127.0.0.1:%v", port)
-}
-
 func connect(path string, port uint16) (net.Conn, error) {
 	pipe, err := net.Dial("tcp", fmt.Sprintf("127.0.0.1:%d", port))
 	if err != nil {

safesocket/safesocket.go

@@ -61,8 +61,12 @@ func LocalTCPPortAndToken() (port int, token string, err error) {
 
 // PlatformUsesPeerCreds reports whether the current platform uses peer credentials
 // to authenticate connections.
-func PlatformUsesPeerCreds() bool {
-	switch runtime.GOOS {
+func PlatformUsesPeerCreds() bool { return GOOSUsesPeerCreds(runtime.GOOS) }
+
+// GOOSUsesPeerCreds is like PlatformUsesPeerCreds but takes a
+// runtime.GOOS value instead of using the current one.
+func GOOSUsesPeerCreds(goos string) bool {
+	switch goos {
 	case "linux", "darwin", "freebsd":
 		return true
 	}

syncs/watchdog_test.go

@@ -6,7 +6,6 @@ package syncs
 
 import (
 	"context"
-	"runtime"
 	"sync"
 	"testing"
 	"time"
@@ -49,11 +48,11 @@ func TestWatchContended(t *testing.T) {
 }
 
 func TestWatchMultipleValues(t *testing.T) {
-	if cibuild.On() && runtime.GOOS == "windows" {
+	if cibuild.On() {
 		// On the CI machine, it sometimes takes 500ms to start a new goroutine.
 		// When this happens, we don't get enough events quickly enough.
 		// Nothing's wrong, and it's not worth working around. Just skip the test.
-		t.Skip("flaky on Windows CI")
+		t.Skip("flaky on CI")
 	}
 	mu := new(sync.Mutex)
 	ctx, cancel := context.WithCancel(context.Background())

tailcfg/tailcfg.go

@@ -7,7 +7,7 @@ package tailcfg
 //go:generate go run tailscale.com/cmd/cloner --type=User,Node,Hostinfo,NetInfo,Login,DNSConfig,DNSResolver,RegisterResponse --clonefunc=true --output=tailcfg_clone.go
 
 import (
-	"bytes"
+	"encoding/hex"
 	"errors"
 	"fmt"
 	"reflect"
@@ -679,7 +679,7 @@ func (et EndpointType) String() string {
 
 // Endpoint is an endpoint IPPort and an associated type.
 // It doesn't currently go over the wire as is but is instead
-// broken up into two parallel slices in MapReqeust, for compatibility
+// broken up into two parallel slices in MapRequest, for compatibility
 // reasons. But this type is used in the codebase.
 type Endpoint struct {
 	Addr netaddr.IPPort
@@ -1027,9 +1027,10 @@ func (k MachineKey) HexString() string                { return fmt.Sprintf("%x",
 func (k *MachineKey) UnmarshalText(text []byte) error { return keyUnmarshalText(k[:], "mkey:", text) }
 
 func keyMarshalText(prefix string, k [32]byte) []byte {
-	buf := bytes.NewBuffer(make([]byte, 0, len(prefix)+64))
-	fmt.Fprintf(buf, "%s%x", prefix, k[:])
-	return buf.Bytes()
+	buf := make([]byte, len(prefix)+64)
+	copy(buf, prefix)
+	hex.Encode(buf[len(prefix):], k[:])
+	return buf
 }
 
 func keyUnmarshalText(dst []byte, prefix string, text []byte) error {

tailcfg/tailcfg_test.go

@@ -518,3 +518,13 @@ func TestEndpointTypeMarshal(t *testing.T) {
 		t.Errorf("got %s; want %s", got, want)
 	}
 }
+
+var sinkBytes []byte
+
+func BenchmarkKeyMarshalText(b *testing.B) {
+	b.ReportAllocs()
+	var k [32]byte
+	for i := 0; i < b.N; i++ {
+		sinkBytes = keyMarshalText("prefix", k)
+	}
+}

tstest/integration/integration_test.go

@@ -7,121 +7,423 @@ package integration
 
 import (
 	"bytes"
+	crand "crypto/rand"
+	"crypto/tls"
 	"encoding/json"
+	"errors"
+	"flag"
 	"fmt"
 	"io"
 	"io/ioutil"
 	"log"
+	"net"
 	"net/http"
 	"net/http/httptest"
 	"os"
 	"os/exec"
 	"path"
 	"path/filepath"
+	"regexp"
 	"runtime"
 	"strings"
 	"sync"
+	"sync/atomic"
 	"testing"
 	"time"
 
 	"go4.org/mem"
+	"tailscale.com/derp"
+	"tailscale.com/derp/derphttp"
+	"tailscale.com/ipn/ipnstate"
+	"tailscale.com/net/stun/stuntest"
+	"tailscale.com/safesocket"
 	"tailscale.com/smallzstd"
+	"tailscale.com/tailcfg"
 	"tailscale.com/tstest"
 	"tailscale.com/tstest/integration/testcontrol"
+	"tailscale.com/types/key"
+	"tailscale.com/types/logger"
+	"tailscale.com/types/nettype"
+	"tailscale.com/version"
 )
 
-func TestIntegration(t *testing.T) {
-	if runtime.GOOS == "windows" {
-		t.Skip("not tested/working on Windows yet")
+var verbose = flag.Bool("verbose", false, "verbose debug logs")
+
+var mainError atomic.Value // of error
+
+func TestMain(m *testing.M) {
+	v := m.Run()
+	if v != 0 {
+		os.Exit(v)
 	}
-	td := t.TempDir()
-	daemonExe := build(t, td, "tailscale.com/cmd/tailscaled")
-	cliExe := build(t, td, "tailscale.com/cmd/tailscale")
-
-	logc := new(logCatcher)
-	ts := httptest.NewServer(logc)
-	defer ts.Close()
-
-	// catchBadTrafficProxy explodes if it gets any traffic.
-	// It's here to catch anything that would otherwise try to leave localhost.
-	catchBadTrafficProxy := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
-		var got bytes.Buffer
-		r.Write(&got)
-		err := fmt.Errorf("unexpected HTTP proxy via proxy: %s", got.Bytes())
-		t.Error(err)
-		go panic(err)
-	}))
-	defer catchBadTrafficProxy.Close()
-
-	controlServer := new(testcontrol.Server)
-	controlHTTPServer := httptest.NewServer(controlServer)
-	defer controlHTTPServer.Close()
-
-	socketPath := filepath.Join(td, "tailscale.sock")
-	dcmd := exec.Command(daemonExe,
-		"--tun=userspace-networking",
-		"--state="+filepath.Join(td, "tailscale.state"),
-		"--socket="+socketPath,
-	)
-	dcmd.Env = append(os.Environ(),
-		"TS_LOG_TARGET="+ts.URL,
-		"HTTP_PROXY="+catchBadTrafficProxy.URL,
-		"HTTPS_PROXY="+catchBadTrafficProxy.URL,
-	)
-	if err := dcmd.Start(); err != nil {
-		t.Fatalf("starting tailscaled: %v", err)
+	if err, ok := mainError.Load().(error); ok {
+		fmt.Fprintf(os.Stderr, "FAIL: %v\n", err)
+		os.Exit(1)
 	}
-	defer dcmd.Process.Kill()
+	os.Exit(0)
+}
 
-	var json []byte
-	if err := tstest.WaitFor(20*time.Second, func() (err error) {
-		json, err = exec.Command(cliExe, "--socket="+socketPath, "status", "--json").CombinedOutput()
-		if err != nil {
-			return fmt.Errorf("running tailscale status: %v, %s", err, json)
-		}
-		return nil
-	}); err != nil {
-		t.Fatal(err)
-	}
+func TestOneNodeUp_NoAuth(t *testing.T) {
+	t.Parallel()
+	bins := buildTestBinaries(t)
+
+	env := newTestEnv(t, bins)
+	defer env.Close()
+
+	n1 := newTestNode(t, env)
+
+	d1 := n1.StartDaemon(t)
+	defer d1.Kill()
+
+	n1.AwaitListening(t)
+
+	st := n1.MustStatus(t)
+	t.Logf("Status: %s", st.BackendState)
 
 	if err := tstest.WaitFor(20*time.Second, func() error {
 		const sub = `Program starting: `
-		if !logc.logsContains(mem.S(sub)) {
-			return fmt.Errorf("log catcher didn't see %#q; got %s", sub, logc.logsString())
+		if !env.LogCatcher.logsContains(mem.S(sub)) {
+			return fmt.Errorf("log catcher didn't see %#q; got %s", sub, env.LogCatcher.logsString())
 		}
 		return nil
 	}); err != nil {
 		t.Error(err)
 	}
 
-	if err := exec.Command(cliExe, "--socket="+socketPath, "up", "--login-server="+controlHTTPServer.URL).Run(); err != nil {
+	n1.MustUp()
+
+	if d, _ := time.ParseDuration(os.Getenv("TS_POST_UP_SLEEP")); d > 0 {
+		t.Logf("Sleeping for %v to give 'up' time to misbehave (https://github.com/tailscale/tailscale/issues/1840) ...", d)
+		time.Sleep(d)
+	}
+
+	t.Logf("Got IP: %v", n1.AwaitIP(t))
+	n1.AwaitRunning(t)
+
+	d1.MustCleanShutdown(t)
+
+	t.Logf("number of HTTP logcatcher requests: %v", env.LogCatcher.numRequests())
+}
+
+func TestOneNodeUp_Auth(t *testing.T) {
+	t.Parallel()
+	bins := buildTestBinaries(t)
+
+	env := newTestEnv(t, bins)
+	defer env.Close()
+	env.Control.RequireAuth = true
+
+	n1 := newTestNode(t, env)
+	d1 := n1.StartDaemon(t)
+	defer d1.Kill()
+
+	n1.AwaitListening(t)
+
+	st := n1.MustStatus(t)
+	t.Logf("Status: %s", st.BackendState)
+
+	t.Logf("Running up --login-server=%s ...", env.ControlServer.URL)
+
+	cmd := n1.Tailscale("up", "--login-server="+env.ControlServer.URL)
+	var authCountAtomic int32
+	cmd.Stdout = &authURLParserWriter{fn: func(urlStr string) error {
+		if env.Control.CompleteAuth(urlStr) {
+			atomic.AddInt32(&authCountAtomic, 1)
+			t.Logf("completed auth path %s", urlStr)
+			return nil
+		}
+		err := fmt.Errorf("Failed to complete auth path to %q", urlStr)
+		t.Log(err)
+		return err
+	}}
+	cmd.Stderr = cmd.Stdout
+	if err := cmd.Run(); err != nil {
 		t.Fatalf("up: %v", err)
 	}
+	t.Logf("Got IP: %v", n1.AwaitIP(t))
 
-	var ip string
-	if err := tstest.WaitFor(20*time.Second, func() error {
-		out, err := exec.Command(cliExe, "--socket="+socketPath, "ip").Output()
-		if err != nil {
-			return err
+	n1.AwaitRunning(t)
+
+	if n := atomic.LoadInt32(&authCountAtomic); n != 1 {
+		t.Errorf("Auth URLs completed = %d; want 1", n)
+	}
+
+	d1.MustCleanShutdown(t)
+
+}
+
+func TestTwoNodes(t *testing.T) {
+	t.Parallel()
+	bins := buildTestBinaries(t)
+
+	env := newTestEnv(t, bins)
+	defer env.Close()
+
+	// Create two nodes:
+	n1 := newTestNode(t, env)
+	d1 := n1.StartDaemon(t)
+	defer d1.Kill()
+
+	n2 := newTestNode(t, env)
+	d2 := n2.StartDaemon(t)
+	defer d2.Kill()
+
+	n1.AwaitListening(t)
+	n2.AwaitListening(t)
+	n1.MustUp()
+	n2.MustUp()
+	n1.AwaitRunning(t)
+	n2.AwaitRunning(t)
+
+	if err := tstest.WaitFor(2*time.Second, func() error {
+		st := n1.MustStatus(t)
+		if len(st.Peer) == 0 {
+			return errors.New("no peers")
+		}
+		if len(st.Peer) > 1 {
+			return fmt.Errorf("got %d peers; want 1", len(st.Peer))
+		}
+		peer := st.Peer[st.Peers()[0]]
+		if peer.ID == st.Self.ID {
+			return errors.New("peer is self")
 		}
-		ip = string(out)
 		return nil
 	}); err != nil {
 		t.Error(err)
 	}
-	t.Logf("Got IP: %v", ip)
 
-	dcmd.Process.Signal(os.Interrupt)
+	d1.MustCleanShutdown(t)
+	d2.MustCleanShutdown(t)
+}
 
-	ps, err := dcmd.Process.Wait()
+// testBinaries are the paths to a tailscaled and tailscale binary.
+// These can be shared by multiple nodes.
+type testBinaries struct {
+	dir    string // temp dir for tailscale & tailscaled
+	daemon string // tailscaled
+	cli    string // tailscale
+}
+
+// buildTestBinaries builds tailscale and tailscaled, failing the test
+// if they fail to compile.
+func buildTestBinaries(t testing.TB) *testBinaries {
+	td := t.TempDir()
+	build(t, td, "tailscale.com/cmd/tailscaled", "tailscale.com/cmd/tailscale")
+	return &testBinaries{
+		dir:    td,
+		daemon: filepath.Join(td, "tailscaled"+exe()),
+		cli:    filepath.Join(td, "tailscale"+exe()),
+	}
+}
+
+// testEnv contains the test environment (set of servers) used by one
+// or more nodes.
+type testEnv struct {
+	t        testing.TB
+	Binaries *testBinaries
+
+	LogCatcher       *logCatcher
+	LogCatcherServer *httptest.Server
+
+	Control       *testcontrol.Server
+	ControlServer *httptest.Server
+
+	TrafficTrap       *trafficTrap
+	TrafficTrapServer *httptest.Server
+
+	derpShutdown func()
+}
+
+// newTestEnv starts a bunch of services and returns a new test
+// environment.
+//
+// Call Close to shut everything down.
+func newTestEnv(t testing.TB, bins *testBinaries) *testEnv {
+	if runtime.GOOS == "windows" {
+		t.Skip("not tested/working on Windows yet")
+	}
+	derpMap, derpShutdown := runDERPAndStun(t, logger.Discard)
+	logc := new(logCatcher)
+	control := &testcontrol.Server{
+		DERPMap: derpMap,
+	}
+	trafficTrap := new(trafficTrap)
+	e := &testEnv{
+		t:                 t,
+		Binaries:          bins,
+		LogCatcher:        logc,
+		LogCatcherServer:  httptest.NewServer(logc),
+		Control:           control,
+		ControlServer:     httptest.NewServer(control),
+		TrafficTrap:       trafficTrap,
+		TrafficTrapServer: httptest.NewServer(trafficTrap),
+		derpShutdown:      derpShutdown,
+	}
+	e.Control.BaseURL = e.ControlServer.URL
+	return e
+}
+
+func (e *testEnv) Close() error {
+	if err := e.TrafficTrap.Err(); err != nil {
+		e.t.Errorf("traffic trap: %v", err)
+		e.t.Logf("logs: %s", e.LogCatcher.logsString())
+	}
+
+	e.LogCatcherServer.Close()
+	e.TrafficTrapServer.Close()
+	e.ControlServer.Close()
+	e.derpShutdown()
+	return nil
+}
+
+// testNode is a machine with a tailscale & tailscaled.
+// Currently, the test is simplistic and user==node==machine.
+// That may grow complexity later to test more.
+type testNode struct {
+	env *testEnv
+
+	dir       string // temp dir for sock & state
+	sockFile  string
+	stateFile string
+}
+
+// newTestNode allocates a temp directory for a new test node.
+// The node is not started automatically.
+func newTestNode(t *testing.T, env *testEnv) *testNode {
+	dir := t.TempDir()
+	return &testNode{
+		env:       env,
+		dir:       dir,
+		sockFile:  filepath.Join(dir, "tailscale.sock"),
+		stateFile: filepath.Join(dir, "tailscale.state"),
+	}
+}
+
+type Daemon struct {
+	Process *os.Process
+}
+
+func (d *Daemon) Kill() {
+	d.Process.Kill()
+}
+
+func (d *Daemon) MustCleanShutdown(t testing.TB) {
+	d.Process.Signal(os.Interrupt)
+	ps, err := d.Process.Wait()
 	if err != nil {
 		t.Fatalf("tailscaled Wait: %v", err)
 	}
 	if ps.ExitCode() != 0 {
 		t.Errorf("tailscaled ExitCode = %d; want 0", ps.ExitCode())
 	}
+}
 
-	t.Logf("number of HTTP logcatcher requests: %v", logc.numRequests())
+// StartDaemon starts the node's tailscaled, failing if it fails to
+// start.
+func (n *testNode) StartDaemon(t testing.TB) *Daemon {
+	cmd := exec.Command(n.env.Binaries.daemon,
+		"--tun=userspace-networking",
+		"--state="+n.stateFile,
+		"--socket="+n.sockFile,
+	)
+	cmd.Env = append(os.Environ(),
+		"TS_LOG_TARGET="+n.env.LogCatcherServer.URL,
+		"HTTP_PROXY="+n.env.TrafficTrapServer.URL,
+		"HTTPS_PROXY="+n.env.TrafficTrapServer.URL,
+	)
+	if err := cmd.Start(); err != nil {
+		t.Fatalf("starting tailscaled: %v", err)
+	}
+	return &Daemon{
+		Process: cmd.Process,
+	}
+}
+
+func (n *testNode) MustUp() {
+	t := n.env.t
+	t.Logf("Running up --login-server=%s ...", n.env.ControlServer.URL)
+	if err := n.Tailscale("up", "--login-server="+n.env.ControlServer.URL).Run(); err != nil {
+		t.Fatalf("up: %v", err)
+	}
+}
+
+// AwaitListening waits for the tailscaled to be serving local clients
+// over its localhost IPC mechanism. (Unix socket, etc)
+func (n *testNode) AwaitListening(t testing.TB) {
+	if err := tstest.WaitFor(20*time.Second, func() (err error) {
+		c, err := safesocket.Connect(n.sockFile, 41112)
+		if err != nil {
+			return err
+		}
+		c.Close()
+		return nil
+	}); err != nil {
+		t.Fatal(err)
+	}
+}
+
+func (n *testNode) AwaitIP(t testing.TB) (ips string) {
+	t.Helper()
+	if err := tstest.WaitFor(20*time.Second, func() error {
+		out, err := n.Tailscale("ip").Output()
+		if err != nil {
+			return err
+		}
+		ips = string(out)
+		return nil
+	}); err != nil {
+		t.Fatalf("awaiting an IP address: %v", err)
+	}
+	if ips == "" {
+		t.Fatalf("returned IP address was blank")
+	}
+	return ips
+}
+
+func (n *testNode) AwaitRunning(t testing.TB) {
+	t.Helper()
+	if err := tstest.WaitFor(20*time.Second, func() error {
+		st, err := n.Status()
+		if err != nil {
+			return err
+		}
+		if st.BackendState != "Running" {
+			return fmt.Errorf("in state %q", st.BackendState)
+		}
+		return nil
+	}); err != nil {
+		t.Fatalf("failure/timeout waiting for transition to Running status: %v", err)
+	}
+}
+
+// Tailscale returns a command that runs the tailscale CLI with the provided arguments.
+// It does not start the process.
+func (n *testNode) Tailscale(arg ...string) *exec.Cmd {
+	cmd := exec.Command(n.env.Binaries.cli, "--socket="+n.sockFile)
+	cmd.Args = append(cmd.Args, arg...)
+	cmd.Dir = n.dir
+	return cmd
+}
+
+func (n *testNode) Status() (*ipnstate.Status, error) {
+	out, err := n.Tailscale("status", "--json").CombinedOutput()
+	if err != nil {
+		return nil, fmt.Errorf("running tailscale status: %v, %s", err, out)
+	}
+	st := new(ipnstate.Status)
+	if err := json.Unmarshal(out, st); err != nil {
+		return nil, fmt.Errorf("decoding tailscale status JSON: %w", err)
+	}
+	return st, nil
+}
+
+func (n *testNode) MustStatus(tb testing.TB) *ipnstate.Status {
+	tb.Helper()
+	st, err := n.Status()
+	if err != nil {
+		tb.Fatal(err)
+	}
+	return st
 }
 
 func exe() string {
@@ -131,7 +433,7 @@ func exe() string {
 	return ""
 }
 
-func findGo(t *testing.T) string {
+func findGo(t testing.TB) string {
 	goBin := filepath.Join(runtime.GOROOT(), "bin", "go"+exe())
 	if fi, err := os.Stat(goBin); err != nil {
 		if os.IsNotExist(err) {
@@ -141,23 +443,47 @@ func findGo(t *testing.T) string {
 	} else if !fi.Mode().IsRegular() {
 		t.Fatalf("%v is unexpected %v", goBin, fi.Mode())
 	}
-	t.Logf("using go binary %v", goBin)
 	return goBin
 }
 
-func build(t *testing.T, outDir, target string) string {
-	exe := ""
-	if runtime.GOOS == "windows" {
-		exe = ".exe"
+// buildMu limits our use of "go build" to one at a time, so we don't
+// fight Go's built-in caching trying to do the same build concurrently.
+var buildMu sync.Mutex
+
+func build(t testing.TB, outDir string, targets ...string) {
+	buildMu.Lock()
+	defer buildMu.Unlock()
+
+	t0 := time.Now()
+	defer func() { t.Logf("built %s in %v", targets, time.Since(t0).Round(time.Millisecond)) }()
+
+	goBin := findGo(t)
+	cmd := exec.Command(goBin, "install")
+	if version.IsRace() {
+		cmd.Args = append(cmd.Args, "-race")
 	}
-	bin := filepath.Join(outDir, path.Base(target)) + exe
-	errOut, err := exec.Command(findGo(t), "build", "-o", bin, target).CombinedOutput()
-	if err != nil {
-		t.Fatalf("failed to build %v: %v, %s", target, err, errOut)
+	cmd.Args = append(cmd.Args, targets...)
+	cmd.Env = append(os.Environ(), "GOARCH="+runtime.GOARCH, "GOBIN="+outDir)
+	errOut, err := cmd.CombinedOutput()
+	if err == nil {
+		return
 	}
-	return bin
+	if strings.Contains(string(errOut), "when GOBIN is set") {
+		// Fallback slow path for cross-compiled binaries.
+		for _, target := range targets {
+			outFile := filepath.Join(outDir, path.Base(target)+exe())
+			cmd := exec.Command(goBin, "build", "-o", outFile, target)
+			cmd.Env = append(os.Environ(), "GOARCH="+runtime.GOARCH)
+			if errOut, err := cmd.CombinedOutput(); err != nil {
+				t.Fatalf("failed to build %v with %v: %v, %s", target, goBin, err, errOut)
+			}
+		}
+		return
+	}
+	t.Fatalf("failed to build %v with %v: %v, %s", targets, goBin, err, errOut)
 }
 
+// logCatcher is a minimal logcatcher for the logtail upload client.
 type logCatcher struct {
 	mu     sync.Mutex
 	buf    bytes.Buffer
@@ -227,7 +553,102 @@ func (lc *logCatcher) ServeHTTP(w http.ResponseWriter, r *http.Request) {
 	} else {
 		for _, ent := range jreq {
 			fmt.Fprintf(&lc.buf, "%s\n", strings.TrimSpace(ent.Text))
+			if *verbose {
+				fmt.Fprintf(os.Stderr, "%s\n", strings.TrimSpace(ent.Text))
+			}
 		}
 	}
 	w.WriteHeader(200) // must have no content, but not a 204
 }
+
+// trafficTrap is an HTTP proxy handler to note whether any
+// HTTP traffic tries to leave localhost from tailscaled. We don't
+// expect any, so any request triggers a failure.
+type trafficTrap struct {
+	atomicErr atomic.Value // of error
+}
+
+func (tt *trafficTrap) Err() error {
+	if err, ok := tt.atomicErr.Load().(error); ok {
+		return err
+	}
+	return nil
+}
+
+func (tt *trafficTrap) ServeHTTP(w http.ResponseWriter, r *http.Request) {
+	var got bytes.Buffer
+	r.Write(&got)
+	err := fmt.Errorf("unexpected HTTP proxy via proxy: %s", got.Bytes())
+	mainError.Store(err)
+	if tt.Err() == nil {
+		// Best effort at remembering the first request.
+		tt.atomicErr.Store(err)
+	}
+	log.Printf("Error: %v", err)
+	w.WriteHeader(403)
+}
+
+func runDERPAndStun(t testing.TB, logf logger.Logf) (derpMap *tailcfg.DERPMap, cleanup func()) {
+	var serverPrivateKey key.Private
+	if _, err := crand.Read(serverPrivateKey[:]); err != nil {
+		t.Fatal(err)
+	}
+	d := derp.NewServer(serverPrivateKey, logf)
+
+	httpsrv := httptest.NewUnstartedServer(derphttp.Handler(d))
+	httpsrv.Config.ErrorLog = logger.StdLogger(logf)
+	httpsrv.Config.TLSNextProto = make(map[string]func(*http.Server, *tls.Conn, http.Handler))
+	httpsrv.StartTLS()
+
+	stunAddr, stunCleanup := stuntest.ServeWithPacketListener(t, nettype.Std{})
+
+	m := &tailcfg.DERPMap{
+		Regions: map[int]*tailcfg.DERPRegion{
+			1: {
+				RegionID:   1,
+				RegionCode: "test",
+				Nodes: []*tailcfg.DERPNode{
+					{
+						Name:         "t1",
+						RegionID:     1,
+						HostName:     "127.0.0.1", // to bypass HTTP proxy
+						IPv4:         "127.0.0.1",
+						IPv6:         "none",
+						STUNPort:     stunAddr.Port,
+						DERPTestPort: httpsrv.Listener.Addr().(*net.TCPAddr).Port,
+						STUNTestIP:   stunAddr.IP.String(),
+					},
+				},
+			},
+		},
+	}
+
+	cleanup = func() {
+		httpsrv.CloseClientConnections()
+		httpsrv.Close()
+		d.Close()
+		stunCleanup()
+	}
+
+	return m, cleanup
+}
+
+type authURLParserWriter struct {
+	buf bytes.Buffer
+	fn  func(urlStr string) error
+}
+
+var authURLRx = regexp.MustCompile(`(https?://\S+/auth/\S+)`)
+
+func (w *authURLParserWriter) Write(p []byte) (n int, err error) {
+	n, err = w.buf.Write(p)
+	m := authURLRx.FindSubmatch(w.buf.Bytes())
+	if m != nil {
+		urlStr := string(m[1])
+		w.buf.Reset() // so it's not matched again
+		if err := w.fn(urlStr); err != nil {
+			return 0, err
+		}
+	}
+	return n, err
+}

tstest/integration/testcontrol/testcontrol.go

@@ -17,6 +17,8 @@ import (
 	"log"
 	"math/rand"
 	"net/http"
+	"net/url"
+	"sort"
 	"strings"
 	"sync"
 	"time"
@@ -34,19 +36,43 @@ import (
 // Server is a control plane server. Its zero value is ready for use.
 // Everything is stored in-memory in one tailnet.
 type Server struct {
-	Logf    logger.Logf      // nil means to use the log package
-	DERPMap *tailcfg.DERPMap // nil means to use prod DERP map
+	Logf        logger.Logf      // nil means to use the log package
+	DERPMap     *tailcfg.DERPMap // nil means to use prod DERP map
+	RequireAuth bool
+	BaseURL     string // must be set to e.g. "http://127.0.0.1:1234" with no trailing URL
+	Verbose     bool
 
 	initMuxOnce sync.Once
 	mux         *http.ServeMux
 
-	mu      sync.Mutex
-	pubKey  wgkey.Key
-	privKey wgkey.Private
-	nodes   map[tailcfg.NodeKey]*tailcfg.Node
-	users   map[tailcfg.NodeKey]*tailcfg.User
-	logins  map[tailcfg.NodeKey]*tailcfg.Login
-	updates map[tailcfg.NodeID]chan updateType
+	mu            sync.Mutex
+	pubKey        wgkey.Key
+	privKey       wgkey.Private
+	nodes         map[tailcfg.NodeKey]*tailcfg.Node
+	users         map[tailcfg.NodeKey]*tailcfg.User
+	logins        map[tailcfg.NodeKey]*tailcfg.Login
+	updates       map[tailcfg.NodeID]chan updateType
+	authPath      map[string]*AuthPath
+	nodeKeyAuthed map[tailcfg.NodeKey]bool // key => true once authenticated
+}
+
+type AuthPath struct {
+	nodeKey tailcfg.NodeKey
+
+	closeOnce sync.Once
+	ch        chan struct{}
+	success   bool
+}
+
+func (ap *AuthPath) completeSuccessfully() {
+	ap.success = true
+	close(ap.ch)
+}
+
+// CompleteSuccessfully completes the login path successfully, as if
+// the user did the whole auth dance.
+func (ap *AuthPath) CompleteSuccessfully() {
+	ap.closeOnce.Do(ap.completeSuccessfully)
 }
 
 func (s *Server) logf(format string, a ...interface{}) {
@@ -142,6 +168,18 @@ func (s *Server) Node(nodeKey tailcfg.NodeKey) *tailcfg.Node {
 	return s.nodes[nodeKey].Clone()
 }
 
+func (s *Server) AllNodes() (nodes []*tailcfg.Node) {
+	s.mu.Lock()
+	defer s.mu.Unlock()
+	for _, n := range s.nodes {
+		nodes = append(nodes, n.Clone())
+	}
+	sort.Slice(nodes, func(i, j int) bool {
+		return nodes[i].StableID < nodes[j].StableID
+	})
+	return nodes
+}
+
 func (s *Server) getUser(nodeKey tailcfg.NodeKey) (*tailcfg.User, *tailcfg.Login) {
 	s.mu.Lock()
 	defer s.mu.Unlock()
@@ -178,6 +216,56 @@ func (s *Server) getUser(nodeKey tailcfg.NodeKey) (*tailcfg.User, *tailcfg.Login
 	return user, login
 }
 
+// authPathDone returns a close-only struct that's closed when the
+// authPath ("/auth/XXXXXX") has authenticated.
+func (s *Server) authPathDone(authPath string) <-chan struct{} {
+	s.mu.Lock()
+	defer s.mu.Unlock()
+	if a, ok := s.authPath[authPath]; ok {
+		return a.ch
+	}
+	return nil
+}
+
+func (s *Server) addAuthPath(authPath string, nodeKey tailcfg.NodeKey) {
+	s.mu.Lock()
+	defer s.mu.Unlock()
+	if s.authPath == nil {
+		s.authPath = map[string]*AuthPath{}
+	}
+	s.authPath[authPath] = &AuthPath{
+		ch:      make(chan struct{}),
+		nodeKey: nodeKey,
+	}
+}
+
+// CompleteAuth marks the provided path or URL (containing
+// "/auth/...")  as successfully authenticated, unblocking any
+// requests blocked on that in serveRegister.
+func (s *Server) CompleteAuth(authPathOrURL string) bool {
+	i := strings.Index(authPathOrURL, "/auth/")
+	if i == -1 {
+		return false
+	}
+	authPath := authPathOrURL[i:]
+
+	s.mu.Lock()
+	defer s.mu.Unlock()
+	ap, ok := s.authPath[authPath]
+	if !ok {
+		return false
+	}
+	if ap.nodeKey.IsZero() {
+		panic("zero AuthPath.NodeKey")
+	}
+	if s.nodeKeyAuthed == nil {
+		s.nodeKeyAuthed = map[tailcfg.NodeKey]bool{}
+	}
+	s.nodeKeyAuthed[ap.nodeKey] = true
+	ap.CompleteSuccessfully()
+	return true
+}
+
 func (s *Server) serveRegister(w http.ResponseWriter, r *http.Request, mkey tailcfg.MachineKey) {
 	var req tailcfg.RegisterRequest
 	if err := s.decode(mkey, r.Body, &req); err != nil {
@@ -189,28 +277,65 @@ func (s *Server) serveRegister(w http.ResponseWriter, r *http.Request, mkey tail
 	if req.NodeKey.IsZero() {
 		panic("serveRegister: request has zero node key")
 	}
+	if s.Verbose {
+		j, _ := json.MarshalIndent(req, "", "\t")
+		log.Printf("Got %T: %s", req, j)
+	}
+
+	// If this is a followup request, wait until interactive followup URL visit complete.
+	if req.Followup != "" {
+		followupURL, err := url.Parse(req.Followup)
+		if err != nil {
+			panic(err)
+		}
+		doneCh := s.authPathDone(followupURL.Path)
+		select {
+		case <-r.Context().Done():
+			return
+		case <-doneCh:
+		}
+		// TODO(bradfitz): support a side test API to mark an
+		// auth as failued so we can send an error response in
+		// some follow-ups? For now all are successes.
+	}
 
 	user, login := s.getUser(req.NodeKey)
 	s.mu.Lock()
 	if s.nodes == nil {
 		s.nodes = map[tailcfg.NodeKey]*tailcfg.Node{}
 	}
+
+	machineAuthorized := true // TODO: add Server.RequireMachineAuth
+
 	s.nodes[req.NodeKey] = &tailcfg.Node{
 		ID:                tailcfg.NodeID(user.ID),
 		StableID:          tailcfg.StableNodeID(fmt.Sprintf("TESTCTRL%08x", int(user.ID))),
 		User:              user.ID,
 		Machine:           mkey,
 		Key:               req.NodeKey,
-		MachineAuthorized: true,
+		MachineAuthorized: machineAuthorized,
+	}
+	requireAuth := s.RequireAuth
+	if requireAuth && s.nodeKeyAuthed[req.NodeKey] {
+		requireAuth = false
 	}
 	s.mu.Unlock()
 
+	authURL := ""
+	if requireAuth {
+		randHex := make([]byte, 10)
+		crand.Read(randHex)
+		authPath := fmt.Sprintf("/auth/%x", randHex)
+		s.addAuthPath(authPath, req.NodeKey)
+		authURL = s.BaseURL + authPath
+	}
+
 	res, err := s.encode(mkey, false, tailcfg.RegisterResponse{
 		User:              *user,
 		Login:             *login,
 		NodeKeyExpired:    false,
-		MachineAuthorized: true,
-		AuthURL:           "", // all good; TODO(bradfitz): add ways to not start all good.
+		MachineAuthorized: machineAuthorized,
+		AuthURL:           authURL,
 	})
 	if err != nil {
 		go panic(fmt.Sprintf("serveRegister: encode: %v", err))
@@ -254,6 +379,21 @@ func sendUpdate(dst chan<- updateType, updateType updateType) {
 	}
 }
 
+func (s *Server) UpdateNode(n *tailcfg.Node) (peersToUpdate []tailcfg.NodeID) {
+	s.mu.Lock()
+	defer s.mu.Unlock()
+	if n.Key.IsZero() {
+		panic("zero nodekey")
+	}
+	s.nodes[n.Key] = n.Clone()
+	for _, n2 := range s.nodes {
+		if n.ID != n2.ID {
+			peersToUpdate = append(peersToUpdate, n2.ID)
+		}
+	}
+	return peersToUpdate
+}
+
 func (s *Server) serveMap(w http.ResponseWriter, r *http.Request, mkey tailcfg.MachineKey) {
 	ctx := r.Context()
 
@@ -279,10 +419,8 @@ func (s *Server) serveMap(w http.ResponseWriter, r *http.Request, mkey tailcfg.M
 	if !req.ReadOnly {
 		endpoints := filterInvalidIPv6Endpoints(req.Endpoints)
 		node.Endpoints = endpoints
-		// TODO: more
-		// TODO: register node,
-		//s.UpdateEndpoint(mkey, req.NodeKey,
-		// XXX
+		node.DiscoKey = req.DiscoKey
+		peersToUpdate = s.UpdateNode(node)
 	}
 
 	nodeID := node.ID
@@ -389,6 +527,12 @@ func (s *Server) MapResponse(req *tailcfg.MapRequest) (res *tailcfg.MapResponse,
 		CollectServices: "true",
 		PacketFilter:    tailcfg.FilterAllowAll,
 	}
+	for _, p := range s.AllNodes() {
+		if p.StableID != node.StableID {
+			res.Peers = append(res.Peers, p)
+		}
+	}
+
 	res.Node.Addresses = []netaddr.IPPrefix{
 		netaddr.MustParseIPPrefix(fmt.Sprintf("100.64.%d.%d/32", uint8(node.ID>>8), uint8(node.ID))),
 	}

tstest/log.go

@@ -107,6 +107,15 @@ func (lt *LogLineTracker) Check() []string {
 	return notSeen
 }
 
+// Reset forgets everything that it's seen.
+func (lt *LogLineTracker) Reset() {
+	lt.mu.Lock()
+	defer lt.mu.Unlock()
+	for _, line := range lt.listenFor {
+		lt.seen[line] = false
+	}
+}
+
 // Close closes lt. After calling Close, calls to Logf become no-ops.
 func (lt *LogLineTracker) Close() {
 	lt.mu.Lock()

types/logger/logger.go

@@ -63,10 +63,15 @@ type limitData struct {
 var disableRateLimit = os.Getenv("TS_DEBUG_LOG_RATE") == "all"
 
 // rateFree are format string substrings that are exempt from rate limiting.
-// Things should not be added to this unless they're already limited otherwise.
+// Things should not be added to this unless they're already limited otherwise
+// or are critical for generating important stats from the logs.
 var rateFree = []string{
 	"magicsock: disco: ",
-	"magicsock: CreateEndpoint:",
+	"magicsock: ParseEndpoint:",
+	// grinder stats lines
+	"SetPrefs: %v",
+	"peer keys: %s",
+	"v%v peers: %v",
 }
 
 // RateLimitedFn is a wrapper for RateLimitedFnWithClock that includes the
@@ -179,7 +184,6 @@ func LogOnChange(logf Logf, maxInterval time.Duration, timeNow func() time.Time)
 		// as it might contain formatting directives.)
 		logf(format, args...)
 	}
-
 }
 
 // ArgWriter is a fmt.Formatter that can be passed to any Logf func to

types/netmap/netmap.go

@@ -31,8 +31,8 @@ type NetworkMap struct {
 	Expiry     time.Time
 	// Name is the DNS name assigned to this node.
 	Name          string
-	Addresses     []netaddr.IPPrefix
-	LocalPort     uint16 // used for debugging
+	Addresses     []netaddr.IPPrefix // same as tailcfg.Node.Addresses (IP addresses of this Node directly)
+	LocalPort     uint16             // used for debugging
 	MachineStatus tailcfg.MachineStatus
 	MachineKey    tailcfg.MachineKey
 	Peers         []*tailcfg.Node // sorted by Node.ID

types/wgkey/key.go

@@ -78,8 +78,16 @@ func (k Key) HexString() string { return hex.EncodeToString(k[:]) }
 func (k Key) Equal(k2 Key) bool { return subtle.ConstantTimeCompare(k[:], k2[:]) == 1 }
 
 func (k *Key) ShortString() string {
-	long := k.Base64()
-	return "[" + long[0:5] + "]"
+	// The goal here is to generate "[" + base64.StdEncoding.EncodeToString(k[:])[:5] + "]".
+	// Since we only care about the first 5 characters, it suffices to encode the first 4 bytes of k.
+	// Encoding those 4 bytes requires 8 bytes.
+	// Make dst have size 9, to fit the leading '[' plus those 8 bytes.
+	// We slice the unused ones away at the end.
+	dst := make([]byte, 9)
+	dst[0] = '['
+	base64.StdEncoding.Encode(dst[1:], k[:4])
+	dst[6] = ']'
+	return string(dst[:7])
 }
 
 func (k *Key) IsZero() bool {
@@ -106,11 +114,10 @@ func (k *Key) UnmarshalJSON(b []byte) error {
 		return errors.New("wgkey.Key: UnmarshalJSON not given a string")
 	}
 	b = b[1 : len(b)-1]
-	key, err := ParseHex(string(b))
-	if err != nil {
-		return fmt.Errorf("wgkey.Key: UnmarshalJSON: %v", err)
+	if len(b) != 2*Size {
+		return fmt.Errorf("wgkey.Key: UnmarshalJSON input wrong size: %d", len(b))
 	}
-	copy(k[:], key[:])
+	hex.Decode(k[:], b)
 	return nil
 }
 

types/wgkey/key_test.go

@@ -156,3 +156,28 @@ func BenchmarkMarshalJSON(b *testing.B) {
 		}
 	}
 }
+
+func BenchmarkUnmarshalJSON(b *testing.B) {
+	b.ReportAllocs()
+	var k Key
+	buf, err := k.MarshalJSON()
+	if err != nil {
+		b.Fatal(err)
+	}
+	for i := 0; i < b.N; i++ {
+		err := k.UnmarshalJSON(buf)
+		if err != nil {
+			b.Fatal(err)
+		}
+	}
+}
+
+var sinkString string
+
+func BenchmarkShortString(b *testing.B) {
+	b.ReportAllocs()
+	var k Key
+	for i := 0; i < b.N; i++ {
+		sinkString = k.ShortString()
+	}
+}

util/dnsname/dnsname.go

@@ -24,13 +24,16 @@ func ToFQDN(s string) (FQDN, error) {
 	if isValidFQDN(s) {
 		return FQDN(s), nil
 	}
-	if len(s) == 0 {
+	if len(s) == 0 || s == "." {
 		return FQDN("."), nil
 	}
 
 	if s[len(s)-1] == '.' {
 		s = s[:len(s)-1]
 	}
+	if s[0] == '.' {
+		s = s[1:]
+	}
 	if len(s) > maxNameLength {
 		return "", fmt.Errorf("%q is too long to be a DNS name", s)
 	}

util/dnsname/dnsname_test.go

@@ -20,11 +20,12 @@ func TestFQDN(t *testing.T) {
 		{".", ".", false, 0},
 		{"foo.com", "foo.com.", false, 2},
 		{"foo.com.", "foo.com.", false, 2},
+		{".foo.com.", "foo.com.", false, 2},
+		{".foo.com", "foo.com.", false, 2},
 		{"com", "com.", false, 1},
 		{"www.tailscale.com", "www.tailscale.com.", false, 3},
 		{"aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa.com", "", true, 0},
 		{strings.Repeat("aaaaa.", 60) + "com", "", true, 0},
-		{".com", "", true, 0},
 		{"foo..com", "", true, 0},
 	}
 

version/race.go

@@ -0,0 +1,11 @@
+// Copyright (c) 2021 Tailscale Inc & AUTHORS All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+// +build race
+
+package version
+
+// IsRace reports whether the current binary was built with the Go
+// race detector enabled.
+func IsRace() bool { return true }

version/race_off.go

@@ -0,0 +1,11 @@
+// Copyright (c) 2021 Tailscale Inc & AUTHORS All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+// +build !race
+
+package version
+
+// IsRace reports whether the current binary was built with the Go
+// race detector enabled.
+func IsRace() bool { return false }

version/version.go

@@ -10,7 +10,7 @@ package version
 // Long is a full version number for this build, of the form
 // "x.y.z-commithash", or "date.yyyymmdd" if no actual version was
 // provided.
-const Long = "date.20210427"
+const Long = "date.20210505"
 
 // Short is a short version number for this build, of the form
 // "x.y.z", or "date.yyyymmdd" if no actual version was provided.

wf/firewall.go

@@ -0,0 +1,510 @@
+// Copyright (c) 2021 Tailscale Inc & AUTHORS All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+// +build windows
+
+package wf
+
+import (
+	"fmt"
+	"os"
+
+	"golang.org/x/sys/windows"
+	"inet.af/netaddr"
+	"inet.af/wf"
+)
+
+// Known addresses.
+var (
+	linkLocalRange           = netaddr.MustParseIPPrefix("ff80::/10")
+	linkLocalDHCPMulticast   = netaddr.MustParseIP("ff02::1:2")
+	siteLocalDHCPMulticast   = netaddr.MustParseIP("ff05::1:3")
+	linkLocalRouterMulticast = netaddr.MustParseIP("ff02::2")
+)
+
+type direction int
+
+const (
+	directionInbound direction = iota
+	directionOutbound
+	directionBoth
+)
+
+type protocol int
+
+const (
+	protocolV4 protocol = iota
+	protocolV6
+	protocolAll
+)
+
+// getLayers returns the wf.LayerIDs where the rules should be added based
+// on the protocol and direction.
+func (p protocol) getLayers(d direction) []wf.LayerID {
+	var layers []wf.LayerID
+	if p == protocolAll || p == protocolV4 {
+		if d == directionBoth || d == directionInbound {
+			layers = append(layers, wf.LayerALEAuthRecvAcceptV4)
+		}
+		if d == directionBoth || d == directionOutbound {
+			layers = append(layers, wf.LayerALEAuthConnectV4)
+		}
+	}
+	if p == protocolAll || p == protocolV6 {
+		if d == directionBoth || d == directionInbound {
+			layers = append(layers, wf.LayerALEAuthRecvAcceptV6)
+		}
+		if d == directionBoth || d == directionOutbound {
+			layers = append(layers, wf.LayerALEAuthConnectV6)
+		}
+	}
+	return layers
+}
+
+func ruleName(action wf.Action, l wf.LayerID, name string) string {
+	switch l {
+	case wf.LayerALEAuthConnectV4:
+		return fmt.Sprintf("%s outbound %s (IPv4)", action, name)
+	case wf.LayerALEAuthConnectV6:
+		return fmt.Sprintf("%s outbound %s (IPv6)", action, name)
+	case wf.LayerALEAuthRecvAcceptV4:
+		return fmt.Sprintf("%s inbound %s (IPv4)", action, name)
+	case wf.LayerALEAuthRecvAcceptV6:
+		return fmt.Sprintf("%s inbound %s (IPv6)", action, name)
+	}
+	return ""
+}
+
+// Firewall uses the Windows Filtering Platform to implement a network firewall.
+type Firewall struct {
+	luid       uint64
+	providerID wf.ProviderID
+	sublayerID wf.SublayerID
+	session    *wf.Session
+
+	permittedRoutes map[netaddr.IPPrefix][]*wf.Rule
+}
+
+// New returns a new Firewall for the provdied interface ID.
+func New(luid uint64) (*Firewall, error) {
+	session, err := wf.New(&wf.Options{
+		Name:    "Tailscale firewall",
+		Dynamic: true,
+	})
+	if err != nil {
+		return nil, err
+	}
+	wguid, err := windows.GenerateGUID()
+	if err != nil {
+		return nil, err
+	}
+	providerID := wf.ProviderID(wguid)
+	if err := session.AddProvider(&wf.Provider{
+		ID:   providerID,
+		Name: "Tailscale provider",
+	}); err != nil {
+		return nil, err
+	}
+	wguid, err = windows.GenerateGUID()
+	if err != nil {
+		return nil, err
+	}
+	sublayerID := wf.SublayerID(wguid)
+	if err := session.AddSublayer(&wf.Sublayer{
+		ID:     sublayerID,
+		Name:   "Tailscale permissive and blocking filters",
+		Weight: 0,
+	}); err != nil {
+		return nil, err
+	}
+	f := &Firewall{
+		luid:            luid,
+		session:         session,
+		providerID:      providerID,
+		sublayerID:      sublayerID,
+		permittedRoutes: make(map[netaddr.IPPrefix][]*wf.Rule),
+	}
+	if err := f.enable(); err != nil {
+		return nil, err
+	}
+	return f, nil
+}
+
+type weight uint64
+
+const (
+	weightTailscaleTraffic weight = 15
+	weightKnownTraffic     weight = 12
+	weightCatchAll         weight = 0
+)
+
+func (f *Firewall) enable() error {
+	if err := f.permitTailscaleService(weightTailscaleTraffic); err != nil {
+		return fmt.Errorf("permitTailscaleService failed: %w", err)
+	}
+
+	if err := f.permitTunInterface(weightTailscaleTraffic); err != nil {
+		return fmt.Errorf("permitTunInterface failed: %w", err)
+	}
+
+	if err := f.permitDNS(weightTailscaleTraffic); err != nil {
+		return fmt.Errorf("permitDNS failed: %w", err)
+	}
+
+	if err := f.permitLoopback(weightKnownTraffic); err != nil {
+		return fmt.Errorf("permitLoopback failed: %w", err)
+	}
+
+	if err := f.permitDHCPv4(weightKnownTraffic); err != nil {
+		return fmt.Errorf("permitDHCPv4 failed: %w", err)
+	}
+
+	if err := f.permitDHCPv6(weightKnownTraffic); err != nil {
+		return fmt.Errorf("permitDHCPv6 failed: %w", err)
+	}
+
+	if err := f.permitNDP(weightKnownTraffic); err != nil {
+		return fmt.Errorf("permitNDP failed: %w", err)
+	}
+
+	/* TODO: actually evaluate if this does anything and if we need this. It's layer 2; our other rules are layer 3.
+	 *  In other words, if somebody complains, try enabling it. For now, keep it off.
+	 * TODO(maisem): implement this.
+	err = permitHyperV(session, baseObjects, weightKnownTraffic)
+	if err != nil {
+		return wrapErr(err)
+	}
+	*/
+
+	if err := f.blockAll(weightCatchAll); err != nil {
+		return fmt.Errorf("blockAll failed: %w", err)
+	}
+	return nil
+}
+
+// UpdatedPermittedRoutes adds rules to allow incoming and outgoing connections
+// from the provided prefixes. It will also remove rules for routes that were
+// previously added but have been removed.
+func (f *Firewall) UpdatePermittedRoutes(newRoutes []netaddr.IPPrefix) error {
+	var routesToAdd []netaddr.IPPrefix
+	routeMap := make(map[netaddr.IPPrefix]bool)
+	for _, r := range newRoutes {
+		routeMap[r] = true
+		if _, ok := f.permittedRoutes[r]; !ok {
+			routesToAdd = append(routesToAdd, r)
+		}
+	}
+	var routesToRemove []netaddr.IPPrefix
+	for r := range f.permittedRoutes {
+		if !routeMap[r] {
+			routesToRemove = append(routesToRemove, r)
+		}
+	}
+	for _, r := range routesToRemove {
+		for _, rule := range f.permittedRoutes[r] {
+			if err := f.session.DeleteRule(rule.ID); err != nil {
+				return err
+			}
+		}
+		delete(f.permittedRoutes, r)
+	}
+	for _, r := range routesToAdd {
+		conditions := []*wf.Match{
+			{
+				Field: wf.FieldIPRemoteAddress,
+				Op:    wf.MatchTypeEqual,
+				Value: r,
+			},
+		}
+		var p protocol
+		if r.IP.Is4() {
+			p = protocolV4
+		} else {
+			p = protocolV6
+		}
+		rules, err := f.addRules("local route", weightKnownTraffic, conditions, wf.ActionPermit, p, directionBoth)
+		if err != nil {
+			return err
+		}
+		f.permittedRoutes[r] = rules
+	}
+	return nil
+}
+
+func (f *Firewall) newRule(name string, w weight, layer wf.LayerID, conditions []*wf.Match, action wf.Action) (*wf.Rule, error) {
+	id, err := windows.GenerateGUID()
+	if err != nil {
+		return nil, err
+	}
+	return &wf.Rule{
+		Name:       ruleName(action, layer, name),
+		ID:         wf.RuleID(id),
+		Provider:   f.providerID,
+		Sublayer:   f.sublayerID,
+		Layer:      layer,
+		Weight:     uint64(w),
+		Conditions: conditions,
+		Action:     action,
+	}, nil
+}
+
+func (f *Firewall) addRules(name string, w weight, conditions []*wf.Match, action wf.Action, p protocol, d direction) ([]*wf.Rule, error) {
+	var rules []*wf.Rule
+	for _, l := range p.getLayers(d) {
+		r, err := f.newRule(name, w, l, conditions, action)
+		if err != nil {
+			return nil, err
+		}
+		if err := f.session.AddRule(r); err != nil {
+			return nil, err
+		}
+		rules = append(rules, r)
+	}
+	return rules, nil
+}
+
+func (f *Firewall) blockAll(w weight) error {
+	_, err := f.addRules("all", w, nil, wf.ActionBlock, protocolAll, directionBoth)
+	return err
+}
+
+func (f *Firewall) permitNDP(w weight) error {
+	// These are aliased according to:
+	// https://social.msdn.microsoft.com/Forums/azure/en-US/eb2aa3cd-5f1c-4461-af86-61e7d43ccc23/filtering-icmp-by-type-code?forum=wfp
+	fieldICMPType := wf.FieldIPLocalPort
+	fieldICMPCode := wf.FieldIPRemotePort
+
+	var icmpConditions = func(t, c uint16, remoteAddress interface{}) []*wf.Match {
+		conditions := []*wf.Match{
+			{
+				Field: wf.FieldIPProtocol,
+				Op:    wf.MatchTypeEqual,
+				Value: wf.IPProtoICMPV6,
+			},
+			{
+				Field: fieldICMPType,
+				Op:    wf.MatchTypeEqual,
+				Value: t,
+			},
+			{
+				Field: fieldICMPCode,
+				Op:    wf.MatchTypeEqual,
+				Value: c,
+			},
+		}
+		if remoteAddress != nil {
+			conditions = append(conditions, &wf.Match{
+				Field: wf.FieldIPRemoteAddress,
+				Op:    wf.MatchTypeEqual,
+				Value: linkLocalRouterMulticast,
+			})
+		}
+		return conditions
+	}
+	/* TODO: actually handle the hop limit somehow! The rules should vaguely be:
+	 *  - icmpv6 133: must be outgoing, dst must be FF02::2/128, hop limit must be 255
+	 *  - icmpv6 134: must be incoming, src must be FE80::/10, hop limit must be 255
+	 *  - icmpv6 135: either incoming or outgoing, hop limit must be 255
+	 *  - icmpv6 136: either incoming or outgoing, hop limit must be 255
+	 *  - icmpv6 137: must be incoming, src must be FE80::/10, hop limit must be 255
+	 */
+
+	//
+	// Router Solicitation Message
+	// ICMP type 133, code 0. Outgoing.
+	//
+	conditions := icmpConditions(133, 0, linkLocalRouterMulticast)
+	if _, err := f.addRules("NDP type 133", w, conditions, wf.ActionPermit, protocolV6, directionOutbound); err != nil {
+		return err
+	}
+
+	//
+	// Router Advertisement Message
+	// ICMP type 134, code 0. Incoming.
+	//
+	conditions = icmpConditions(134, 0, linkLocalRange)
+	if _, err := f.addRules("NDP type 134", w, conditions, wf.ActionPermit, protocolV6, directionInbound); err != nil {
+		return err
+	}
+
+	//
+	// Neighbor Solicitation Message
+	// ICMP type 135, code 0. Bi-directional.
+	//
+	conditions = icmpConditions(135, 0, nil)
+	if _, err := f.addRules("NDP type 135", w, conditions, wf.ActionPermit, protocolV6, directionBoth); err != nil {
+		return err
+	}
+
+	//
+	// Neighbor Advertisement Message
+	// ICMP type 136, code 0. Bi-directional.
+	//
+	conditions = icmpConditions(136, 0, nil)
+	if _, err := f.addRules("NDP type 136", w, conditions, wf.ActionPermit, protocolV6, directionBoth); err != nil {
+		return err
+	}
+
+	//
+	// Redirect Message
+	// ICMP type 137, code 0. Incoming.
+	//
+	conditions = icmpConditions(137, 0, linkLocalRange)
+	if _, err := f.addRules("NDP type 137", w, conditions, wf.ActionPermit, protocolV6, directionInbound); err != nil {
+		return err
+	}
+	return nil
+}
+
+func (f *Firewall) permitDHCPv6(w weight) error {
+	var dhcpConditions = func(remoteAddrs ...interface{}) []*wf.Match {
+		conditions := []*wf.Match{
+			{
+				Field: wf.FieldIPProtocol,
+				Op:    wf.MatchTypeEqual,
+				Value: wf.IPProtoUDP,
+			},
+			{
+				Field: wf.FieldIPLocalAddress,
+				Op:    wf.MatchTypeEqual,
+				Value: linkLocalRange,
+			},
+			{
+				Field: wf.FieldIPLocalPort,
+				Op:    wf.MatchTypeEqual,
+				Value: uint16(546),
+			},
+			{
+				Field: wf.FieldIPRemotePort,
+				Op:    wf.MatchTypeEqual,
+				Value: uint16(547),
+			},
+		}
+		for _, a := range remoteAddrs {
+			conditions = append(conditions, &wf.Match{
+				Field: wf.FieldIPRemoteAddress,
+				Op:    wf.MatchTypeEqual,
+				Value: a,
+			})
+		}
+		return conditions
+	}
+	conditions := dhcpConditions(linkLocalDHCPMulticast, siteLocalDHCPMulticast)
+	if _, err := f.addRules("DHCP request", w, conditions, wf.ActionPermit, protocolV6, directionOutbound); err != nil {
+		return err
+	}
+	conditions = dhcpConditions(linkLocalRange)
+	if _, err := f.addRules("DHCP response", w, conditions, wf.ActionPermit, protocolV6, directionInbound); err != nil {
+		return err
+	}
+	return nil
+}
+
+func (f *Firewall) permitDHCPv4(w weight) error {
+	var dhcpConditions = func(remoteAddrs ...interface{}) []*wf.Match {
+		conditions := []*wf.Match{
+			{
+				Field: wf.FieldIPProtocol,
+				Op:    wf.MatchTypeEqual,
+				Value: wf.IPProtoUDP,
+			},
+			{
+				Field: wf.FieldIPLocalPort,
+				Op:    wf.MatchTypeEqual,
+				Value: uint16(68),
+			},
+			{
+				Field: wf.FieldIPRemotePort,
+				Op:    wf.MatchTypeEqual,
+				Value: uint16(67),
+			},
+		}
+		for _, a := range remoteAddrs {
+			conditions = append(conditions, &wf.Match{
+				Field: wf.FieldIPRemoteAddress,
+				Op:    wf.MatchTypeEqual,
+				Value: a,
+			})
+		}
+		return conditions
+	}
+	conditions := dhcpConditions(netaddr.IPv4(255, 255, 255, 255))
+	if _, err := f.addRules("DHCP request", w, conditions, wf.ActionPermit, protocolV4, directionOutbound); err != nil {
+		return err
+	}
+
+	conditions = dhcpConditions()
+	if _, err := f.addRules("DHCP response", w, conditions, wf.ActionPermit, protocolV4, directionInbound); err != nil {
+		return err
+	}
+	return nil
+}
+
+func (f *Firewall) permitTunInterface(w weight) error {
+	condition := []*wf.Match{
+		{
+			Field: wf.FieldIPLocalInterface,
+			Op:    wf.MatchTypeEqual,
+			Value: f.luid,
+		},
+	}
+	_, err := f.addRules("on TUN", w, condition, wf.ActionPermit, protocolAll, directionBoth)
+	return err
+}
+
+func (f *Firewall) permitLoopback(w weight) error {
+	condition := []*wf.Match{
+		{
+			Field: wf.FieldFlags,
+			Op:    wf.MatchTypeEqual,
+			Value: wf.ConditionFlagIsLoopback,
+		},
+	}
+	_, err := f.addRules("on loopback", w, condition, wf.ActionPermit, protocolAll, directionBoth)
+	return err
+}
+
+func (f *Firewall) permitDNS(w weight) error {
+	conditions := []*wf.Match{
+		{
+			Field: wf.FieldIPRemotePort,
+			Op:    wf.MatchTypeEqual,
+			Value: uint16(53),
+		},
+		// Repeat the condition type for logical OR.
+		{
+			Field: wf.FieldIPProtocol,
+			Op:    wf.MatchTypeEqual,
+			Value: wf.IPProtoUDP,
+		},
+		{
+			Field: wf.FieldIPProtocol,
+			Op:    wf.MatchTypeEqual,
+			Value: wf.IPProtoTCP,
+		},
+	}
+	_, err := f.addRules("DNS", w, conditions, wf.ActionPermit, protocolAll, directionBoth)
+	return err
+}
+
+func (f *Firewall) permitTailscaleService(w weight) error {
+	currentFile, err := os.Executable()
+	if err != nil {
+		return err
+	}
+
+	appID, err := wf.AppID(currentFile)
+	if err != nil {
+		return fmt.Errorf("could not get app id for %q: %w", currentFile, err)
+	}
+	conditions := []*wf.Match{
+		{
+			Field: wf.FieldALEAppID,
+			Op:    wf.MatchTypeEqual,
+			Value: appID,
+		},
+	}
+	_, err = f.addRules("unrestricted traffic for Tailscale service", w, conditions, wf.ActionPermit, protocolAll, directionBoth)
+	return err
+}

wgengine/bench/bench.go

@@ -80,7 +80,7 @@ func main() {
 
 	// tx=134236 rx=133166 (1070 = 0.80% loss) (1088.9 Mbits/sec)
 	case 101:
-		setupWGTest(logf, traf, Addr1, Addr2)
+		setupWGTest(nil, logf, traf, Addr1, Addr2)
 
 	default:
 		log.Fatalf("provide a valid test number (0..n)")

wgengine/bench/bench_test.go

@@ -43,7 +43,7 @@ func BenchmarkBatchTCP(b *testing.B) {
 
 func BenchmarkWireGuardTest(b *testing.B) {
 	run(b, func(logf logger.Logf, traf *TrafficGen) {
-		setupWGTest(logf, traf, Addr1, Addr2)
+		setupWGTest(b, logf, traf, Addr1, Addr2)
 	})
 }
 

wgengine/bench/trafficgen.go

@@ -180,6 +180,7 @@ func (t *TrafficGen) Generate(b []byte, ofs int) int {
 // GotPacket processes a packet that came back on the receive side.
 func (t *TrafficGen) GotPacket(b []byte, ofs int) {
 	t.mu.Lock()
+	defer t.mu.Unlock()
 
 	s := &t.cur
 	seq := int64(binary.BigEndian.Uint64(
@@ -203,9 +204,6 @@ func (t *TrafficGen) GotPacket(b []byte, ofs int) {
 
 	f := t.onFirstPacket
 	t.onFirstPacket = nil
-
-	t.mu.Unlock()
-
 	if f != nil {
 		f()
 	}

wgengine/bench/wg.go

@@ -5,11 +5,12 @@
 package main
 
 import (
+	"errors"
 	"io"
 	"log"
 	"os"
-	"strings"
 	"sync"
+	"testing"
 
 	"github.com/tailscale/wireguard-go/tun"
 	"inet.af/netaddr"
@@ -25,7 +26,7 @@ import (
 	"tailscale.com/wgengine/wgcfg"
 )
 
-func setupWGTest(logf logger.Logf, traf *TrafficGen, a1, a2 netaddr.IPPrefix) {
+func setupWGTest(b *testing.B, logf logger.Logf, traf *TrafficGen, a1, a2 netaddr.IPPrefix) {
 	l1 := logger.WithPrefix(logf, "e1: ")
 	k1, err := wgkey.NewPrivate()
 	if err != nil {
@@ -49,6 +50,9 @@ func setupWGTest(logf logger.Logf, traf *TrafficGen, a1, a2 netaddr.IPPrefix) {
 	if err != nil {
 		log.Fatalf("e1 init: %v", err)
 	}
+	if b != nil {
+		b.Cleanup(e1.Close)
+	}
 
 	l2 := logger.WithPrefix(logf, "e2: ")
 	k2, err := wgkey.NewPrivate()
@@ -73,6 +77,9 @@ func setupWGTest(logf logger.Logf, traf *TrafficGen, a1, a2 netaddr.IPPrefix) {
 	if err != nil {
 		log.Fatalf("e2 init: %v", err)
 	}
+	if b != nil {
+		b.Cleanup(e2.Close)
+	}
 
 	e1.SetFilter(filter.NewAllowAllForTest(l1))
 	e2.SetFilter(filter.NewAllowAllForTest(l2))
@@ -80,15 +87,25 @@ func setupWGTest(logf logger.Logf, traf *TrafficGen, a1, a2 netaddr.IPPrefix) {
 	var wait sync.WaitGroup
 	wait.Add(2)
 
+	var e1waitDoneOnce sync.Once
 	e1.SetStatusCallback(func(st *wgengine.Status, err error) {
+		if errors.Is(err, wgengine.ErrEngineClosing) {
+			return
+		}
 		if err != nil {
 			log.Fatalf("e1 status err: %v", err)
 		}
 		logf("e1 status: %v", *st)
 
 		var eps []string
+		var ipps []netaddr.IPPort
 		for _, ep := range st.LocalAddrs {
 			eps = append(eps, ep.Addr.String())
+			ipps = append(ipps, ep.Addr)
+		}
+		endpoint := wgcfg.Endpoints{
+			PublicKey: c1.PrivateKey.Public(),
+			IPPorts:   wgcfg.NewIPPortSet(ipps...),
 		}
 
 		n := tailcfg.Node{
@@ -107,22 +124,32 @@ func setupWGTest(logf logger.Logf, traf *TrafficGen, a1, a2 netaddr.IPPrefix) {
 		p := wgcfg.Peer{
 			PublicKey:  c1.PrivateKey.Public(),
 			AllowedIPs: []netaddr.IPPrefix{a1},
-			Endpoints:  strings.Join(eps, ","),
+			Endpoints:  endpoint,
 		}
 		c2.Peers = []wgcfg.Peer{p}
 		e2.Reconfig(&c2, &router.Config{}, new(dns.Config))
-		wait.Done()
+		e1waitDoneOnce.Do(wait.Done)
 	})
 
+	var e2waitDoneOnce sync.Once
 	e2.SetStatusCallback(func(st *wgengine.Status, err error) {
+		if errors.Is(err, wgengine.ErrEngineClosing) {
+			return
+		}
 		if err != nil {
 			log.Fatalf("e2 status err: %v", err)
 		}
 		logf("e2 status: %v", *st)
 
 		var eps []string
+		var ipps []netaddr.IPPort
 		for _, ep := range st.LocalAddrs {
 			eps = append(eps, ep.Addr.String())
+			ipps = append(ipps, ep.Addr)
+		}
+		endpoint := wgcfg.Endpoints{
+			PublicKey: c2.PrivateKey.Public(),
+			IPPorts:   wgcfg.NewIPPortSet(ipps...),
 		}
 
 		n := tailcfg.Node{
@@ -141,11 +168,11 @@ func setupWGTest(logf logger.Logf, traf *TrafficGen, a1, a2 netaddr.IPPrefix) {
 		p := wgcfg.Peer{
 			PublicKey:  c2.PrivateKey.Public(),
 			AllowedIPs: []netaddr.IPPrefix{a2},
-			Endpoints:  strings.Join(eps, ","),
+			Endpoints:  endpoint,
 		}
 		c1.Peers = []wgcfg.Peer{p}
 		e1.Reconfig(&c1, &router.Config{}, new(dns.Config))
-		wait.Done()
+		e2waitDoneOnce.Do(wait.Done)
 	})
 
 	// Not using DERP in this test (for now?).

wgengine/magicsock/legacy.go

@@ -10,7 +10,6 @@ import (
 	"crypto/subtle"
 	"encoding/binary"
 	"errors"
-	"fmt"
 	"hash"
 	"net"
 	"strings"
@@ -27,6 +26,7 @@ import (
 	"tailscale.com/types/key"
 	"tailscale.com/types/logger"
 	"tailscale.com/types/wgkey"
+	"tailscale.com/wgengine/wgcfg"
 )
 
 var (
@@ -34,7 +34,11 @@ var (
 	errDisabled       = errors.New("magicsock: legacy networking disabled")
 )
 
-func (c *Conn) createLegacyEndpointLocked(pk key.Public, addrs string) (conn.Endpoint, error) {
+// createLegacyEndpointLocked creates a new wireguard-go endpoint for a legacy connection.
+// pk is the public key of the remote peer. addrs is the ordered set of addresses for the remote peer.
+// rawDest is the encoded wireguard-go endpoint string. It should be treated as a black box.
+// It is provided so that addrSet.DstToString can return it when requested by wireguard-go.
+func (c *Conn) createLegacyEndpointLocked(pk key.Public, addrs wgcfg.IPPortSet, rawDest string) (conn.Endpoint, error) {
 	if c.disableLegacy {
 		return nil, errDisabled
 	}
@@ -43,17 +47,9 @@ func (c *Conn) createLegacyEndpointLocked(pk key.Public, addrs string) (conn.End
 		Logf:      c.logf,
 		publicKey: pk,
 		curAddr:   -1,
+		rawdst:    rawDest,
 	}
-
-	if addrs != "" {
-		for _, ep := range strings.Split(addrs, ",") {
-			ipp, err := netaddr.ParseIPPort(ep)
-			if err != nil {
-				return nil, fmt.Errorf("bogus address %q", ep)
-			}
-			a.ipPorts = append(a.ipPorts, ipp)
-		}
-	}
+	a.ipPorts = append(a.ipPorts, addrs.IPPorts()...)
 
 	// If this endpoint is being updated, remember its old set of
 	// endpoints so we can remove any (from c.addrsByUDP) that are
@@ -384,6 +380,9 @@ type addrSet struct {
 	// set to a better one. This is only to suppress some
 	// redundant logs.
 	loggedLogPriMask uint32
+
+	// rawdst is the destination string from/for wireguard-go.
+	rawdst string
 }
 
 // derpID returns this addrSet's home DERP node, or 0 if none is found.
@@ -426,17 +425,7 @@ func (a *addrSet) DstToBytes() []byte {
 	return packIPPort(a.dst())
 }
 func (a *addrSet) DstToString() string {
-	var addrs []string
-	for _, addr := range a.ipPorts {
-		addrs = append(addrs, addr.String())
-	}
-
-	a.mu.Lock()
-	defer a.mu.Unlock()
-	if a.roamAddr != nil {
-		addrs = append(addrs, a.roamAddr.String())
-	}
-	return strings.Join(addrs, ",")
+	return a.rawdst
 }
 func (a *addrSet) DstIP() net.IP {
 	return a.dst().IP.IPAddr().IP // TODO: add netaddr accessor to cut an alloc here?

wgengine/magicsock/magicsock.go

@@ -11,6 +11,7 @@ import (
 	"context"
 	crand "crypto/rand"
 	"encoding/binary"
+	"encoding/json"
 	"errors"
 	"fmt"
 	"hash/fnv"
@@ -27,7 +28,6 @@ import (
 	"time"
 
 	"github.com/tailscale/wireguard-go/conn"
-	"go4.org/mem"
 	"golang.org/x/crypto/nacl/box"
 	"golang.org/x/time/rate"
 	"inet.af/netaddr"
@@ -401,7 +401,7 @@ type Options struct {
 	// and 10 seconds seems like a good trade-off between often
 	// enough and not too often.) The provided func is called
 	// while holding userspaceEngine.wgLock and likely calls
-	// Conn.CreateEndpoint, which acquires Conn.mu. As such, you
+	// Conn.ParseEndpoint, which acquires Conn.mu. As such, you
 	// should not hold Conn.mu while calling it.
 	NoteRecvActivity func(tailcfg.DiscoKey)
 
@@ -1696,7 +1696,7 @@ func (c *Conn) processDERPReadResult(dm derpReadResult, b []byte) (n int, ep con
 		if discoEp == nil && c.noteRecvActivity != nil {
 			didNoteRecvActivity = true
 			c.mu.Unlock()          // release lock before calling noteRecvActivity
-			c.noteRecvActivity(dk) // (calls back into CreateEndpoint)
+			c.noteRecvActivity(dk) // (calls back into ParseEndpoint)
 			// Now require the lock. No invariants need to be rechecked; just
 			// 1-2 map lookups follow that are harmless if, say, the peer has
 			// been deleted during this time.
@@ -1837,7 +1837,7 @@ func (c *Conn) handleDiscoMessage(msg []byte, src netaddr.IPPort) (isDiscoMsg bo
 		// We don't have an active endpoint for this sender but we knew about the node, so
 		// it's an idle endpoint that doesn't yet exist in the wireguard config. We now have
 		// to notify the userspace engine (via noteRecvActivity) so wireguard-go can create
-		// an Endpoint (ultimately calling our CreateEndpoint).
+		// an Endpoint (ultimately calling our ParseEndpoint).
 		c.logf("magicsock: got disco message from idle peer, starting lazy conf for %v, %v", peerNode.Key.ShortString(), sender.ShortString())
 		if c.noteRecvActivity == nil {
 			c.logf("magicsock: [unexpected] have node without endpoint, without c.noteRecvActivity hook")
@@ -1851,7 +1851,7 @@ func (c *Conn) handleDiscoMessage(msg []byte, src netaddr.IPPort) (isDiscoMsg bo
 		// We can't hold Conn.mu while calling noteRecvActivity.
 		// noteRecvActivity acquires userspaceEngine.wgLock (and per our
 		// lock ordering rules: wgLock must come first), and also calls
-		// back into our Conn.CreateEndpoint, which would double-acquire
+		// back into our Conn.ParseEndpoint, which would double-acquire
 		// Conn.mu.
 		c.mu.Unlock()
 		c.noteRecvActivity(sender)
@@ -2736,44 +2736,32 @@ func packIPPort(ua netaddr.IPPort) []byte {
 }
 
 // ParseEndpoint is called by WireGuard to connect to an endpoint.
-//
-// keyAddrs is the 32 byte public key of the peer followed by addrs.
-// Addrs is either:
-//
-//  1) a comma-separated list of UDP ip:ports (the peer doesn't have a discovery key)
-//  2) "<hex-discovery-key>.disco.tailscale:12345", a magic value that means the peer
-//     is running code that supports active discovery, so CreateEndpoint returns
-//     a discoEndpoint.
-func (c *Conn) ParseEndpoint(keyAddrs string) (conn.Endpoint, error) {
-	if len(keyAddrs) < 32 {
-		c.logf("[unexpected] ParseEndpoint keyAddrs too short: %q", keyAddrs)
-		return nil, errors.New("endpoint string too short")
+// endpointStr is a json-serialized wgcfg.Endpoints struct.
+// If those Endpoints contain an active discovery key, ParseEndpoint returns a discoEndpoint.
+// Otherwise it returns a legacy endpoint.
+func (c *Conn) ParseEndpoint(endpointStr string) (conn.Endpoint, error) {
+	var endpoints wgcfg.Endpoints
+	err := json.Unmarshal([]byte(endpointStr), &endpoints)
+	if err != nil {
+		return nil, fmt.Errorf("magicsock: ParseEndpoint: json.Unmarshal failed on %q: %w", endpointStr, err)
 	}
-	var pk key.Public
-	copy(pk[:], keyAddrs)
-	addrs := keyAddrs[len(pk):]
+	pk := key.Public(endpoints.PublicKey)
+	discoKey := endpoints.DiscoKey
+	c.logf("magicsock: ParseEndpoint: key=%s: disco=%s ipps=%s", pk.ShortString(), discoKey.ShortString(), derpStr(endpoints.IPPorts.String()))
+
 	c.mu.Lock()
 	defer c.mu.Unlock()
-
-	c.logf("magicsock: ParseEndpoint: key=%s: %s", pk.ShortString(), derpStr(addrs))
-
-	if !strings.HasSuffix(addrs, wgcfg.EndpointDiscoSuffix) {
-		return c.createLegacyEndpointLocked(pk, addrs)
-	}
-
-	discoHex := strings.TrimSuffix(addrs, wgcfg.EndpointDiscoSuffix)
-	discoKey, err := key.NewPublicFromHexMem(mem.S(discoHex))
-	if err != nil {
-		return nil, fmt.Errorf("magicsock: invalid discokey endpoint %q for %v: %w", addrs, pk.ShortString(), err)
+	if discoKey.IsZero() {
+		return c.createLegacyEndpointLocked(pk, endpoints.IPPorts, endpointStr)
 	}
 	de := &discoEndpoint{
-		c:                  c,
-		publicKey:          tailcfg.NodeKey(pk),        // peer public key (for WireGuard + DERP)
-		discoKey:           tailcfg.DiscoKey(discoKey), // for discovery mesages
-		discoShort:         tailcfg.DiscoKey(discoKey).ShortString(),
-		wgEndpointHostPort: addrs,
-		sentPing:           map[stun.TxID]sentPing{},
-		endpointState:      map[netaddr.IPPort]*endpointState{},
+		c:             c,
+		publicKey:     tailcfg.NodeKey(pk),        // peer public key (for WireGuard + DERP)
+		discoKey:      tailcfg.DiscoKey(discoKey), // for discovery mesages
+		discoShort:    tailcfg.DiscoKey(discoKey).ShortString(),
+		wgEndpoint:    endpointStr,
+		sentPing:      map[stun.TxID]sentPing{},
+		endpointState: map[netaddr.IPPort]*endpointState{},
 	}
 	de.initFakeUDPAddr()
 	de.updateFromNode(c.nodeOfDisco[de.discoKey])
@@ -3110,12 +3098,12 @@ type discoEndpoint struct {
 	numStopAndResetAtomic int64
 
 	// These fields are initialized once and never modified.
-	c                  *Conn
-	publicKey          tailcfg.NodeKey  // peer public key (for WireGuard + DERP)
-	discoKey           tailcfg.DiscoKey // for discovery mesages
-	discoShort         string           // ShortString of discoKey
-	fakeWGAddr         netaddr.IPPort   // the UDP address we tell wireguard-go we're using
-	wgEndpointHostPort string           // string from CreateEndpoint: "<hex-discovery-key>.disco.tailscale:12345"
+	c          *Conn
+	publicKey  tailcfg.NodeKey  // peer public key (for WireGuard + DERP)
+	discoKey   tailcfg.DiscoKey // for discovery mesages
+	discoShort string           // ShortString of discoKey
+	fakeWGAddr netaddr.IPPort   // the UDP address we tell wireguard-go we're using
+	wgEndpoint string           // string from ParseEndpoint, holds a JSON-serialized wgcfg.Endpoints
 
 	// Owned by Conn.mu:
 	lastPingFrom netaddr.IPPort
@@ -3295,7 +3283,7 @@ func (de *discoEndpoint) String() string {
 func (de *discoEndpoint) ClearSrc()           {}
 func (de *discoEndpoint) SrcToString() string { panic("unused") } // unused by wireguard-go
 func (de *discoEndpoint) SrcIP() net.IP       { panic("unused") } // unused by wireguard-go
-func (de *discoEndpoint) DstToString() string { return de.wgEndpointHostPort }
+func (de *discoEndpoint) DstToString() string { return de.wgEndpoint }
 func (de *discoEndpoint) DstIP() net.IP       { panic("unused") }
 func (de *discoEndpoint) DstToBytes() []byte  { return packIPPort(de.fakeWGAddr) }
 

wgengine/magicsock/magicsock_test.go

@@ -26,7 +26,6 @@ import (
 	"time"
 	"unsafe"
 
-	"github.com/google/go-cmp/cmp"
 	"github.com/tailscale/wireguard-go/device"
 	"github.com/tailscale/wireguard-go/tun/tuntest"
 	"golang.org/x/crypto/nacl/box"
@@ -169,7 +168,7 @@ func newMagicStack(t testing.TB, logf logger.Logf, l nettype.PacketListener, der
 	tsTun.SetFilter(filter.NewAllowAllForTest(logf))
 
 	wgLogger := wglog.NewLogger(logf)
-	dev := device.NewDevice(tsTun, conn.Bind(), wgLogger.DeviceLogger, new(device.DeviceOptions))
+	dev := device.NewDevice(tsTun, conn.Bind(), wgLogger.DeviceLogger)
 	dev.Up()
 
 	// Wait for magicsock to connect up to DERP.
@@ -455,7 +454,7 @@ func makeConfigs(t *testing.T, addrs []netaddr.IPPort) []wgcfg.Config {
 		privKeys = append(privKeys, wgkey.Private(privKey))
 
 		addresses = append(addresses, []netaddr.IPPrefix{
-			parseCIDR(t, fmt.Sprintf("1.0.0.%d/32", i+1)),
+			netaddr.MustParseIPPrefix(fmt.Sprintf("1.0.0.%d/32", i+1)),
 		})
 	}
 
@@ -470,10 +469,14 @@ func makeConfigs(t *testing.T, addrs []netaddr.IPPort) []wgcfg.Config {
 			if peerNum == i {
 				continue
 			}
+			publicKey := privKeys[peerNum].Public()
 			peer := wgcfg.Peer{
-				PublicKey:           privKeys[peerNum].Public(),
-				AllowedIPs:          addresses[peerNum],
-				Endpoints:           addr.String(),
+				PublicKey:  publicKey,
+				AllowedIPs: addresses[peerNum],
+				Endpoints: wgcfg.Endpoints{
+					PublicKey: publicKey,
+					IPPorts:   wgcfg.NewIPPortSet(addr),
+				},
 				PersistentKeepalive: 25,
 			}
 			cfg.Peers = append(cfg.Peers, peer)
@@ -483,15 +486,6 @@ func makeConfigs(t *testing.T, addrs []netaddr.IPPort) []wgcfg.Config {
 	return cfgs
 }
 
-func parseCIDR(t *testing.T, addr string) netaddr.IPPrefix {
-	t.Helper()
-	cidr, err := netaddr.ParseIPPrefix(addr)
-	if err != nil {
-		t.Fatal(err)
-	}
-	return cidr
-}
-
 // TestDeviceStartStop exercises the startup and shutdown logic of
 // wireguard-go, which is intimately intertwined with magicsock's own
 // lifecycle. We seem to be good at generating deadlocks here, so if
@@ -515,7 +509,7 @@ func TestDeviceStartStop(t *testing.T) {
 
 	tun := tuntest.NewChannelTUN()
 	wgLogger := wglog.NewLogger(t.Logf)
-	dev := device.NewDevice(tun.TUN(), conn.Bind(), wgLogger.DeviceLogger, new(device.DeviceOptions))
+	dev := device.NewDevice(tun.TUN(), conn.Bind(), wgLogger.DeviceLogger)
 	dev.Up()
 	dev.Close()
 }
@@ -996,148 +990,6 @@ func testTwoDevicePing(t *testing.T, d *devices) {
 		ping1(t)
 		ping2(t)
 	})
-
-	// TODO: Remove this once the following tests are reliable.
-	if run, _ := strconv.ParseBool(os.Getenv("RUN_CURSED_TESTS")); !run {
-		t.Skip("skipping following tests because RUN_CURSED_TESTS is not set.")
-	}
-
-	pingSeq := func(t *testing.T, count int, totalTime time.Duration, strict bool) {
-		msg := func(i int) []byte {
-			b := tuntest.Ping(net.ParseIP("1.0.0.2"), net.ParseIP("1.0.0.1"))
-			b[len(b)-1] = byte(i) // set seq num
-			return b
-		}
-
-		// Space out ping transmissions so that the overall
-		// transmission happens in totalTime.
-		//
-		// We do this because the packet spray logic in magicsock is
-		// time-based to allow for reliable NAT traversal. However,
-		// for the packet spraying test further down, there needs to
-		// be at least 1 sprayed packet that is not the handshake, in
-		// case the handshake gets eaten by the race resolution logic.
-		//
-		// This is an inherent "race by design" in our current
-		// magicsock+wireguard-go codebase: sometimes, racing
-		// handshakes will result in a sub-optimal path for a few
-		// hundred milliseconds, until a subsequent spray corrects the
-		// issue. In order for the test to reflect that magicsock
-		// works as designed, we have to space out packet transmission
-		// here.
-		interPacketGap := totalTime / time.Duration(count)
-		if interPacketGap < 1*time.Millisecond {
-			interPacketGap = 0
-		}
-
-		for i := 0; i < count; i++ {
-			b := msg(i)
-			m1.tun.Outbound <- b
-			time.Sleep(interPacketGap)
-		}
-
-		for i := 0; i < count; i++ {
-			b := msg(i)
-			select {
-			case msgRecv := <-m2.tun.Inbound:
-				if !bytes.Equal(b, msgRecv) {
-					if strict {
-						t.Errorf("return ping %d did not transit correctly: %s", i, cmp.Diff(b, msgRecv))
-					}
-				}
-			case <-time.After(pingTimeout):
-				if strict {
-					t.Errorf("return ping %d did not transit", i)
-				}
-			}
-		}
-	}
-
-	t.Run("ping 1.0.0.1 x50", func(t *testing.T) {
-		setT(t)
-		defer setT(outerT)
-		pingSeq(t, 50, 0, true)
-	})
-
-	// Add DERP relay.
-	derpEp := "127.3.3.40:1"
-	ep0 := cfgs[0].Peers[0].Endpoints
-	ep0 = derpEp + "," + ep0
-	cfgs[0].Peers[0].Endpoints = ep0
-	ep1 := cfgs[1].Peers[0].Endpoints
-	ep1 = derpEp + "," + ep1
-	cfgs[1].Peers[0].Endpoints = ep1
-	if err := m1.Reconfig(&cfgs[0]); err != nil {
-		t.Fatal(err)
-	}
-	if err := m2.Reconfig(&cfgs[1]); err != nil {
-		t.Fatal(err)
-	}
-
-	t.Run("add DERP", func(t *testing.T) {
-		setT(t)
-		defer setT(outerT)
-		pingSeq(t, 20, 0, true)
-	})
-
-	// Disable real route.
-	cfgs[0].Peers[0].Endpoints = derpEp
-	cfgs[1].Peers[0].Endpoints = derpEp
-	if err := m1.Reconfig(&cfgs[0]); err != nil {
-		t.Fatal(err)
-	}
-	if err := m2.Reconfig(&cfgs[1]); err != nil {
-		t.Fatal(err)
-	}
-	time.Sleep(250 * time.Millisecond) // TODO remove
-
-	t.Run("all traffic over DERP", func(t *testing.T) {
-		setT(t)
-		defer setT(outerT)
-		defer func() {
-			if t.Failed() || true {
-				logf("cfg0: %v", stringifyConfig(cfgs[0]))
-				logf("cfg1: %v", stringifyConfig(cfgs[1]))
-			}
-		}()
-		pingSeq(t, 20, 0, true)
-	})
-
-	m1.dev.RemoveAllPeers()
-	m2.dev.RemoveAllPeers()
-
-	// Give one peer a non-DERP endpoint. We expect the other to
-	// accept it via roamAddr.
-	cfgs[0].Peers[0].Endpoints = ep0
-	if ep2 := cfgs[1].Peers[0].Endpoints; len(ep2) != 1 {
-		t.Errorf("unexpected peer endpoints in dev2: %v", ep2)
-	}
-	if err := m2.Reconfig(&cfgs[1]); err != nil {
-		t.Fatal(err)
-	}
-	if err := m1.Reconfig(&cfgs[0]); err != nil {
-		t.Fatal(err)
-	}
-	// Dear future human debugging a test failure here: this test is
-	// flaky, and very infrequently will drop 1-2 of the 50 ping
-	// packets. This does not affect normal operation of tailscaled,
-	// but makes this test fail.
-	//
-	// TODO(danderson): finish root-causing and de-flake this test.
-	t.Run("one real route is enough thanks to spray", func(t *testing.T) {
-		setT(t)
-		defer setT(outerT)
-		pingSeq(t, 50, 700*time.Millisecond, false)
-
-		cfg, err := wgcfg.DeviceConfig(m2.dev)
-		if err != nil {
-			t.Fatal(err)
-		}
-		ep2 := cfg.Peers[0].Endpoints
-		if len(ep2) != 2 {
-			t.Error("handshake spray failed to find real route")
-		}
-	})
 }
 
 // TestAddrSet tests addrSet appendDests and updateDst.
@@ -1362,14 +1214,6 @@ func TestDiscoStringLogRace(t *testing.T) {
 	wg.Wait()
 }
 
-func stringifyConfig(cfg wgcfg.Config) string {
-	j, err := json.Marshal(cfg)
-	if err != nil {
-		panic(err)
-	}
-	return string(j)
-}
-
 func Test32bitAlignment(t *testing.T) {
 	var de discoEndpoint
 
@@ -1403,6 +1247,19 @@ func newNonLegacyTestConn(t testing.TB) *Conn {
 	return conn
 }
 
+func makeEndpoint(tb testing.TB, public tailcfg.NodeKey, disco tailcfg.DiscoKey) string {
+	tb.Helper()
+	ep := wgcfg.Endpoints{
+		PublicKey: wgkey.Key(public),
+		DiscoKey:  disco,
+	}
+	buf, err := json.Marshal(ep)
+	if err != nil {
+		tb.Fatal(err)
+	}
+	return string(buf)
+}
+
 // addTestEndpoint sets conn's network map to a single peer expected
 // to receive packets from sendConn (or DERP), and returns that peer's
 // nodekey and discokey.
@@ -1422,7 +1279,7 @@ func addTestEndpoint(tb testing.TB, conn *Conn, sendConn net.PacketConn) (tailcf
 		},
 	})
 	conn.SetPrivateKey(wgkey.Private{0: 1})
-	_, err := conn.ParseEndpoint(string(nodeKey[:]) + "0000000000000000000000000000000000000000000000000000000000000001.disco.tailscale:12345")
+	_, err := conn.ParseEndpoint(makeEndpoint(tb, nodeKey, discoKey))
 	if err != nil {
 		tb.Fatal(err)
 	}
@@ -1596,7 +1453,7 @@ func TestSetNetworkMapChangingNodeKey(t *testing.T) {
 			},
 		},
 	})
-	_, err := conn.ParseEndpoint(string(nodeKey1[:]) + "0000000000000000000000000000000000000000000000000000000000000001.disco.tailscale:12345")
+	_, err := conn.ParseEndpoint(makeEndpoint(t, nodeKey1, discoKey))
 	if err != nil {
 		t.Fatal(err)
 	}

wgengine/netstack/netstack.go

@@ -12,9 +12,11 @@ import (
 	"io"
 	"log"
 	"net"
+	"os"
 	"strconv"
 	"strings"
 	"sync"
+	"sync/atomic"
 	"time"
 
 	"inet.af/netaddr"
@@ -55,6 +57,12 @@ type Impl struct {
 	logf        logger.Logf
 	onlySubnets bool // whether we only want to handle subnet relaying
 
+	// atomicIsLocalIPFunc holds a func that reports whether an IP
+	// is a local (non-subnet) Tailscale IP address of this
+	// machine. It's always a non-nil func. It's changed on netmap
+	// updates.
+	atomicIsLocalIPFunc atomic.Value // of func(netaddr.IP) bool
+
 	mu  sync.Mutex
 	dns DNSMap
 	// connsOpenBySubnetIP keeps track of number of connections open
@@ -119,6 +127,7 @@ func Create(logf logger.Logf, tundev *tstun.Wrapper, e wgengine.Engine, mc *magi
 		connsOpenBySubnetIP: make(map[netaddr.IP]int),
 		onlySubnets:         onlySubnets,
 	}
+	ns.atomicIsLocalIPFunc.Store(tsaddr.NewContainsIPFunc(nil))
 	return ns, nil
 }
 
@@ -145,7 +154,7 @@ func (ns *Impl) Start() error {
 			ns.logf("netstack: could not parse local address %s for incoming TCP connection", ip)
 			return false
 		}
-		if !tsaddr.IsTailscaleIP(ip) {
+		if !ns.isLocalIP(ip) {
 			ns.addSubnetAddress(pn, ip)
 		}
 		return tcpFwd.HandlePacket(tei, pb)
@@ -219,6 +228,7 @@ func ipPrefixToAddressWithPrefix(ipp netaddr.IPPrefix) tcpip.AddressWithPrefix {
 }
 
 func (ns *Impl) updateIPs(nm *netmap.NetworkMap) {
+	ns.atomicIsLocalIPFunc.Store(tsaddr.NewContainsIPFunc(nm.Addresses))
 	ns.updateDNS(nm)
 
 	oldIPs := make(map[tcpip.AddressWithPrefix]bool)
@@ -373,7 +383,19 @@ func (ns *Impl) injectOutbound() {
 	}
 }
 
+// isLocalIP reports whether ip is a Tailscale IP assigned to this
+// node directly (but not a subnet-routed IP).
+func (ns *Impl) isLocalIP(ip netaddr.IP) bool {
+	return ns.atomicIsLocalIPFunc.Load().(func(netaddr.IP) bool)(ip)
+}
+
 func (ns *Impl) injectInbound(p *packet.Parsed, t *tstun.Wrapper) filter.Response {
+	if ns.onlySubnets && ns.isLocalIP(p.Dst.IP) {
+		// In hybrid ("only subnets") mode, bail out early if
+		// the traffic is destined for an actual Tailscale
+		// address. The real host OS interface will handle it.
+		return filter.Accept
+	}
 	var pn tcpip.NetworkProtocolNumber
 	switch p.IPVersion {
 	case 4:
@@ -389,7 +411,14 @@ func (ns *Impl) injectInbound(p *packet.Parsed, t *tstun.Wrapper) filter.Respons
 		Data: vv,
 	})
 	ns.linkEP.InjectInbound(pn, packetBuf)
-	return filter.Accept
+
+	// We've now delivered this to netstack, so we're done.
+	// Instead of returning a filter.Accept here (which would also
+	// potentially deliver it to the host OS), and instead of
+	// filter.Drop (which would log about rejected traffic),
+	// instead return filter.DropSilently which just quietly stops
+	// processing it in the tstun TUN wrapper.
+	return filter.DropSilently
 }
 
 func (ns *Impl) acceptTCP(r *tcp.ForwarderRequest) {
@@ -424,6 +453,9 @@ func (ns *Impl) acceptTCP(r *tcp.ForwarderRequest) {
 func (ns *Impl) forwardTCP(client *gonet.TCPConn, wq *waiter.Queue, dialAddr tcpip.Address, dialPort uint16) {
 	defer client.Close()
 	dialAddrStr := net.JoinHostPort(dialAddr.String(), strconv.Itoa(int(dialPort)))
+	if alt := os.Getenv(fmt.Sprintf("TAILSCALE_INCOMING_REMAP_%s_%d", dialAddr, dialPort)); alt != "" {
+		dialAddrStr = alt
+	}
 	ns.logf("[v2] netstack: forwarding incoming connection to %s", dialAddrStr)
 	ctx, cancel := context.WithCancel(context.Background())
 	defer cancel()

wgengine/pendopen.go

@@ -104,6 +104,7 @@ func (e *userspaceEngine) trackOpenPostFilterOut(pp *packet.Parsed, t *tstun.Wra
 
 	if pp.IPVersion == 0 ||
 		pp.IPProto != ipproto.TCP ||
+		pp.TCPFlags&packet.TCPAck != 0 ||
 		pp.TCPFlags&packet.TCPSyn == 0 {
 		return
 	}
@@ -219,14 +220,32 @@ func (e *userspaceEngine) onOpenTimeout(flow flowtrack.Tuple) {
 	// handshake completed, which is what I want.
 	_ = ps.LastHandshake
 
-	e.logf("open-conn-track: timeout opening %v to node %v; lastSeen=%v, lastRecv=%v",
+	online := "?"
+	if n.Online != nil {
+		if *n.Online {
+			online = "yes"
+		} else {
+			online = "no"
+		}
+	}
+	e.logf("open-conn-track: timeout opening %v to node %v; lastSeen=%v, online=%v, lastRecv=%v",
 		flow, n.Key.ShortString(),
-		agoOrNever(lastSeen), agoOrNever(e.magicConn.LastRecvActivityOfDisco(n.DiscoKey)))
+		durFmt(lastSeen),
+		online,
+		durFmt(e.magicConn.LastRecvActivityOfDisco(n.DiscoKey)))
 }
 
-func agoOrNever(t time.Time) string {
+func durFmt(t time.Time) string {
 	if t.IsZero() {
 		return "never"
 	}
-	return time.Since(t).Round(time.Second).String()
+	d := time.Since(t).Round(time.Second)
+	if d < 10*time.Minute {
+		// node.LastSeen times are rounded very coarsely, and
+		// we compare times from different clocks (server vs
+		// local), so negative is common when close. Format as
+		// "recent" if negative or actually recent.
+		return "recent"
+	}
+	return d.String()
 }

wgengine/router/ifconfig_windows.go

@@ -359,7 +359,7 @@ func configureInterface(cfg *Config, tun *tun.NativeTun) (retErr error) {
 			firstGateway6 = &ipnet.IP
 		} else if route.IP.Is4() && firstGateway4 == nil {
 			// TODO: do same dummy behavior as v6?
-			return errors.New("Due to a Windows limitation, one cannot have interface routes without an interface address")
+			return errors.New("due to a Windows limitation, one cannot have interface routes without an interface address")
 		}
 
 		ipn := route.IPNet()
@@ -377,7 +377,7 @@ func configureInterface(cfg *Config, tun *tun.NativeTun) (retErr error) {
 			NextHop: gateway,
 			Metric:  0,
 		}
-		if bytes.Compare(r.Destination.IP, gateway) == 0 {
+		if net.IP.Equal(r.Destination.IP, gateway) {
 			// no need to add a route for the interface's
 			// own IP. The kernel does that for us.
 			// If we try to replace it, we'll fail to
@@ -411,7 +411,7 @@ func configureInterface(cfg *Config, tun *tun.NativeTun) (retErr error) {
 		// There's only one way to get to a given IP+Mask, so delete
 		// all matches after the first.
 		if i > 0 &&
-			bytes.Equal(routes[i].Destination.IP, routes[i-1].Destination.IP) &&
+			net.IP.Equal(routes[i].Destination.IP, routes[i-1].Destination.IP) &&
 			bytes.Equal(routes[i].Destination.Mask, routes[i-1].Destination.Mask) {
 			continue
 		}

wgengine/router/router_windows.go

@@ -30,15 +30,6 @@ type winRouter struct {
 	nativeTun           *tun.NativeTun
 	routeChangeCallback *winipcfg.RouteChangeCallback
 	firewall            *firewallTweaker
-
-	// firewallSubproc is a subprocess that runs a tweaked version of
-	// wireguard-windows's "default route killswitch" code. We run it
-	// as a subprocess because it does unsafe callouts to the WFP API,
-	// and we want to defend against memory corruption in our main
-	// process. Owned and mutated only by Set, and doesn't need a lock
-	// because Set is only called with wgengine's lock held,
-	// preventing concurrent reconfigs.
-	firewallSubproc *exec.Cmd
 }
 
 func newUserspaceRouter(logf logger.Logf, tundev tun.Device) (Router, error) {

wgengine/userspace.go

@@ -7,12 +7,10 @@ package wgengine
 import (
 	"bufio"
 	"bytes"
-	"context"
 	crand "crypto/rand"
 	"errors"
 	"fmt"
 	"io"
-	"net"
 	"os"
 	"reflect"
 	"runtime"
@@ -28,7 +26,7 @@ import (
 	"inet.af/netaddr"
 	"tailscale.com/control/controlclient"
 	"tailscale.com/health"
-	"tailscale.com/internal/deepprint"
+	"tailscale.com/internal/deephash"
 	"tailscale.com/ipn/ipnstate"
 	"tailscale.com/net/dns"
 	"tailscale.com/net/dns/resolver"
@@ -120,7 +118,6 @@ type userspaceEngine struct {
 	statusCallback      StatusCallback
 	peerSequence        []wgkey.Key
 	endpoints           []tailcfg.Endpoint
-	pingers             map[wgkey.Key]*pinger                // legacy pingers for pre-discovery peers
 	pendOpen            map[flowtrack.Tuple]*pendingOpenFlow // see pendopen.go
 	networkMapCallbacks map[*someHandle]NetworkMapCallback
 	tsIPByIPPort        map[netaddr.IPPort]netaddr.IP          // allows registration of IP:ports as belonging to a certain Tailscale IP for whois lookups
@@ -241,9 +238,8 @@ func NewUserspaceEngine(logf logger.Logf, conf Config) (_ Engine, reterr error)
 		waitCh:  make(chan struct{}),
 		tundev:  tsTUNDev,
 		router:  conf.Router,
-		pingers: make(map[wgkey.Key]*pinger),
 	}
-	e.isLocalAddr.Store(genLocalAddrFunc(nil))
+	e.isLocalAddr.Store(tsaddr.NewContainsIPFunc(nil))
 
 	if conf.LinkMonitor != nil {
 		e.linkMon = conf.LinkMonitor
@@ -310,52 +306,6 @@ func NewUserspaceEngine(logf logger.Logf, conf Config) (_ Engine, reterr error)
 	}
 
 	e.wgLogger = wglog.NewLogger(logf)
-	opts := &device.DeviceOptions{
-		HandshakeDone: func(peerKey device.NoisePublicKey, peer *device.Peer, deviceAllowedIPs *device.AllowedIPs) {
-			// Send an unsolicited status event every time a
-			// handshake completes. This makes sure our UI can
-			// update quickly as soon as it connects to a peer.
-			//
-			// We use a goroutine here to avoid deadlocking
-			// wireguard, since RequestStatus() will call back
-			// into it, and wireguard is what called us to get
-			// here.
-			go e.RequestStatus()
-
-			peerWGKey := wgkey.Key(peerKey)
-			if e.magicConn.PeerHasDiscoKey(tailcfg.NodeKey(peerKey)) {
-				e.logf("wireguard handshake complete for %v", peerWGKey.ShortString())
-				// This is a modern peer with discovery support. No need to send pings.
-				return
-			}
-
-			e.logf("wireguard handshake complete for %v; sending legacy pings", peerWGKey.ShortString())
-
-			// Ping every single-IP that peer routes.
-			// These synthetic packets are used to traverse NATs.
-			var ips []netaddr.IP
-			var allowedIPs []netaddr.IPPrefix
-			deviceAllowedIPs.EntriesForPeer(peer, func(stdIP net.IP, cidr uint) bool {
-				ip, ok := netaddr.FromStdIP(stdIP)
-				if !ok {
-					logf("[unexpected] bad IP from deviceAllowedIPs.EntriesForPeer: %v", stdIP)
-					return true
-				}
-				ipp := netaddr.IPPrefix{IP: ip, Bits: uint8(cidr)}
-				allowedIPs = append(allowedIPs, ipp)
-				if ipp.IsSingleIP() {
-					ips = append(ips, ip)
-				}
-				return true
-			})
-			if len(ips) > 0 {
-				go e.pinger(peerWGKey, ips)
-			} else {
-				logf("[unexpected] peer %s has no single-IP routes: %v", peerWGKey.ShortString(), allowedIPs)
-			}
-		},
-	}
-
 	e.tundev.OnTSMPPongReceived = func(pong packet.TSMPPongReply) {
 		e.mu.Lock()
 		defer e.mu.Unlock()
@@ -368,7 +318,7 @@ func NewUserspaceEngine(logf logger.Logf, conf Config) (_ Engine, reterr error)
 
 	// wgdev takes ownership of tundev, will close it when closed.
 	e.logf("Creating wireguard device...")
-	e.wgdev = device.NewDevice(e.tundev, e.magicConn.Bind(), e.wgLogger.DeviceLogger, opts)
+	e.wgdev = device.NewDevice(e.tundev, e.magicConn.Bind(), e.wgLogger.DeviceLogger)
 	closePool.addFunc(e.wgdev.Close)
 	closePool.addFunc(func() {
 		if err := e.magicConn.Close(); err != nil {
@@ -507,132 +457,6 @@ func (e *userspaceEngine) pollResolver() {
 	}
 }
 
-// pinger sends ping packets for a few seconds.
-//
-// These generated packets are used to ensure we trigger the spray logic in
-// the magicsock package for NAT traversal.
-//
-// These are only used with legacy peers (before 0.100.0) that don't
-// have advertised discovery keys.
-type pinger struct {
-	e      *userspaceEngine
-	done   chan struct{} // closed after shutdown (not the ctx.Done() chan)
-	cancel context.CancelFunc
-}
-
-// close cleans up pinger and removes it from the userspaceEngine.pingers map.
-// It cannot be called while p.e.mu is held.
-func (p *pinger) close() {
-	p.cancel()
-	<-p.done
-}
-
-func (p *pinger) run(ctx context.Context, peerKey wgkey.Key, ips []netaddr.IP, srcIP netaddr.IP) {
-	defer func() {
-		p.e.mu.Lock()
-		if p.e.pingers[peerKey] == p {
-			delete(p.e.pingers, peerKey)
-		}
-		p.e.mu.Unlock()
-
-		close(p.done)
-	}()
-
-	header := packet.ICMP4Header{
-		IP4Header: packet.IP4Header{
-			Src: srcIP,
-		},
-		Type: packet.ICMP4EchoRequest,
-		Code: packet.ICMP4NoCode,
-	}
-
-	// sendFreq is slightly longer than sprayFreq in magicsock to ensure
-	// that if these ping packets are the only source of early packets
-	// sent to the peer, that each one will be sprayed.
-	const sendFreq = 300 * time.Millisecond
-	const stopAfter = 3 * time.Second
-
-	start := time.Now()
-	var dstIPs []netaddr.IP
-	for _, ip := range ips {
-		if ip.Is6() {
-			// This code is only used for legacy (pre-discovery)
-			// peers. They're not going to work right with IPv6 on the
-			// overlay anyway, so don't bother trying to make ping
-			// work.
-			continue
-		}
-		dstIPs = append(dstIPs, ip)
-	}
-
-	payload := []byte("magicsock_spray") // no meaning
-
-	header.IPID = 1
-	t := time.NewTicker(sendFreq)
-	defer t.Stop()
-	for {
-		select {
-		case <-ctx.Done():
-			return
-		case <-t.C:
-		}
-		if time.Since(start) > stopAfter {
-			return
-		}
-		for _, dstIP := range dstIPs {
-			header.Dst = dstIP
-			// InjectOutbound take ownership of the packet, so we allocate.
-			b := packet.Generate(&header, payload)
-			p.e.tundev.InjectOutbound(b)
-		}
-		header.IPID++
-	}
-}
-
-// pinger sends ping packets for a few seconds.
-//
-// These generated packets are used to ensure we trigger the spray logic in
-// the magicsock package for NAT traversal.
-//
-// This is only used with legacy peers (before 0.100.0) that don't
-// have advertised discovery keys.
-func (e *userspaceEngine) pinger(peerKey wgkey.Key, ips []netaddr.IP) {
-	e.logf("[v1] generating initial ping traffic to %s (%v)", peerKey.ShortString(), ips)
-	var srcIP netaddr.IP
-
-	e.wgLock.Lock()
-	if len(e.lastCfgFull.Addresses) > 0 {
-		srcIP = e.lastCfgFull.Addresses[0].IP
-	}
-	e.wgLock.Unlock()
-
-	if srcIP.IsZero() {
-		e.logf("generating initial ping traffic: no source IP")
-		return
-	}
-
-	ctx, cancel := context.WithCancel(context.Background())
-	p := &pinger{
-		e:      e,
-		done:   make(chan struct{}),
-		cancel: cancel,
-	}
-
-	e.mu.Lock()
-	if e.closing {
-		e.mu.Unlock()
-		return
-	}
-	oldPinger := e.pingers[peerKey]
-	e.pingers[peerKey] = p
-	e.mu.Unlock()
-
-	if oldPinger != nil {
-		oldPinger.close()
-	}
-	p.run(ctx, peerKey, ips, srcIP)
-}
-
 var (
 	debugTrimWireguardEnv = os.Getenv("TS_DEBUG_TRIM_WIREGUARD")
 	debugTrimWireguard, _ = strconv.ParseBool(debugTrimWireguardEnv)
@@ -675,15 +499,7 @@ func isTrimmablePeer(p *wgcfg.Peer, numPeers int) bool {
 	if forceFullWireguardConfig(numPeers) {
 		return false
 	}
-	if !isSingleEndpoint(p.Endpoints) {
-		return false
-	}
-
-	host, _, err := net.SplitHostPort(p.Endpoints)
-	if err != nil {
-		return false
-	}
-	if !strings.HasSuffix(host, ".disco.tailscale") {
+	if p.Endpoints.DiscoKey.IsZero() {
 		return false
 	}
 
@@ -753,26 +569,6 @@ func (e *userspaceEngine) isActiveSince(dk tailcfg.DiscoKey, ip netaddr.IP, t ti
 	return unixTime >= t.Unix()
 }
 
-// discoKeyFromPeer returns the DiscoKey for a wireguard config's Peer.
-//
-// Invariant: isTrimmablePeer(p) == true, so it should have 1 endpoint with
-// Host of form "<64-hex-digits>.disco.tailscale". If invariant is violated,
-// we return the zero value.
-func discoKeyFromPeer(p *wgcfg.Peer) tailcfg.DiscoKey {
-	if len(p.Endpoints) < 64 {
-		return tailcfg.DiscoKey{}
-	}
-	host, rest := p.Endpoints[:64], p.Endpoints[64:]
-	if !strings.HasPrefix(rest, ".disco.tailscale") {
-		return tailcfg.DiscoKey{}
-	}
-	k, err := key.NewPublicFromHexMem(mem.S(host))
-	if err != nil {
-		return tailcfg.DiscoKey{}
-	}
-	return tailcfg.DiscoKey(k)
-}
-
 // discoChanged are the set of peers whose disco keys have changed, implying they've restarted.
 // If a peer is in this set and was previously in the live wireguard config,
 // it needs to be first removed and then re-added to flush out its wireguard session key.
@@ -820,7 +616,7 @@ func (e *userspaceEngine) maybeReconfigWireguardLocked(discoChanged map[key.Publ
 			}
 			continue
 		}
-		dk := discoKeyFromPeer(p)
+		dk := p.Endpoints.DiscoKey
 		trackDisco = append(trackDisco, dk)
 		recentlyActive := false
 		for _, cidr := range p.AllowedIPs {
@@ -837,7 +633,7 @@ func (e *userspaceEngine) maybeReconfigWireguardLocked(discoChanged map[key.Publ
 		}
 	}
 
-	if !deepprint.UpdateHash(&e.lastEngineSigTrim, min, trimmedDisco, trackDisco, trackIPs) {
+	if !deephash.UpdateHash(&e.lastEngineSigTrim, min, trimmedDisco, trackDisco, trackIPs) {
 		// No changes
 		return nil
 	}
@@ -936,28 +732,6 @@ func (e *userspaceEngine) updateActivityMapsLocked(trackDisco []tailcfg.DiscoKey
 	e.tundev.SetDestIPActivityFuncs(e.destIPActivityFuncs)
 }
 
-// genLocalAddrFunc returns a func that reports whether an IP is in addrs.
-// addrs is assumed to be all /32 or /128 entries.
-func genLocalAddrFunc(addrs []netaddr.IPPrefix) func(netaddr.IP) bool {
-	// Specialize the three common cases: no address, just IPv4
-	// (or just IPv6), and both IPv4 and IPv6.
-	if len(addrs) == 0 {
-		return func(netaddr.IP) bool { return false }
-	}
-	if len(addrs) == 1 {
-		return func(t netaddr.IP) bool { return t == addrs[0].IP }
-	}
-	if len(addrs) == 2 {
-		return func(t netaddr.IP) bool { return t == addrs[0].IP || t == addrs[1].IP }
-	}
-	// Otherwise, the general implementation: a map lookup.
-	m := map[netaddr.IP]bool{}
-	for _, a := range addrs {
-		m[a.IP] = true
-	}
-	return func(t netaddr.IP) bool { return m[t] }
-}
-
 func (e *userspaceEngine) Reconfig(cfg *wgcfg.Config, routerCfg *router.Config, dnsCfg *dns.Config) error {
 	if routerCfg == nil {
 		panic("routerCfg must not be nil")
@@ -966,7 +740,7 @@ func (e *userspaceEngine) Reconfig(cfg *wgcfg.Config, routerCfg *router.Config,
 		panic("dnsCfg must not be nil")
 	}
 
-	e.isLocalAddr.Store(genLocalAddrFunc(routerCfg.LocalAddrs))
+	e.isLocalAddr.Store(tsaddr.NewContainsIPFunc(routerCfg.LocalAddrs))
 
 	e.wgLock.Lock()
 	defer e.wgLock.Unlock()
@@ -980,8 +754,8 @@ func (e *userspaceEngine) Reconfig(cfg *wgcfg.Config, routerCfg *router.Config,
 	}
 	e.mu.Unlock()
 
-	engineChanged := deepprint.UpdateHash(&e.lastEngineSigFull, cfg)
-	routerChanged := deepprint.UpdateHash(&e.lastRouterSig, routerCfg, dnsCfg)
+	engineChanged := deephash.UpdateHash(&e.lastEngineSigFull, cfg)
+	routerChanged := deephash.UpdateHash(&e.lastRouterSig, routerCfg, dnsCfg)
 	if !engineChanged && !routerChanged {
 		return ErrNoChanges
 	}
@@ -992,26 +766,26 @@ func (e *userspaceEngine) Reconfig(cfg *wgcfg.Config, routerCfg *router.Config,
 	// and a second time with it.
 	discoChanged := make(map[key.Public]bool)
 	{
-		prevEP := make(map[key.Public]string)
+		prevEP := make(map[key.Public]tailcfg.DiscoKey)
 		for i := range e.lastCfgFull.Peers {
-			if p := &e.lastCfgFull.Peers[i]; isSingleEndpoint(p.Endpoints) {
-				prevEP[key.Public(p.PublicKey)] = p.Endpoints
+			if p := &e.lastCfgFull.Peers[i]; !p.Endpoints.DiscoKey.IsZero() {
+				prevEP[key.Public(p.PublicKey)] = p.Endpoints.DiscoKey
 			}
 		}
 		for i := range cfg.Peers {
 			p := &cfg.Peers[i]
-			if !isSingleEndpoint(p.Endpoints) {
+			if p.Endpoints.DiscoKey.IsZero() {
 				continue
 			}
 			pub := key.Public(p.PublicKey)
-			if old, ok := prevEP[pub]; ok && old != p.Endpoints {
+			if old, ok := prevEP[pub]; ok && old != p.Endpoints.DiscoKey {
 				discoChanged[pub] = true
 				e.logf("wgengine: Reconfig: %s changed from %q to %q", pub.ShortString(), old, p.Endpoints)
 			}
 		}
 	}
 
-	e.lastCfgFull = cfg.Copy()
+	e.lastCfgFull = *cfg.Clone()
 
 	// Tell magicsock about the new (or initial) private key
 	// (which is needed by DERP) before wgdev gets it, as wgdev
@@ -1048,11 +822,6 @@ func (e *userspaceEngine) Reconfig(cfg *wgcfg.Config, routerCfg *router.Config,
 	return nil
 }
 
-// isSingleEndpoint reports whether endpoints contains exactly one host:port pair.
-func isSingleEndpoint(s string) bool {
-	return s != "" && !strings.Contains(s, ",")
-}
-
 func (e *userspaceEngine) GetFilter() *filter.Filter {
 	return e.tundev.GetFilter()
 }
@@ -1075,6 +844,8 @@ func (e *userspaceEngine) getStatusCallback() StatusCallback {
 
 var singleNewline = []byte{'\n'}
 
+var ErrEngineClosing = errors.New("engine closing; no status")
+
 func (e *userspaceEngine) getStatus() (*Status, error) {
 	// Grab derpConns before acquiring wgLock to not violate lock ordering;
 	// the DERPs method acquires magicsock.Conn.mu.
@@ -1088,7 +859,7 @@ func (e *userspaceEngine) getStatus() (*Status, error) {
 	closing := e.closing
 	e.mu.Unlock()
 	if closing {
-		return nil, errors.New("engine closing; no status")
+		return nil, ErrEngineClosing
 	}
 
 	if e.wgdev == nil {
@@ -1235,17 +1006,12 @@ func (e *userspaceEngine) RequestStatus() {
 }
 
 func (e *userspaceEngine) Close() {
-	var pingers []*pinger
-
 	e.mu.Lock()
 	if e.closing {
 		e.mu.Unlock()
 		return
 	}
 	e.closing = true
-	for _, pinger := range e.pingers {
-		pingers = append(pingers, pinger)
-	}
 	e.mu.Unlock()
 
 	r := bufio.NewReader(strings.NewReader(""))
@@ -1259,13 +1025,6 @@ func (e *userspaceEngine) Close() {
 	e.router.Close()
 	e.wgdev.Close()
 	e.tundev.Close()
-
-	// Shut down pingers after tundev is closed (by e.wgdev.Close) so the
-	// synchronous close does not get stuck on InjectOutbound.
-	for _, pinger := range pingers {
-		pinger.close()
-	}
-
 	close(e.waitCh)
 }
 

wgengine/userspace_test.go

@@ -104,7 +104,7 @@ func TestUserspaceEngineReconfig(t *testing.T) {
 					AllowedIPs: []netaddr.IPPrefix{
 						{IP: netaddr.IPv4(100, 100, 99, 1), Bits: 32},
 					},
-					Endpoints: discoHex + ".disco.tailscale:12345",
+					Endpoints: wgcfg.Endpoints{DiscoKey: dkFromHex(discoHex)},
 				},
 			},
 		}

wgengine/wgcfg/clone.go

@@ -0,0 +1,101 @@
+// Copyright (c) 2020 Tailscale Inc & AUTHORS All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+// Code generated by tailscale.com/cmd/cloner -type Config,Peer,Endpoints,IPPortSet; DO NOT EDIT.
+
+package wgcfg
+
+import (
+	"inet.af/netaddr"
+	"tailscale.com/tailcfg"
+	"tailscale.com/types/wgkey"
+)
+
+// Clone makes a deep copy of Config.
+// The result aliases no memory with the original.
+func (src *Config) Clone() *Config {
+	if src == nil {
+		return nil
+	}
+	dst := new(Config)
+	*dst = *src
+	dst.Addresses = append(src.Addresses[:0:0], src.Addresses...)
+	dst.DNS = append(src.DNS[:0:0], src.DNS...)
+	dst.Peers = make([]Peer, len(src.Peers))
+	for i := range dst.Peers {
+		dst.Peers[i] = *src.Peers[i].Clone()
+	}
+	return dst
+}
+
+// A compilation failure here means this code must be regenerated, with command:
+//   tailscale.com/cmd/cloner -type Config,Peer,Endpoints,IPPortSet
+var _ConfigNeedsRegeneration = Config(struct {
+	Name       string
+	PrivateKey wgkey.Private
+	Addresses  []netaddr.IPPrefix
+	MTU        uint16
+	DNS        []netaddr.IP
+	Peers      []Peer
+}{})
+
+// Clone makes a deep copy of Peer.
+// The result aliases no memory with the original.
+func (src *Peer) Clone() *Peer {
+	if src == nil {
+		return nil
+	}
+	dst := new(Peer)
+	*dst = *src
+	dst.AllowedIPs = append(src.AllowedIPs[:0:0], src.AllowedIPs...)
+	dst.Endpoints = *src.Endpoints.Clone()
+	return dst
+}
+
+// A compilation failure here means this code must be regenerated, with command:
+//   tailscale.com/cmd/cloner -type Config,Peer,Endpoints,IPPortSet
+var _PeerNeedsRegeneration = Peer(struct {
+	PublicKey           wgkey.Key
+	AllowedIPs          []netaddr.IPPrefix
+	Endpoints           Endpoints
+	PersistentKeepalive uint16
+}{})
+
+// Clone makes a deep copy of Endpoints.
+// The result aliases no memory with the original.
+func (src *Endpoints) Clone() *Endpoints {
+	if src == nil {
+		return nil
+	}
+	dst := new(Endpoints)
+	*dst = *src
+	dst.IPPorts = *src.IPPorts.Clone()
+	return dst
+}
+
+// A compilation failure here means this code must be regenerated, with command:
+//   tailscale.com/cmd/cloner -type Config,Peer,Endpoints,IPPortSet
+var _EndpointsNeedsRegeneration = Endpoints(struct {
+	PublicKey wgkey.Key
+	DiscoKey  tailcfg.DiscoKey
+	IPPorts   IPPortSet
+}{})
+
+// Clone makes a deep copy of IPPortSet.
+// The result aliases no memory with the original.
+func (src *IPPortSet) Clone() *IPPortSet {
+	if src == nil {
+		return nil
+	}
+	dst := new(IPPortSet)
+	*dst = *src
+	dst.ipp = append(src.ipp[:0:0], src.ipp...)
+	return dst
+}
+
+// A compilation failure here means this code must be regenerated, with command:
+//   tailscale.com/cmd/cloner -type Config,Peer,Endpoints,IPPortSet
+var _IPPortSetNeedsRegeneration = IPPortSet(struct {
+	ipp []netaddr.IPPort
+}{})

wgengine/wgcfg/config.go

@@ -6,14 +6,15 @@
 package wgcfg
 
 import (
+	"encoding/json"
+	"strings"
+
 	"inet.af/netaddr"
+	"tailscale.com/tailcfg"
 	"tailscale.com/types/wgkey"
 )
 
-// EndpointDiscoSuffix is appended to the hex representation of a peer's discovery key
-// and is then the sole wireguard endpoint for peers with a non-zero discovery key.
-// This form is then recognize by magicsock's CreateEndpoint.
-const EndpointDiscoSuffix = ".disco.tailscale:12345"
+//go:generate go run tailscale.com/cmd/cloner -type=Config,Peer,Endpoints,IPPortSet -output=clone.go
 
 // Config is a WireGuard configuration.
 // It only supports the set of things Tailscale uses.
@@ -29,36 +30,90 @@ type Config struct {
 type Peer struct {
 	PublicKey           wgkey.Key
 	AllowedIPs          []netaddr.IPPrefix
-	Endpoints           string // comma-separated host/port pairs: "1.2.3.4:56,[::]:80"
+	Endpoints           Endpoints
 	PersistentKeepalive uint16
 }
 
-// Copy makes a deep copy of Config.
-// The result aliases no memory with the original.
-func (cfg Config) Copy() Config {
-	res := cfg
-	if res.Addresses != nil {
-		res.Addresses = append([]netaddr.IPPrefix{}, res.Addresses...)
-	}
-	if res.DNS != nil {
-		res.DNS = append([]netaddr.IP{}, res.DNS...)
-	}
-	peers := make([]Peer, 0, len(res.Peers))
-	for _, peer := range res.Peers {
-		peers = append(peers, peer.Copy())
-	}
-	res.Peers = peers
-	return res
+// Endpoints represents the routes to reach a remote node.
+// It is serialized and provided to wireguard-go as a conn.Endpoint.
+type Endpoints struct {
+	// PublicKey is the public key for the remote node.
+	PublicKey wgkey.Key `json:"pk"`
+	// DiscoKey is the disco key associated with the remote node.
+	DiscoKey tailcfg.DiscoKey `json:"dk,omitempty"`
+	// IPPorts is a set of possible ip+ports the remote node can be reached at.
+	// This is used only for legacy connections to pre-disco (pre-0.100) peers.
+	IPPorts IPPortSet `json:"ipp,omitempty"`
 }
 
-// Copy makes a deep copy of Peer.
-// The result aliases no memory with the original.
-func (peer Peer) Copy() Peer {
-	res := peer
-	if res.AllowedIPs != nil {
-		res.AllowedIPs = append([]netaddr.IPPrefix{}, res.AllowedIPs...)
+func (e Endpoints) Equal(f Endpoints) bool {
+	if e.PublicKey != f.PublicKey {
+		return false
 	}
-	return res
+	if e.DiscoKey != f.DiscoKey {
+		return false
+	}
+	return e.IPPorts.EqualUnordered(f.IPPorts)
+}
+
+// IPPortSet is an immutable slice of netaddr.IPPorts.
+type IPPortSet struct {
+	ipp []netaddr.IPPort
+}
+
+// NewIPPortSet returns an IPPortSet containing the ports in ipp.
+func NewIPPortSet(ipps ...netaddr.IPPort) IPPortSet {
+	return IPPortSet{ipp: append(ipps[:0:0], ipps...)}
+}
+
+// String returns a comma-separated list of all IPPorts in s.
+func (s IPPortSet) String() string {
+	buf := new(strings.Builder)
+	for i, ipp := range s.ipp {
+		if i > 0 {
+			buf.WriteByte(',')
+		}
+		buf.WriteString(ipp.String())
+	}
+	return buf.String()
+}
+
+// IPPorts returns a slice of netaddr.IPPorts containing the IPPorts in s.
+func (s IPPortSet) IPPorts() []netaddr.IPPort {
+	return append(s.ipp[:0:0], s.ipp...)
+}
+
+// EqualUnordered reports whether s and t contain the same IPPorts, regardless of order.
+func (s IPPortSet) EqualUnordered(t IPPortSet) bool {
+	if len(s.ipp) != len(t.ipp) {
+		return false
+	}
+	// Check whether the endpoints are the same, regardless of order.
+	ipps := make(map[netaddr.IPPort]int, len(s.ipp))
+	for _, ipp := range s.ipp {
+		ipps[ipp]++
+	}
+	for _, ipp := range t.ipp {
+		ipps[ipp]--
+	}
+	for _, n := range ipps {
+		if n != 0 {
+			return false
+		}
+	}
+	return true
+}
+
+// MarshalJSON marshals s into JSON.
+// It is necessary so that IPPortSet's fields can be unexported, to guarantee immutability.
+func (s IPPortSet) MarshalJSON() ([]byte, error) {
+	return json.Marshal(s.ipp)
+}
+
+// UnmarshalJSON unmarshals s from JSON.
+// It is necessary so that IPPortSet's fields can be unexported, to guarantee immutability.
+func (s *IPPortSet) UnmarshalJSON(b []byte) error {
+	return json.Unmarshal(b, &s.ipp)
 }
 
 // PeerWithKey returns the Peer with key k and reports whether it was found.

wgengine/wgcfg/device.go

@@ -20,13 +20,14 @@ func DeviceConfig(d *device.Device) (*Config, error) {
 		w.Close()
 	}()
 	cfg, err := FromUAPI(r)
+	// Prefer errors from IpcGetOperation.
+	if setErr := <-errc; setErr != nil {
+		return nil, setErr
+	}
+	// Check FromUAPI error.
 	if err != nil {
 		return nil, err
 	}
-	if err := <-errc; err != nil {
-		return nil, err
-	}
-
 	sort.Slice(cfg.Peers, func(i, j int) bool {
 		return cfg.Peers[i].PublicKey.LessThan(&cfg.Peers[j].PublicKey)
 	})
@@ -47,15 +48,17 @@ func ReconfigDevice(d *device.Device, cfg *Config, logf logger.Logf) (err error)
 	}
 
 	r, w := io.Pipe()
-	errc := make(chan error)
+	errc := make(chan error, 1)
 	go func() {
 		errc <- d.IpcSetOperation(r)
+		w.Close()
 	}()
 
 	err = cfg.ToUAPI(w, prev)
-	if err != nil {
-		return err
-	}
 	w.Close()
-	return <-errc
+	// Prefer errors from IpcSetOperation.
+	if setErr := <-errc; setErr != nil {
+		return setErr
+	}
+	return err // err (if any) from cfg.ToUAPI
 }

wgengine/wgcfg/device_test.go

@@ -8,7 +8,9 @@ import (
 	"bufio"
 	"bytes"
 	"io"
+	"net"
 	"os"
+	"reflect"
 	"sort"
 	"strings"
 	"sync"
@@ -56,8 +58,8 @@ func TestDeviceConfig(t *testing.T) {
 		}},
 	}
 
-	device1 := device.NewDevice(newNilTun(), conn.NewDefaultBind(), device.NewLogger(device.LogLevelError, "device1"))
-	device2 := device.NewDevice(newNilTun(), conn.NewDefaultBind(), device.NewLogger(device.LogLevelError, "device2"))
+	device1 := device.NewDevice(newNilTun(), new(noopBind), device.NewLogger(device.LogLevelError, "device1"))
+	device2 := device.NewDevice(newNilTun(), new(noopBind), device.NewLogger(device.LogLevelError, "device2"))
 	defer device1.Close()
 	defer device2.Close()
 
@@ -89,7 +91,7 @@ func TestDeviceConfig(t *testing.T) {
 				t.Errorf("on error, could not IpcGetOperation: %v", err)
 			}
 			w.Flush()
-			t.Errorf("cfg:\n%s\n---- want:\n%s\n---- uapi:\n%s", gotStr, wantStr, buf.String())
+			t.Errorf("config mismatch:\n---- got:\n%s\n---- want:\n%s\n---- uapi:\n%s", gotStr, wantStr, buf.String())
 		}
 	}
 
@@ -126,7 +128,7 @@ func TestDeviceConfig(t *testing.T) {
 	})
 
 	t.Run("device1 modify peer", func(t *testing.T) {
-		cfg1.Peers[0].Endpoints = "1.2.3.4:12345"
+		cfg1.Peers[0].Endpoints.IPPorts = NewIPPortSet(netaddr.MustParseIPPort("1.2.3.4:12345"))
 		if err := ReconfigDevice(device1, cfg1, t.Logf); err != nil {
 			t.Fatal(err)
 		}
@@ -134,7 +136,7 @@ func TestDeviceConfig(t *testing.T) {
 	})
 
 	t.Run("device1 replace endpoint", func(t *testing.T) {
-		cfg1.Peers[0].Endpoints = "1.1.1.1:123"
+		cfg1.Peers[0].Endpoints.IPPorts = NewIPPortSet(netaddr.MustParseIPPort("1.1.1.1:123"))
 		if err := ReconfigDevice(device1, cfg1, t.Logf); err != nil {
 			t.Fatal(err)
 		}
@@ -175,7 +177,7 @@ func TestDeviceConfig(t *testing.T) {
 		}
 		peersEqual := func(p, q Peer) bool {
 			return p.PublicKey == q.PublicKey && p.PersistentKeepalive == q.PersistentKeepalive &&
-				p.Endpoints == q.Endpoints && cidrsEqual(p.AllowedIPs, q.AllowedIPs)
+				reflect.DeepEqual(p.Endpoints, q.Endpoints) && cidrsEqual(p.AllowedIPs, q.AllowedIPs)
 		}
 		if !peersEqual(peer0(origCfg), peer0(newCfg)) {
 			t.Error("reconfig modified old peer")
@@ -237,3 +239,26 @@ func (t *nilTun) Close() error {
 	close(t.closed)
 	return nil
 }
+
+// A noopBind is a conn.Bind that does no actual binding work.
+type noopBind struct{}
+
+func (noopBind) Open(port uint16) (fns []conn.ReceiveFunc, actualPort uint16, err error) {
+	return nil, 1, nil
+}
+func (noopBind) Close() error                          { return nil }
+func (noopBind) SetMark(mark uint32) error             { return nil }
+func (noopBind) Send(b []byte, ep conn.Endpoint) error { return nil }
+func (noopBind) ParseEndpoint(s string) (conn.Endpoint, error) {
+	return dummyEndpoint(s), nil
+}
+
+// A dummyEndpoint is a string holding the endpoint destination.
+type dummyEndpoint string
+
+func (e dummyEndpoint) ClearSrc()           {}
+func (e dummyEndpoint) SrcToString() string { return "" }
+func (e dummyEndpoint) DstToString() string { return string(e) }
+func (e dummyEndpoint) DstToBytes() []byte  { return nil }
+func (e dummyEndpoint) DstIP() net.IP       { return nil }
+func (dummyEndpoint) SrcIP() net.IP         { return nil }

wgengine/wgcfg/nmcfg/nmcfg.go

@@ -8,8 +8,6 @@ package nmcfg
 import (
 	"bytes"
 	"fmt"
-	"net"
-	"strconv"
 	"strings"
 
 	"inet.af/netaddr"
@@ -79,17 +77,19 @@ func WGCfg(nm *netmap.NetworkMap, logf logger.Logf, flags netmap.WGConfigFlags,
 			cpeer.PersistentKeepalive = 25 // seconds
 		}
 
-		if !peer.DiscoKey.IsZero() {
-			cpeer.Endpoints = fmt.Sprintf("%x.disco.tailscale:12345", peer.DiscoKey[:])
-		} else {
-			if err := appendEndpoint(cpeer, peer.DERP); err != nil {
+		cpeer.Endpoints = wgcfg.Endpoints{PublicKey: wgkey.Key(peer.Key), DiscoKey: peer.DiscoKey}
+		if peer.DiscoKey.IsZero() {
+			// Legacy connection. Add IP+port endpoints.
+			var ipps []netaddr.IPPort
+			if err := appendEndpoint(cpeer, &ipps, peer.DERP); err != nil {
 				return nil, err
 			}
 			for _, ep := range peer.Endpoints {
-				if err := appendEndpoint(cpeer, ep); err != nil {
+				if err := appendEndpoint(cpeer, &ipps, ep); err != nil {
 					return nil, err
 				}
 			}
+			cpeer.Endpoints.IPPorts = wgcfg.NewIPPortSet(ipps...)
 		}
 		didExitNodeWarn := false
 		for _, allowedIP := range peer.AllowedIPs {
@@ -136,21 +136,14 @@ func WGCfg(nm *netmap.NetworkMap, logf logger.Logf, flags netmap.WGConfigFlags,
 	return cfg, nil
 }
 
-func appendEndpoint(peer *wgcfg.Peer, epStr string) error {
+func appendEndpoint(peer *wgcfg.Peer, ipps *[]netaddr.IPPort, epStr string) error {
 	if epStr == "" {
 		return nil
 	}
-	_, port, err := net.SplitHostPort(epStr)
+	ipp, err := netaddr.ParseIPPort(epStr)
 	if err != nil {
 		return fmt.Errorf("malformed endpoint %q for peer %v", epStr, peer.PublicKey.ShortString())
 	}
-	_, err = strconv.ParseUint(port, 10, 16)
-	if err != nil {
-		return fmt.Errorf("invalid port in endpoint %q for peer %v", epStr, peer.PublicKey.ShortString())
-	}
-	if peer.Endpoints != "" {
-		peer.Endpoints += ","
-	}
-	peer.Endpoints += epStr
+	*ipps = append(*ipps, ipp)
 	return nil
 }

wgengine/wgcfg/parser.go

@@ -7,6 +7,7 @@ package wgcfg
 import (
 	"bufio"
 	"encoding/hex"
+	"encoding/json"
 	"fmt"
 	"io"
 	"net"
@@ -26,21 +27,6 @@ func (e *ParseError) Error() string {
 	return fmt.Sprintf("%s: %q", e.why, e.offender)
 }
 
-func validateEndpoints(s string) error {
-	if s == "" {
-		// Otherwise strings.Split of the empty string produces [""].
-		return nil
-	}
-	vals := strings.Split(s, ",")
-	for _, val := range vals {
-		_, _, err := parseEndpoint(val)
-		if err != nil {
-			return err
-		}
-	}
-	return nil
-}
-
 func parseEndpoint(s string) (host string, port uint16, err error) {
 	i := strings.LastIndexByte(s, ':')
 	if i < 0 {
@@ -103,6 +89,7 @@ func FromUAPI(r io.Reader) (*Config, error) {
 		}
 		key := parts[0]
 		value := parts[1]
+		valueBytes := scanner.Bytes()[len(key)+1:]
 
 		if key == "public_key" {
 			if deviceConfig {
@@ -121,7 +108,7 @@ func FromUAPI(r io.Reader) (*Config, error) {
 		if deviceConfig {
 			err = cfg.handleDeviceLine(key, value)
 		} else {
-			err = cfg.handlePeerLine(peer, key, value)
+			err = cfg.handlePeerLine(peer, key, value, valueBytes)
 		}
 		if err != nil {
 			return nil, err
@@ -165,14 +152,13 @@ func (cfg *Config) handlePublicKeyLine(value string) (*Peer, error) {
 	return peer, nil
 }
 
-func (cfg *Config) handlePeerLine(peer *Peer, key, value string) error {
+func (cfg *Config) handlePeerLine(peer *Peer, key, value string, valueBytes []byte) error {
 	switch key {
 	case "endpoint":
-		err := validateEndpoints(value)
+		err := json.Unmarshal(valueBytes, &peer.Endpoints)
 		if err != nil {
 			return err
 		}
-		peer.Endpoints = value
 	case "persistent_keepalive_interval":
 		n, err := strconv.ParseUint(value, 10, 16)
 		if err != nil {

wgengine/wgcfg/parser_test.go

@@ -53,21 +53,3 @@ func TestParseEndpoint(t *testing.T) {
 		t.Error("Error was expected")
 	}
 }
-
-func TestValidateEndpoints(t *testing.T) {
-	tests := []struct {
-		in   string
-		want error
-	}{
-		{"", nil},
-		{"1.2.3.4:5", nil},
-		{"1.2.3.4:5,6.7.8.9:10", nil},
-		{",", &ParseError{why: "Missing port from endpoint", offender: ""}},
-	}
-	for _, tt := range tests {
-		got := validateEndpoints(tt.in)
-		if !reflect.DeepEqual(got, tt.want) {
-			t.Errorf("%q = %#v (%s); want %#v (%s)", tt.in, got, got, tt.want, tt.want)
-		}
-	}
-}

wgengine/wgcfg/writer.go

@@ -5,11 +5,10 @@
 package wgcfg
 
 import (
+	"encoding/json"
 	"fmt"
 	"io"
-	"sort"
 	"strconv"
-	"strings"
 
 	"inet.af/netaddr"
 	"tailscale.com/types/wgkey"
@@ -53,8 +52,12 @@ func (cfg *Config) ToUAPI(w io.Writer, prev *Config) error {
 		setPeer(p)
 		set("protocol_version", "1")
 
-		if !endpointsEqual(oldPeer.Endpoints, p.Endpoints) {
-			set("endpoint", p.Endpoints)
+		if !oldPeer.Endpoints.Equal(p.Endpoints) {
+			buf, err := json.Marshal(p.Endpoints)
+			if err != nil {
+				return err
+			}
+			set("endpoint", string(buf))
 		}
 
 		// TODO: replace_allowed_ips is expensive.
@@ -90,24 +93,6 @@ func (cfg *Config) ToUAPI(w io.Writer, prev *Config) error {
 	return stickyErr
 }
 
-func endpointsEqual(x, y string) bool {
-	// Cheap comparisons.
-	if x == y {
-		return true
-	}
-	xs := strings.Split(x, ",")
-	ys := strings.Split(y, ",")
-	if len(xs) != len(ys) {
-		return false
-	}
-	// Otherwise, see if they're the same, but out of order.
-	sort.Strings(xs)
-	sort.Strings(ys)
-	x = strings.Join(xs, ",")
-	y = strings.Join(ys, ",")
-	return x == y
-}
-
 func cidrsEqual(x, y []netaddr.IPPrefix) bool {
 	// TODO: re-implement using netaddr.IPSet.Equal.
 	if len(x) != len(y) {

wgengine/wglog/wglog.go

@@ -21,7 +21,7 @@ import (
 // It can be modified at run time to adjust to new wireguard-go configurations.
 type Logger struct {
 	DeviceLogger *device.Logger
-	replacer     atomic.Value // of *strings.Replacer
+	replace      atomic.Value // of map[string]string
 }
 
 // NewLogger creates a new logger for use with wireguard-go.
@@ -29,41 +29,48 @@ type Logger struct {
 // and rewrites peer keys from wireguard-go into Tailscale format.
 func NewLogger(logf logger.Logf) *Logger {
 	ret := new(Logger)
-
 	wrapper := func(format string, args ...interface{}) {
-		msg := fmt.Sprintf(format, args...)
-		if strings.Contains(msg, "Routine:") && !strings.Contains(msg, "receive incoming") {
+		if strings.Contains(format, "Routine:") && !strings.Contains(format, "receive incoming") {
 			// wireguard-go logs as it starts and stops routines.
 			// Drop those; there are a lot of them, and they're just noise.
 			return
 		}
-		if strings.Contains(msg, "Failed to send data packet") {
+		if strings.Contains(format, "Failed to send data packet") {
 			// Drop. See https://github.com/tailscale/tailscale/issues/1239.
 			return
 		}
-		if strings.Contains(msg, "Interface up requested") || strings.Contains(msg, "Interface down requested") {
+		if strings.Contains(format, "Interface up requested") || strings.Contains(format, "Interface down requested") {
 			// Drop. Logs 1/s constantly while the tun device is open.
 			// See https://github.com/tailscale/tailscale/issues/1388.
 			return
 		}
-		r := ret.replacer.Load()
-		if r == nil {
+		replace, _ := ret.replace.Load().(map[string]string)
+		if replace == nil {
 			// No replacements specified; log as originally planned.
 			logf(format, args...)
 			return
 		}
-		// Do the replacements.
-		new := r.(*strings.Replacer).Replace(msg)
-		if new == msg {
-			// No replacements. Log as originally planned.
-			logf(format, args...)
-			return
+		// Duplicate the args slice so that we can modify it.
+		// This is not always required, but the code required to avoid it is not worth the complexity.
+		newargs := make([]interface{}, len(args))
+		copy(newargs, args)
+		for i, arg := range newargs {
+			// We want to replace *device.Peer args with the Tailscale-formatted version of themselves.
+			// Using *device.Peer directly makes this hard to test, so we string any fmt.Stringers,
+			// and if the string ends up looking exactly like a known Peer, we replace it.
+			// This is slightly imprecise, in that we don't check the formatting verb. Oh well.
+			s, ok := arg.(fmt.Stringer)
+			if !ok {
+				continue
+			}
+			wgStr := s.String()
+			tsStr, ok := replace[wgStr]
+			if !ok {
+				continue
+			}
+			newargs[i] = tsStr
 		}
-		// We made some replacements. Log the new version.
-		// This changes the format string,
-		// which is somewhat unfortunate as it impacts rate limiting,
-		// but there's not much we can do about that.
-		logf("%s", new)
+		logf(format, newargs...)
 	}
 	ret.DeviceLogger = &device.Logger{
 		Verbosef: logger.WithPrefix(wrapper, "[v2] "),
@@ -76,22 +83,28 @@ func NewLogger(logf logger.Logf) *Logger {
 // SetPeers is safe for concurrent use.
 func (x *Logger) SetPeers(peers []wgcfg.Peer) {
 	// Construct a new peer public key log rewriter.
-	var replace []string
+	replace := make(map[string]string)
 	for _, peer := range peers {
-		old := "peer(" + wireguardGoString(peer.PublicKey) + ")"
+		old := wireguardGoString(peer.PublicKey)
 		new := peer.PublicKey.ShortString()
-		replace = append(replace, old, new)
+		replace[old] = new
 	}
-	r := strings.NewReplacer(replace...)
-	x.replacer.Store(r)
+	x.replace.Store(replace)
 }
 
 // wireguardGoString prints p in the same format used by wireguard-go.
 func wireguardGoString(k wgkey.Key) string {
-	base64Key := base64.StdEncoding.EncodeToString(k[:])
-	abbreviatedKey := "invalid"
-	if len(base64Key) == 44 {
-		abbreviatedKey = base64Key[0:4] + "…" + base64Key[39:43]
-	}
-	return abbreviatedKey
+	const prefix = "peer("
+	b := make([]byte, len(prefix)+44)
+	copy(b, prefix)
+	r := b[len(prefix):]
+	base64.StdEncoding.Encode(r, k[:])
+	r = r[4:]
+	copy(r, "…")
+	r = r[len("…"):]
+	copy(r, b[len(prefix)+39:len(prefix)+43])
+	r = r[4:]
+	r[0] = ')'
+	r = r[1:]
+	return string(b[:len(b)-len(r)])
 }

wgengine/wglog/wglog_test.go

@@ -8,6 +8,7 @@ import (
 	"fmt"
 	"testing"
 
+	"tailscale.com/types/logger"
 	"tailscale.com/types/wgkey"
 	"tailscale.com/wgengine/wgcfg"
 	"tailscale.com/wgengine/wglog"
@@ -15,22 +16,27 @@ import (
 
 func TestLogger(t *testing.T) {
 	tests := []struct {
-		in   string
-		want string
-		omit bool
+		format string
+		args   []interface{}
+		want   string
+		omit   bool
 	}{
-		{"hi", "hi", false},
-		{"Routine: starting", "", true},
-		{"peer(IMTB…r7lM) says it misses you", "[IMTBr] says it misses you", false},
+		{"hi", nil, "hi", false},
+		{"Routine: starting", nil, "", true},
+		{"%v says it misses you", []interface{}{stringer("peer(IMTB…r7lM)")}, "[IMTBr] says it misses you", false},
 	}
 
-	c := make(chan string, 1)
+	type log struct {
+		format string
+		args   []interface{}
+	}
+
+	c := make(chan log, 1)
 	logf := func(format string, args ...interface{}) {
-		s := fmt.Sprintf(format, args...)
 		select {
-		case c <- s:
+		case c <- log{format, args}:
 		default:
-			t.Errorf("wrote %q, but shouldn't have", s)
+			t.Errorf("wrote %q, but shouldn't have", fmt.Sprintf(format, args...))
 		}
 	}
 
@@ -45,15 +51,50 @@ func TestLogger(t *testing.T) {
 		if tt.omit {
 			// Write a message ourselves into the channel.
 			// Then if logf also attempts to write into the channel, it'll fail.
-			c <- ""
+			c <- log{}
 		}
-		x.DeviceLogger.Errorf(tt.in)
-		got := <-c
+		x.DeviceLogger.Errorf(tt.format, tt.args...)
+		gotLog := <-c
 		if tt.omit {
 			continue
 		}
-		if got != tt.want {
-			t.Errorf("Println(%q) = %q want %q", tt.in, got, tt.want)
+		if got := fmt.Sprintf(gotLog.format, gotLog.args...); got != tt.want {
+			t.Errorf("Printf(%q, %v) = %q want %q", tt.format, tt.args, got, tt.want)
 		}
 	}
 }
+
+func stringer(s string) stringerString {
+	return stringerString(s)
+}
+
+type stringerString string
+
+func (s stringerString) String() string { return string(s) }
+
+func BenchmarkSetPeers(b *testing.B) {
+	b.ReportAllocs()
+	x := wglog.NewLogger(logger.Discard)
+	peers := [][]wgcfg.Peer{genPeers(0), genPeers(15), genPeers(16), genPeers(15)}
+	for i := 0; i < b.N; i++ {
+		for _, p := range peers {
+			x.SetPeers(p)
+		}
+	}
+}
+
+func genPeers(n int) []wgcfg.Peer {
+	if n > 32 {
+		panic("too many peers")
+	}
+	if n == 0 {
+		return nil
+	}
+	peers := make([]wgcfg.Peer, n)
+	for i := range peers {
+		var k wgkey.Key
+		k[n] = byte(n)
+		peers[i].PublicKey = k
+	}
+	return peers
+}