More logging :(

The cyolosecurity fork of certstore did not update its module name and
thus can only be used with a replace directive. This interferes with
installing using `go install` so I created a tailscale fork with an
updated module name.

Signed-off-by: Adrian Dewhurst <adrian@tailscale.com>

.github/ISSUE_TEMPLATE/bug_report.md

@@ -2,36 +2,7 @@
 name: Bug report
 about: Create a bug report
 title: ''
-labels: ''
+labels: 'needs-triage'
 assignees: ''
 
 ---
-
-<!-- Please note, this template is for definite bugs, not requests for
-support. If you need help with Tailscale, please email
-support@tailscale.com. We don't provide support via Github issues. -->
-
-**Describe the bug**
-A clear and concise description of what the bug is.
-
-**To Reproduce**
-Steps to reproduce the behavior:
-1. Go to '...'
-2. Click on '....'
-3. Scroll down to '....'
-4. See error
-
-**Expected behavior**
-A clear and concise description of what you expected to happen.
-
-**Screenshots**
-If applicable, add screenshots to help explain your problem.
-
-**Version information:**
- - Device: [e.g. iPhone X, laptop]
- - OS: [e.g. Windows, MacOS]
- - OS version: [e.g. Windows 10, Ubuntu 18.04]
- - Tailscale version: [e.g. 0.95-0]
-
-**Additional context**
-Add any other context about the problem here.

.github/ISSUE_TEMPLATE/feature_request.md

@@ -2,25 +2,6 @@
 name: Feature request
 about: Suggest an idea for this project
 title: ''
-labels: ''
+labels: 'needs-triage'
 assignees: ''
-
 ---
-
-**Is your feature request related to a problem? Please describe.**
-
-A clear and concise description of what the problem is. Ex. I'm always
-frustrated when [...]
-
-**Describe the solution you'd like**
-
-A clear and concise description of what you want to happen.
-
-**Describe alternatives you've considered**
-
-A clear and concise description of any alternative solutions or
-features you've considered.
-
-**Additional context**
-
-Add any other context or screenshots about the feature request here.

VERSION.txt

@@ -1 +1 @@
-1.7.0
+1.9.0

cmd/hello/hello.go

@@ -112,13 +112,13 @@ func tailscaleIP(who *apitype.WhoIsResponse) string {
 		return ""
 	}
 	for _, nodeIP := range who.Node.Addresses {
-		if nodeIP.IP.Is4() && nodeIP.IsSingleIP() {
-			return nodeIP.IP.String()
+		if nodeIP.IP().Is4() && nodeIP.IsSingleIP() {
+			return nodeIP.IP().String()
 		}
 	}
 	for _, nodeIP := range who.Node.Addresses {
 		if nodeIP.IsSingleIP() {
-			return nodeIP.IP.String()
+			return nodeIP.IP().String()
 		}
 	}
 	return ""

cmd/tailscale/cli/cli_test.go

@@ -16,126 +16,91 @@ import (
 	"inet.af/netaddr"
 	"tailscale.com/ipn"
 	"tailscale.com/ipn/ipnstate"
-	"tailscale.com/types/logger"
 	"tailscale.com/types/preftype"
 )
 
+// geese is a collection of gooses. It need not be complete.
+// But it should include anything handled specially (e.g. linux, windows)
+// and at least one thing that's not (darwin, freebsd).
+var geese = []string{"linux", "darwin", "windows", "freebsd"}
+
 // Test that checkForAccidentalSettingReverts's updateMaskedPrefsFromUpFlag can handle
 // all flags. This will panic if a new flag creeps in that's unhandled.
+//
+// Also, issue 1880: advertise-exit-node was being ignored. Verify that all flags cause an edit.
 func TestUpdateMaskedPrefsFromUpFlag(t *testing.T) {
-	mp := new(ipn.MaskedPrefs)
-	upFlagSet.VisitAll(func(f *flag.Flag) {
-		updateMaskedPrefsFromUpFlag(mp, f.Name)
-	})
+	for _, goos := range geese {
+		var upArgs upArgsT
+		fs := newUpFlagSet(goos, &upArgs)
+		fs.VisitAll(func(f *flag.Flag) {
+			mp := new(ipn.MaskedPrefs)
+			updateMaskedPrefsFromUpFlag(mp, f.Name)
+			got := mp.Pretty()
+			wantEmpty := preflessFlag(f.Name)
+			isEmpty := got == "MaskedPrefs{}"
+			if isEmpty != wantEmpty {
+				t.Errorf("flag %q created MaskedPrefs %s; want empty=%v", f.Name, got, wantEmpty)
+			}
+		})
+	}
 }
 
 func TestCheckForAccidentalSettingReverts(t *testing.T) {
-	f := func(flags ...string) map[string]bool {
-		m := make(map[string]bool)
-		for _, f := range flags {
-			m[f] = true
-		}
-		return m
-	}
 	tests := []struct {
 		name     string
-		flagSet  map[string]bool
+		flags    []string // argv to be parsed by FlagSet
 		curPrefs *ipn.Prefs
-		curUser  string // os.Getenv("USER") on the client side
-		goos     string // empty means "linux"
-		mp       *ipn.MaskedPrefs
-		want     string
+
+		curExitNodeIP netaddr.IP
+		curUser       string // os.Getenv("USER") on the client side
+		goos          string // empty means "linux"
+
+		want string
 	}{
 		{
-			name:    "bare_up_means_up",
-			flagSet: f(),
+			name:  "bare_up_means_up",
+			flags: []string{},
 			curPrefs: &ipn.Prefs{
 				ControlURL:  ipn.DefaultControlURL,
 				WantRunning: false,
 				Hostname:    "foo",
 			},
-			mp: &ipn.MaskedPrefs{
-				Prefs: ipn.Prefs{
-					WantRunning: true,
-				},
-				WantRunningSet: true,
-			},
 			want: "",
 		},
 		{
-			name:    "losing_hostname",
-			flagSet: f("accept-dns"),
+			name:  "losing_hostname",
+			flags: []string{"--accept-dns"},
 			curPrefs: &ipn.Prefs{
-				ControlURL:  ipn.DefaultControlURL,
-				WantRunning: false,
-				Hostname:    "foo",
-				CorpDNS:     true,
-			},
-			mp: &ipn.MaskedPrefs{
-				Prefs: ipn.Prefs{
-					ControlURL:  ipn.DefaultControlURL,
-					WantRunning: true,
-					CorpDNS:     true,
-				},
-				ControlURLSet:  true,
-				WantRunningSet: true,
-				CorpDNSSet:     true,
+				ControlURL:       ipn.DefaultControlURL,
+				WantRunning:      false,
+				Hostname:         "foo",
+				CorpDNS:          true,
+				NetfilterMode:    preftype.NetfilterOn,
+				AllowSingleHosts: true,
 			},
 			want: accidentalUpPrefix + " --accept-dns --hostname=foo",
 		},
 		{
-			name:    "hostname_changing_explicitly",
-			flagSet: f("hostname"),
+			name:  "hostname_changing_explicitly",
+			flags: []string{"--hostname=bar"},
 			curPrefs: &ipn.Prefs{
-				ControlURL:  ipn.DefaultControlURL,
-				WantRunning: false,
-				Hostname:    "foo",
-			},
-			mp: &ipn.MaskedPrefs{
-				Prefs: ipn.Prefs{
-					ControlURL:  ipn.DefaultControlURL,
-					WantRunning: true,
-					Hostname:    "bar",
-				},
-				ControlURLSet:  true,
-				WantRunningSet: true,
-				HostnameSet:    true,
+				ControlURL:       ipn.DefaultControlURL,
+				CorpDNS:          true,
+				NetfilterMode:    preftype.NetfilterOn,
+				AllowSingleHosts: true,
+				Hostname:         "foo",
 			},
 			want: "",
 		},
 		{
-			name:    "hostname_changing_empty_explicitly",
-			flagSet: f("hostname"),
+			name:  "hostname_changing_empty_explicitly",
+			flags: []string{"--hostname="},
 			curPrefs: &ipn.Prefs{
-				ControlURL:  ipn.DefaultControlURL,
-				WantRunning: false,
-				Hostname:    "foo",
-			},
-			mp: &ipn.MaskedPrefs{
-				Prefs: ipn.Prefs{
-					ControlURL:  ipn.DefaultControlURL,
-					WantRunning: true,
-					Hostname:    "",
-				},
-				ControlURLSet:  true,
-				WantRunningSet: true,
-				HostnameSet:    true,
-			},
-			want: "",
-		},
-		{
-			name:    "empty_slice_equals_nil_slice",
-			flagSet: f("hostname"),
-			curPrefs: &ipn.Prefs{
-				ControlURL:      ipn.DefaultControlURL,
-				AdvertiseRoutes: []netaddr.IPPrefix{},
-			},
-			mp: &ipn.MaskedPrefs{
-				Prefs: ipn.Prefs{
-					ControlURL:      ipn.DefaultControlURL,
-					AdvertiseRoutes: nil,
-				},
-				ControlURLSet: true,
+				ControlURL:       ipn.DefaultControlURL,
+				CorpDNS:          true,
+				NetfilterMode:    preftype.NetfilterOn,
+				AllowSingleHosts: true,
+				Hostname:         "foo",
 			},
 			want: "",
 		},
@@ -143,228 +108,174 @@ func TestCheckForAccidentalSettingReverts(t *testing.T) {
 			// Issue 1725: "tailscale up --authkey=..." (or other non-empty flags) works from
 			// a fresh server's initial prefs.
 			name:     "up_with_default_prefs",
-			flagSet:  f("authkey"),
+			flags:    []string{"--authkey=foosdlkfjskdljf"},
 			curPrefs: ipn.NewPrefs(),
-			mp: &ipn.MaskedPrefs{
-				Prefs:          *defaultPrefsFromUpArgs(t, "linux"),
-				WantRunningSet: true,
-			},
-			want: "",
+			want:     "",
 		},
 		{
-			name:    "implicit_operator_change",
-			flagSet: f("hostname"),
+			name:  "implicit_operator_change",
+			flags: []string{"--hostname=foo"},
 			curPrefs: &ipn.Prefs{
-				ControlURL:   ipn.DefaultControlURL,
-				OperatorUser: "alice",
+				ControlURL:       ipn.DefaultControlURL,
+				OperatorUser:     "alice",
+				AllowSingleHosts: true,
+				CorpDNS:          true,
+				NetfilterMode:    preftype.NetfilterOn,
 			},
 			curUser: "eve",
-			mp: &ipn.MaskedPrefs{
-				Prefs: ipn.Prefs{
-					ControlURL: ipn.DefaultControlURL,
-				},
-				ControlURLSet: true,
-			},
-			want: accidentalUpPrefix + " --hostname= --operator=alice",
+			want:    accidentalUpPrefix + " --hostname=foo --operator=alice",
 		},
 		{
-			name:    "implicit_operator_matches_shell_user",
-			flagSet: f("hostname"),
+			name:  "implicit_operator_matches_shell_user",
+			flags: []string{"--hostname=foo"},
 			curPrefs: &ipn.Prefs{
-				ControlURL:   ipn.DefaultControlURL,
-				OperatorUser: "alice",
+				ControlURL:       ipn.DefaultControlURL,
+				AllowSingleHosts: true,
+				CorpDNS:          true,
+				NetfilterMode:    preftype.NetfilterOn,
+				OperatorUser:     "alice",
 			},
 			curUser: "alice",
-			mp: &ipn.MaskedPrefs{
-				Prefs: ipn.Prefs{
-					ControlURL: ipn.DefaultControlURL,
-				},
-				ControlURLSet: true,
-			},
-			want: "",
+			want:    "",
 		},
 		{
-			name:    "error_advertised_routes_exit_node_removed",
-			flagSet: f("advertise-routes"),
+			name:  "error_advertised_routes_exit_node_removed",
+			flags: []string{"--advertise-routes=10.0.42.0/24"},
 			curPrefs: &ipn.Prefs{
-				ControlURL: ipn.DefaultControlURL,
+				ControlURL:       ipn.DefaultControlURL,
+				AllowSingleHosts: true,
+				CorpDNS:          true,
+				NetfilterMode:    preftype.NetfilterOn,
 				AdvertiseRoutes: []netaddr.IPPrefix{
 					netaddr.MustParseIPPrefix("10.0.42.0/24"),
 					netaddr.MustParseIPPrefix("0.0.0.0/0"),
 					netaddr.MustParseIPPrefix("::/0"),
 				},
 			},
-			mp: &ipn.MaskedPrefs{
-				Prefs: ipn.Prefs{
-					ControlURL: ipn.DefaultControlURL,
-					AdvertiseRoutes: []netaddr.IPPrefix{
-						netaddr.MustParseIPPrefix("10.0.42.0/24"),
-					},
-				},
-				AdvertiseRoutesSet: true,
-			},
 			want: accidentalUpPrefix + " --advertise-routes=10.0.42.0/24 --advertise-exit-node",
 		},
 		{
-			name:    "advertised_routes_exit_node_removed",
-			flagSet: f("advertise-routes", "advertise-exit-node"),
+			name:  "advertised_routes_exit_node_removed_explicit",
+			flags: []string{"--advertise-routes=10.0.42.0/24", "--advertise-exit-node=false"},
 			curPrefs: &ipn.Prefs{
-				ControlURL: ipn.DefaultControlURL,
+				ControlURL:       ipn.DefaultControlURL,
+				AllowSingleHosts: true,
+				CorpDNS:          true,
+				NetfilterMode:    preftype.NetfilterOn,
 				AdvertiseRoutes: []netaddr.IPPrefix{
 					netaddr.MustParseIPPrefix("10.0.42.0/24"),
 					netaddr.MustParseIPPrefix("0.0.0.0/0"),
 					netaddr.MustParseIPPrefix("::/0"),
 				},
 			},
-			mp: &ipn.MaskedPrefs{
-				Prefs: ipn.Prefs{
-					ControlURL: ipn.DefaultControlURL,
-					AdvertiseRoutes: []netaddr.IPPrefix{
-						netaddr.MustParseIPPrefix("10.0.42.0/24"),
-					},
-				},
-				AdvertiseRoutesSet: true,
-			},
 			want: "",
 		},
 		{
-			name:    "advertised_routes_includes_the_0_routes", // but no --advertise-exit-node
-			flagSet: f("advertise-routes"),
+			name:  "advertised_routes_includes_the_0_routes", // but no --advertise-exit-node
+			flags: []string{"--advertise-routes=11.1.43.0/24,0.0.0.0/0,::/0"},
 			curPrefs: &ipn.Prefs{
-				ControlURL: ipn.DefaultControlURL,
+				ControlURL:       ipn.DefaultControlURL,
+				AllowSingleHosts: true,
+				CorpDNS:          true,
+				NetfilterMode:    preftype.NetfilterOn,
 				AdvertiseRoutes: []netaddr.IPPrefix{
 					netaddr.MustParseIPPrefix("10.0.42.0/24"),
 					netaddr.MustParseIPPrefix("0.0.0.0/0"),
 					netaddr.MustParseIPPrefix("::/0"),
 				},
 			},
-			mp: &ipn.MaskedPrefs{
-				Prefs: ipn.Prefs{
-					ControlURL: ipn.DefaultControlURL,
-					AdvertiseRoutes: []netaddr.IPPrefix{
-						netaddr.MustParseIPPrefix("11.1.43.0/24"),
-						netaddr.MustParseIPPrefix("0.0.0.0/0"),
-						netaddr.MustParseIPPrefix("::/0"),
-					},
-				},
-				AdvertiseRoutesSet: true,
+			want: "",
+		},
+		{
+			name:  "advertise_exit_node", // Issue 1859
+			flags: []string{"--advertise-exit-node"},
+			curPrefs: &ipn.Prefs{
+				ControlURL:       ipn.DefaultControlURL,
+				AllowSingleHosts: true,
+				CorpDNS:          true,
+				NetfilterMode:    preftype.NetfilterOn,
 			},
 			want: "",
 		},
 		{
-			name:    "advertised_routes_includes_only_one_0_route", // and no --advertise-exit-node
-			flagSet: f("advertise-routes"),
+			name:  "advertise_exit_node_over_existing_routes",
+			flags: []string{"--advertise-exit-node"},
 			curPrefs: &ipn.Prefs{
-				ControlURL: ipn.DefaultControlURL,
-				AdvertiseRoutes: []netaddr.IPPrefix{
-					netaddr.MustParseIPPrefix("10.0.42.0/24"),
-					netaddr.MustParseIPPrefix("0.0.0.0/0"),
-					netaddr.MustParseIPPrefix("::/0"),
-				},
-			},
-			mp: &ipn.MaskedPrefs{
-				Prefs: ipn.Prefs{
-					ControlURL: ipn.DefaultControlURL,
-					AdvertiseRoutes: []netaddr.IPPrefix{
-						netaddr.MustParseIPPrefix("11.1.43.0/24"),
-						netaddr.MustParseIPPrefix("0.0.0.0/0"),
-					},
-				},
-				AdvertiseRoutesSet: true,
-			},
-			want: accidentalUpPrefix + " --advertise-routes=11.1.43.0/24,0.0.0.0/0 --advertise-exit-node",
-		},
-		{
-			name:    "advertise_exit_node", // Issue 1859
-			flagSet: f("advertise-exit-node"),
-			curPrefs: &ipn.Prefs{
-				ControlURL: ipn.DefaultControlURL,
-			},
-			mp: &ipn.MaskedPrefs{
-				Prefs: ipn.Prefs{
-					ControlURL: ipn.DefaultControlURL,
-					AdvertiseRoutes: []netaddr.IPPrefix{
-						netaddr.MustParseIPPrefix("0.0.0.0/0"),
-						netaddr.MustParseIPPrefix("::/0"),
-					},
-				},
-				// Note: without setting "AdvertiseRoutesSet", as
-				// updateMaskedPrefsFromUpFlag doesn't set that.
-			},
-			want: "",
-		},
-		{
-			name:    "advertise_exit_node_over_existing_routes",
-			flagSet: f("advertise-exit-node"),
-			curPrefs: &ipn.Prefs{
-				ControlURL: ipn.DefaultControlURL,
+				ControlURL:       ipn.DefaultControlURL,
+				AllowSingleHosts: true,
+				CorpDNS:          true,
+				NetfilterMode:    preftype.NetfilterOn,
+
 				AdvertiseRoutes: []netaddr.IPPrefix{
 					netaddr.MustParseIPPrefix("1.2.0.0/16"),
 				},
 			},
-			mp: &ipn.MaskedPrefs{
-				Prefs: ipn.Prefs{
-					ControlURL: ipn.DefaultControlURL,
-					AdvertiseRoutes: []netaddr.IPPrefix{
-						netaddr.MustParseIPPrefix("0.0.0.0/0"),
-						netaddr.MustParseIPPrefix("::/0"),
-					},
-				},
-				// Note: without setting "AdvertiseRoutesSet", as
-				// updateMaskedPrefsFromUpFlag doesn't set that.
-			},
 			want: accidentalUpPrefix + " --advertise-exit-node --advertise-routes=1.2.0.0/16",
 		},
 		{
-			name:    "advertise_exit_node_over_existing_routes_and_exit_node",
-			flagSet: f("advertise-exit-node"),
+			name:  "advertise_exit_node_over_existing_routes_and_exit_node",
+			flags: []string{"--advertise-exit-node"},
 			curPrefs: &ipn.Prefs{
-				ControlURL: ipn.DefaultControlURL,
+				ControlURL:       ipn.DefaultControlURL,
+				AllowSingleHosts: true,
+				CorpDNS:          true,
+				NetfilterMode:    preftype.NetfilterOn,
 				AdvertiseRoutes: []netaddr.IPPrefix{
 					netaddr.MustParseIPPrefix("0.0.0.0/0"),
 					netaddr.MustParseIPPrefix("::/0"),
 					netaddr.MustParseIPPrefix("1.2.0.0/16"),
 				},
 			},
-			mp: &ipn.MaskedPrefs{
-				Prefs: ipn.Prefs{
-					ControlURL: ipn.DefaultControlURL,
-					AdvertiseRoutes: []netaddr.IPPrefix{
-						netaddr.MustParseIPPrefix("0.0.0.0/0"),
-						netaddr.MustParseIPPrefix("::/0"),
-					},
-				},
-				// Note: without setting "AdvertiseRoutesSet", as
-				// updateMaskedPrefsFromUpFlag doesn't set that.
-			},
 			want: accidentalUpPrefix + " --advertise-exit-node --advertise-routes=1.2.0.0/16",
 		},
 		{
-			name:    "exit_node_clearing", // Issue 1777
-			flagSet: f("exit-node"),
+			name:  "exit_node_clearing", // Issue 1777
+			flags: []string{"--exit-node="},
 			curPrefs: &ipn.Prefs{
-				ControlURL: ipn.DefaultControlURL,
+				ControlURL:       ipn.DefaultControlURL,
+				AllowSingleHosts: true,
+				CorpDNS:          true,
+				NetfilterMode:    preftype.NetfilterOn,
+
 				ExitNodeID: "fooID",
 			},
-			mp: &ipn.MaskedPrefs{
-				Prefs: ipn.Prefs{
-					ControlURL: ipn.DefaultControlURL,
-					ExitNodeIP: netaddr.IP{},
-				},
-				ExitNodeIPSet: true,
-			},
 			want: "",
 		},
 		{
-			name:    "remove_all_implicit",
-			flagSet: f("force-reauth"),
+			name:  "remove_all_implicit",
+			flags: []string{"--force-reauth"},
 			curPrefs: &ipn.Prefs{
 				WantRunning:      true,
 				ControlURL:       ipn.DefaultControlURL,
 				RouteAll:         true,
 				AllowSingleHosts: false,
 				ExitNodeIP:       netaddr.MustParseIP("100.64.5.6"),
-				CorpDNS:          true,
+				CorpDNS:          false,
+				ShieldsUp:        true,
+				AdvertiseTags:    []string{"tag:foo", "tag:bar"},
+				Hostname:         "myhostname",
+				ForceDaemon:      true,
+				AdvertiseRoutes: []netaddr.IPPrefix{
+					netaddr.MustParseIPPrefix("10.0.0.0/16"),
+					netaddr.MustParseIPPrefix("0.0.0.0/0"),
+					netaddr.MustParseIPPrefix("::/0"),
+				},
+				NetfilterMode: preftype.NetfilterNoDivert,
+				OperatorUser:  "alice",
+			},
+			curUser: "eve",
+			want:    accidentalUpPrefix + " --force-reauth --accept-dns=false --accept-routes --advertise-exit-node --advertise-routes=10.0.0.0/16 --advertise-tags=tag:foo,tag:bar --exit-node=100.64.5.6 --host-routes=false --hostname=myhostname --netfilter-mode=nodivert --operator=alice --shields-up",
+		},
+		{
+			name:  "remove_all_implicit_except_hostname",
+			flags: []string{"--hostname=newhostname"},
+			curPrefs: &ipn.Prefs{
+				WantRunning:      true,
+				ControlURL:       ipn.DefaultControlURL,
+				RouteAll:         true,
+				AllowSingleHosts: false,
+				ExitNodeIP:       netaddr.MustParseIP("100.64.5.6"),
+				CorpDNS:          false,
 				ShieldsUp:        true,
 				AdvertiseTags:    []string{"tag:foo", "tag:bar"},
 				Hostname:         "myhostname",
@@ -376,102 +287,121 @@ func TestCheckForAccidentalSettingReverts(t *testing.T) {
 				OperatorUser:  "alice",
 			},
 			curUser: "eve",
-			mp: &ipn.MaskedPrefs{
-				Prefs: ipn.Prefs{
-					ControlURL:  ipn.DefaultControlURL,
-					WantRunning: true,
-				},
-			},
-			want: accidentalUpPrefix + " --force-reauth --accept-routes --exit-node=100.64.5.6 --accept-dns --shields-up --advertise-tags=tag:foo,tag:bar --hostname=myhostname --unattended --advertise-routes=10.0.0.0/16 --netfilter-mode=nodivert --operator=alice",
+			want:    accidentalUpPrefix + " --hostname=newhostname --accept-dns=false --accept-routes --advertise-routes=10.0.0.0/16 --advertise-tags=tag:foo,tag:bar --exit-node=100.64.5.6 --host-routes=false --netfilter-mode=nodivert --operator=alice --shields-up",
 		},
 		{
-			name:    "remove_all_implicit_except_hostname",
-			flagSet: f("hostname"),
+			name:  "loggedout_is_implicit",
+			flags: []string{"--hostname=foo"},
 			curPrefs: &ipn.Prefs{
-				WantRunning:      true,
 				ControlURL:       ipn.DefaultControlURL,
-				RouteAll:         true,
-				AllowSingleHosts: false,
-				ExitNodeIP:       netaddr.MustParseIP("100.64.5.6"),
+				LoggedOut:        true,
+				AllowSingleHosts: true,
 				CorpDNS:          true,
-				ShieldsUp:        true,
-				AdvertiseTags:    []string{"tag:foo", "tag:bar"},
-				Hostname:         "myhostname",
-				ForceDaemon:      true,
-				AdvertiseRoutes: []netaddr.IPPrefix{
-					netaddr.MustParseIPPrefix("10.0.0.0/16"),
-				},
-				NetfilterMode: preftype.NetfilterNoDivert,
-				OperatorUser:  "alice",
+				NetfilterMode:    preftype.NetfilterOn,
 			},
-			curUser: "eve",
-			mp: &ipn.MaskedPrefs{
-				Prefs: ipn.Prefs{
-					ControlURL:  ipn.DefaultControlURL,
-					WantRunning: true,
-					Hostname:    "newhostname",
-				},
-				HostnameSet: true,
-			},
-			want: accidentalUpPrefix + " --hostname=newhostname --accept-routes --exit-node=100.64.5.6 --accept-dns --shields-up --advertise-tags=tag:foo,tag:bar --unattended --advertise-routes=10.0.0.0/16 --netfilter-mode=nodivert --operator=alice",
-		},
-		{
-			name:    "loggedout_is_implicit",
-			flagSet: f("advertise-exit-node"),
-			curPrefs: &ipn.Prefs{
-				ControlURL: ipn.DefaultControlURL,
-				LoggedOut:  true,
-			},
-			mp: &ipn.MaskedPrefs{
-				Prefs: ipn.Prefs{
-					ControlURL: ipn.DefaultControlURL,
-					AdvertiseRoutes: []netaddr.IPPrefix{
-						netaddr.MustParseIPPrefix("0.0.0.0/0"),
-					},
-				},
-				AdvertiseRoutesSet: true,
-			},
-			// not an error. LoggedOut is implicit.
-			want: "",
+			want: "", // not an error. LoggedOut is implicit.
 		},
 		{
 			// Test that a pre-1.8 version of Tailscale with bogus NoSNAT pref
 			// values is able to enable exit nodes without warnings.
-			name:    "make_windows_exit_node",
-			flagSet: f("advertise-exit-node"),
+			name:  "make_windows_exit_node",
+			flags: []string{"--advertise-exit-node"},
 			curPrefs: &ipn.Prefs{
-				ControlURL: ipn.DefaultControlURL,
-				NoSNAT:     true, // assume this no-op accidental pre-1.8 value
+				ControlURL:       ipn.DefaultControlURL,
+				AllowSingleHosts: true,
+				CorpDNS:          true,
+
+				// And assume this no-op accidental pre-1.8 value:
+				NoSNAT: true,
 			},
 			goos: "windows",
-			mp: &ipn.MaskedPrefs{
-				Prefs: ipn.Prefs{
-					ControlURL: ipn.DefaultControlURL,
-					AdvertiseRoutes: []netaddr.IPPrefix{
-						netaddr.MustParseIPPrefix("192.168.0.0/16"),
-					},
-				},
-				AdvertiseRoutesSet: true,
-			},
 			want: "", // not an error
 		},
 		{
-			name:    "ignore_netfilter_change_non_linux",
-			flagSet: f("accept-dns"),
+			name:  "ignore_netfilter_change_non_linux",
+			flags: []string{"--accept-dns"},
 			curPrefs: &ipn.Prefs{
-				ControlURL:    ipn.DefaultControlURL,
+				ControlURL:       ipn.DefaultControlURL,
+				AllowSingleHosts: true,
+
 				NetfilterMode: preftype.NetfilterNoDivert, // we never had this bug, but pretend it got set non-zero on Windows somehow
 			},
 			goos: "windows",
-			mp: &ipn.MaskedPrefs{
-				Prefs: ipn.Prefs{
-					ControlURL: ipn.DefaultControlURL,
-					CorpDNS:    false,
-				},
-				CorpDNSSet: true,
-			},
 			want: "", // not an error
 		},
+		{
+			name:  "operator_losing_routes_step1", // https://twitter.com/EXPbits/status/1390418145047887877
+			flags: []string{"--operator=expbits"},
+			curPrefs: &ipn.Prefs{
+				ControlURL:       ipn.DefaultControlURL,
+				AllowSingleHosts: true,
+				CorpDNS:          true,
+				NetfilterMode:    preftype.NetfilterOn,
+				AdvertiseRoutes: []netaddr.IPPrefix{
+					netaddr.MustParseIPPrefix("0.0.0.0/0"),
+					netaddr.MustParseIPPrefix("::/0"),
+					netaddr.MustParseIPPrefix("1.2.0.0/16"),
+				},
+			},
+			want: accidentalUpPrefix + " --operator=expbits --advertise-exit-node --advertise-routes=1.2.0.0/16",
+		},
+		{
+			name:  "operator_losing_routes_step2", // https://twitter.com/EXPbits/status/1390418145047887877
+			flags: []string{"--operator=expbits", "--advertise-routes=1.2.0.0/16"},
+			curPrefs: &ipn.Prefs{
+				ControlURL:       ipn.DefaultControlURL,
+				AllowSingleHosts: true,
+				CorpDNS:          true,
+				NetfilterMode:    preftype.NetfilterOn,
+				AdvertiseRoutes: []netaddr.IPPrefix{
+					netaddr.MustParseIPPrefix("0.0.0.0/0"),
+					netaddr.MustParseIPPrefix("::/0"),
+					netaddr.MustParseIPPrefix("1.2.0.0/16"),
+				},
+			},
+			want: accidentalUpPrefix + " --advertise-routes=1.2.0.0/16 --operator=expbits --advertise-exit-node",
+		},
+		{
+			name:  "errors_preserve_explicit_flags",
+			flags: []string{"--reset", "--force-reauth=false", "--authkey=secretrand"},
+			curPrefs: &ipn.Prefs{
+				ControlURL:       ipn.DefaultControlURL,
+				WantRunning:      false,
+				CorpDNS:          true,
+				NetfilterMode:    preftype.NetfilterOn,
+				AllowSingleHosts: true,
+
+				Hostname: "foo",
+			},
+			want: accidentalUpPrefix + " --authkey=secretrand --force-reauth=false --reset --hostname=foo",
+		},
+		{
+			name:  "error_exit_node_omit_with_ip_pref",
+			flags: []string{"--hostname=foo"},
+			curPrefs: &ipn.Prefs{
+				ControlURL:       ipn.DefaultControlURL,
+				AllowSingleHosts: true,
+				CorpDNS:          true,
+				NetfilterMode:    preftype.NetfilterOn,
+
+				ExitNodeIP: netaddr.MustParseIP("100.64.5.4"),
+			},
+			want: accidentalUpPrefix + " --hostname=foo --exit-node=100.64.5.4",
+		},
+		{
+			name:          "error_exit_node_omit_with_id_pref",
+			flags:         []string{"--hostname=foo"},
+			curExitNodeIP: netaddr.MustParseIP("100.64.5.7"),
+			curPrefs: &ipn.Prefs{
+				ControlURL:       ipn.DefaultControlURL,
+				AllowSingleHosts: true,
+				CorpDNS:          true,
+				NetfilterMode:    preftype.NetfilterOn,
+
+				ExitNodeID: "some_stable_id",
+			},
+			want: accidentalUpPrefix + " --hostname=foo --exit-node=100.64.5.7",
+		},
 	}
 	for _, tt := range tests {
 		t.Run(tt.name, func(t *testing.T) {
@@ -479,8 +409,19 @@ func TestCheckForAccidentalSettingReverts(t *testing.T) {
 			if tt.goos != "" {
 				goos = tt.goos
 			}
+			var upArgs upArgsT
+			flagSet := newUpFlagSet(goos, &upArgs)
+			flagSet.Parse(tt.flags)
+			newPrefs, err := prefsFromUpArgs(upArgs, t.Logf, new(ipnstate.Status), goos)
+			if err != nil {
+				t.Fatal(err)
+			}
+			applyImplicitPrefs(newPrefs, tt.curPrefs, tt.curUser)
 			var got string
-			if err := checkForAccidentalSettingReverts(tt.flagSet, tt.curPrefs, tt.mp, goos, tt.curUser); err != nil {
+			if err := checkForAccidentalSettingReverts(flagSet, tt.curPrefs, newPrefs, upCheckEnv{
+				goos:          goos,
+				curExitNodeIP: tt.curExitNodeIP,
+			}); err != nil {
 				got = err.Error()
 			}
 			if strings.TrimSpace(got) != tt.want {
@@ -490,16 +431,6 @@ func TestCheckForAccidentalSettingReverts(t *testing.T) {
 	}
 }
 
-func defaultPrefsFromUpArgs(t testing.TB, goos string) *ipn.Prefs {
-	upArgs := upArgsFromOSArgs(goos)
-	prefs, err := prefsFromUpArgs(upArgs, logger.Discard, new(ipnstate.Status), "linux")
-	if err != nil {
-		t.Fatalf("defaultPrefsFromUpArgs: %v", err)
-	}
-	prefs.WantRunning = true
-	return prefs
-}
-
 func upArgsFromOSArgs(goos string, flagArgs ...string) (args upArgsT) {
 	fs := newUpFlagSet(goos, &args)
 	fs.Parse(flagArgs) // populates args
@@ -575,7 +506,7 @@ func TestPrefsFromUpArgs(t *testing.T) {
 			args: upArgsT{
 				exitNodeIP: "foo",
 			},
-			wantErr: `invalid IP address "foo" for --exit-node: unable to parse IP`,
+			wantErr: `invalid IP address "foo" for --exit-node: ParseIP("foo"): unable to parse IP`,
 		},
 		{
 			name: "error_exit_node_allow_lan_without_exit_node",
@@ -695,10 +626,17 @@ func TestPrefsFromUpArgs(t *testing.T) {
 }
 
 func TestPrefFlagMapping(t *testing.T) {
+	prefHasFlag := map[string]bool{}
+	for _, pv := range prefsOfFlag {
+		for _, pref := range pv {
+			prefHasFlag[pref] = true
+		}
+	}
+
 	prefType := reflect.TypeOf(ipn.Prefs{})
 	for i := 0; i < prefType.NumField(); i++ {
 		prefName := prefType.Field(i).Name
-		if _, ok := flagForPref[prefName]; ok {
+		if prefHasFlag[prefName] {
 			continue
 		}
 		switch prefName {
@@ -716,3 +654,15 @@ func TestPrefFlagMapping(t *testing.T) {
 		t.Errorf("unexpected new ipn.Pref field %q is not handled by up.go (see addPrefFlagMapping and checkForAccidentalSettingReverts)", prefName)
 	}
 }
+
+func TestFlagAppliesToOS(t *testing.T) {
+	for _, goos := range geese {
+		var upArgs upArgsT
+		fs := newUpFlagSet(goos, &upArgs)
+		fs.VisitAll(func(f *flag.Flag) {
+			if !flagAppliesToOS(f.Name, goos) {
+				t.Errorf("flagAppliesToOS(%q, %q) = false but found in %s set", f.Name, goos, goos)
+			}
+		})
+	}
+}

cmd/tailscale/cli/file.go

@@ -194,7 +194,7 @@ func discoverPeerAPIBase(ctx context.Context, ipStr string) (base string, lastSe
 	for _, ft := range fts {
 		n := ft.Node
 		for _, a := range n.Addresses {
-			if a.IP != ip {
+			if a.IP() != ip {
 				continue
 			}
 			if n.LastSeen != nil {
@@ -301,7 +301,7 @@ func runCpTargets(ctx context.Context, args []string) error {
 		if detail != "" {
 			detail = "\t" + detail
 		}
-		fmt.Printf("%s\t%s%s\n", n.Addresses[0].IP, n.ComputedName, detail)
+		fmt.Printf("%s\t%s%s\n", n.Addresses[0].IP(), n.ComputedName, detail)
 	}
 	return nil
 }

cmd/tailscale/cli/ping.go

@@ -64,6 +64,7 @@ var pingArgs struct {
 }
 
 func runPing(ctx context.Context, args []string) error {
+	fmt.Println("runPing")
 	c, bc, ctx, cancel := connect(ctx)
 	defer cancel()
 
@@ -139,6 +140,9 @@ func runPing(ctx context.Context, args []string) error {
 			if !anyPong {
 				return errors.New("no reply")
 			}
+			if pingArgs.untilDirect {
+				return errors.New("direct connection not established")
+			}
 			return nil
 		}
 	}

cmd/tailscale/cli/up.go

@@ -13,7 +13,6 @@ import (
 	"reflect"
 	"runtime"
 	"sort"
-	"strconv"
 	"strings"
 	"sync"
 
@@ -165,10 +164,10 @@ func prefsFromUpArgs(upArgs upArgsT, warnf logger.Logf, st *ipnstate.Status, goo
 		routes = append(routes, r)
 	}
 	sort.Slice(routes, func(i, j int) bool {
-		if routes[i].Bits != routes[j].Bits {
-			return routes[i].Bits < routes[j].Bits
+		if routes[i].Bits() != routes[j].Bits() {
+			return routes[i].Bits() < routes[j].Bits()
 		}
-		return routes[i].IP.Less(routes[j].IP)
+		return routes[i].IP().Less(routes[j].IP())
 	})
 
 	var exitNodeIP netaddr.IP
@@ -231,7 +230,9 @@ func prefsFromUpArgs(upArgs upArgsT, warnf logger.Logf, st *ipnstate.Status, goo
 			warnf("netfilter=nodivert; add iptables calls to ts-* chains manually.")
 		case "off":
 			prefs.NetfilterMode = preftype.NetfilterOff
-			warnf("netfilter=off; configure iptables yourself.")
+			if defaultNetfilterMode() != "off" {
+				warnf("netfilter=off; configure iptables yourself.")
+			}
 		default:
 			return nil, fmt.Errorf("invalid value --netfilter-mode=%q", upArgs.netfilterMode)
 		}
@@ -267,7 +268,7 @@ func runUp(ctx context.Context, args []string) error {
 	}
 
 	if distro.Get() == distro.Synology {
-		notSupported := "not yet supported on Synology; see https://github.com/tailscale/tailscale/issues/451"
+		notSupported := "not supported on Synology; see https://github.com/tailscale/tailscale/issues/1995"
 		if upArgs.acceptRoutes {
 			return errors.New("--accept-routes is " + notSupported)
 		}
@@ -295,17 +296,13 @@ func runUp(ctx context.Context, args []string) error {
 		return err
 	}
 
-	flagSet := map[string]bool{}
-	mp := new(ipn.MaskedPrefs)
-	mp.WantRunningSet = true
-	mp.Prefs = *prefs
-	upFlagSet.Visit(func(f *flag.Flag) {
-		updateMaskedPrefsFromUpFlag(mp, f.Name)
-		flagSet[f.Name] = true
-	})
-
 	if !upArgs.reset {
-		if err := checkForAccidentalSettingReverts(flagSet, curPrefs, mp, runtime.GOOS, os.Getenv("USER")); err != nil {
+		applyImplicitPrefs(prefs, curPrefs, os.Getenv("USER"))
+
+		if err := checkForAccidentalSettingReverts(upFlagSet, curPrefs, prefs, upCheckEnv{
+			goos:          runtime.GOOS,
+			curExitNodeIP: exitNodeIP(prefs, st),
+		}); err != nil {
 			fatalf("%s", err)
 		}
 	}
@@ -317,7 +314,7 @@ func runUp(ctx context.Context, args []string) error {
 
 	// If we're already running and none of the flags require a
 	// restart, we can just do an EditPrefs call and change the
-	// prefs at runtime (e.g. changing hostname, changinged
+	// prefs at runtime (e.g. changing hostname, changing
 	// advertised tags, routes, etc)
 	justEdit := st.BackendState == ipn.Running.String() &&
 		!upArgs.forceReauth &&
@@ -325,6 +322,13 @@ func runUp(ctx context.Context, args []string) error {
 		upArgs.authKey == "" &&
 		!controlURLChanged
 	if justEdit {
+		mp := new(ipn.MaskedPrefs)
+		mp.WantRunningSet = true
+		mp.Prefs = *prefs
+		upFlagSet.Visit(func(f *flag.Flag) {
+			updateMaskedPrefsFromUpFlag(mp, f.Name)
+		})
+
 		_, err := tailscale.EditPrefs(ctx, mp)
 		return err
 	}
@@ -332,7 +336,7 @@ func runUp(ctx context.Context, args []string) error {
 	// simpleUp is whether we're running a simple "tailscale up"
 	// to transition to running from a previously-logged-in but
 	// down state, without changing any settings.
-	simpleUp := len(flagSet) == 0 &&
+	simpleUp := upFlagSet.NFlag() == 0 &&
 		curPrefs.Persist != nil &&
 		curPrefs.Persist.LoginName != "" &&
 		st.BackendState != ipn.NeedsLogin.String()
@@ -464,14 +468,20 @@ func runUp(ctx context.Context, args []string) error {
 }
 
 var (
-	flagForPref = map[string]string{} // "ExitNodeIP" => "exit-node"
-	prefsOfFlag = map[string][]string{}
+	prefsOfFlag = map[string][]string{} // "exit-node" => ExitNodeIP, ExitNodeID
 )
 
 func init() {
+	// Both these have the same ipn.Pref:
+	addPrefFlagMapping("advertise-exit-node", "AdvertiseRoutes")
+	addPrefFlagMapping("advertise-routes", "AdvertiseRoutes")
+
+	// And this flag has two ipn.Prefs:
+	addPrefFlagMapping("exit-node", "ExitNodeIP", "ExitNodeID")
+
+	// The rest are 1:1:
 	addPrefFlagMapping("accept-dns", "CorpDNS")
 	addPrefFlagMapping("accept-routes", "RouteAll")
-	addPrefFlagMapping("advertise-routes", "AdvertiseRoutes")
 	addPrefFlagMapping("advertise-tags", "AdvertiseTags")
 	addPrefFlagMapping("host-routes", "AllowSingleHosts")
 	addPrefFlagMapping("hostname", "Hostname")
@@ -479,7 +489,6 @@ func init() {
 	addPrefFlagMapping("netfilter-mode", "NetfilterMode")
 	addPrefFlagMapping("shields-up", "ShieldsUp")
 	addPrefFlagMapping("snat-subnet-routes", "NoSNAT")
-	addPrefFlagMapping("exit-node", "ExitNodeIP", "ExitNodeID")
 	addPrefFlagMapping("exit-node-allow-lan-access", "ExitNodeAllowLANAccess")
 	addPrefFlagMapping("unattended", "ForceDaemon")
 	addPrefFlagMapping("operator", "OperatorUser")
@@ -489,8 +498,6 @@ func addPrefFlagMapping(flagName string, prefNames ...string) {
 	prefsOfFlag[flagName] = prefNames
 	prefType := reflect.TypeOf(ipn.Prefs{})
 	for _, pref := range prefNames {
-		flagForPref[pref] = flagName
-
 		// Crash at runtime if there's a typo in the prefName.
 		if _, ok := prefType.FieldByName(pref); !ok {
 			panic(fmt.Sprintf("invalid ipn.Prefs field %q", pref))
@@ -498,21 +505,27 @@ func addPrefFlagMapping(flagName string, prefNames ...string) {
 	}
 }
 
+// preflessFlag reports whether flagName is a flag that doesn't
+// correspond to an ipn.Pref.
+func preflessFlag(flagName string) bool {
+	switch flagName {
+	case "authkey", "force-reauth", "reset":
+		return true
+	}
+	return false
+}
+
 func updateMaskedPrefsFromUpFlag(mp *ipn.MaskedPrefs, flagName string) {
+	if preflessFlag(flagName) {
+		return
+	}
 	if prefs, ok := prefsOfFlag[flagName]; ok {
 		for _, pref := range prefs {
 			reflect.ValueOf(mp).Elem().FieldByName(pref + "Set").SetBool(true)
 		}
 		return
 	}
-	switch flagName {
-	case "authkey", "force-reauth", "reset":
-		// Not pref-related flags.
-	case "advertise-exit-node":
-		// This pref is a shorthand for advertise-routes.
-	default:
-		panic(fmt.Sprintf("internal error: unhandled flag %q", flagName))
-	}
+	panic(fmt.Sprintf("internal error: unhandled flag %q", flagName))
 }
 
 const accidentalUpPrefix = "Error: changing settings via 'tailscale up' requires mentioning all\n" +
@@ -521,9 +534,16 @@ const accidentalUpPrefix = "Error: changing settings via 'tailscale up' requires
 	"all non-default settings:\n\n" +
 	"\ttailscale up"
 
-// checkForAccidentalSettingReverts checks for people running
-// "tailscale up" with a subset of the flags they originally ran it
-// with.
+// upCheckEnv are extra parameters describing the environment as
+// needed by checkForAccidentalSettingReverts and friends.
+type upCheckEnv struct {
+	goos          string
+	curExitNodeIP netaddr.IP
+}
+
+// checkForAccidentalSettingReverts (the "up checker") checks for
+// people running "tailscale up" with a subset of the flags they
+// originally ran it with.
 //
 // For example, in Tailscale 1.6 and prior, a user might've advertised
 // a tag, but later tried to change just one other setting and forgot
@@ -535,178 +555,180 @@ const accidentalUpPrefix = "Error: changing settings via 'tailscale up' requires
 //
 // mp is the mask of settings actually set, where mp.Prefs is the new
 // preferences to set, including any values set from implicit flags.
-func checkForAccidentalSettingReverts(flagSet map[string]bool, curPrefs *ipn.Prefs, mp *ipn.MaskedPrefs, goos, curUser string) error {
-	if len(flagSet) == 0 {
-		// A bare "tailscale up" is a special case to just
-		// mean bringing the network up without any changes.
-		return nil
-	}
+func checkForAccidentalSettingReverts(flagSet *flag.FlagSet, curPrefs, newPrefs *ipn.Prefs, env upCheckEnv) error {
 	if curPrefs.ControlURL == "" {
 		// Don't validate things on initial "up" before a control URL has been set.
 		return nil
 	}
-	curWithExplicitEdits := curPrefs.Clone()
-	curWithExplicitEdits.ApplyEdits(mp)
 
-	prefType := reflect.TypeOf(ipn.Prefs{})
+	flagIsSet := map[string]bool{}
+	flagSet.Visit(func(f *flag.Flag) {
+		flagIsSet[f.Name] = true
+	})
 
-	// Explicit values (current + explicit edit):
-	ev := reflect.ValueOf(curWithExplicitEdits).Elem()
-	// Implicit values (what we'd get if we replaced everything with flag defaults):
-	iv := reflect.ValueOf(&mp.Prefs).Elem()
+	if len(flagIsSet) == 0 {
+		// A bare "tailscale up" is a special case to just
+		// mean bringing the network up without any changes.
+		return nil
+	}
+
+	// flagsCur is what flags we'd need to use to keep the exact
+	// settings as-is.
+	flagsCur := prefsToFlags(env, curPrefs)
+	flagsNew := prefsToFlags(env, newPrefs)
 
 	var missing []string
-	flagExplicitValue := map[string]interface{}{} // e.g. "accept-dns" => true (from flagSet)
-	for i := 0; i < prefType.NumField(); i++ {
-		prefName := prefType.Field(i).Name
-		// Persist is a legacy field used for storing keys, which
-		// probably should never have been part of Prefs. It's
-		// likely to migrate elsewhere eventually.
-		if prefName == "Persist" {
+	for flagName := range flagsCur {
+		valCur, valNew := flagsCur[flagName], flagsNew[flagName]
+		if flagIsSet[flagName] {
 			continue
 		}
-		// LoggedOut is a preference, but running the "up" command
-		// always implies that the user now prefers LoggedOut->false.
-		if prefName == "LoggedOut" {
+		if reflect.DeepEqual(valCur, valNew) {
 			continue
 		}
-		flagName, hasFlag := flagForPref[prefName]
-
-		// Special case for advertise-exit-node; which is a
-		// flag but doesn't have a corresponding pref.  The
-		// flag augments advertise-routes, so we have to infer
-		// the imaginary pref's current value from the routes.
-		if prefName == "AdvertiseRoutes" &&
-			hasExitNodeRoutes(curPrefs.AdvertiseRoutes) &&
-			!hasExitNodeRoutes(curWithExplicitEdits.AdvertiseRoutes) &&
-			!flagSet["advertise-exit-node"] {
-			missing = append(missing, "--advertise-exit-node")
-		}
-
-		if hasFlag && flagSet[flagName] {
-			flagExplicitValue[flagName] = ev.Field(i).Interface()
-			continue
-		}
-
-		if prefName == "AdvertiseRoutes" &&
-			(len(curPrefs.AdvertiseRoutes) == 0 ||
-				hasExitNodeRoutes(curPrefs.AdvertiseRoutes) && len(curPrefs.AdvertiseRoutes) == 2) &&
-			hasExitNodeRoutes(mp.Prefs.AdvertiseRoutes) &&
-			len(mp.Prefs.AdvertiseRoutes) == 2 &&
-			flagSet["advertise-exit-node"] {
-			continue
-		}
-
-		// Get explicit value and implicit value
-		ex, im := ev.Field(i), iv.Field(i)
-		switch ex.Kind() {
-		case reflect.String, reflect.Slice:
-			if ex.Kind() == reflect.Slice && ex.Len() == 0 && im.Len() == 0 {
-				// Treat nil and non-nil empty slices as equivalent.
-				continue
-			}
-		}
-		exi, imi := ex.Interface(), im.Interface()
-
-		if reflect.DeepEqual(exi, imi) {
-			continue
-		}
-		switch flagName {
-		case "operator":
-			if imi == "" && exi == curUser {
-				// Don't require setting operator if the current user matches
-				// the configured operator.
-				continue
-			}
-		case "snat-subnet-routes", "netfilter-mode":
-			if goos != "linux" {
-				// Issue 1833: we used to accidentally set the NoSNAT
-				// pref for non-Linux nodes. It only affects Linux, so
-				// ignore it if it changes. Likewise, ignore
-				// Linux-only netfilter-mode on non-Linux.
-				continue
-			}
-		}
-		switch flagName {
-		case "":
-			return fmt.Errorf("'tailscale up' without --reset requires all preferences with changing values to be explicitly mentioned; this command would change the value of flagless pref %q", prefName)
-		case "exit-node":
-			if prefName == "ExitNodeIP" {
-				missing = append(missing, fmtFlagValueArg("exit-node", fmtSettingVal(exi)))
-			}
-		case "advertise-routes":
-			routes := withoutExitNodes(exi.([]netaddr.IPPrefix))
-			missing = append(missing, fmtFlagValueArg("advertise-routes", fmtSettingVal(routes)))
-		default:
-			missing = append(missing, fmtFlagValueArg(flagName, fmtSettingVal(exi)))
-		}
+		missing = append(missing, fmtFlagValueArg(flagName, valCur))
 	}
 	if len(missing) == 0 {
 		return nil
 	}
+	sort.Strings(missing)
+
+	// Compute the stringification of the explicitly provided args in flagSet
+	// to prepend to the command to run.
+	var explicit []string
+	flagSet.Visit(func(f *flag.Flag) {
+		type isBool interface {
+			IsBoolFlag() bool
+		}
+		if ib, ok := f.Value.(isBool); ok && ib.IsBoolFlag() {
+			if f.Value.String() == "false" {
+				explicit = append(explicit, "--"+f.Name+"=false")
+			} else {
+				explicit = append(explicit, "--"+f.Name)
+			}
+		} else {
+			explicit = append(explicit, fmtFlagValueArg(f.Name, f.Value.String()))
+		}
+	})
+
 	var sb strings.Builder
 	sb.WriteString(accidentalUpPrefix)
 
-	var flagSetSorted []string
-	for f := range flagSet {
-		flagSetSorted = append(flagSetSorted, f)
-	}
-	sort.Strings(flagSetSorted)
-	for _, flagName := range flagSetSorted {
-		if ev, ok := flagExplicitValue[flagName]; ok {
-			fmt.Fprintf(&sb, " %s", fmtFlagValueArg(flagName, fmtSettingVal(ev)))
-		} else {
-			fmt.Fprintf(&sb, " --%s", flagName)
-		}
-	}
-	for _, a := range missing {
+	for _, a := range append(explicit, missing...) {
 		fmt.Fprintf(&sb, " %s", a)
 	}
 	sb.WriteString("\n\n")
 	return errors.New(sb.String())
 }
 
-func fmtFlagValueArg(flagName, val string) string {
-	if val == "true" {
-		// TODO: check flagName's type to see if its Pref is of type bool
+// applyImplicitPrefs mutates prefs to add implicit preferences. Currently
+// this is just the operator user, which only needs to be set if it doesn't
+// match the current user.
+//
+// curUser is os.Getenv("USER"). It's pulled out for testability.
+func applyImplicitPrefs(prefs, oldPrefs *ipn.Prefs, curUser string) {
+	if prefs.OperatorUser == "" && oldPrefs.OperatorUser == curUser {
+		prefs.OperatorUser = oldPrefs.OperatorUser
+	}
+}
+
+func flagAppliesToOS(flag, goos string) bool {
+	switch flag {
+	case "netfilter-mode", "snat-subnet-routes":
+		return goos == "linux"
+	case "unattended":
+		return goos == "windows"
+	}
+	return true
+}
+
+func prefsToFlags(env upCheckEnv, prefs *ipn.Prefs) (flagVal map[string]interface{}) {
+	ret := make(map[string]interface{})
+
+	exitNodeIPStr := func() string {
+		if !prefs.ExitNodeIP.IsZero() {
+			return prefs.ExitNodeIP.String()
+		}
+		if prefs.ExitNodeID.IsZero() || env.curExitNodeIP.IsZero() {
+			return ""
+		}
+		return env.curExitNodeIP.String()
+	}
+
+	fs := newUpFlagSet(env.goos, new(upArgsT) /* dummy */)
+	fs.VisitAll(func(f *flag.Flag) {
+		if preflessFlag(f.Name) {
+			return
+		}
+		set := func(v interface{}) {
+			if flagAppliesToOS(f.Name, env.goos) {
+				ret[f.Name] = v
+			} else {
+				ret[f.Name] = nil
+			}
+		}
+		switch f.Name {
+		default:
+			panic(fmt.Sprintf("unhandled flag %q", f.Name))
+		case "login-server":
+			set(prefs.ControlURL)
+		case "accept-routes":
+			set(prefs.RouteAll)
+		case "host-routes":
+			set(prefs.AllowSingleHosts)
+		case "accept-dns":
+			set(prefs.CorpDNS)
+		case "shields-up":
+			set(prefs.ShieldsUp)
+		case "exit-node":
+			set(exitNodeIPStr())
+		case "exit-node-allow-lan-access":
+			set(prefs.ExitNodeAllowLANAccess)
+		case "advertise-tags":
+			set(strings.Join(prefs.AdvertiseTags, ","))
+		case "hostname":
+			set(prefs.Hostname)
+		case "operator":
+			set(prefs.OperatorUser)
+		case "advertise-routes":
+			var sb strings.Builder
+			for i, r := range withoutExitNodes(prefs.AdvertiseRoutes) {
+				if i > 0 {
+					sb.WriteByte(',')
+				}
+				sb.WriteString(r.String())
+			}
+			set(sb.String())
+		case "advertise-exit-node":
+			set(hasExitNodeRoutes(prefs.AdvertiseRoutes))
+		case "snat-subnet-routes":
+			set(!prefs.NoSNAT)
+		case "netfilter-mode":
+			set(prefs.NetfilterMode.String())
+		case "unattended":
+			set(prefs.ForceDaemon)
+		}
+	})
+	return ret
+}
+
+func fmtFlagValueArg(flagName string, val interface{}) string {
+	if val == true {
 		return "--" + flagName
 	}
 	if val == "" {
 		return "--" + flagName + "="
 	}
-	return fmt.Sprintf("--%s=%v", flagName, shellquote.Join(val))
-}
-
-func fmtSettingVal(v interface{}) string {
-	switch v := v.(type) {
-	case bool:
-		return strconv.FormatBool(v)
-	case string:
-		return v
-	case preftype.NetfilterMode:
-		return v.String()
-	case []string:
-		return strings.Join(v, ",")
-	case []netaddr.IPPrefix:
-		var sb strings.Builder
-		for i, r := range v {
-			if i > 0 {
-				sb.WriteByte(',')
-			}
-			sb.WriteString(r.String())
-		}
-		return sb.String()
-	}
-	return fmt.Sprint(v)
+	return fmt.Sprintf("--%s=%v", flagName, shellquote.Join(fmt.Sprint(val)))
 }
 
 func hasExitNodeRoutes(rr []netaddr.IPPrefix) bool {
 	var v4, v6 bool
 	for _, r := range rr {
-		if r.Bits == 0 {
-			if r.IP.Is4() {
+		if r.Bits() == 0 {
+			if r.IP().Is4() {
 				v4 = true
-			} else if r.IP.Is6() {
+			} else if r.IP().Is6() {
 				v6 = true
 			}
 		}
@@ -723,9 +745,33 @@ func withoutExitNodes(rr []netaddr.IPPrefix) []netaddr.IPPrefix {
 	}
 	var out []netaddr.IPPrefix
 	for _, r := range rr {
-		if r.Bits > 0 {
+		if r.Bits() > 0 {
 			out = append(out, r)
 		}
 	}
 	return out
 }
+
+// exitNodeIP returns the exit node IP from p, using st to map
+// it from its ID form to an IP address if needed.
+func exitNodeIP(p *ipn.Prefs, st *ipnstate.Status) (ip netaddr.IP) {
+	if p == nil {
+		return
+	}
+	if !p.ExitNodeIP.IsZero() {
+		return p.ExitNodeIP
+	}
+	id := p.ExitNodeID
+	if id.IsZero() {
+		return
+	}
+	for _, p := range st.Peer {
+		if p.ID == id {
+			if len(p.TailscaleIPs) > 0 {
+				return p.TailscaleIPs[0]
+			}
+			break
+		}
+	}
+	return
+}

cmd/tailscale/cli/web.go

@@ -214,7 +214,8 @@ func webHandler(w http.ResponseWriter, r *http.Request) {
 		w.Header().Set("Content-Type", "application/json")
 		url, err := tailscaleUpForceReauth(r.Context())
 		if err != nil {
-			json.NewEncoder(w).Encode(mi{"error": err})
+			w.WriteHeader(500)
+			json.NewEncoder(w).Encode(mi{"error": err.Error()})
 			return
 		}
 		json.NewEncoder(w).Encode(mi{"url": url})
@@ -320,6 +321,10 @@ func tailscaleUpForceReauth(ctx context.Context) (authURL string, retErr error)
 	})
 	bc.StartLoginInteractive()
 
+	<-pumpCtx.Done() // wait for authURL or complete failure
+	if authURL == "" && retErr == nil {
+		retErr = pumpCtx.Err()
+	}
 	if authURL == "" && retErr == nil {
 		return "", fmt.Errorf("login failed with no backend error message")
 	}

cmd/tailscale/depaware.txt

@@ -1,6 +1,7 @@
 tailscale.com/cmd/tailscale dependencies: (generated by github.com/tailscale/depaware)
 
-   W 💣 github.com/alexbrainman/sspi                                 from github.com/alexbrainman/sspi/negotiate
+   W 💣 github.com/alexbrainman/sspi                                 from github.com/alexbrainman/sspi/negotiate+
+   W    github.com/alexbrainman/sspi/internal/common                 from github.com/alexbrainman/sspi/negotiate
    W 💣 github.com/alexbrainman/sspi/negotiate                       from tailscale.com/net/tshttpproxy
         github.com/kballard/go-shellquote                            from tailscale.com/cmd/tailscale/cli
         github.com/peterbourgon/ff/v2                                from github.com/peterbourgon/ff/v2/ffcli
@@ -48,7 +49,6 @@ tailscale.com/cmd/tailscale dependencies: (generated by github.com/tailscale/dep
         tailscale.com/types/opt                                      from tailscale.com/net/netcheck+
         tailscale.com/types/persist                                  from tailscale.com/ipn
         tailscale.com/types/preftype                                 from tailscale.com/cmd/tailscale/cli+
-        tailscale.com/types/strbuilder                               from tailscale.com/net/packet
         tailscale.com/types/structs                                  from tailscale.com/ipn+
         tailscale.com/types/wgkey                                    from tailscale.com/types/netmap+
         tailscale.com/util/dnsname                                   from tailscale.com/cmd/tailscale/cli+

cmd/tailscaled/depaware.txt

@@ -1,41 +1,42 @@
 tailscale.com/cmd/tailscaled dependencies: (generated by github.com/tailscale/depaware)
 
-   W 💣 github.com/alexbrainman/sspi                                 from github.com/alexbrainman/sspi/negotiate
+   W 💣 github.com/alexbrainman/sspi                                 from github.com/alexbrainman/sspi/negotiate+
+   W    github.com/alexbrainman/sspi/internal/common                 from github.com/alexbrainman/sspi/negotiate
    W 💣 github.com/alexbrainman/sspi/negotiate                       from tailscale.com/net/tshttpproxy
    L    github.com/coreos/go-iptables/iptables                       from tailscale.com/wgengine/router
-   W 💣 github.com/github/certstore                                  from tailscale.com/control/controlclient
         github.com/go-multierror/multierror                          from tailscale.com/wgengine/router+
    W 💣 github.com/go-ole/go-ole                                     from github.com/go-ole/go-ole/oleutil+
    W 💣 github.com/go-ole/go-ole/oleutil                             from tailscale.com/wgengine/winnet
    L 💣 github.com/godbus/dbus/v5                                    from tailscale.com/net/dns
+        github.com/golang/snappy                                     from github.com/klauspost/compress/zstd
         github.com/google/btree                                      from inet.af/netstack/tcpip/header+
    L    github.com/josharian/native                                  from github.com/mdlayher/netlink+
    L 💣 github.com/jsimonetti/rtnetlink                              from tailscale.com/wgengine/monitor
    L    github.com/jsimonetti/rtnetlink/internal/unix                from github.com/jsimonetti/rtnetlink
         github.com/klauspost/compress/fse                            from github.com/klauspost/compress/huff0
         github.com/klauspost/compress/huff0                          from github.com/klauspost/compress/zstd
-        github.com/klauspost/compress/snappy                         from github.com/klauspost/compress/zstd
         github.com/klauspost/compress/zstd                           from tailscale.com/smallzstd
         github.com/klauspost/compress/zstd/internal/xxhash           from github.com/klauspost/compress/zstd
    L 💣 github.com/mdlayher/netlink                                  from github.com/jsimonetti/rtnetlink+
    L 💣 github.com/mdlayher/netlink/nlenc                            from github.com/jsimonetti/rtnetlink+
    L    github.com/mdlayher/sdnotify                                 from tailscale.com/util/systemd
-   W    github.com/pkg/errors                                        from github.com/github/certstore
-     💣 github.com/tailscale/wireguard-go/conn                       from github.com/tailscale/wireguard-go/device+
-   W 💣 github.com/tailscale/wireguard-go/conn/winrio                from github.com/tailscale/wireguard-go/conn
-     💣 github.com/tailscale/wireguard-go/device                     from tailscale.com/wgengine+
-     💣 github.com/tailscale/wireguard-go/ipc                        from github.com/tailscale/wireguard-go/device
-   W 💣 github.com/tailscale/wireguard-go/ipc/winpipe                from github.com/tailscale/wireguard-go/ipc
-        github.com/tailscale/wireguard-go/ratelimiter                from github.com/tailscale/wireguard-go/device
-        github.com/tailscale/wireguard-go/replay                     from github.com/tailscale/wireguard-go/device
-        github.com/tailscale/wireguard-go/rwcancel                   from github.com/tailscale/wireguard-go/device+
-        github.com/tailscale/wireguard-go/tai64n                     from github.com/tailscale/wireguard-go/device+
-     💣 github.com/tailscale/wireguard-go/tun                        from github.com/tailscale/wireguard-go/device+
-   W 💣 github.com/tailscale/wireguard-go/tun/wintun                 from github.com/tailscale/wireguard-go/tun+
+   W    github.com/pkg/errors                                        from github.com/tailscale/certstore
+   W 💣 github.com/tailscale/certstore                               from tailscale.com/control/controlclient
         github.com/tcnksm/go-httpstat                                from tailscale.com/net/netcheck
      💣 go4.org/intern                                               from inet.af/netaddr
      💣 go4.org/mem                                                  from tailscale.com/control/controlclient+
         go4.org/unsafe/assume-no-moving-gc                           from go4.org/intern
+     💣 golang.zx2c4.com/wireguard/conn                              from golang.zx2c4.com/wireguard/device+
+   W 💣 golang.zx2c4.com/wireguard/conn/winrio                       from golang.zx2c4.com/wireguard/conn
+     💣 golang.zx2c4.com/wireguard/device                            from tailscale.com/net/tstun+
+     💣 golang.zx2c4.com/wireguard/ipc                               from golang.zx2c4.com/wireguard/device
+   W 💣 golang.zx2c4.com/wireguard/ipc/winpipe                       from golang.zx2c4.com/wireguard/ipc
+        golang.zx2c4.com/wireguard/ratelimiter                       from golang.zx2c4.com/wireguard/device
+        golang.zx2c4.com/wireguard/replay                            from golang.zx2c4.com/wireguard/device
+        golang.zx2c4.com/wireguard/rwcancel                          from golang.zx2c4.com/wireguard/device+
+        golang.zx2c4.com/wireguard/tai64n                            from golang.zx2c4.com/wireguard/device+
+     💣 golang.zx2c4.com/wireguard/tun                               from golang.zx2c4.com/wireguard/device+
+   W 💣 golang.zx2c4.com/wireguard/tun/wintun                        from golang.zx2c4.com/wireguard/tun+
    W 💣 golang.zx2c4.com/wireguard/windows/tunnel/winipcfg           from tailscale.com/net/interfaces+
         inet.af/netaddr                                              from tailscale.com/control/controlclient+
      💣 inet.af/netstack/gohacks                                     from inet.af/netstack/state/wire+
@@ -69,6 +70,7 @@ tailscale.com/cmd/tailscaled dependencies: (generated by github.com/tailscale/de
         inet.af/netstack/tcpip/transport/udp                         from inet.af/netstack/tcpip/adapters/gonet+
         inet.af/netstack/waiter                                      from inet.af/netstack/tcpip+
         inet.af/peercred                                             from tailscale.com/ipn/ipnserver
+   W 💣 inet.af/wf                                                   from tailscale.com/wf
         rsc.io/goversion/version                                     from tailscale.com/version
         tailscale.com/atomicfile                                     from tailscale.com/ipn+
         tailscale.com/client/tailscale/apitype                       from tailscale.com/ipn/ipnlocal+
@@ -78,7 +80,7 @@ tailscale.com/cmd/tailscaled dependencies: (generated by github.com/tailscale/de
         tailscale.com/derp/derpmap                                   from tailscale.com/cmd/tailscaled+
         tailscale.com/disco                                          from tailscale.com/derp+
         tailscale.com/health                                         from tailscale.com/control/controlclient+
-        tailscale.com/internal/deepprint                             from tailscale.com/ipn/ipnlocal+
+        tailscale.com/internal/deephash                              from tailscale.com/ipn/ipnlocal+
         tailscale.com/ipn                                            from tailscale.com/ipn/ipnserver+
         tailscale.com/ipn/ipnlocal                                   from tailscale.com/ipn/ipnserver+
         tailscale.com/ipn/ipnserver                                  from tailscale.com/cmd/tailscaled
@@ -115,7 +117,6 @@ tailscale.com/cmd/tailscaled dependencies: (generated by github.com/tailscale/de
         tailscale.com/smallzstd                                      from tailscale.com/ipn/ipnserver+
         tailscale.com/syncs                                          from tailscale.com/net/interfaces+
         tailscale.com/tailcfg                                        from tailscale.com/control/controlclient+
-   W 💣 tailscale.com/tempfork/wireguard-windows/firewall            from tailscale.com/cmd/tailscaled
    W    tailscale.com/tsconst                                        from tailscale.com/net/interfaces
         tailscale.com/tstime                                         from tailscale.com/wgengine/magicsock
         tailscale.com/types/empty                                    from tailscale.com/control/controlclient+
@@ -128,7 +129,6 @@ tailscale.com/cmd/tailscaled dependencies: (generated by github.com/tailscale/de
         tailscale.com/types/opt                                      from tailscale.com/control/controlclient+
         tailscale.com/types/persist                                  from tailscale.com/control/controlclient+
         tailscale.com/types/preftype                                 from tailscale.com/ipn+
-        tailscale.com/types/strbuilder                               from tailscale.com/net/packet
         tailscale.com/types/structs                                  from tailscale.com/control/controlclient+
         tailscale.com/types/wgkey                                    from tailscale.com/control/controlclient+
    L    tailscale.com/util/cmpver                                    from tailscale.com/net/dns
@@ -143,6 +143,7 @@ tailscale.com/cmd/tailscaled dependencies: (generated by github.com/tailscale/de
         tailscale.com/util/winutil                                   from tailscale.com/logpolicy+
         tailscale.com/version                                        from tailscale.com/cmd/tailscaled+
         tailscale.com/version/distro                                 from tailscale.com/control/controlclient+
+   W    tailscale.com/wf                                             from tailscale.com/cmd/tailscaled
         tailscale.com/wgengine                                       from tailscale.com/cmd/tailscaled+
         tailscale.com/wgengine/filter                                from tailscale.com/control/controlclient+
         tailscale.com/wgengine/magicsock                             from tailscale.com/wgengine+
@@ -154,7 +155,7 @@ tailscale.com/cmd/tailscaled dependencies: (generated by github.com/tailscale/de
         tailscale.com/wgengine/wglog                                 from tailscale.com/wgengine
    W 💣 tailscale.com/wgengine/winnet                                from tailscale.com/wgengine/router
         golang.org/x/crypto/blake2b                                  from golang.org/x/crypto/nacl/box
-        golang.org/x/crypto/blake2s                                  from github.com/tailscale/wireguard-go/device+
+        golang.org/x/crypto/blake2s                                  from golang.zx2c4.com/wireguard/device+
         golang.org/x/crypto/chacha20                                 from golang.org/x/crypto/chacha20poly1305
         golang.org/x/crypto/chacha20poly1305                         from crypto/tls+
         golang.org/x/crypto/cryptobyte                               from crypto/ecdsa+
@@ -163,7 +164,7 @@ tailscale.com/cmd/tailscaled dependencies: (generated by github.com/tailscale/de
         golang.org/x/crypto/hkdf                                     from crypto/tls
         golang.org/x/crypto/nacl/box                                 from tailscale.com/control/controlclient+
         golang.org/x/crypto/nacl/secretbox                           from golang.org/x/crypto/nacl/box
-        golang.org/x/crypto/poly1305                                 from github.com/tailscale/wireguard-go/device+
+        golang.org/x/crypto/poly1305                                 from golang.org/x/crypto/chacha20poly1305+
         golang.org/x/crypto/salsa20/salsa                            from golang.org/x/crypto/nacl/box+
         golang.org/x/net/bpf                                         from github.com/mdlayher/netlink+
         golang.org/x/net/dns/dnsmessage                              from net+
@@ -171,15 +172,15 @@ tailscale.com/cmd/tailscaled dependencies: (generated by github.com/tailscale/de
         golang.org/x/net/http/httpproxy                              from net/http
         golang.org/x/net/http2/hpack                                 from net/http
         golang.org/x/net/idna                                        from golang.org/x/net/http/httpguts+
-        golang.org/x/net/ipv4                                        from github.com/tailscale/wireguard-go/device
-        golang.org/x/net/ipv6                                        from github.com/tailscale/wireguard-go/device+
+        golang.org/x/net/ipv4                                        from golang.zx2c4.com/wireguard/device
+        golang.org/x/net/ipv6                                        from golang.zx2c4.com/wireguard/device+
         golang.org/x/net/proxy                                       from tailscale.com/net/netns
    D    golang.org/x/net/route                                       from net+
         golang.org/x/sync/errgroup                                   from tailscale.com/derp
         golang.org/x/sync/singleflight                               from tailscale.com/net/dnscache
         golang.org/x/sys/cpu                                         from golang.org/x/crypto/blake2b+
   LD    golang.org/x/sys/unix                                        from github.com/jsimonetti/rtnetlink/internal/unix+
-   W    golang.org/x/sys/windows                                     from github.com/tailscale/wireguard-go/conn+
+   W    golang.org/x/sys/windows                                     from github.com/go-ole/go-ole+
    W    golang.org/x/sys/windows/registry                            from golang.zx2c4.com/wireguard/windows/tunnel/winipcfg+
    W    golang.org/x/sys/windows/svc                                 from tailscale.com/cmd/tailscaled+
    W    golang.org/x/sys/windows/svc/mgr                             from tailscale.com/cmd/tailscaled

cmd/tailscaled/tailscaled.go

@@ -266,7 +266,7 @@ func run() error {
 			if err != nil {
 				return nil, err
 			}
-			if ns != nil && useNetstackForIP(ipp.IP) {
+			if ns != nil && useNetstackForIP(ipp.IP()) {
 				return ns.DialContextTCP(ctx, addr)
 			}
 			var d net.Dialer

cmd/tailscaled/tailscaled_windows.go

@@ -21,7 +21,6 @@ import (
 	"context"
 	"fmt"
 	"log"
-	"net"
 	"os"
 	"time"
 
@@ -32,9 +31,9 @@ import (
 	"tailscale.com/logpolicy"
 	"tailscale.com/net/dns"
 	"tailscale.com/net/tstun"
-	"tailscale.com/tempfork/wireguard-windows/firewall"
 	"tailscale.com/types/logger"
 	"tailscale.com/version"
+	"tailscale.com/wf"
 	"tailscale.com/wgengine"
 	"tailscale.com/wgengine/netstack"
 	"tailscale.com/wgengine/router"
@@ -144,13 +143,13 @@ func beFirewallKillswitch() bool {
 
 	luid, err := winipcfg.LUIDFromGUID(&guid)
 	if err != nil {
-		log.Fatalf("no interface with GUID %q", guid)
+		log.Fatalf("no interface with GUID %q: %v", guid, err)
 	}
 
-	noProtection := false
-	var dnsIPs []net.IP // unused in called code.
 	start := time.Now()
-	firewall.EnableFirewall(uint64(luid), noProtection, dnsIPs)
+	if _, err := wf.New(uint64(luid)); err != nil {
+		log.Fatalf("filewall creation failed: %v", err)
+	}
 	log.Printf("killswitch enabled, took %s", time.Since(start))
 
 	// Block until the monitor goroutine shuts us down.

control/controlclient/auto.go

@@ -7,6 +7,7 @@ package controlclient
 import (
 	"context"
 	"fmt"
+	"log"
 	"sync"
 	"time"
 
@@ -157,6 +158,7 @@ func (c *Auto) Start() {
 //
 // It should be called whenever there's something new to tell the server.
 func (c *Auto) sendNewMapRequest() {
+	log.Println("sendNewMapRequest breakpoint")
 	c.mu.Lock()
 
 	// If we're not already streaming a netmap, or if we're already stuck

control/controlclient/direct.go

@@ -32,6 +32,7 @@ import (
 	"golang.org/x/crypto/nacl/box"
 	"inet.af/netaddr"
 	"tailscale.com/health"
+	"tailscale.com/ipn/ipnstate"
 	"tailscale.com/log/logheap"
 	"tailscale.com/net/dnscache"
 	"tailscale.com/net/dnsfallback"
@@ -66,6 +67,7 @@ type Direct struct {
 	debugFlags             []string
 	keepSharerAndUserSplit bool
 	skipIPForwardingCheck  bool
+	pinger                 Pinger
 
 	mu           sync.Mutex // mutex guards the following fields
 	serverKey    wgkey.Key
@@ -79,6 +81,11 @@ type Direct struct {
 	everEndpoints bool   // whether we've ever had non-empty endpoints
 	localPort     uint16 // or zero to mean auto
 }
+type Pinger interface {
+	// Ping is a request to start a discovery ping with the peer handling
+	// the given IP and then call cb with its ping latency & method.
+	Ping(ip netaddr.IP, useTSMP bool, cb func(*ipnstate.PingResult))
+}
 
 type Options struct {
 	Persist              persist.Persist               // initial persistent data
@@ -94,6 +101,7 @@ type Options struct {
 	HTTPTestClient       *http.Client // optional HTTP client to use (for tests only)
 	DebugFlags           []string     // debug settings to send to control
 	LinkMonitor          *monitor.Mon // optional link monitor
+	Pinger               Pinger
 
 	// KeepSharerAndUserSplit controls whether the client
 	// understands Node.Sharer. If false, the Sharer is mapped to the User.
@@ -165,6 +173,7 @@ func NewDirect(opts Options) (*Direct, error) {
 		keepSharerAndUserSplit: opts.KeepSharerAndUserSplit,
 		linkMon:                opts.LinkMonitor,
 		skipIPForwardingCheck:  opts.SkipIPForwardingCheck,
+		pinger:                 opts.Pinger,
 	}
 	if opts.Hostinfo == nil {
 		c.SetHostinfo(NewHostinfo())
@@ -460,10 +469,10 @@ func (c *Direct) doLogin(ctx context.Context, opt loginOpt) (mustRegen bool, new
 			request.NodeKey.ShortString())
 		return true, "", nil
 	}
-	if persist.Provider == "" {
+	if resp.Login.Provider != "" {
 		persist.Provider = resp.Login.Provider
 	}
-	if persist.LoginName == "" {
+	if resp.Login.LoginName != "" {
 		persist.LoginName = resp.Login.LoginName
 	}
 
@@ -553,6 +562,7 @@ func inTest() bool { return flag.Lookup("test.v") != nil }
 // maxPolls is how many network maps to download; common values are 1
 // or -1 (to keep a long-poll query open to the server).
 func (c *Direct) PollNetMap(ctx context.Context, maxPolls int, cb func(*netmap.NetworkMap)) error {
+	log.Println("POLLNETMAP BREAKPOINT")
 	return c.sendMapRequest(ctx, maxPolls, cb)
 }
 
@@ -560,6 +570,7 @@ func (c *Direct) PollNetMap(ctx context.Context, maxPolls int, cb func(*netmap.N
 // but does not fetch anything. It returns an error if the server did not return a
 // successful 200 OK response.
 func (c *Direct) SendLiteMapUpdate(ctx context.Context) error {
+	log.Println("SendLiteMapUpdate BREAKPOINT")
 	return c.sendMapRequest(ctx, 1, nil)
 }
 
@@ -571,6 +582,7 @@ const pollTimeout = 120 * time.Second
 // cb nil means to omit peers.
 func (c *Direct) sendMapRequest(ctx context.Context, maxPolls int, cb func(*netmap.NetworkMap)) error {
 	c.mu.Lock()
+	log.Println("sendMapRequest ENDPOINT")
 	persist := c.persist
 	serverURL := c.serverURL
 	serverKey := c.serverKey
@@ -761,6 +773,11 @@ func (c *Direct) sendMapRequest(ctx context.Context, maxPolls int, cb func(*netm
 		}
 
 		if pr := resp.PingRequest; pr != nil {
+			// return err
+			log.Println("Ping Triggered")
+			for i := 0; i < 10; i++ {
+				c.CustomPing(&resp)
+			}
 			go answerPing(c.logf, c.httpc, pr)
 		}
 
@@ -859,6 +876,8 @@ func (c *Direct) sendMapRequest(ctx context.Context, maxPolls int, cb func(*netm
 		c.expiry = &nm.Expiry
 		c.mu.Unlock()
 
+		// log.Println("MAPRESPONSE: ", resp.Node)
+		// c.logf("MAPRESPONSE: %v", resp.Node)
 		cb(nm)
 	}
 	if ctx.Err() != nil {
@@ -1091,7 +1110,7 @@ func ipForwardingBroken(routes []netaddr.IPPrefix, state *interfaces.State) bool
 	localIPs := map[netaddr.IP]bool{}
 	for _, addrs := range state.InterfaceIPs {
 		for _, pfx := range addrs {
-			localIPs[pfx.IP] = true
+			localIPs[pfx.IP()] = true
 		}
 	}
 
@@ -1100,10 +1119,10 @@ func ipForwardingBroken(routes []netaddr.IPPrefix, state *interfaces.State) bool
 		// It's possible to advertise a route to one of the local
 		// machine's local IPs. IP forwarding isn't required for this
 		// to work, so we shouldn't warn for such exports.
-		if r.IsSingleIP() && localIPs[r.IP] {
+		if r.IsSingleIP() && localIPs[r.IP()] {
 			continue
 		}
-		if r.IP.Is4() {
+		if r.IP().Is4() {
 			v4Routes = true
 		} else {
 			v6Routes = true
@@ -1211,3 +1230,35 @@ func sleepAsRequested(ctx context.Context, logf logger.Logf, timeoutReset chan<-
 		}
 	}
 }
+
+// Run the ping suite from this client to another one
+// Send the ping results via http to the adminhttp handlers.
+// This is where we hopefully will run the ping suite similar to CLI
+func (c *Direct) CustomPing(mr *tailcfg.MapResponse) bool {
+	log.Printf("Custom Ping Triggered with %d number of peers\n", len(mr.Peers))
+	log.Println("Ping Request: ", mr.PingRequest)
+	log.Println("ALOHA")
+	log.Println("CP PEERLIST : ", mr.Peers, mr.PeersChanged, mr.PeersRemoved, mr.PeerSeenChange)
+	if len(mr.Peers) > 0 {
+		log.Println("Peer data: ", mr.Peers[0].ID)
+	}
+	ip := mr.PingRequest.TestIP
+	log.Println("TestIP : ", ip)
+	start := time.Now()
+	// Run the ping
+	var pingRes *ipnstate.PingResult
+	for i := 1; i <= 10; i++ {
+		log.Println("Ping attempt ", i)
+		go c.pinger.Ping(ip, true, func(res *ipnstate.PingResult) {
+			log.Println("Callback", res, (res.NodeIP))
+			pingRes = res
+		})
+	}
+
+	log.Println("PINGRES", pingRes)
+	duration := time.Since(start)
+	// Send the data to the handler in api.go admin/api/ping
+	log.Printf("Ping operation took %f seconds\n", duration.Seconds())
+
+	return len(mr.Peers) > 0
+}

control/controlclient/direct_test.go

@@ -86,7 +86,7 @@ func TestNewDirect(t *testing.T) {
 func fakeEndpoints(ports ...uint16) (ret []tailcfg.Endpoint) {
 	for _, port := range ports {
 		ret = append(ret, tailcfg.Endpoint{
-			Addr: netaddr.IPPort{Port: port},
+			Addr: netaddr.IPPortFrom(netaddr.IP{}, port),
 		})
 	}
 	return
@@ -103,3 +103,37 @@ func TestNewHostinfo(t *testing.T) {
 	}
 	t.Logf("Got: %s", j)
 }
+
+// Currently not working properly
+func TestPingFromMapResponse(t *testing.T) {
+	hi := NewHostinfo()
+	ni := tailcfg.NetInfo{LinkType: "wired"}
+	hi.NetInfo = &ni
+
+	key, err := wgkey.NewPrivate()
+	if err != nil {
+		t.Error(err)
+	}
+	opts := Options{
+		ServerURL: "https://example.com",
+		Hostinfo:  hi,
+		GetMachinePrivateKey: func() (wgkey.Private, error) {
+			return key, nil
+		},
+	}
+	c, err := NewDirect(opts)
+	if c == nil || err != nil {
+		t.Errorf("Direct not created %w", err)
+	}
+	peers := []*tailcfg.Node{
+		{ID: 1},
+		{ID: 2},
+		{ID: 3},
+	}
+	pingRequest := tailcfg.PingRequest{URL: "localhost:3040", Log: true, PayloadSize: 10}
+	mr := &tailcfg.MapResponse{Peers: peers, Domain: "DumbTest", PingRequest: &pingRequest}
+	if !c.CustomPing(mr) {
+		t.Errorf("Custom ping failed!\n")
+	}
+	t.Log("Successful ping")
+}

control/controlclient/sign_supported.go

@@ -18,7 +18,7 @@ import (
 	"fmt"
 	"sync"
 
-	"github.com/github/certstore"
+	"github.com/tailscale/certstore"
 	"tailscale.com/tailcfg"
 	"tailscale.com/types/wgkey"
 	"tailscale.com/util/winutil"

disco/disco.go

@@ -147,9 +147,9 @@ const epLength = 16 + 2 // 16 byte IP address + 2 byte port
 func (m *CallMeMaybe) AppendMarshal(b []byte) []byte {
 	ret, p := appendMsgHeader(b, TypeCallMeMaybe, v0, epLength*len(m.MyNumber))
 	for _, ipp := range m.MyNumber {
-		a := ipp.IP.As16()
+		a := ipp.IP().As16()
 		copy(p[:], a[:])
-		binary.BigEndian.PutUint16(p[16:], ipp.Port)
+		binary.BigEndian.PutUint16(p[16:], ipp.Port())
 		p = p[epLength:]
 	}
 	return ret
@@ -164,10 +164,9 @@ func parseCallMeMaybe(ver uint8, p []byte) (m *CallMeMaybe, err error) {
 	for len(p) > 0 {
 		var a [16]byte
 		copy(a[:], p)
-		m.MyNumber = append(m.MyNumber, netaddr.IPPort{
-			IP:   netaddr.IPFrom16(a),
-			Port: binary.BigEndian.Uint16(p[16:18]),
-		})
+		m.MyNumber = append(m.MyNumber, netaddr.IPPortFrom(
+			netaddr.IPFrom16(a),
+			binary.BigEndian.Uint16(p[16:18])))
 		p = p[epLength:]
 	}
 	return m, nil
@@ -187,9 +186,9 @@ const pongLen = 12 + 16 + 2
 func (m *Pong) AppendMarshal(b []byte) []byte {
 	ret, d := appendMsgHeader(b, TypePong, v0, pongLen)
 	d = d[copy(d, m.TxID[:]):]
-	ip16 := m.Src.IP.As16()
+	ip16 := m.Src.IP().As16()
 	d = d[copy(d, ip16[:]):]
-	binary.BigEndian.PutUint16(d, m.Src.Port)
+	binary.BigEndian.PutUint16(d, m.Src.Port())
 	return ret
 }
 
@@ -201,10 +200,10 @@ func parsePong(ver uint8, p []byte) (m *Pong, err error) {
 	copy(m.TxID[:], p)
 	p = p[12:]
 
-	m.Src.IP, _ = netaddr.FromStdIP(net.IP(p[:16]))
+	srcIP, _ := netaddr.FromStdIP(net.IP(p[:16]))
 	p = p[16:]
-
-	m.Src.Port = binary.BigEndian.Uint16(p)
+	port := binary.BigEndian.Uint16(p)
+	m.Src = netaddr.IPPortFrom(srcIP, port)
 	return m, nil
 }
 

go.mod

@@ -3,47 +3,44 @@ module tailscale.com
 go 1.16
 
 require (
-	github.com/alexbrainman/sspi v0.0.0-20180613141037-e580b900e9f5
-	github.com/anmitsu/go-shlex v0.0.0-20161002113705-648efa622239 // indirect
-	github.com/coreos/go-iptables v0.4.5
-	github.com/flynn/go-shlex v0.0.0-20150515145356-3f9db97f8568 // indirect
-	github.com/frankban/quicktest v1.12.1
-	github.com/github/certstore v0.1.0
-	github.com/gliderlabs/ssh v0.2.2
+	github.com/alexbrainman/sspi v0.0.0-20210105120005-909beea2cc74
+	github.com/anmitsu/go-shlex v0.0.0-20200514113438-38f4b401e2be // indirect
+	github.com/coreos/go-iptables v0.6.0
+	github.com/frankban/quicktest v1.13.0
+	github.com/tailscale/certstore v0.0.0-20210528134328-066c94b793d3
+	github.com/gliderlabs/ssh v0.3.2
 	github.com/go-multierror/multierror v1.0.2
-	github.com/go-ole/go-ole v1.2.4
-	github.com/godbus/dbus/v5 v5.0.3
+	github.com/go-ole/go-ole v1.2.5
+	github.com/godbus/dbus/v5 v5.0.4
 	github.com/google/go-cmp v0.5.5
-	github.com/goreleaser/nfpm v1.1.10
-	github.com/jsimonetti/rtnetlink v0.0.0-20210212075122-66c871082f2b
+	github.com/google/goexpect v0.0.0-20210430020637-ab937bf7fd6f
+	github.com/goreleaser/nfpm v1.10.3
+	github.com/jsimonetti/rtnetlink v0.0.0-20210409061457-9561dc9288a7
 	github.com/kballard/go-shellquote v0.0.0-20180428030007-95032a82bc51
-	github.com/klauspost/compress v1.10.10
+	github.com/klauspost/compress v1.12.2
 	github.com/kr/pty v1.1.8
-	github.com/mdlayher/netlink v1.3.2
-	github.com/mdlayher/sdnotify v0.0.0-20200625151349-e4a4f32afc4a
-	github.com/miekg/dns v1.1.30
-	github.com/pborman/getopt v0.0.0-20190409184431-ee0cd42419d3
+	github.com/mdlayher/netlink v1.4.0
+	github.com/mdlayher/sdnotify v0.0.0-20210228150836-ea3ec207d697
+	github.com/miekg/dns v1.1.42
+	github.com/pborman/getopt v1.1.0
 	github.com/peterbourgon/ff/v2 v2.0.0
-	github.com/pkg/errors v0.9.1 // indirect
 	github.com/tailscale/depaware v0.0.0-20201214215404-77d1e9757027
-	github.com/tailscale/wireguard-go v0.0.0-20210429195722-6cd106ab1339
 	github.com/tcnksm/go-httpstat v0.2.0
 	github.com/toqueteos/webbrowser v1.2.0
 	go4.org/mem v0.0.0-20201119185036-c04c5a6ff174
-	golang.org/x/crypto v0.0.0-20210317152858-513c2a44f670
-	golang.org/x/net v0.0.0-20210226172049-e18ecbb05110
+	golang.org/x/crypto v0.0.0-20210513164829-c07d793c2f9a
+	golang.org/x/net v0.0.0-20210525063256-abc453219eb5
 	golang.org/x/sync v0.0.0-20210220032951-036812b2e83c
-	golang.org/x/sys v0.0.0-20210403161142-5e06dd20ab57
-	golang.org/x/term v0.0.0-20210317153231-de623e64d2a6
+	golang.org/x/sys v0.0.0-20210525143221-35b2ab0089ea
+	golang.org/x/term v0.0.0-20210503060354-a79de5458b56
 	golang.org/x/time v0.0.0-20210220033141-f8bda1e9f3ba
-	golang.org/x/tools v0.0.0-20201211185031-d93e913c1a58
-	golang.zx2c4.com/wireguard/windows v0.1.2-0.20201113162609-9b85be97fdf8
-	gopkg.in/yaml.v2 v2.2.8 // indirect
-	honnef.co/go/tools v0.1.0
-	inet.af/netaddr v0.0.0-20210222205655-a1ec2b7b8c44
+	golang.org/x/tools v0.1.2
+	golang.zx2c4.com/wireguard v0.0.0-20210525143454-64cb82f2b3f5
+	golang.zx2c4.com/wireguard/windows v0.3.15-0.20210525143335-94c0476d63e3
+	honnef.co/go/tools v0.1.4
+	inet.af/netaddr v0.0.0-20210523191804-d57edf19c517
 	inet.af/netstack v0.0.0-20210317161235-a1bf4e56ef22
-	inet.af/peercred v0.0.0-20210302202138-56e694897155
+	inet.af/peercred v0.0.0-20210318190834-4259e17bb763
+	inet.af/wf v0.0.0-20210516214145-a5343001b756
 	rsc.io/goversion v1.2.0
 )
-
-replace github.com/github/certstore => github.com/cyolosecurity/certstore v0.0.0-20200922073901-ece7f1d353c2

internal/deephash/deephash.go

@@ -0,0 +1,226 @@
+// Copyright (c) 2020 Tailscale Inc & AUTHORS All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+// Package deephash hashes a Go value recursively, in a predictable
+// order, without looping.
+package deephash
+
+import (
+	"bufio"
+	"crypto/sha256"
+	"encoding/hex"
+	"fmt"
+	"hash"
+	"reflect"
+	"strconv"
+	"sync"
+)
+
+func calcHash(v interface{}) string {
+	h := sha256.New()
+	b := bufio.NewWriterSize(h, h.BlockSize())
+	scratch := make([]byte, 0, 128)
+	printTo(b, v, scratch)
+	b.Flush()
+	scratch = h.Sum(scratch[:0])
+	hex.Encode(scratch[:cap(scratch)], scratch[:sha256.Size])
+	return string(scratch[:sha256.Size*2])
+}
+
+// UpdateHash sets last to the hash of v and reports whether its value changed.
+func UpdateHash(last *string, v ...interface{}) (changed bool) {
+	sig := calcHash(v)
+	if *last != sig {
+		*last = sig
+		return true
+	}
+	return false
+}
+
+func printTo(w *bufio.Writer, v interface{}, scratch []byte) {
+	print(w, reflect.ValueOf(v), make(map[uintptr]bool), scratch)
+}
+
+var appenderToType = reflect.TypeOf((*appenderTo)(nil)).Elem()
+
+type appenderTo interface {
+	AppendTo([]byte) []byte
+}
+
+// print hashes v into w.
+// It reports whether it was able to do so without hitting a cycle.
+func print(w *bufio.Writer, v reflect.Value, visited map[uintptr]bool, scratch []byte) (acyclic bool) {
+	if !v.IsValid() {
+		return true
+	}
+
+	if v.CanInterface() {
+		// Use AppendTo methods, if available and cheap.
+		if v.CanAddr() && v.Type().Implements(appenderToType) {
+			a := v.Addr().Interface().(appenderTo)
+			scratch = a.AppendTo(scratch[:0])
+			w.Write(scratch)
+			return true
+		}
+	}
+
+	// Generic handling.
+	switch v.Kind() {
+	default:
+		panic(fmt.Sprintf("unhandled kind %v for type %v", v.Kind(), v.Type()))
+	case reflect.Ptr:
+		ptr := v.Pointer()
+		if visited[ptr] {
+			return false
+		}
+		visited[ptr] = true
+		return print(w, v.Elem(), visited, scratch)
+	case reflect.Struct:
+		acyclic = true
+		w.WriteString("struct{\n")
+		for i, n := 0, v.NumField(); i < n; i++ {
+			fmt.Fprintf(w, " [%d]: ", i)
+			if !print(w, v.Field(i), visited, scratch) {
+				acyclic = false
+			}
+			w.WriteString("\n")
+		}
+		w.WriteString("}\n")
+		return acyclic
+	case reflect.Slice, reflect.Array:
+		if v.Type().Elem().Kind() == reflect.Uint8 && v.CanInterface() {
+			fmt.Fprintf(w, "%q", v.Interface())
+			return true
+		}
+		fmt.Fprintf(w, "[%d]{\n", v.Len())
+		acyclic = true
+		for i, ln := 0, v.Len(); i < ln; i++ {
+			fmt.Fprintf(w, " [%d]: ", i)
+			if !print(w, v.Index(i), visited, scratch) {
+				acyclic = false
+			}
+			w.WriteString("\n")
+		}
+		w.WriteString("}\n")
+		return acyclic
+	case reflect.Interface:
+		return print(w, v.Elem(), visited, scratch)
+	case reflect.Map:
+		if hashMapAcyclic(w, v, visited, scratch) {
+			return true
+		}
+		return hashMapFallback(w, v, visited, scratch)
+	case reflect.String:
+		w.WriteString(v.String())
+	case reflect.Bool:
+		fmt.Fprintf(w, "%v", v.Bool())
+	case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64:
+		fmt.Fprintf(w, "%v", v.Int())
+	case reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64, reflect.Uintptr:
+		scratch = strconv.AppendUint(scratch[:0], v.Uint(), 10)
+		w.Write(scratch)
+	case reflect.Float32, reflect.Float64:
+		fmt.Fprintf(w, "%v", v.Float())
+	case reflect.Complex64, reflect.Complex128:
+		fmt.Fprintf(w, "%v", v.Complex())
+	}
+	return true
+}
+
+type mapHasher struct {
+	xbuf [sha256.Size]byte // XOR'ed accumulated buffer
+	ebuf [sha256.Size]byte // scratch buffer
+	s256 hash.Hash         // sha256 hash.Hash
+	bw   *bufio.Writer     // to hasher into ebuf
+	val  valueCache        // re-usable values for map iteration
+	iter *reflect.MapIter  // re-usable map iterator
+}
+
+func (mh *mapHasher) Reset() {
+	for i := range mh.xbuf {
+		mh.xbuf[i] = 0
+	}
+}
+
+func (mh *mapHasher) startEntry() {
+	for i := range mh.ebuf {
+		mh.ebuf[i] = 0
+	}
+	mh.bw.Flush()
+	mh.s256.Reset()
+}
+
+func (mh *mapHasher) endEntry() {
+	mh.bw.Flush()
+	for i, b := range mh.s256.Sum(mh.ebuf[:0]) {
+		mh.xbuf[i] ^= b
+	}
+}
+
+var mapHasherPool = &sync.Pool{
+	New: func() interface{} {
+		mh := new(mapHasher)
+		mh.s256 = sha256.New()
+		mh.bw = bufio.NewWriter(mh.s256)
+		mh.val = make(valueCache)
+		mh.iter = new(reflect.MapIter)
+		return mh
+	},
+}
+
+type valueCache map[reflect.Type]reflect.Value
+
+func (c valueCache) get(t reflect.Type) reflect.Value {
+	v, ok := c[t]
+	if !ok {
+		v = reflect.New(t).Elem()
+		c[t] = v
+	}
+	return v
+}
+
+// hashMapAcyclic is the faster sort-free version of map hashing. If
+// it detects a cycle it returns false and guarantees that nothing was
+// written to w.
+func hashMapAcyclic(w *bufio.Writer, v reflect.Value, visited map[uintptr]bool, scratch []byte) (acyclic bool) {
+	mh := mapHasherPool.Get().(*mapHasher)
+	defer mapHasherPool.Put(mh)
+	mh.Reset()
+	iter := mapIter(mh.iter, v)
+	defer mapIter(mh.iter, reflect.Value{}) // avoid pinning v from mh.iter when we return
+	k := mh.val.get(v.Type().Key())
+	e := mh.val.get(v.Type().Elem())
+	for iter.Next() {
+		key := iterKey(iter, k)
+		val := iterVal(iter, e)
+		mh.startEntry()
+		if !print(mh.bw, key, visited, scratch) {
+			return false
+		}
+		if !print(mh.bw, val, visited, scratch) {
+			return false
+		}
+		mh.endEntry()
+	}
+	w.Write(mh.xbuf[:])
+	return true
+}
+
+func hashMapFallback(w *bufio.Writer, v reflect.Value, visited map[uintptr]bool, scratch []byte) (acyclic bool) {
+	acyclic = true
+	sm := newSortedMap(v)
+	fmt.Fprintf(w, "map[%d]{\n", len(sm.Key))
+	for i, k := range sm.Key {
+		if !print(w, k, visited, scratch) {
+			acyclic = false
+		}
+		w.WriteString(": ")
+		if !print(w, sm.Value[i], visited, scratch) {
+			acyclic = false
+		}
+		w.WriteString("\n")
+	}
+	w.WriteString("}\n")
+	return acyclic
+}

internal/deephash/deephash_test.go

@@ -0,0 +1,136 @@
+// Copyright (c) 2020 Tailscale Inc & AUTHORS All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package deephash
+
+import (
+	"bufio"
+	"bytes"
+	"fmt"
+	"reflect"
+	"testing"
+
+	"inet.af/netaddr"
+	"tailscale.com/tailcfg"
+	"tailscale.com/util/dnsname"
+	"tailscale.com/wgengine/router"
+	"tailscale.com/wgengine/wgcfg"
+)
+
+func TestDeepHash(t *testing.T) {
+	// v contains the types of values we care about for our current callers.
+	// Mostly we're just testing that we don't panic on handled types.
+	v := getVal()
+
+	hash1 := calcHash(v)
+	t.Logf("hash: %v", hash1)
+	for i := 0; i < 20; i++ {
+		hash2 := calcHash(getVal())
+		if hash1 != hash2 {
+			t.Error("second hash didn't match")
+		}
+	}
+}
+
+func getVal() []interface{} {
+	return []interface{}{
+		&wgcfg.Config{
+			Name:      "foo",
+			Addresses: []netaddr.IPPrefix{netaddr.IPPrefixFrom(netaddr.IPFrom16([16]byte{3: 3}), 5)},
+			Peers: []wgcfg.Peer{
+				{
+					Endpoints: wgcfg.Endpoints{
+						IPPorts: wgcfg.NewIPPortSet(netaddr.MustParseIPPort("42.42.42.42:5")),
+					},
+				},
+			},
+		},
+		&router.Config{
+			Routes: []netaddr.IPPrefix{
+				netaddr.MustParseIPPrefix("1.2.3.0/24"),
+				netaddr.MustParseIPPrefix("1234::/64"),
+			},
+		},
+		map[dnsname.FQDN][]netaddr.IP{
+			dnsname.FQDN("a."): {netaddr.MustParseIP("1.2.3.4"), netaddr.MustParseIP("4.3.2.1")},
+			dnsname.FQDN("b."): {netaddr.MustParseIP("8.8.8.8"), netaddr.MustParseIP("9.9.9.9")},
+			dnsname.FQDN("c."): {netaddr.MustParseIP("6.6.6.6"), netaddr.MustParseIP("7.7.7.7")},
+			dnsname.FQDN("d."): {netaddr.MustParseIP("6.7.6.6"), netaddr.MustParseIP("7.7.7.8")},
+			dnsname.FQDN("e."): {netaddr.MustParseIP("6.8.6.6"), netaddr.MustParseIP("7.7.7.9")},
+			dnsname.FQDN("f."): {netaddr.MustParseIP("6.9.6.6"), netaddr.MustParseIP("7.7.7.0")},
+		},
+		map[dnsname.FQDN][]netaddr.IPPort{
+			dnsname.FQDN("a."): {netaddr.MustParseIPPort("1.2.3.4:11"), netaddr.MustParseIPPort("4.3.2.1:22")},
+			dnsname.FQDN("b."): {netaddr.MustParseIPPort("8.8.8.8:11"), netaddr.MustParseIPPort("9.9.9.9:22")},
+			dnsname.FQDN("c."): {netaddr.MustParseIPPort("8.8.8.8:12"), netaddr.MustParseIPPort("9.9.9.9:23")},
+			dnsname.FQDN("d."): {netaddr.MustParseIPPort("8.8.8.8:13"), netaddr.MustParseIPPort("9.9.9.9:24")},
+			dnsname.FQDN("e."): {netaddr.MustParseIPPort("8.8.8.8:14"), netaddr.MustParseIPPort("9.9.9.9:25")},
+		},
+		map[tailcfg.DiscoKey]bool{
+			{1: 1}: true,
+			{1: 2}: false,
+			{2: 3}: true,
+			{3: 4}: false,
+		},
+	}
+}
+
+func BenchmarkHash(b *testing.B) {
+	b.ReportAllocs()
+	v := getVal()
+	for i := 0; i < b.N; i++ {
+		calcHash(v)
+	}
+}
+
+func TestHashMapAcyclic(t *testing.T) {
+	m := map[int]string{}
+	for i := 0; i < 100; i++ {
+		m[i] = fmt.Sprint(i)
+	}
+	got := map[string]bool{}
+
+	var buf bytes.Buffer
+	bw := bufio.NewWriter(&buf)
+
+	for i := 0; i < 20; i++ {
+		visited := map[uintptr]bool{}
+		scratch := make([]byte, 0, 64)
+		v := reflect.ValueOf(m)
+		buf.Reset()
+		bw.Reset(&buf)
+		if !hashMapAcyclic(bw, v, visited, scratch) {
+			t.Fatal("returned false")
+		}
+		if got[string(buf.Bytes())] {
+			continue
+		}
+		got[string(buf.Bytes())] = true
+	}
+	if len(got) != 1 {
+		t.Errorf("got %d results; want 1", len(got))
+	}
+}
+
+func BenchmarkHashMapAcyclic(b *testing.B) {
+	b.ReportAllocs()
+	m := map[int]string{}
+	for i := 0; i < 100; i++ {
+		m[i] = fmt.Sprint(i)
+	}
+
+	var buf bytes.Buffer
+	bw := bufio.NewWriter(&buf)
+	visited := map[uintptr]bool{}
+	scratch := make([]byte, 0, 64)
+	v := reflect.ValueOf(m)
+
+	for i := 0; i < b.N; i++ {
+		buf.Reset()
+		bw.Reset(&buf)
+		if !hashMapAcyclic(bw, v, visited, scratch) {
+			b.Fatal("returned false")
+		}
+	}
+}

internal/deephash/fmtsort.go

@@ -10,7 +10,7 @@
 
 // This is a slightly modified fork of Go's src/internal/fmtsort/sort.go
 
-package deepprint
+package deephash
 
 import (
 	"reflect"

internal/deephash/mapiter.go

@@ -0,0 +1,37 @@
+// Copyright (c) 2021 Tailscale Inc & AUTHORS All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+// +build !tailscale_go
+
+package deephash
+
+import "reflect"
+
+// iterKey returns the current iter key.
+// scratch is a re-usable reflect.Value.
+// iterKey may store the iter key in scratch and return scratch,
+// or it may allocate and return a new reflect.Value.
+func iterKey(iter *reflect.MapIter, _ reflect.Value) reflect.Value {
+	return iter.Key()
+}
+
+// iterVal returns the current iter val.
+// scratch is a re-usable reflect.Value.
+// iterVal may store the iter val in scratch and return scratch,
+// or it may allocate and return a new reflect.Value.
+func iterVal(iter *reflect.MapIter, _ reflect.Value) reflect.Value {
+	return iter.Value()
+}
+
+// mapIter returns a map iterator for mapVal.
+// scratch is a re-usable reflect.MapIter.
+// mapIter may re-use scratch and return it,
+// or it may allocate and return a new *reflect.MapIter.
+// If mapVal is the zero reflect.Value, mapIter may return nil.
+func mapIter(_ *reflect.MapIter, mapVal reflect.Value) *reflect.MapIter {
+	if !mapVal.IsValid() {
+		return nil
+	}
+	return mapVal.MapRange()
+}

internal/deephash/mapiter_future.go

@@ -0,0 +1,42 @@
+// Copyright (c) 2021 Tailscale Inc & AUTHORS All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+// +build tailscale_go
+
+package deephash
+
+import "reflect"
+
+// iterKey returns the current iter key.
+// scratch is a re-usable reflect.Value.
+// iterKey may store the iter key in scratch and return scratch,
+// or it may allocate and return a new reflect.Value.
+func iterKey(iter *reflect.MapIter, scratch reflect.Value) reflect.Value {
+	iter.SetKey(scratch)
+	return scratch
+}
+
+// iterVal returns the current iter val.
+// scratch is a re-usable reflect.Value.
+// iterVal may store the iter val in scratch and return scratch,
+// or it may allocate and return a new reflect.Value.
+func iterVal(iter *reflect.MapIter, scratch reflect.Value) reflect.Value {
+	iter.SetValue(scratch)
+	return scratch
+}
+
+// mapIter returns a map iterator for mapVal.
+// scratch is a re-usable reflect.MapIter.
+// mapIter may re-use scratch and return it,
+// or it may allocate and return a new *reflect.MapIter.
+// If mapVal is the zero reflect.Value, mapIter may return nil.
+func mapIter(scratch *reflect.MapIter, mapVal reflect.Value) *reflect.MapIter {
+	scratch.Reset(mapVal) // always Reset, to allow the caller to avoid pinning memory
+	if !mapVal.IsValid() {
+		// Returning scratch would also be OK.
+		// Do this for consistency with the non-optimized version.
+		return nil
+	}
+	return scratch
+}

internal/deepprint/deepprint.go

@@ -1,103 +0,0 @@
-// Copyright (c) 2020 Tailscale Inc & AUTHORS All rights reserved.
-// Use of this source code is governed by a BSD-style
-// license that can be found in the LICENSE file.
-
-// Package deepprint walks a Go value recursively, in a predictable
-// order, without looping, and prints each value out to a given
-// Writer, which is assumed to be a hash.Hash, as this package doesn't
-// format things nicely.
-//
-// This is intended as a lighter version of go-spew, etc. We don't need its
-// features when our writer is just a hash.
-package deepprint
-
-import (
-	"crypto/sha256"
-	"fmt"
-	"io"
-	"reflect"
-)
-
-func Hash(v ...interface{}) string {
-	h := sha256.New()
-	Print(h, v)
-	return fmt.Sprintf("%x", h.Sum(nil))
-}
-
-// UpdateHash sets last to the hash of v and reports whether its value changed.
-func UpdateHash(last *string, v ...interface{}) (changed bool) {
-	sig := Hash(v)
-	if *last != sig {
-		*last = sig
-		return true
-	}
-	return false
-}
-
-func Print(w io.Writer, v ...interface{}) {
-	print(w, reflect.ValueOf(v), make(map[uintptr]bool))
-}
-
-func print(w io.Writer, v reflect.Value, visited map[uintptr]bool) {
-	if !v.IsValid() {
-		return
-	}
-	switch v.Kind() {
-	default:
-		panic(fmt.Sprintf("unhandled kind %v for type %v", v.Kind(), v.Type()))
-	case reflect.Ptr:
-		ptr := v.Pointer()
-		if visited[ptr] {
-			return
-		}
-		visited[ptr] = true
-		print(w, v.Elem(), visited)
-		return
-	case reflect.Struct:
-		fmt.Fprintf(w, "struct{\n")
-		t := v.Type()
-		for i, n := 0, v.NumField(); i < n; i++ {
-			sf := t.Field(i)
-			fmt.Fprintf(w, "%s: ", sf.Name)
-			print(w, v.Field(i), visited)
-			fmt.Fprintf(w, "\n")
-		}
-	case reflect.Slice, reflect.Array:
-		if v.Type().Elem().Kind() == reflect.Uint8 && v.CanInterface() {
-			fmt.Fprintf(w, "%q", v.Interface())
-			return
-		}
-		fmt.Fprintf(w, "[%d]{\n", v.Len())
-		for i, ln := 0, v.Len(); i < ln; i++ {
-			fmt.Fprintf(w, " [%d]: ", i)
-			print(w, v.Index(i), visited)
-			fmt.Fprintf(w, "\n")
-		}
-		fmt.Fprintf(w, "}\n")
-	case reflect.Interface:
-		print(w, v.Elem(), visited)
-	case reflect.Map:
-		sm := newSortedMap(v)
-		fmt.Fprintf(w, "map[%d]{\n", len(sm.Key))
-		for i, k := range sm.Key {
-			print(w, k, visited)
-			fmt.Fprintf(w, ": ")
-			print(w, sm.Value[i], visited)
-			fmt.Fprintf(w, "\n")
-		}
-		fmt.Fprintf(w, "}\n")
-
-	case reflect.String:
-		fmt.Fprintf(w, "%s", v.String())
-	case reflect.Bool:
-		fmt.Fprintf(w, "%v", v.Bool())
-	case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64:
-		fmt.Fprintf(w, "%v", v.Int())
-	case reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64, reflect.Uintptr:
-		fmt.Fprintf(w, "%v", v.Uint())
-	case reflect.Float32, reflect.Float64:
-		fmt.Fprintf(w, "%v", v.Float())
-	case reflect.Complex64, reflect.Complex128:
-		fmt.Fprintf(w, "%v", v.Complex())
-	}
-}

internal/deepprint/deepprint_test.go

@@ -1,64 +0,0 @@
-// Copyright (c) 2020 Tailscale Inc & AUTHORS All rights reserved.
-// Use of this source code is governed by a BSD-style
-// license that can be found in the LICENSE file.
-
-package deepprint
-
-import (
-	"bytes"
-	"testing"
-
-	"inet.af/netaddr"
-	"tailscale.com/wgengine/router"
-	"tailscale.com/wgengine/wgcfg"
-)
-
-func TestDeepPrint(t *testing.T) {
-	// v contains the types of values we care about for our current callers.
-	// Mostly we're just testing that we don't panic on handled types.
-	v := getVal()
-
-	var buf bytes.Buffer
-	Print(&buf, v)
-	t.Logf("Got: %s", buf.Bytes())
-
-	hash1 := Hash(v)
-	t.Logf("hash: %v", hash1)
-	for i := 0; i < 20; i++ {
-		hash2 := Hash(getVal())
-		if hash1 != hash2 {
-			t.Error("second hash didn't match")
-		}
-	}
-}
-
-func getVal() []interface{} {
-	return []interface{}{
-		&wgcfg.Config{
-			Name:      "foo",
-			Addresses: []netaddr.IPPrefix{{Bits: 5, IP: netaddr.IPFrom16([16]byte{3: 3})}},
-			Peers: []wgcfg.Peer{
-				{
-					Endpoints: "foo:5",
-				},
-			},
-		},
-		&router.Config{
-			Routes: []netaddr.IPPrefix{
-				netaddr.MustParseIPPrefix("1.2.3.0/24"),
-				netaddr.MustParseIPPrefix("1234::/64"),
-			},
-		},
-		map[string]string{
-			"key1": "val1",
-			"key2": "val2",
-			"key3": "val3",
-			"key4": "val4",
-			"key5": "val5",
-			"key6": "val6",
-			"key7": "val7",
-			"key8": "val8",
-			"key9": "val9",
-		},
-	}
-}

ipn/ipnlocal/local.go

@@ -10,6 +10,7 @@ import (
 	"errors"
 	"fmt"
 	"io"
+	"log"
 	"net"
 	"net/http"
 	"os"
@@ -29,7 +30,7 @@ import (
 	"tailscale.com/client/tailscale/apitype"
 	"tailscale.com/control/controlclient"
 	"tailscale.com/health"
-	"tailscale.com/internal/deepprint"
+	"tailscale.com/internal/deephash"
 	"tailscale.com/ipn"
 	"tailscale.com/ipn/ipnstate"
 	"tailscale.com/ipn/policy"
@@ -44,11 +45,13 @@ import (
 	"tailscale.com/types/logger"
 	"tailscale.com/types/netmap"
 	"tailscale.com/types/persist"
+	"tailscale.com/types/preftype"
 	"tailscale.com/types/wgkey"
 	"tailscale.com/util/dnsname"
 	"tailscale.com/util/osshare"
 	"tailscale.com/util/systemd"
 	"tailscale.com/version"
+	"tailscale.com/version/distro"
 	"tailscale.com/wgengine"
 	"tailscale.com/wgengine/filter"
 	"tailscale.com/wgengine/router"
@@ -292,6 +295,7 @@ func (b *LocalBackend) Prefs() *ipn.Prefs {
 // Status returns the latest status of the backend and its
 // sub-components.
 func (b *LocalBackend) Status() *ipnstate.Status {
+	log.Println("Status ENDPOINT")
 	sb := new(ipnstate.StatusBuilder)
 	b.UpdateStatus(sb)
 	return sb.Status()
@@ -326,6 +330,9 @@ func (b *LocalBackend) updateStatus(sb *ipnstate.StatusBuilder, extraLocked func
 		}
 	})
 	sb.MutateSelfStatus(func(ss *ipnstate.PeerStatus) {
+		if b.netMap != nil && b.netMap.SelfNode != nil {
+			ss.ID = b.netMap.SelfNode.StableID
+		}
 		for _, pln := range b.peerAPIListeners {
 			ss.PeerAPIURL = append(ss.PeerAPIURL, pln.urlStr)
 		}
@@ -353,18 +360,19 @@ func (b *LocalBackend) populatePeerStatusLocked(sb *ipnstate.StatusBuilder) {
 		var tailAddr4 string
 		var tailscaleIPs = make([]netaddr.IP, 0, len(p.Addresses))
 		for _, addr := range p.Addresses {
-			if addr.IsSingleIP() && tsaddr.IsTailscaleIP(addr.IP) {
-				if addr.IP.Is4() && tailAddr4 == "" {
+			if addr.IsSingleIP() && tsaddr.IsTailscaleIP(addr.IP()) {
+				if addr.IP().Is4() && tailAddr4 == "" {
 					// The peer struct previously only allowed a single
 					// Tailscale IP address. For compatibility for a few releases starting
 					// with 1.8, keep it pulled out as IPv4-only for a bit.
-					tailAddr4 = addr.IP.String()
+					tailAddr4 = addr.IP().String()
 				}
-				tailscaleIPs = append(tailscaleIPs, addr.IP)
+				tailscaleIPs = append(tailscaleIPs, addr.IP())
 			}
 		}
 		sb.AddPeer(key.Public(p.Key), &ipnstate.PeerStatus{
 			InNetworkMap:       true,
+			ID:                 p.StableID,
 			UserID:             p.User,
 			TailAddrDeprecated: tailAddr4,
 			TailscaleIPs:       tailscaleIPs,
@@ -386,10 +394,10 @@ func (b *LocalBackend) populatePeerStatusLocked(sb *ipnstate.StatusBuilder) {
 func (b *LocalBackend) WhoIs(ipp netaddr.IPPort) (n *tailcfg.Node, u tailcfg.UserProfile, ok bool) {
 	b.mu.Lock()
 	defer b.mu.Unlock()
-	n, ok = b.nodeByAddr[ipp.IP]
+	n, ok = b.nodeByAddr[ipp.IP()]
 	if !ok {
 		var ip netaddr.IP
-		if ipp.Port != 0 {
+		if ipp.Port() != 0 {
 			ip, ok = b.e.WhoIsIPPort(ipp)
 		}
 		if !ok {
@@ -430,14 +438,15 @@ func (b *LocalBackend) setClientStatus(st controlclient.Status) {
 		}
 		return
 	}
-	if st.LoginFinished != nil {
+
+	b.mu.Lock()
+	wasBlocked := b.blocked
+	b.mu.Unlock()
+
+	if st.LoginFinished != nil && wasBlocked {
 		// Auth completed, unblock the engine
 		b.blockEngineUpdates(false)
 		b.authReconfig()
-		b.EditPrefs(&ipn.MaskedPrefs{
-			LoggedOutSet: true,
-			Prefs:        ipn.Prefs{LoggedOut: false},
-		})
 		b.send(ipn.Notify{LoginFinished: &empty.Message{}})
 	}
 
@@ -476,11 +485,15 @@ func (b *LocalBackend) setClientStatus(st controlclient.Status) {
 		b.authURL = st.URL
 		b.authURLSticky = st.URL
 	}
-	if b.state == ipn.NeedsLogin {
-		if !b.prefs.WantRunning {
+	if wasBlocked && st.LoginFinished != nil {
+		// Interactive login finished successfully (URL visited).
+		// After an interactive login, the user always wants
+		// WantRunning.
+		if !b.prefs.WantRunning || b.prefs.LoggedOut {
 			prefsChanged = true
 		}
 		b.prefs.WantRunning = true
+		b.prefs.LoggedOut = false
 	}
 	// Prefs will be written out; this is not safe unless locked or cloned.
 	if prefsChanged {
@@ -543,7 +556,7 @@ func (b *LocalBackend) findExitNodeIDLocked(nm *netmap.NetworkMap) (prefsChanged
 
 	for _, peer := range nm.Peers {
 		for _, addr := range peer.Addresses {
-			if !addr.IsSingleIP() || addr.IP != b.prefs.ExitNodeIP {
+			if !addr.IsSingleIP() || addr.IP() != b.prefs.ExitNodeIP {
 				continue
 			}
 			// Found the node being referenced, upgrade prefs to
@@ -562,10 +575,18 @@ func (b *LocalBackend) findExitNodeIDLocked(nm *netmap.NetworkMap) (prefsChanged
 func (b *LocalBackend) setWgengineStatus(s *wgengine.Status, err error) {
 	if err != nil {
 		b.logf("wgengine status error: %v", err)
+
+		b.statusLock.Lock()
+		b.statusChanged.Broadcast()
+		b.statusLock.Unlock()
 		return
 	}
 	if s == nil {
 		b.logf("[unexpected] non-error wgengine update with status=nil: %v", s)
+
+		b.statusLock.Lock()
+		b.statusChanged.Broadcast()
+		b.statusLock.Unlock()
 		return
 	}
 
@@ -804,6 +825,7 @@ func (b *LocalBackend) Start(opts ipn.Options) error {
 		DiscoPublicKey:       discoPublic,
 		DebugFlags:           debugFlags,
 		LinkMonitor:          b.e.GetLinkMonitor(),
+		Pinger:               b.e,
 
 		// Don't warn about broken Linux IP forwading when
 		// netstack is being used.
@@ -874,7 +896,7 @@ func (b *LocalBackend) updateFilter(netMap *netmap.NetworkMap, prefs *ipn.Prefs)
 	}
 	if prefs != nil {
 		for _, r := range prefs.AdvertiseRoutes {
-			if r.Bits == 0 {
+			if r.Bits() == 0 {
 				// When offering a default route to the world, we
 				// filter out locally reachable LANs, so that the
 				// default route effectively appears to be a "guest
@@ -899,7 +921,7 @@ func (b *LocalBackend) updateFilter(netMap *netmap.NetworkMap, prefs *ipn.Prefs)
 	localNets := localNetsB.IPSet()
 	logNets := logNetsB.IPSet()
 
-	changed := deepprint.UpdateHash(&b.filterHash, haveNetmap, addrs, packetFilter, localNets.Ranges(), logNets.Ranges(), shieldsUp)
+	changed := deephash.UpdateHash(&b.filterHash, haveNetmap, addrs, packetFilter, localNets.Ranges(), logNets.Ranges(), shieldsUp)
 	if !changed {
 		return
 	}
@@ -942,13 +964,13 @@ var removeFromDefaultRoute = []netaddr.IPPrefix{
 func interfaceRoutes() (ips *netaddr.IPSet, hostIPs []netaddr.IP, err error) {
 	var b netaddr.IPSetBuilder
 	if err := interfaces.ForeachInterfaceAddress(func(_ interfaces.Interface, pfx netaddr.IPPrefix) {
-		if tsaddr.IsTailscaleIP(pfx.IP) {
+		if tsaddr.IsTailscaleIP(pfx.IP()) {
 			return
 		}
 		if pfx.IsSingleIP() {
 			return
 		}
-		hostIPs = append(hostIPs, pfx.IP)
+		hostIPs = append(hostIPs, pfx.IP())
 		b.AddPrefix(pfx)
 	}); err != nil {
 		return nil, nil, err
@@ -1677,9 +1699,45 @@ func (b *LocalBackend) authReconfig() {
 
 	rcfg := b.routerConfig(cfg, uc)
 
-	var dcfg dns.Config
+	dcfg := dns.Config{
+		Routes: map[dnsname.FQDN][]netaddr.IPPort{},
+		Hosts:  map[dnsname.FQDN][]netaddr.IP{},
+	}
+
+	// Populate MagicDNS records. We do this unconditionally so that
+	// quad-100 can always respond to MagicDNS queries, even if the OS
+	// isn't configured to make MagicDNS resolution truly
+	// magic. Details in
+	// https://github.com/tailscale/tailscale/issues/1886.
+	set := func(name string, addrs []netaddr.IPPrefix) {
+		if len(addrs) == 0 || name == "" {
+			return
+		}
+		fqdn, err := dnsname.ToFQDN(name)
+		if err != nil {
+			return // TODO: propagate error?
+		}
+		var ips []netaddr.IP
+		for _, addr := range addrs {
+			// Remove IPv6 addresses for now, as we don't
+			// guarantee that the peer node actually can speak
+			// IPv6 correctly.
+			//
+			// https://github.com/tailscale/tailscale/issues/1152
+			// tracks adding the right capability reporting to
+			// enable AAAA in MagicDNS.
+			if addr.IP().Is6() {
+				continue
+			}
+			ips = append(ips, addr.IP())
+		}
+		dcfg.Hosts[fqdn] = ips
+	}
+	set(nm.Name, nm.Addresses)
+	for _, peer := range nm.Peers {
+		set(peer.Name, peer.Addresses)
+	}
 
-	// If CorpDNS is false, dcfg remains the zero value.
 	if uc.CorpDNS {
 		addDefault := func(resolvers []tailcfg.DNSResolver) {
 			for _, resolver := range resolvers {
@@ -1693,9 +1751,6 @@ func (b *LocalBackend) authReconfig() {
 		}
 
 		addDefault(nm.DNS.Resolvers)
-		if len(nm.DNS.Routes) > 0 {
-			dcfg.Routes = map[dnsname.FQDN][]netaddr.IPPort{}
-		}
 		for suffix, resolvers := range nm.DNS.Routes {
 			fqdn, err := dnsname.ToFQDN(suffix)
 			if err != nil {
@@ -1717,36 +1772,9 @@ func (b *LocalBackend) authReconfig() {
 			}
 			dcfg.SearchDomains = append(dcfg.SearchDomains, fqdn)
 		}
-		set := func(name string, addrs []netaddr.IPPrefix) {
-			if len(addrs) == 0 || name == "" {
-				return
-			}
-			fqdn, err := dnsname.ToFQDN(name)
-			if err != nil {
-				return // TODO: propagate error?
-			}
-			var ips []netaddr.IP
-			for _, addr := range addrs {
-				// Remove IPv6 addresses for now, as we don't
-				// guarantee that the peer node actually can speak
-				// IPv6 correctly.
-				//
-				// https://github.com/tailscale/tailscale/issues/1152
-				// tracks adding the right capability reporting to
-				// enable AAAA in MagicDNS.
-				if addr.IP.Is6() {
-					continue
-				}
-				ips = append(ips, addr.IP)
-			}
-			dcfg.Hosts[fqdn] = ips
-		}
 		if nm.DNS.Proxied { // actually means "enable MagicDNS"
-			dcfg.AuthoritativeSuffixes = magicDNSRootDomains(nm)
-			dcfg.Hosts = map[dnsname.FQDN][]netaddr.IP{}
-			set(nm.Name, nm.Addresses)
-			for _, peer := range nm.Peers {
-				set(peer.Name, peer.Addresses)
+			for _, dom := range magicDNSRootDomains(nm) {
+				dcfg.Routes[dom] = nil // resolve internally with dcfg.Hosts
 			}
 		}
 
@@ -1770,7 +1798,7 @@ func (b *LocalBackend) authReconfig() {
 			//
 			// https://github.com/tailscale/tailscale/issues/1713
 			addDefault(nm.DNS.FallbackResolvers)
-		case len(dcfg.Routes) == 0 && len(dcfg.Hosts) == 0 && len(dcfg.AuthoritativeSuffixes) == 0:
+		case len(dcfg.Routes) == 0:
 			// No settings requiring split DNS, no problem.
 		case version.OS() == "android":
 			// We don't support split DNS at all on Android yet.
@@ -1792,17 +1820,15 @@ func parseResolver(cfg tailcfg.DNSResolver) (netaddr.IPPort, error) {
 	if err != nil {
 		return netaddr.IPPort{}, fmt.Errorf("[unexpected] non-IP resolver %q", cfg.Addr)
 	}
-	return netaddr.IPPort{
-		IP:   ip,
-		Port: 53,
-	}, nil
+	return netaddr.IPPortFrom(ip, 53), nil
 }
 
 // tailscaleVarRoot returns the root directory of Tailscale's writable
 // storage area. (e.g. "/var/lib/tailscale")
 func tailscaleVarRoot() string {
-	if runtime.GOOS == "ios" {
-		dir, _ := paths.IOSSharedDir.Load().(string)
+	switch runtime.GOOS {
+	case "ios", "android":
+		dir, _ := paths.AppSharedDir.Load().(string)
 		return dir
 	}
 	stateFile := paths.DefaultTailscaledStateFile()
@@ -1853,7 +1879,7 @@ func (b *LocalBackend) initPeerAPIListener() {
 	if len(b.netMap.Addresses) == len(b.peerAPIListeners) {
 		allSame := true
 		for i, pln := range b.peerAPIListeners {
-			if pln.ip != b.netMap.Addresses[i].IP {
+			if pln.ip != b.netMap.Addresses[i].IP() {
 				allSame = false
 				break
 			}
@@ -1898,7 +1924,7 @@ func (b *LocalBackend) initPeerAPIListener() {
 		var err error
 		skipListen := i > 0 && isNetstack
 		if !skipListen {
-			ln, err = ps.listen(a.IP, b.prevIfState)
+			ln, err = ps.listen(a.IP(), b.prevIfState)
 			if err != nil {
 				if runtime.GOOS == "windows" {
 					// Expected for now. See Issue 1620.
@@ -1906,13 +1932,13 @@ func (b *LocalBackend) initPeerAPIListener() {
 					// ("peerAPIListeners too low").
 					continue
 				}
-				b.logf("[unexpected] peerapi listen(%q) error: %v", a.IP, err)
+				b.logf("[unexpected] peerapi listen(%q) error: %v", a.IP(), err)
 				continue
 			}
 		}
 		pln := &peerAPIListener{
 			ps: ps,
-			ip: a.IP,
+			ip: a.IP(),
 			ln: ln, // nil for 2nd+ on netstack
 			lb: b,
 		}
@@ -1921,7 +1947,7 @@ func (b *LocalBackend) initPeerAPIListener() {
 		} else {
 			pln.port = ln.Addr().(*net.TCPAddr).Port
 		}
-		pln.urlStr = "http://" + net.JoinHostPort(a.IP.String(), strconv.Itoa(pln.port))
+		pln.urlStr = "http://" + net.JoinHostPort(a.IP().String(), strconv.Itoa(pln.port))
 		b.logf("peerapi: serving on %s", pln.urlStr)
 		go pln.serve()
 		b.peerAPIListeners = append(b.peerAPIListeners, pln)
@@ -1972,14 +1998,14 @@ func peerRoutes(peers []wgcfg.Peer, cgnatThreshold int) (routes []netaddr.IPPref
 		for _, aip := range peer.AllowedIPs {
 			aip = unmapIPPrefix(aip)
 			// Only add the Tailscale IPv6 ULA once, if we see anybody using part of it.
-			if aip.IP.Is6() && aip.IsSingleIP() && tsULA.Contains(aip.IP) {
+			if aip.IP().Is6() && aip.IsSingleIP() && tsULA.Contains(aip.IP()) {
 				if !didULA {
 					didULA = true
 					routes = append(routes, tsULA)
 				}
 				continue
 			}
-			if aip.IsSingleIP() && cgNAT.Contains(aip.IP) {
+			if aip.IsSingleIP() && cgNAT.Contains(aip.IP()) {
 				cgNATIPs = append(cgNATIPs, aip)
 			} else {
 				routes = append(routes, aip)
@@ -2005,6 +2031,11 @@ func (b *LocalBackend) routerConfig(cfg *wgcfg.Config, prefs *ipn.Prefs) *router
 		Routes:           peerRoutes(cfg.Peers, 10_000),
 	}
 
+	if distro.Get() == distro.Synology {
+		// Issue 1995: we don't use iptables on Synology.
+		rs.NetfilterMode = preftype.NetfilterOff
+	}
+
 	// Sanity check: we expect the control server to program both a v4
 	// and a v6 default route, if default routing is on. Fill in
 	// blackhole routes appropriately if we're missing some. This is
@@ -2046,16 +2077,13 @@ func (b *LocalBackend) routerConfig(cfg *wgcfg.Config, prefs *ipn.Prefs) *router
 		}
 	}
 
-	rs.Routes = append(rs.Routes, netaddr.IPPrefix{
-		IP:   tsaddr.TailscaleServiceIP(),
-		Bits: 32,
-	})
+	rs.Routes = append(rs.Routes, netaddr.IPPrefixFrom(tsaddr.TailscaleServiceIP(), 32))
 
 	return rs
 }
 
 func unmapIPPrefix(ipp netaddr.IPPrefix) netaddr.IPPrefix {
-	return netaddr.IPPrefix{IP: ipp.IP.Unmap(), Bits: ipp.Bits}
+	return netaddr.IPPrefixFrom(ipp.IP().Unmap(), ipp.Bits())
 }
 
 func unmapIPPrefixes(ippsList ...[]netaddr.IPPrefix) (ret []netaddr.IPPrefix) {
@@ -2109,8 +2137,8 @@ func (b *LocalBackend) enterState(newState ipn.State) {
 	if oldState == newState {
 		return
 	}
-	b.logf("Switching ipn state %v -> %v (WantRunning=%v)",
-		oldState, newState, prefs.WantRunning)
+	b.logf("Switching ipn state %v -> %v (WantRunning=%v, nm=%v)",
+		oldState, newState, prefs.WantRunning, netMap != nil)
 	health.SetIPNState(newState.String(), prefs.WantRunning)
 	b.send(ipn.Notify{State: &newState})
 
@@ -2139,7 +2167,7 @@ func (b *LocalBackend) enterState(newState ipn.State) {
 	case ipn.Running:
 		var addrs []string
 		for _, addr := range b.netMap.Addresses {
-			addrs = append(addrs, addr.IP.String())
+			addrs = append(addrs, addr.IP().String())
 		}
 		systemd.Status("Connected; %s; %s", activeLogin, strings.Join(addrs, " "))
 	default:
@@ -2166,13 +2194,14 @@ func (b *LocalBackend) nextState() ipn.State {
 		cc          = b.cc
 		netMap      = b.netMap
 		state       = b.state
+		blocked     = b.blocked
 		wantRunning = b.prefs.WantRunning
 		loggedOut   = b.prefs.LoggedOut
 	)
 	b.mu.Unlock()
 
 	switch {
-	case !wantRunning && !loggedOut && b.hasNodeKey():
+	case !wantRunning && !loggedOut && !blocked && b.hasNodeKey():
 		return ipn.Stopped
 	case netMap == nil:
 		if cc.AuthCantContinue() || loggedOut {
@@ -2406,7 +2435,7 @@ func (b *LocalBackend) setNetMapLocked(nm *netmap.NetworkMap) {
 	addNode := func(n *tailcfg.Node) {
 		for _, ipp := range n.Addresses {
 			if ipp.IsSingleIP() {
-				b.nodeByAddr[ipp.IP] = n
+				b.nodeByAddr[ipp.IP()] = n
 			}
 		}
 	}
@@ -2558,9 +2587,9 @@ func peerAPIBase(nm *netmap.NetworkMap, peer *tailcfg.Node) string {
 			continue
 		}
 		switch {
-		case a.IP.Is4():
+		case a.IP().Is4():
 			have4 = true
-		case a.IP.Is6():
+		case a.IP().Is6():
 			have6 = true
 		}
 	}
@@ -2576,11 +2605,11 @@ func peerAPIBase(nm *netmap.NetworkMap, peer *tailcfg.Node) string {
 	var ipp netaddr.IPPort
 	switch {
 	case have4 && p4 != 0:
-		ipp = netaddr.IPPort{IP: nodeIP(peer, netaddr.IP.Is4), Port: p4}
+		ipp = netaddr.IPPortFrom(nodeIP(peer, netaddr.IP.Is4), p4)
 	case have6 && p6 != 0:
-		ipp = netaddr.IPPort{IP: nodeIP(peer, netaddr.IP.Is6), Port: p6}
+		ipp = netaddr.IPPortFrom(nodeIP(peer, netaddr.IP.Is6), p6)
 	}
-	if ipp.IP.IsZero() {
+	if ipp.IP().IsZero() {
 		return ""
 	}
 	return fmt.Sprintf("http://%v", ipp)
@@ -2588,8 +2617,8 @@ func peerAPIBase(nm *netmap.NetworkMap, peer *tailcfg.Node) string {
 
 func nodeIP(n *tailcfg.Node, pred func(netaddr.IP) bool) netaddr.IP {
 	for _, a := range n.Addresses {
-		if a.IsSingleIP() && pred(a.IP) {
-			return a.IP
+		if a.IsSingleIP() && pred(a.IP()) {
+			return a.IP()
 		}
 	}
 	return netaddr.IP{}

ipn/ipnlocal/local_test.go

@@ -171,7 +171,7 @@ func TestShrinkDefaultRoute(t *testing.T) {
 			out: []string{
 				"fe80::1",
 				"ff00::1",
-				tsaddr.TailscaleULARange().IP.String(),
+				tsaddr.TailscaleULARange().IP().String(),
 			},
 			localIPFn: func(ip netaddr.IP) bool { return !inRemove(ip) && ip.Is6() },
 		},

ipn/ipnlocal/peerapi.go

@@ -510,7 +510,7 @@ func (h *peerAPIHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
 <body>
 <h1>Hello, %s (%v)</h1>
 This is my Tailscale device. Your device is %v.
-`, html.EscapeString(who), h.remoteAddr.IP, html.EscapeString(h.peerNode.ComputedName))
+`, html.EscapeString(who), h.remoteAddr.IP(), html.EscapeString(h.peerNode.ComputedName))
 
 	if h.isSelf {
 		fmt.Fprintf(w, "<p>You are the owner of this node.\n")

ipn/ipnlocal/state_test.go

@@ -368,13 +368,11 @@ func TestStateMachine(t *testing.T) {
 	{
 		c.Assert(cc.getCalls(), qt.DeepEquals, []string{"Login"})
 		notifies.drain(0)
-		// BUG: this should immediately set WantRunning to true.
-		// Users don't log in if they don't want to also connect.
-		// (Generally, we're inconsistent about who is supposed to
-		// update Prefs at what time. But the overall philosophy is:
-		// update it when the user's intent changes. This is clearly
-		// at the time the user *requests* Login, not at the time
-		// the login finishes.)
+		// Note: WantRunning isn't true yet. It'll switch to true
+		// after a successful login finishes.
+		// (This behaviour is needed so that b.Login() won't
+		// start connecting to an old account right away, if one
+		// exists when you launch another login.)
 	}
 
 	// Attempted non-interactive login with no key; indicate that
@@ -384,18 +382,16 @@ func TestStateMachine(t *testing.T) {
 	url1 := "http://localhost:1/1"
 	cc.send(nil, url1, false, nil)
 	{
-		c.Assert(cc.getCalls(), qt.HasLen, 0)
+		c.Assert(cc.getCalls(), qt.DeepEquals, []string{})
 
 		// ...but backend eats that notification, because the user
 		// didn't explicitly request interactive login yet, and
 		// we're already in NeedsLogin state.
 		nn := notifies.drain(1)
 
-		// Trying to log in automatically sets WantRunning.
-		// BUG: that should have happened right after Login().
 		c.Assert(nn[0].Prefs, qt.Not(qt.IsNil))
 		c.Assert(nn[0].Prefs.LoggedOut, qt.IsFalse)
-		c.Assert(nn[0].Prefs.WantRunning, qt.IsTrue)
+		c.Assert(nn[0].Prefs.WantRunning, qt.IsFalse)
 	}
 
 	// Now we'll try an interactive login.
@@ -451,11 +447,12 @@ func TestStateMachine(t *testing.T) {
 	// same time.
 	// The backend should propagate this upward for the UI.
 	t.Logf("\n\nLoginFinished")
-	notifies.expect(2)
+	notifies.expect(3)
 	cc.setAuthBlocked(false)
+	cc.persist.LoginName = "user1"
 	cc.send(nil, "", true, &netmap.NetworkMap{})
 	{
-		nn := notifies.drain(2)
+		nn := notifies.drain(3)
 		// BUG: still too soon for UpdateEndpoints.
 		//
 		// Arguably it makes sense to unpause now, since the machine
@@ -468,15 +465,12 @@ func TestStateMachine(t *testing.T) {
 		// it's visible in the logs)
 		c.Assert([]string{"unpause", "UpdateEndpoints"}, qt.DeepEquals, cc.getCalls())
 		c.Assert(nn[0].LoginFinished, qt.Not(qt.IsNil))
-		c.Assert(nn[1].State, qt.Not(qt.IsNil))
-		c.Assert(ipn.NeedsMachineAuth, qt.Equals, *nn[1].State)
+		c.Assert(nn[1].Prefs, qt.Not(qt.IsNil))
+		c.Assert(nn[2].State, qt.Not(qt.IsNil))
+		c.Assert(nn[1].Prefs.Persist.LoginName, qt.Equals, "user1")
+		c.Assert(ipn.NeedsMachineAuth, qt.Equals, *nn[2].State)
 	}
 
-	// TODO: check that the logged-in username propagates from control
-	// through to the UI notifications. I think it's used as a hint
-	// for future logins, to pre-fill the username box? Not really sure
-	// how it works.
-
 	// Pretend that the administrator has authorized our machine.
 	t.Logf("\n\nMachineAuthorized")
 	notifies.expect(1)
@@ -581,77 +575,72 @@ func TestStateMachine(t *testing.T) {
 
 	// Let's make the logout succeed.
 	t.Logf("\n\nLogout (async) - succeed")
-	notifies.expect(1)
+	notifies.expect(0)
 	cc.setAuthBlocked(true)
 	cc.send(nil, "", false, nil)
 	{
-		nn := notifies.drain(1)
+		notifies.drain(0)
 		c.Assert(cc.getCalls(), qt.HasLen, 0)
-		c.Assert(nn[0].Prefs, qt.Not(qt.IsNil))
-		c.Assert(nn[0].Prefs.LoggedOut, qt.IsTrue)
-		// BUG: WantRunning should be false after manual logout.
-		c.Assert(nn[0].Prefs.WantRunning, qt.IsTrue)
+		c.Assert(b.Prefs().LoggedOut, qt.IsTrue)
+		c.Assert(b.Prefs().WantRunning, qt.IsFalse)
 		c.Assert(ipn.NeedsLogin, qt.Equals, b.State())
 	}
 
 	// A second logout should do nothing, since the prefs haven't changed.
 	t.Logf("\n\nLogout2 (async)")
-	notifies.expect(1)
+	notifies.expect(0)
 	b.Logout()
 	{
-		nn := notifies.drain(1)
+		notifies.drain(0)
 		// BUG: the backend has already called StartLogout, and we're
 		// still logged out. So it shouldn't call it again.
 		c.Assert([]string{"StartLogout"}, qt.DeepEquals, cc.getCalls())
-		// BUG: Prefs should not change here. Already logged out.
-		c.Assert(nn[0].Prefs, qt.Not(qt.IsNil))
-		c.Assert(nn[0].Prefs.LoggedOut, qt.IsTrue)
-		c.Assert(nn[0].Prefs.WantRunning, qt.IsFalse)
+		c.Assert(cc.getCalls(), qt.HasLen, 0)
+		c.Assert(b.Prefs().LoggedOut, qt.IsTrue)
+		c.Assert(b.Prefs().WantRunning, qt.IsFalse)
 		c.Assert(ipn.NeedsLogin, qt.Equals, b.State())
 	}
 
 	// Let's acknowledge the second logout too.
 	t.Logf("\n\nLogout2 (async) - succeed")
-	notifies.expect(1)
+	notifies.expect(0)
 	cc.setAuthBlocked(true)
 	cc.send(nil, "", false, nil)
 	{
-		nn := notifies.drain(1)
+		notifies.drain(0)
 		c.Assert(cc.getCalls(), qt.HasLen, 0)
-		c.Assert(nn[0].Prefs, qt.Not(qt.IsNil))
-		c.Assert(nn[0].Prefs.LoggedOut, qt.IsTrue)
-		// BUG: second logout shouldn't cause WantRunning->true !!
-		c.Assert(nn[0].Prefs.WantRunning, qt.IsTrue)
+		c.Assert(cc.getCalls(), qt.HasLen, 0)
+		c.Assert(b.Prefs().LoggedOut, qt.IsTrue)
+		c.Assert(b.Prefs().WantRunning, qt.IsFalse)
 		c.Assert(ipn.NeedsLogin, qt.Equals, b.State())
 	}
 
 	// Try the synchronous logout feature.
 	t.Logf("\n\nLogout3 (sync)")
-	notifies.expect(1)
+	notifies.expect(0)
 	b.LogoutSync(context.Background())
 	// NOTE: This returns as soon as cc.Logout() returns, which is okay
 	// I guess, since that's supposed to be synchronous.
 	{
-		nn := notifies.drain(1)
+		notifies.drain(0)
 		c.Assert([]string{"Logout"}, qt.DeepEquals, cc.getCalls())
-		c.Assert(nn[0].Prefs, qt.Not(qt.IsNil))
-		c.Assert(nn[0].Prefs.LoggedOut, qt.IsTrue)
-		c.Assert(nn[0].Prefs.WantRunning, qt.IsFalse)
+		c.Assert(cc.getCalls(), qt.HasLen, 0)
+		c.Assert(b.Prefs().LoggedOut, qt.IsTrue)
+		c.Assert(b.Prefs().WantRunning, qt.IsFalse)
 		c.Assert(ipn.NeedsLogin, qt.Equals, b.State())
 	}
 
 	// Generate the third logout event.
 	t.Logf("\n\nLogout3 (sync) - succeed")
-	notifies.expect(1)
+	notifies.expect(0)
 	cc.setAuthBlocked(true)
 	cc.send(nil, "", false, nil)
 	{
-		nn := notifies.drain(1)
+		notifies.drain(0)
 		c.Assert(cc.getCalls(), qt.HasLen, 0)
-		c.Assert(nn[0].Prefs, qt.Not(qt.IsNil))
-		c.Assert(nn[0].Prefs.LoggedOut, qt.IsTrue)
-		// BUG: third logout shouldn't cause WantRunning->true !!
-		c.Assert(nn[0].Prefs.WantRunning, qt.IsTrue)
+		c.Assert(cc.getCalls(), qt.HasLen, 0)
+		c.Assert(b.Prefs().LoggedOut, qt.IsTrue)
+		c.Assert(b.Prefs().WantRunning, qt.IsFalse)
 		c.Assert(ipn.NeedsLogin, qt.Equals, b.State())
 	}
 
@@ -669,10 +658,6 @@ func TestStateMachine(t *testing.T) {
 	// happens if the user exits and restarts while logged out.
 	// Note that it's explicitly okay to call b.Start() over and over
 	// again, every time the frontend reconnects.
-	//
-	// BUG: WantRunning is true here (because of the bug above).
-	//  We'll have to adjust the following test's expectations if we
-	//  fix that.
 
 	// TODO: test user switching between statekeys.
 
@@ -691,7 +676,7 @@ func TestStateMachine(t *testing.T) {
 		c.Assert(nn[0].Prefs, qt.Not(qt.IsNil))
 		c.Assert(nn[1].State, qt.Not(qt.IsNil))
 		c.Assert(nn[0].Prefs.LoggedOut, qt.IsTrue)
-		c.Assert(nn[0].Prefs.WantRunning, qt.IsTrue)
+		c.Assert(nn[0].Prefs.WantRunning, qt.IsFalse)
 		c.Assert(ipn.NeedsLogin, qt.Equals, *nn[1].State)
 		c.Assert(ipn.NeedsLogin, qt.Equals, b.State())
 	}
@@ -703,16 +688,20 @@ func TestStateMachine(t *testing.T) {
 	t.Logf("\n\nLoginFinished3")
 	notifies.expect(3)
 	cc.setAuthBlocked(false)
+	cc.persist.LoginName = "user2"
 	cc.send(nil, "", true, &netmap.NetworkMap{
 		MachineStatus: tailcfg.MachineAuthorized,
 	})
 	{
 		nn := notifies.drain(3)
 		c.Assert([]string{"unpause"}, qt.DeepEquals, cc.getCalls())
-		c.Assert(nn[0].Prefs, qt.Not(qt.IsNil))
-		c.Assert(nn[1].LoginFinished, qt.Not(qt.IsNil))
+		c.Assert(nn[0].LoginFinished, qt.Not(qt.IsNil))
+		c.Assert(nn[1].Prefs, qt.Not(qt.IsNil))
 		c.Assert(nn[2].State, qt.Not(qt.IsNil))
-		c.Assert(nn[0].Prefs.LoggedOut, qt.IsFalse)
+		// Prefs after finishing the login, so LoginName updated.
+		c.Assert(nn[1].Prefs.Persist.LoginName, qt.Equals, "user2")
+		c.Assert(nn[1].Prefs.LoggedOut, qt.IsFalse)
+		c.Assert(nn[1].Prefs.WantRunning, qt.IsTrue)
 		c.Assert(ipn.Starting, qt.Equals, *nn[2].State)
 	}
 
@@ -773,6 +762,63 @@ func TestStateMachine(t *testing.T) {
 		c.Assert(ipn.Starting, qt.Equals, *nn[0].State)
 	}
 
+	// Disconnect.
+	t.Logf("\n\nStop")
+	notifies.expect(2)
+	b.EditPrefs(&ipn.MaskedPrefs{
+		WantRunningSet: true,
+		Prefs:          ipn.Prefs{WantRunning: false},
+	})
+	{
+		nn := notifies.drain(2)
+		c.Assert([]string{"unpause"}, qt.DeepEquals, cc.getCalls())
+		// BUG: I would expect Prefs to change first, and state after.
+		c.Assert(nn[0].State, qt.Not(qt.IsNil))
+		c.Assert(nn[1].Prefs, qt.Not(qt.IsNil))
+		c.Assert(ipn.Stopped, qt.Equals, *nn[0].State)
+	}
+
+	// We want to try logging in as a different user, while Stopped.
+	// First, start the login process (without logging out first).
+	t.Logf("\n\nLoginDifferent")
+	notifies.expect(2)
+	b.StartLoginInteractive()
+	url3 := "http://localhost:1/3"
+	cc.send(nil, url3, false, nil)
+	{
+		nn := notifies.drain(2)
+		// It might seem like WantRunning should switch to true here,
+		// but that would be risky since we already have a valid
+		// user account. It might try to reconnect to the old account
+		// before the new one is ready. So no change yet.
+		c.Assert([]string{"Login", "unpause"}, qt.DeepEquals, cc.getCalls())
+		c.Assert(nn[0].BrowseToURL, qt.Not(qt.IsNil))
+		c.Assert(nn[1].State, qt.Not(qt.IsNil))
+		c.Assert(*nn[0].BrowseToURL, qt.Equals, url3)
+		c.Assert(ipn.NeedsLogin, qt.Equals, *nn[1].State)
+	}
+
+	// Now, let's say the interactive login completed, using a different
+	// user account than before.
+	t.Logf("\n\nLoginDifferent URL visited")
+	notifies.expect(3)
+	cc.persist.LoginName = "user3"
+	cc.send(nil, "", true, &netmap.NetworkMap{
+		MachineStatus: tailcfg.MachineAuthorized,
+	})
+	{
+		nn := notifies.drain(3)
+		c.Assert([]string{"unpause"}, qt.DeepEquals, cc.getCalls())
+		c.Assert(nn[0].LoginFinished, qt.Not(qt.IsNil))
+		c.Assert(nn[1].Prefs, qt.Not(qt.IsNil))
+		c.Assert(nn[2].State, qt.Not(qt.IsNil))
+		// Prefs after finishing the login, so LoginName updated.
+		c.Assert(nn[1].Prefs.Persist.LoginName, qt.Equals, "user3")
+		c.Assert(nn[1].Prefs.LoggedOut, qt.IsFalse)
+		c.Assert(nn[1].Prefs.WantRunning, qt.IsTrue)
+		c.Assert(ipn.Starting, qt.Equals, *nn[2].State)
+	}
+
 	// The last test case is the most common one: restarting when both
 	// logged in and WantRunning.
 	t.Logf("\n\nStart5")
@@ -793,17 +839,18 @@ func TestStateMachine(t *testing.T) {
 
 	// Control server accepts our valid key from before.
 	t.Logf("\n\nLoginFinished5")
-	notifies.expect(2)
+	notifies.expect(1)
 	cc.setAuthBlocked(false)
 	cc.send(nil, "", true, &netmap.NetworkMap{
 		MachineStatus: tailcfg.MachineAuthorized,
 	})
 	{
-		nn := notifies.drain(2)
+		nn := notifies.drain(1)
 		c.Assert([]string{"unpause"}, qt.DeepEquals, cc.getCalls())
-		c.Assert(nn[0].LoginFinished, qt.Not(qt.IsNil))
-		c.Assert(nn[1].State, qt.Not(qt.IsNil))
-		c.Assert(ipn.Starting, qt.Equals, *nn[1].State)
+		// NOTE: No LoginFinished message since no interactive
+		// login was needed.
+		c.Assert(nn[0].State, qt.Not(qt.IsNil))
+		c.Assert(ipn.Starting, qt.Equals, *nn[0].State)
 		// NOTE: No prefs change this time. WantRunning stays true.
 		// We were in Starting in the first place, so that doesn't
 		// change either.

ipn/ipnserver/server.go

@@ -6,7 +6,9 @@ package ipnserver
 
 import (
 	"bufio"
+	"bytes"
 	"context"
+	"encoding/json"
 	"errors"
 	"fmt"
 	"io"
@@ -145,7 +147,7 @@ func (s *server) getConnIdentity(c net.Conn) (ci connIdentity, err error) {
 	if err != nil {
 		return ci, fmt.Errorf("parsing local remote: %w", err)
 	}
-	if !la.IP.IsLoopback() || !ra.IP.IsLoopback() {
+	if !la.IP().IsLoopback() || !ra.IP().IsLoopback() {
 		return ci, errors.New("non-loopback connection")
 	}
 	tab, err := netstat.Get()
@@ -251,8 +253,7 @@ func (s *server) serveConn(ctx context.Context, c net.Conn, logf logger.Logf) {
 			return
 		}
 		defer c.Close()
-		serverToClient := func(b []byte) { ipn.WriteMsg(c, b) }
-		bs := ipn.NewBackendServer(logf, nil, serverToClient)
+		bs := ipn.NewBackendServer(logf, nil, jsonNotifier(c, s.logf))
 		_, occupied := err.(inUseOtherUserError)
 		if occupied {
 			bs.SendInUseOtherUserErrorMessage(err.Error())
@@ -567,7 +568,9 @@ func (s *server) setServerModeUserLocked() {
 	}
 }
 
-func (s *server) writeToClients(b []byte) {
+var jsonEscapedZero = []byte(`\u0000`)
+
+func (s *server) writeToClients(n ipn.Notify) {
 	inServerMode := s.b.InServerMode()
 
 	s.mu.Lock()
@@ -584,8 +587,17 @@ func (s *server) writeToClients(b []byte) {
 		}
 	}
 
-	for c := range s.clients {
-		ipn.WriteMsg(c, b)
+	if len(s.clients) == 0 {
+		// Common case (at least on busy servers): nobody
+		// connected (no GUI, etc), so return before
+		// serializing JSON.
+		return
+	}
+
+	if b, ok := marshalNotify(n, s.logf); ok {
+		for c := range s.clients {
+			ipn.WriteMsg(c, b)
+		}
 	}
 }
 
@@ -671,8 +683,7 @@ func Run(ctx context.Context, logf logger.Logf, logid string, getEngine func() (
 			errMsg := err.Error()
 			go func() {
 				defer c.Close()
-				serverToClient := func(b []byte) { ipn.WriteMsg(c, b) }
-				bs := ipn.NewBackendServer(logf, nil, serverToClient)
+				bs := ipn.NewBackendServer(logf, nil, jsonNotifier(c, logf))
 				bs.SendErrorMessage(errMsg)
 				time.Sleep(time.Second)
 			}()
@@ -962,3 +973,25 @@ func peerPid(entries []netstat.Entry, la, ra netaddr.IPPort) int {
 	}
 	return 0
 }
+
+// jsonNotifier returns a notify-writer func that writes ipn.Notify
+// messages to w.
+func jsonNotifier(w io.Writer, logf logger.Logf) func(ipn.Notify) {
+	return func(n ipn.Notify) {
+		if b, ok := marshalNotify(n, logf); ok {
+			ipn.WriteMsg(w, b)
+		}
+	}
+}
+
+func marshalNotify(n ipn.Notify, logf logger.Logf) (b []byte, ok bool) {
+	b, err := json.Marshal(n)
+	if err != nil {
+		logf("ipnserver: [unexpected] error serializing JSON: %v", err)
+		return nil, false
+	}
+	if bytes.Contains(b, jsonEscapedZero) {
+		logf("[unexpected] zero byte in BackendServer.send notify message: %q", b)
+	}
+	return b, true
+}

ipn/ipnstate/ipnstate.go

@@ -65,6 +65,7 @@ type PeerStatusLite struct {
 }
 
 type PeerStatus struct {
+	ID        tailcfg.StableNodeID
 	PublicKey key.Public
 	HostName  string // HostInfo's Hostname (not a DNS name or necessarily unique)
 	DNSName   string
@@ -203,6 +204,9 @@ func (sb *StatusBuilder) AddPeer(peer key.Public, st *PeerStatus) {
 		return
 	}
 
+	if v := st.ID; v != "" {
+		e.ID = v
+	}
 	if v := st.HostName; v != "" {
 		e.HostName = v
 	}

ipn/message.go

@@ -88,9 +88,9 @@ type Command struct {
 
 type BackendServer struct {
 	logf          logger.Logf
-	b             Backend              // the Backend we are serving up
-	sendNotifyMsg func(jsonMsg []byte) // send a notification message
-	GotQuit       bool                 // a Quit command was received
+	b             Backend      // the Backend we are serving up
+	sendNotifyMsg func(Notify) // send a notification message
+	GotQuit       bool         // a Quit command was received
 }
 
 // NewBackendServer creates a new BackendServer using b.
@@ -98,13 +98,15 @@ type BackendServer struct {
 // If sendNotifyMsg is non-nil, it additionally sets the Backend's
 // notification callback to call the func with ipn.Notify messages in
 // JSON form. If nil, it does not change the notification callback.
-func NewBackendServer(logf logger.Logf, b Backend, sendNotifyMsg func(b []byte)) *BackendServer {
+func NewBackendServer(logf logger.Logf, b Backend, sendNotifyMsg func(Notify)) *BackendServer {
 	bs := &BackendServer{
 		logf:          logf,
 		b:             b,
 		sendNotifyMsg: sendNotifyMsg,
 	}
-	if sendNotifyMsg != nil {
+	// b may be nil if the BackendServer is being created just to
+	// encapsulate and send an error message.
+	if sendNotifyMsg != nil && b != nil {
 		b.SetNotifyCallback(bs.send)
 	}
 	return bs
@@ -115,14 +117,7 @@ func (bs *BackendServer) send(n Notify) {
 		return
 	}
 	n.Version = version.Long
-	b, err := json.Marshal(n)
-	if err != nil {
-		log.Fatalf("Failed json.Marshal(notify): %v\n%#v", err, n)
-	}
-	if bytes.Contains(b, jsonEscapedZero) {
-		log.Printf("[unexpected] zero byte in BackendServer.send notify message: %q", b)
-	}
-	bs.sendNotifyMsg(b)
+	bs.sendNotifyMsg(n)
 }
 
 func (bs *BackendServer) SendErrorMessage(msg string) {

ipn/message_test.go

@@ -7,6 +7,7 @@ package ipn
 import (
 	"bytes"
 	"context"
+	"encoding/json"
 	"testing"
 	"time"
 
@@ -74,7 +75,11 @@ func TestClientServer(t *testing.T) {
 			bc.GotNotifyMsg(b)
 		}
 	}()
-	serverToClient := func(b []byte) {
+	serverToClient := func(n Notify) {
+		b, err := json.Marshal(n)
+		if err != nil {
+			panic(err.Error())
+		}
 		serverToClientCh <- append([]byte{}, b...)
 	}
 	clientToServer := func(b []byte) {
@@ -182,3 +187,17 @@ func TestClientServer(t *testing.T) {
 	})
 	flushUntil(Running)
 }
+
+func TestNilBackend(t *testing.T) {
+	var called *Notify
+	bs := NewBackendServer(t.Logf, nil, func(n Notify) {
+		called = &n
+	})
+	bs.SendErrorMessage("Danger, Will Robinson!")
+	if called == nil {
+		t.Errorf("expect callback to be called, wasn't")
+	}
+	if called.ErrMessage == nil || *called.ErrMessage != "Danger, Will Robinson!" {
+		t.Errorf("callback got wrong error: %v", called.ErrMessage)
+	}
+}

logtail/logtail.go

@@ -15,6 +15,7 @@ import (
 	"net/http"
 	"os"
 	"strconv"
+	"sync/atomic"
 	"time"
 
 	"tailscale.com/logtail/backoff"
@@ -72,7 +73,7 @@ func NewLogger(cfg Config, logf tslogger.Logf) *Logger {
 	}
 	l := &Logger{
 		stderr:         cfg.Stderr,
-		stderrLevel:    cfg.StderrLevel,
+		stderrLevel:    int64(cfg.StderrLevel),
 		httpc:          cfg.HTTPC,
 		url:            cfg.BaseURL + "/c/" + cfg.Collection + "/" + cfg.PrivateID.String(),
 		lowMem:         cfg.LowMemory,
@@ -103,7 +104,7 @@ func NewLogger(cfg Config, logf tslogger.Logf) *Logger {
 // logging facilities and uploading to a log server.
 type Logger struct {
 	stderr         io.Writer
-	stderrLevel    int
+	stderrLevel    int64 // accessed atomically
 	httpc          *http.Client
 	url            string
 	lowMem         bool
@@ -125,10 +126,8 @@ type Logger struct {
 // SetVerbosityLevel controls the verbosity level that should be
 // written to stderr. 0 is the default (not verbose). Levels 1 or higher
 // are increasingly verbose.
-//
-// It should not be changed concurrently with log writes.
 func (l *Logger) SetVerbosityLevel(level int) {
-	l.stderrLevel = level
+	atomic.StoreInt64(&l.stderrLevel, int64(level))
 }
 
 // SetLinkMonitor sets the optional the link monitor.
@@ -514,7 +513,7 @@ func (l *Logger) Write(buf []byte) (int, error) {
 		return 0, nil
 	}
 	level, buf := parseAndRemoveLogLevel(buf)
-	if l.stderr != nil && l.stderr != ioutil.Discard && level <= l.stderrLevel {
+	if l.stderr != nil && l.stderr != ioutil.Discard && int64(level) <= atomic.LoadInt64(&l.stderrLevel) {
 		if buf[len(buf)-1] == '\n' {
 			l.stderr.Write(buf)
 		} else {

net/dns/config.go

@@ -22,27 +22,26 @@ type Config struct {
 	// for queries that fall within that suffix.
 	// If a query doesn't match any entry in Routes, the
 	// DefaultResolvers are used.
+	// A Routes entry with no resolvers means the route should be
+	// authoritatively answered using the contents of Hosts.
 	Routes map[dnsname.FQDN][]netaddr.IPPort
 	// SearchDomains are DNS suffixes to try when expanding
 	// single-label queries.
 	SearchDomains []dnsname.FQDN
 	// Hosts maps DNS FQDNs to their IPs, which can be a mix of IPv4
 	// and IPv6.
-	// Queries matching entries in Hosts are resolved locally without
-	// recursing off-machine.
+	// Queries matching entries in Hosts are resolved locally by
+	// 100.100.100.100 without leaving the machine.
+	// Adding an entry to Hosts merely creates the record. If you want
+	// it to resolve, you also need to add appropriate routes to
+	// Routes.
 	Hosts map[dnsname.FQDN][]netaddr.IP
-	// AuthoritativeSuffixes is a list of fully-qualified DNS suffixes
-	// for which the in-process Tailscale resolver is authoritative.
-	// Queries for names within AuthoritativeSuffixes can only be
-	// fulfilled by entries in Hosts. Queries with no match in Hosts
-	// return NXDOMAIN.
-	AuthoritativeSuffixes []dnsname.FQDN
 }
 
 // needsAnyResolvers reports whether c requires a resolver to be set
 // at the OS level.
 func (c Config) needsOSResolver() bool {
-	return c.hasDefaultResolvers() || c.hasRoutes() || c.hasHosts()
+	return c.hasDefaultResolvers() || c.hasRoutes()
 }
 
 func (c Config) hasRoutes() bool {
@@ -52,7 +51,7 @@ func (c Config) hasRoutes() bool {
 // hasDefaultResolversOnly reports whether the only resolvers in c are
 // DefaultResolvers.
 func (c Config) hasDefaultResolversOnly() bool {
-	return c.hasDefaultResolvers() && !c.hasRoutes() && !c.hasHosts()
+	return c.hasDefaultResolvers() && !c.hasRoutes()
 }
 
 func (c Config) hasDefaultResolvers() bool {
@@ -63,44 +62,28 @@ func (c Config) hasDefaultResolvers() bool {
 // routes use the same resolvers, or nil if multiple sets of resolvers
 // are specified.
 func (c Config) singleResolverSet() []netaddr.IPPort {
-	var first []netaddr.IPPort
+	var (
+		prev            []netaddr.IPPort
+		prevInitialized bool
+	)
 	for _, resolvers := range c.Routes {
-		if first == nil {
-			first = resolvers
+		if !prevInitialized {
+			prev = resolvers
+			prevInitialized = true
 			continue
 		}
-		if !sameIPPorts(first, resolvers) {
+		if !sameIPPorts(prev, resolvers) {
 			return nil
 		}
 	}
-	return first
+	return prev
 }
 
-// hasHosts reports whether c requires resolution of MagicDNS hosts or
-// domains.
-func (c Config) hasHosts() bool {
-	return len(c.Hosts) > 0 || len(c.AuthoritativeSuffixes) > 0
-}
-
-// matchDomains returns the list of match suffixes needed by Routes,
-// AuthoritativeSuffixes. Hosts is not considered as we assume that
-// they're covered by AuthoritativeSuffixes for now.
+// matchDomains returns the list of match suffixes needed by Routes.
 func (c Config) matchDomains() []dnsname.FQDN {
-	ret := make([]dnsname.FQDN, 0, len(c.Routes)+len(c.AuthoritativeSuffixes))
-	seen := map[dnsname.FQDN]bool{}
-	for _, suffix := range c.AuthoritativeSuffixes {
-		if seen[suffix] {
-			continue
-		}
-		ret = append(ret, suffix)
-		seen[suffix] = true
-	}
+	ret := make([]dnsname.FQDN, 0, len(c.Routes))
 	for suffix := range c.Routes {
-		if seen[suffix] {
-			continue
-		}
 		ret = append(ret, suffix)
-		seen[suffix] = true
 	}
 	sort.Slice(ret, func(i, j int) bool {
 		return ret[i].WithTrailingDot() < ret[j].WithTrailingDot()

net/dns/manager.go

@@ -6,7 +6,6 @@ package dns
 
 import (
 	"runtime"
-	"strings"
 	"time"
 
 	"inet.af/netaddr"
@@ -75,40 +74,40 @@ func (m *Manager) Set(cfg Config) error {
 
 // compileConfig converts cfg into a quad-100 resolver configuration
 // and an OS-level configuration.
-func (m *Manager) compileConfig(cfg Config) (resolver.Config, OSConfig, error) {
+func (m *Manager) compileConfig(cfg Config) (rcfg resolver.Config, ocfg OSConfig, err error) {
+	// The internal resolver always gets MagicDNS hosts and
+	// authoritative suffixes, even if we don't propagate MagicDNS to
+	// the OS.
+	rcfg.Hosts = cfg.Hosts
+	routes := map[dnsname.FQDN][]netaddr.IPPort{} // assigned conditionally to rcfg.Routes below.
+	for suffix, resolvers := range cfg.Routes {
+		if len(resolvers) == 0 {
+			rcfg.LocalDomains = append(rcfg.LocalDomains, suffix)
+		} else {
+			routes[suffix] = resolvers
+		}
+	}
+	// Similarly, the OS always gets search paths.
+	ocfg.SearchDomains = cfg.SearchDomains
+
 	// Deal with trivial configs first.
 	switch {
 	case !cfg.needsOSResolver():
 		// Set search domains, but nothing else. This also covers the
 		// case where cfg is entirely zero, in which case these
 		// configs clear all Tailscale DNS settings.
-		return resolver.Config{}, OSConfig{
-			SearchDomains: cfg.SearchDomains,
-		}, nil
+		return rcfg, ocfg, nil
 	case cfg.hasDefaultResolversOnly():
 		// Trivial CorpDNS configuration, just override the OS
 		// resolver.
-		return resolver.Config{}, OSConfig{
-			Nameservers:   toIPsOnly(cfg.DefaultResolvers),
-			SearchDomains: cfg.SearchDomains,
-		}, nil
+		ocfg.Nameservers = toIPsOnly(cfg.DefaultResolvers)
+		return rcfg, ocfg, nil
 	case cfg.hasDefaultResolvers():
 		// Default resolvers plus other stuff always ends up proxying
 		// through quad-100.
-		rcfg := resolver.Config{
-			Routes: map[dnsname.FQDN][]netaddr.IPPort{
-				".": cfg.DefaultResolvers,
-			},
-			Hosts:        cfg.Hosts,
-			LocalDomains: cfg.AuthoritativeSuffixes,
-		}
-		for suffix, resolvers := range cfg.Routes {
-			rcfg.Routes[suffix] = resolvers
-		}
-		ocfg := OSConfig{
-			Nameservers:   []netaddr.IP{tsaddr.TailscaleServiceIP()},
-			SearchDomains: cfg.SearchDomains,
-		}
+		rcfg.Routes = routes
+		rcfg.Routes["."] = cfg.DefaultResolvers
+		ocfg.Nameservers = []netaddr.IP{tsaddr.TailscaleServiceIP()}
 		return rcfg, ocfg, nil
 	}
 
@@ -116,8 +115,6 @@ func (m *Manager) compileConfig(cfg Config) (resolver.Config, OSConfig, error) {
 	// configurations. The possible cases don't return directly any
 	// more, because as a final step we have to handle the case where
 	// the OS can't do split DNS.
-	var rcfg resolver.Config
-	var ocfg OSConfig
 
 	// Workaround for
 	// https://github.com/tailscale/corp/issues/1662. Even though
@@ -135,35 +132,19 @@ func (m *Manager) compileConfig(cfg Config) (resolver.Config, OSConfig, error) {
 	// This bool is used in a couple of places below to implement this
 	// workaround.
 	isWindows := runtime.GOOS == "windows"
-
-	// The windows check is for
-	// . See also below
-	// for further routing workarounds there.
-	if !cfg.hasHosts() && cfg.singleResolverSet() != nil && m.os.SupportsSplitDNS() && !isWindows {
+	if cfg.singleResolverSet() != nil && m.os.SupportsSplitDNS() && !isWindows {
 		// Split DNS configuration requested, where all split domains
 		// go to the same resolvers. We can let the OS do it.
-		return resolver.Config{}, OSConfig{
-			Nameservers:   toIPsOnly(cfg.singleResolverSet()),
-			SearchDomains: cfg.SearchDomains,
-			MatchDomains:  cfg.matchDomains(),
-		}, nil
+		ocfg.Nameservers = toIPsOnly(cfg.singleResolverSet())
+		ocfg.MatchDomains = cfg.matchDomains()
+		return rcfg, ocfg, nil
 	}
 
 	// Split DNS configuration with either multiple upstream routes,
 	// or routes + MagicDNS, or just MagicDNS, or on an OS that cannot
 	// split-DNS. Install a split config pointing at quad-100.
-	rcfg = resolver.Config{
-		Hosts:        cfg.Hosts,
-		LocalDomains: cfg.AuthoritativeSuffixes,
-		Routes:       map[dnsname.FQDN][]netaddr.IPPort{},
-	}
-	for suffix, resolvers := range cfg.Routes {
-		rcfg.Routes[suffix] = resolvers
-	}
-	ocfg = OSConfig{
-		Nameservers:   []netaddr.IP{tsaddr.TailscaleServiceIP()},
-		SearchDomains: cfg.SearchDomains,
-	}
+	rcfg.Routes = routes
+	ocfg.Nameservers = []netaddr.IP{tsaddr.TailscaleServiceIP()}
 
 	// If the OS can't do native split-dns, read out the underlying
 	// resolver config and blend it into our config.
@@ -173,28 +154,7 @@ func (m *Manager) compileConfig(cfg Config) (resolver.Config, OSConfig, error) {
 	if !m.os.SupportsSplitDNS() || isWindows {
 		bcfg, err := m.os.GetBaseConfig()
 		if err != nil {
-			// Temporary hack to make OSes where split-DNS isn't fully
-			// implemented yet not completely crap out, but instead
-			// fall back to quad-9 as a hardcoded "backup resolver".
-			//
-			// This codepath currently only triggers when opted into
-			// the split-DNS feature server side, and when at least
-			// one search domain is something within tailscale.com, so
-			// we don't accidentally leak unstable user DNS queries to
-			// quad-9 if we accidentally go down this codepath.
-			canUseHack := false
-			for _, dom := range cfg.SearchDomains {
-				if strings.HasSuffix(dom.WithoutTrailingDot(), ".tailscale.com") {
-					canUseHack = true
-					break
-				}
-			}
-			if !canUseHack {
-				return resolver.Config{}, OSConfig{}, err
-			}
-			bcfg = OSConfig{
-				Nameservers: []netaddr.IP{netaddr.IPv4(9, 9, 9, 9)},
-			}
+			return resolver.Config{}, OSConfig{}, err
 		}
 		rcfg.Routes["."] = toIPPorts(bcfg.Nameservers)
 		ocfg.SearchDomains = append(ocfg.SearchDomains, bcfg.SearchDomains...)
@@ -211,7 +171,7 @@ func (m *Manager) compileConfig(cfg Config) (resolver.Config, OSConfig, error) {
 func toIPsOnly(ipps []netaddr.IPPort) (ret []netaddr.IP) {
 	ret = make([]netaddr.IP, 0, len(ipps))
 	for _, ipp := range ipps {
-		ret = append(ret, ipp.IP)
+		ret = append(ret, ipp.IP())
 	}
 	return ret
 }
@@ -219,7 +179,7 @@ func toIPsOnly(ipps []netaddr.IPPort) (ret []netaddr.IP) {
 func toIPPorts(ips []netaddr.IP) (ret []netaddr.IPPort) {
 	ret = make([]netaddr.IPPort, 0, len(ips))
 	for _, ip := range ips {
-		ret = append(ret, netaddr.IPPort{IP: ip, Port: 53})
+		ret = append(ret, netaddr.IPPortFrom(ip, 53))
 	}
 	return ret
 }

net/dns/manager_linux.go

@@ -57,12 +57,12 @@ func NewOSConfigurator(logf logger.Logf, interfaceName string) (ret OSConfigurat
 		}
 		if err := dbusPing("org.freedesktop.NetworkManager", "/org/freedesktop/NetworkManager/DnsManager"); err != nil {
 			dbg("nm", "no")
-			return newResolvedManager(logf)
+			return newResolvedManager(logf, interfaceName)
 		}
 		dbg("nm", "yes")
 		if err := nmIsUsingResolved(); err != nil {
 			dbg("nm-resolved", "no")
-			return newResolvedManager(logf)
+			return newResolvedManager(logf, interfaceName)
 		}
 		dbg("nm-resolved", "yes")
 
@@ -90,7 +90,7 @@ func NewOSConfigurator(logf logger.Logf, interfaceName string) (ret OSConfigurat
 			return newNMManager(interfaceName)
 		}
 		dbg("nm-old", "no")
-		return newResolvedManager(logf)
+		return newResolvedManager(logf, interfaceName)
 	case "resolvconf":
 		dbg("rc", "resolvconf")
 		if err := resolvconfSourceIsNM(bs); err == nil {

net/dns/manager_test.go

@@ -76,6 +76,20 @@ func TestManager(t *testing.T) {
 				SearchDomains: fqdns("tailscale.com", "universe.tf"),
 			},
 		},
+		{
+			// Regression test for https://github.com/tailscale/tailscale/issues/1886
+			name: "hosts-only",
+			in: Config{
+				Hosts: hosts(
+					"dave.ts.com.", "1.2.3.4",
+					"bradfitz.ts.com.", "2.3.4.5"),
+			},
+			rs: resolver.Config{
+				Hosts: hosts(
+					"dave.ts.com.", "1.2.3.4",
+					"bradfitz.ts.com.", "2.3.4.5"),
+			},
+		},
 		{
 			name: "corp",
 			in: Config{
@@ -104,10 +118,10 @@ func TestManager(t *testing.T) {
 			in: Config{
 				DefaultResolvers: mustIPPs("1.1.1.1:53", "9.9.9.9:53"),
 				SearchDomains:    fqdns("tailscale.com", "universe.tf"),
+				Routes:           upstreams("ts.com", ""),
 				Hosts: hosts(
 					"dave.ts.com.", "1.2.3.4",
 					"bradfitz.ts.com.", "2.3.4.5"),
-				AuthoritativeSuffixes: fqdns("ts.com"),
 			},
 			os: OSConfig{
 				Nameservers:   mustIPs("100.100.100.100"),
@@ -126,10 +140,10 @@ func TestManager(t *testing.T) {
 			in: Config{
 				DefaultResolvers: mustIPPs("1.1.1.1:53", "9.9.9.9:53"),
 				SearchDomains:    fqdns("tailscale.com", "universe.tf"),
+				Routes:           upstreams("ts.com", ""),
 				Hosts: hosts(
 					"dave.ts.com.", "1.2.3.4",
 					"bradfitz.ts.com.", "2.3.4.5"),
-				AuthoritativeSuffixes: fqdns("ts.com"),
 			},
 			split: true,
 			os: OSConfig{
@@ -261,8 +275,8 @@ func TestManager(t *testing.T) {
 				Hosts: hosts(
 					"dave.ts.com.", "1.2.3.4",
 					"bradfitz.ts.com.", "2.3.4.5"),
-				AuthoritativeSuffixes: fqdns("ts.com"),
-				SearchDomains:         fqdns("tailscale.com", "universe.tf"),
+				Routes:        upstreams("ts.com", ""),
+				SearchDomains: fqdns("tailscale.com", "universe.tf"),
 			},
 			bs: OSConfig{
 				Nameservers:   mustIPs("8.8.8.8"),
@@ -286,8 +300,8 @@ func TestManager(t *testing.T) {
 				Hosts: hosts(
 					"dave.ts.com.", "1.2.3.4",
 					"bradfitz.ts.com.", "2.3.4.5"),
-				AuthoritativeSuffixes: fqdns("ts.com"),
-				SearchDomains:         fqdns("tailscale.com", "universe.tf"),
+				Routes:        upstreams("ts.com", ""),
+				SearchDomains: fqdns("tailscale.com", "universe.tf"),
 			},
 			split: true,
 			os: OSConfig{
@@ -305,12 +319,11 @@ func TestManager(t *testing.T) {
 		{
 			name: "routes-magic",
 			in: Config{
-				Routes: upstreams("corp.com", "2.2.2.2:53"),
+				Routes: upstreams("corp.com", "2.2.2.2:53", "ts.com", ""),
 				Hosts: hosts(
 					"dave.ts.com.", "1.2.3.4",
 					"bradfitz.ts.com.", "2.3.4.5"),
-				AuthoritativeSuffixes: fqdns("ts.com"),
-				SearchDomains:         fqdns("tailscale.com", "universe.tf"),
+				SearchDomains: fqdns("tailscale.com", "universe.tf"),
 			},
 			bs: OSConfig{
 				Nameservers:   mustIPs("8.8.8.8"),
@@ -333,12 +346,13 @@ func TestManager(t *testing.T) {
 		{
 			name: "routes-magic-split",
 			in: Config{
-				Routes: upstreams("corp.com", "2.2.2.2:53"),
+				Routes: upstreams(
+					"corp.com", "2.2.2.2:53",
+					"ts.com", ""),
 				Hosts: hosts(
 					"dave.ts.com.", "1.2.3.4",
 					"bradfitz.ts.com.", "2.3.4.5"),
-				AuthoritativeSuffixes: fqdns("ts.com"),
-				SearchDomains:         fqdns("tailscale.com", "universe.tf"),
+				SearchDomains: fqdns("tailscale.com", "universe.tf"),
 			},
 			split: true,
 			os: OSConfig{
@@ -368,11 +382,12 @@ func TestManager(t *testing.T) {
 			if err := m.Set(test.in); err != nil {
 				t.Fatalf("m.Set: %v", err)
 			}
-			tr := cmp.Transformer("ipStr", func(ip netaddr.IP) string { return ip.String() })
-			if diff := cmp.Diff(f.OSConfig, test.os, tr, cmpopts.EquateEmpty()); diff != "" {
+			trIP := cmp.Transformer("ipStr", func(ip netaddr.IP) string { return ip.String() })
+			trIPPort := cmp.Transformer("ippStr", func(ipp netaddr.IPPort) string { return ipp.String() })
+			if diff := cmp.Diff(f.OSConfig, test.os, trIP, trIPPort, cmpopts.EquateEmpty()); diff != "" {
 				t.Errorf("wrong OSConfig (-got+want)\n%s", diff)
 			}
-			if diff := cmp.Diff(f.ResolverConfig, test.rs, tr, cmpopts.EquateEmpty()); diff != "" {
+			if diff := cmp.Diff(f.ResolverConfig, test.rs, trIP, trIPPort, cmpopts.EquateEmpty()); diff != "" {
 				t.Errorf("wrong resolver.Config (-got+want)\n%s", diff)
 			}
 		})
@@ -428,7 +443,12 @@ func upstreams(strs ...string) (ret map[dnsname.FQDN][]netaddr.IPPort) {
 	var key dnsname.FQDN
 	ret = map[dnsname.FQDN][]netaddr.IPPort{}
 	for _, s := range strs {
-		if ipp, err := netaddr.ParseIPPort(s); err == nil {
+		if s == "" {
+			if key == "" {
+				panic("IPPort provided before suffix")
+			}
+			ret[key] = nil
+		} else if ipp, err := netaddr.ParseIPPort(s); err == nil {
 			if key == "" {
 				panic("IPPort provided before suffix")
 			}

net/dns/nm.go

@@ -175,9 +175,12 @@ func (m *nmManager) trySet(ctx context.Context, config OSConfig) error {
 		search = append(search, "~.")
 	}
 
-	general := settings["connection"]
-	general["llmnr"] = dbus.MakeVariant(0)
-	general["mdns"] = dbus.MakeVariant(0)
+	// Ideally we would like to disable LLMNR and mdns on the
+	// interface here, but older NetworkManagers don't understand
+	// those settings and choke on them, so we don't. Both LLMNR and
+	// mdns will fail since tailscale0 doesn't do multicast, so it's
+	// effectively fine. We used to try and enforce LLMNR and mdns
+	// settings here, but that led to #1870.
 
 	ipv4Map := settings["ipv4"]
 	ipv4Map["dns"] = dbus.MakeVariant(dnsv4)
@@ -247,7 +250,7 @@ func (m *nmManager) trySet(ctx context.Context, config OSConfig) error {
 	}
 
 	if call := device.CallWithContext(ctx, "org.freedesktop.NetworkManager.Device.Reapply", 0, settings, version, uint32(0)); call.Err != nil {
-		return fmt.Errorf("reapply: %w", err)
+		return fmt.Errorf("reapply: %w", call.Err)
 	}
 
 	return nil

net/dns/resolved.go

@@ -12,11 +12,11 @@ import (
 	"context"
 	"errors"
 	"fmt"
+	"net"
 
 	"github.com/godbus/dbus/v5"
 	"golang.org/x/sys/unix"
 	"inet.af/netaddr"
-	"tailscale.com/net/interfaces"
 	"tailscale.com/types/logger"
 	"tailscale.com/util/dnsname"
 )
@@ -85,17 +85,24 @@ func isResolvedActive() bool {
 // resolvedManager uses the systemd-resolved DBus API.
 type resolvedManager struct {
 	logf     logger.Logf
+	ifidx    int
 	resolved dbus.BusObject
 }
 
-func newResolvedManager(logf logger.Logf) (*resolvedManager, error) {
+func newResolvedManager(logf logger.Logf, interfaceName string) (*resolvedManager, error) {
 	conn, err := dbus.SystemBus()
 	if err != nil {
 		return nil, err
 	}
 
+	iface, err := net.InterfaceByName(interfaceName)
+	if err != nil {
+		return nil, err
+	}
+
 	return &resolvedManager{
 		logf:     logf,
+		ifidx:    iface.Index,
 		resolved: conn.Object("org.freedesktop.resolve1", dbus.ObjectPath("/org/freedesktop/resolve1")),
 	}, nil
 }
@@ -105,16 +112,6 @@ func (m *resolvedManager) SetDNS(config OSConfig) error {
 	ctx, cancel := context.WithTimeout(context.Background(), reconfigTimeout)
 	defer cancel()
 
-	// In principle, we could persist this in the manager struct
-	// if we knew that interface indices are persistent. This does not seem to be the case.
-	_, iface, err := interfaces.Tailscale()
-	if err != nil {
-		return fmt.Errorf("getting interface index: %w", err)
-	}
-	if iface == nil {
-		return errNotReady
-	}
-
 	var linkNameservers = make([]resolvedLinkNameserver, len(config.Nameservers))
 	for i, server := range config.Nameservers {
 		ip := server.As16()
@@ -131,9 +128,9 @@ func (m *resolvedManager) SetDNS(config OSConfig) error {
 		}
 	}
 
-	err = m.resolved.CallWithContext(
+	err := m.resolved.CallWithContext(
 		ctx, "org.freedesktop.resolve1.Manager.SetLinkDNS", 0,
-		iface.Index, linkNameservers,
+		m.ifidx, linkNameservers,
 	).Store()
 	if err != nil {
 		return fmt.Errorf("setLinkDNS: %w", err)
@@ -174,13 +171,13 @@ func (m *resolvedManager) SetDNS(config OSConfig) error {
 
 	err = m.resolved.CallWithContext(
 		ctx, "org.freedesktop.resolve1.Manager.SetLinkDomains", 0,
-		iface.Index, linkDomains,
+		m.ifidx, linkDomains,
 	).Store()
 	if err != nil {
 		return fmt.Errorf("setLinkDomains: %w", err)
 	}
 
-	if call := m.resolved.CallWithContext(ctx, "org.freedesktop.resolve1.Manager.SetLinkDefaultRoute", 0, iface.Index, len(config.MatchDomains) == 0); call.Err != nil {
+	if call := m.resolved.CallWithContext(ctx, "org.freedesktop.resolve1.Manager.SetLinkDefaultRoute", 0, m.ifidx, len(config.MatchDomains) == 0); call.Err != nil {
 		return fmt.Errorf("setLinkDefaultRoute: %w", err)
 	}
 
@@ -189,22 +186,22 @@ func (m *resolvedManager) SetDNS(config OSConfig) error {
 	// or something).
 
 	// Disable LLMNR, we don't do multicast.
-	if call := m.resolved.CallWithContext(ctx, "org.freedesktop.resolve1.Manager.SetLinkLLMNR", 0, iface.Index, "no"); call.Err != nil {
+	if call := m.resolved.CallWithContext(ctx, "org.freedesktop.resolve1.Manager.SetLinkLLMNR", 0, m.ifidx, "no"); call.Err != nil {
 		m.logf("[v1] failed to disable LLMNR: %v", call.Err)
 	}
 
 	// Disable mdns.
-	if call := m.resolved.CallWithContext(ctx, "org.freedesktop.resolve1.Manager.SetLinkMulticastDNS", 0, iface.Index, "no"); call.Err != nil {
+	if call := m.resolved.CallWithContext(ctx, "org.freedesktop.resolve1.Manager.SetLinkMulticastDNS", 0, m.ifidx, "no"); call.Err != nil {
 		m.logf("[v1] failed to disable mdns: %v", call.Err)
 	}
 
 	// We don't support dnssec consistently right now, force it off to
 	// avoid partial failures when we split DNS internally.
-	if call := m.resolved.CallWithContext(ctx, "org.freedesktop.resolve1.Manager.SetLinkDNSSEC", 0, iface.Index, "no"); call.Err != nil {
+	if call := m.resolved.CallWithContext(ctx, "org.freedesktop.resolve1.Manager.SetLinkDNSSEC", 0, m.ifidx, "no"); call.Err != nil {
 		m.logf("[v1] failed to disable DNSSEC: %v", call.Err)
 	}
 
-	if call := m.resolved.CallWithContext(ctx, "org.freedesktop.resolve1.Manager.SetLinkDNSOverTLS", 0, iface.Index, "no"); call.Err != nil {
+	if call := m.resolved.CallWithContext(ctx, "org.freedesktop.resolve1.Manager.SetLinkDNSOverTLS", 0, m.ifidx, "no"); call.Err != nil {
 		m.logf("[v1] failed to disable DoT: %v", call.Err)
 	}
 
@@ -227,15 +224,7 @@ func (m *resolvedManager) Close() error {
 	ctx, cancel := context.WithTimeout(context.Background(), reconfigTimeout)
 	defer cancel()
 
-	_, iface, err := interfaces.Tailscale()
-	if err != nil {
-		return fmt.Errorf("getting interface index: %w", err)
-	}
-	if iface == nil {
-		return errNotReady
-	}
-
-	if call := m.resolved.CallWithContext(ctx, "org.freedesktop.resolve1.Manager.RevertLink", 0, iface.Index); call.Err != nil {
+	if call := m.resolved.CallWithContext(ctx, "org.freedesktop.resolve1.Manager.RevertLink", 0, m.ifidx); call.Err != nil {
 		return fmt.Errorf("RevertLink: %w", call.Err)
 	}
 

net/dns/resolver/tsdns_test.go

@@ -433,8 +433,8 @@ func TestDelegateCollision(t *testing.T) {
 		qtype dns.Type
 		addr  netaddr.IPPort
 	}{
-		{"test.site.", dns.TypeA, netaddr.IPPort{IP: netaddr.IPv4(1, 1, 1, 1), Port: 1001}},
-		{"test.site.", dns.TypeAAAA, netaddr.IPPort{IP: netaddr.IPv4(1, 1, 1, 1), Port: 1002}},
+		{"test.site.", dns.TypeA, netaddr.IPPortFrom(netaddr.IPv4(1, 1, 1, 1), 1001)},
+		{"test.site.", dns.TypeAAAA, netaddr.IPPortFrom(netaddr.IPv4(1, 1, 1, 1), 1002)},
 	}
 
 	// packets will have the same dns txid.

net/interfaces/interfaces.go

@@ -195,7 +195,7 @@ func ForeachInterface(fn func(Interface, []netaddr.IPPrefix)) error {
 			}
 		}
 		sort.Slice(pfxs, func(i, j int) bool {
-			return pfxs[i].IP.Less(pfxs[j].IP)
+			return pfxs[i].IP().Less(pfxs[j].IP())
 		})
 		fn(Interface{iface}, pfxs)
 	}
@@ -264,7 +264,7 @@ func (s *State) String() string {
 			fmt.Fprintf(&sb, "%s:[", ifName)
 			needSpace := false
 			for _, pfx := range s.InterfaceIPs[ifName] {
-				if !isInterestingIP(pfx.IP) {
+				if !isInterestingIP(pfx.IP()) {
 					continue
 				}
 				if needSpace {
@@ -367,7 +367,7 @@ func (s *State) AnyInterfaceUp() bool {
 
 func hasTailscaleIP(pfxs []netaddr.IPPrefix) bool {
 	for _, pfx := range pfxs {
-		if tsaddr.IsTailscaleIP(pfx.IP) {
+		if tsaddr.IsTailscaleIP(pfx.IP()) {
 			return true
 		}
 	}
@@ -407,11 +407,11 @@ func GetState() (*State, error) {
 			return
 		}
 		for _, pfx := range pfxs {
-			if pfx.IP.IsLoopback() || pfx.IP.IsLinkLocalUnicast() {
+			if pfx.IP().IsLoopback() || pfx.IP().IsLinkLocalUnicast() {
 				continue
 			}
-			s.HaveV6Global = s.HaveV6Global || isGlobalV6(pfx.IP)
-			s.HaveV4 = s.HaveV4 || pfx.IP.Is4()
+			s.HaveV6Global = s.HaveV6Global || isGlobalV6(pfx.IP())
+			s.HaveV4 = s.HaveV4 || pfx.IP().Is4()
 		}
 	}); err != nil {
 		return nil, err
@@ -447,7 +447,7 @@ func HTTPOfListener(ln net.Listener) string {
 	var goodIP string
 	var privateIP string
 	ForeachInterfaceAddress(func(i Interface, pfx netaddr.IPPrefix) {
-		ip := pfx.IP
+		ip := pfx.IP()
 		if isPrivateIP(ip) {
 			if privateIP == "" {
 				privateIP = ip.String()
@@ -484,7 +484,7 @@ func LikelyHomeRouterIP() (gateway, myIP netaddr.IP, ok bool) {
 		return
 	}
 	ForeachInterfaceAddress(func(i Interface, pfx netaddr.IPPrefix) {
-		ip := pfx.IP
+		ip := pfx.IP()
 		if !i.IsUp() || ip.IsZero() || !myIP.IsZero() {
 			return
 		}
@@ -528,7 +528,7 @@ var (
 // isInterestingIP.
 func anyInterestingIP(pfxs []netaddr.IPPrefix) bool {
 	for _, pfx := range pfxs {
-		if isInterestingIP(pfx.IP) {
+		if isInterestingIP(pfx.IP()) {
 			return true
 		}
 	}

net/interfaces/interfaces_darwin.go

@@ -10,7 +10,6 @@ import (
 	"log"
 	"net"
 	"syscall"
-	"time"
 
 	"golang.org/x/net/route"
 	"golang.org/x/sys/unix"
@@ -29,32 +28,9 @@ func DefaultRouteInterface() (string, error) {
 	return iface.Name, nil
 }
 
-// fetchRoutingTable is a retry loop around route.FetchRIB, fetching NET_RT_DUMP2.
-//
-// The retry loop is due to a bug in the BSDs (or Go?). See
-// https://github.com/tailscale/tailscale/issues/1345
+// fetchRoutingTable calls route.FetchRIB, fetching NET_RT_DUMP2.
 func fetchRoutingTable() (rib []byte, err error) {
-	fails := 0
-	for {
-		rib, err := route.FetchRIB(syscall.AF_UNSPEC, syscall.NET_RT_DUMP2, 0)
-		if err == nil {
-			return rib, nil
-		}
-		fails++
-		if fails < 10 {
-			// Empirically, 1 retry is enough. In a long
-			// stress test while toggling wifi on & off, I
-			// only saw a few occurrences of 2 and one 3.
-			// So 10 should be more plenty.
-			if fails > 5 {
-				time.Sleep(5 * time.Millisecond)
-			}
-			continue
-		}
-		if err != nil {
-			return nil, fmt.Errorf("route.FetchRIB: %w", err)
-		}
-	}
+	return route.FetchRIB(syscall.AF_UNSPEC, syscall.NET_RT_DUMP2, 0)
 }
 
 func DefaultRouteInterfaceIndex() (int, error) {

net/netcheck/netcheck.go

@@ -625,7 +625,7 @@ func (rs *reportState) stopTimers() {
 func (rs *reportState) addNodeLatency(node *tailcfg.DERPNode, ipp netaddr.IPPort, d time.Duration) {
 	var ipPortStr string
 	if ipp != (netaddr.IPPort{}) {
-		ipPortStr = net.JoinHostPort(ipp.IP.String(), fmt.Sprint(ipp.Port))
+		ipPortStr = net.JoinHostPort(ipp.IP().String(), fmt.Sprint(ipp.Port()))
 	}
 
 	rs.mu.Lock()
@@ -650,13 +650,13 @@ func (rs *reportState) addNodeLatency(node *tailcfg.DERPNode, ipp netaddr.IPPort
 	}
 
 	switch {
-	case ipp.IP.Is6():
+	case ipp.IP().Is6():
 		updateLatency(ret.RegionV6Latency, node.RegionID, d)
 		ret.IPv6 = true
 		ret.GlobalV6 = ipPortStr
 		// TODO: track MappingVariesByDestIP for IPv6
 		// too? Would be sad if so, but who knows.
-	case ipp.IP.Is4():
+	case ipp.IP().Is4():
 		updateLatency(ret.RegionV4Latency, node.RegionID, d)
 		ret.IPv4 = true
 		if rs.gotEP4 == "" {
@@ -1172,7 +1172,7 @@ func (c *Client) nodeAddr(ctx context.Context, n *tailcfg.DERPNode, proto probeP
 		if proto == probeIPv6 && ip.Is4() {
 			return nil
 		}
-		return netaddr.IPPort{IP: ip, Port: uint16(port)}.UDPAddr()
+		return netaddr.IPPortFrom(ip, uint16(port)).UDPAddr()
 	}
 
 	switch proto {
@@ -1182,7 +1182,7 @@ func (c *Client) nodeAddr(ctx context.Context, n *tailcfg.DERPNode, proto probeP
 			if !ip.Is4() {
 				return nil
 			}
-			return netaddr.IPPort{IP: ip, Port: uint16(port)}.UDPAddr()
+			return netaddr.IPPortFrom(ip, uint16(port)).UDPAddr()
 		}
 	case probeIPv6:
 		if n.IPv6 != "" {
@@ -1190,7 +1190,7 @@ func (c *Client) nodeAddr(ctx context.Context, n *tailcfg.DERPNode, proto probeP
 			if !ip.Is6() {
 				return nil
 			}
-			return netaddr.IPPort{IP: ip, Port: uint16(port)}.UDPAddr()
+			return netaddr.IPPortFrom(ip, uint16(port)).UDPAddr()
 		}
 	default:
 		return nil

net/netstat/netstat_windows.go

@@ -157,10 +157,9 @@ func ipport4(addr uint32, port uint16) netaddr.IPPort {
 	if !endian.Big {
 		addr = bits.ReverseBytes32(addr)
 	}
-	return netaddr.IPPort{
-		IP:   netaddr.IPv4(byte(addr>>24), byte(addr>>16), byte(addr>>8), byte(addr)),
-		Port: port,
-	}
+	return netaddr.IPPortFrom(
+		netaddr.IPv4(byte(addr>>24), byte(addr>>16), byte(addr>>8), byte(addr)),
+		port)
 }
 
 func ipport6(addr [16]byte, scope uint32, port uint16) netaddr.IPPort {
@@ -169,10 +168,7 @@ func ipport6(addr [16]byte, scope uint32, port uint16) netaddr.IPPort {
 		// TODO: something better here?
 		ip = ip.WithZone(fmt.Sprint(scope))
 	}
-	return netaddr.IPPort{
-		IP:   ip,
-		Port: port,
-	}
+	return netaddr.IPPortFrom(ip, port)
 }
 
 func port(v *uint32) uint16 {

net/packet/packet.go

@@ -12,7 +12,6 @@ import (
 
 	"inet.af/netaddr"
 	"tailscale.com/types/ipproto"
-	"tailscale.com/types/strbuilder"
 )
 
 const unknown = ipproto.Unknown
@@ -62,36 +61,17 @@ func (p *Parsed) String() string {
 		return "Unknown{???}"
 	}
 
-	sb := strbuilder.Get()
-	sb.WriteString(p.IPProto.String())
-	sb.WriteByte('{')
-	writeIPPort(sb, p.Src)
-	sb.WriteString(" > ")
-	writeIPPort(sb, p.Dst)
-	sb.WriteByte('}')
-	return sb.String()
-}
-
-// writeIPPort writes ipp.String() into sb, with fewer allocations.
-//
-// TODO: make netaddr more efficient in this area, and retire this func.
-func writeIPPort(sb *strbuilder.Builder, ipp netaddr.IPPort) {
-	if ipp.IP.Is4() {
-		raw := ipp.IP.As4()
-		sb.WriteUint(uint64(raw[0]))
-		sb.WriteByte('.')
-		sb.WriteUint(uint64(raw[1]))
-		sb.WriteByte('.')
-		sb.WriteUint(uint64(raw[2]))
-		sb.WriteByte('.')
-		sb.WriteUint(uint64(raw[3]))
-		sb.WriteByte(':')
-	} else {
-		sb.WriteByte('[')
-		sb.WriteString(ipp.IP.String()) // TODO: faster?
-		sb.WriteString("]:")
-	}
-	sb.WriteUint(uint64(ipp.Port))
+	// max is the maximum reasonable length of the string we are constructing.
+	// It's OK to overshoot, as the temp buffer is allocated on the stack.
+	const max = len("ICMPv6{[ffff:ffff:ffff:ffff:ffff:ffff:ffff:ffff%enp5s0]:65535 > [ffff:ffff:ffff:ffff:ffff:ffff:ffff:ffff%enp5s0]:65535}")
+	b := make([]byte, 0, max)
+	b = append(b, p.IPProto.String()...)
+	b = append(b, '{')
+	b = p.Src.AppendTo(b)
+	b = append(b, ' ', '>', ' ')
+	b = p.Dst.AppendTo(b)
+	b = append(b, '}')
+	return string(b)
 }
 
 // Decode extracts data from the packet in b into q.
@@ -142,8 +122,8 @@ func (q *Parsed) decode4(b []byte) {
 	}
 
 	// If it's valid IPv4, then the IP addresses are valid
-	q.Src.IP = netaddr.IPv4(b[12], b[13], b[14], b[15])
-	q.Dst.IP = netaddr.IPv4(b[16], b[17], b[18], b[19])
+	q.Src = q.Src.WithIP(netaddr.IPv4(b[12], b[13], b[14], b[15]))
+	q.Dst = q.Dst.WithIP(netaddr.IPv4(b[16], b[17], b[18], b[19]))
 
 	q.subofs = int((b[0] & 0x0F) << 2)
 	if q.subofs > q.length {
@@ -185,8 +165,8 @@ func (q *Parsed) decode4(b []byte) {
 				q.IPProto = unknown
 				return
 			}
-			q.Src.Port = 0
-			q.Dst.Port = 0
+			q.Src = q.Src.WithPort(0)
+			q.Dst = q.Dst.WithPort(0)
 			q.dataofs = q.subofs + icmp4HeaderLength
 			return
 		case ipproto.IGMP:
@@ -198,8 +178,8 @@ func (q *Parsed) decode4(b []byte) {
 				q.IPProto = unknown
 				return
 			}
-			q.Src.Port = binary.BigEndian.Uint16(sub[0:2])
-			q.Dst.Port = binary.BigEndian.Uint16(sub[2:4])
+			q.Src = q.Src.WithPort(binary.BigEndian.Uint16(sub[0:2]))
+			q.Dst = q.Dst.WithPort(binary.BigEndian.Uint16(sub[2:4]))
 			q.TCPFlags = TCPFlag(sub[13]) & 0x3F
 			headerLength := (sub[12] & 0xF0) >> 2
 			q.dataofs = q.subofs + int(headerLength)
@@ -209,8 +189,8 @@ func (q *Parsed) decode4(b []byte) {
 				q.IPProto = unknown
 				return
 			}
-			q.Src.Port = binary.BigEndian.Uint16(sub[0:2])
-			q.Dst.Port = binary.BigEndian.Uint16(sub[2:4])
+			q.Src = q.Src.WithPort(binary.BigEndian.Uint16(sub[0:2]))
+			q.Dst = q.Dst.WithPort(binary.BigEndian.Uint16(sub[2:4]))
 			q.dataofs = q.subofs + udpHeaderLength
 			return
 		case ipproto.SCTP:
@@ -218,8 +198,8 @@ func (q *Parsed) decode4(b []byte) {
 				q.IPProto = unknown
 				return
 			}
-			q.Src.Port = binary.BigEndian.Uint16(sub[0:2])
-			q.Dst.Port = binary.BigEndian.Uint16(sub[2:4])
+			q.Src = q.Src.WithPort(binary.BigEndian.Uint16(sub[0:2]))
+			q.Dst = q.Dst.WithPort(binary.BigEndian.Uint16(sub[2:4]))
 			return
 		case ipproto.TSMP:
 			// Inter-tailscale messages.
@@ -265,8 +245,10 @@ func (q *Parsed) decode6(b []byte) {
 
 	// okay to ignore `ok` here, because IPs pulled from packets are
 	// always well-formed stdlib IPs.
-	q.Src.IP, _ = netaddr.FromStdIP(net.IP(b[8:24]))
-	q.Dst.IP, _ = netaddr.FromStdIP(net.IP(b[24:40]))
+	srcIP, _ := netaddr.FromStdIP(net.IP(b[8:24]))
+	dstIP, _ := netaddr.FromStdIP(net.IP(b[24:40]))
+	q.Src = q.Src.WithIP(srcIP)
+	q.Dst = q.Dst.WithIP(dstIP)
 
 	// We don't support any IPv6 extension headers. Don't try to
 	// be clever. Therefore, the IP subprotocol always starts at
@@ -290,16 +272,16 @@ func (q *Parsed) decode6(b []byte) {
 			q.IPProto = unknown
 			return
 		}
-		q.Src.Port = 0
-		q.Dst.Port = 0
+		q.Src = q.Src.WithPort(0)
+		q.Dst = q.Dst.WithPort(0)
 		q.dataofs = q.subofs + icmp6HeaderLength
 	case ipproto.TCP:
 		if len(sub) < tcpHeaderLength {
 			q.IPProto = unknown
 			return
 		}
-		q.Src.Port = binary.BigEndian.Uint16(sub[0:2])
-		q.Dst.Port = binary.BigEndian.Uint16(sub[2:4])
+		q.Src = q.Src.WithPort(binary.BigEndian.Uint16(sub[0:2]))
+		q.Dst = q.Dst.WithPort(binary.BigEndian.Uint16(sub[2:4]))
 		q.TCPFlags = TCPFlag(sub[13]) & 0x3F
 		headerLength := (sub[12] & 0xF0) >> 2
 		q.dataofs = q.subofs + int(headerLength)
@@ -309,16 +291,16 @@ func (q *Parsed) decode6(b []byte) {
 			q.IPProto = unknown
 			return
 		}
-		q.Src.Port = binary.BigEndian.Uint16(sub[0:2])
-		q.Dst.Port = binary.BigEndian.Uint16(sub[2:4])
+		q.Src = q.Src.WithPort(binary.BigEndian.Uint16(sub[0:2]))
+		q.Dst = q.Dst.WithPort(binary.BigEndian.Uint16(sub[2:4]))
 		q.dataofs = q.subofs + udpHeaderLength
 	case ipproto.SCTP:
 		if len(sub) < sctpHeaderLength {
 			q.IPProto = unknown
 			return
 		}
-		q.Src.Port = binary.BigEndian.Uint16(sub[0:2])
-		q.Dst.Port = binary.BigEndian.Uint16(sub[2:4])
+		q.Src = q.Src.WithPort(binary.BigEndian.Uint16(sub[0:2]))
+		q.Dst = q.Dst.WithPort(binary.BigEndian.Uint16(sub[2:4]))
 		return
 	case ipproto.TSMP:
 		// Inter-tailscale messages.
@@ -338,8 +320,8 @@ func (q *Parsed) IP4Header() IP4Header {
 	return IP4Header{
 		IPID:    ipid,
 		IPProto: q.IPProto,
-		Src:     q.Src.IP,
-		Dst:     q.Dst.IP,
+		Src:     q.Src.IP(),
+		Dst:     q.Dst.IP(),
 	}
 }
 
@@ -351,8 +333,8 @@ func (q *Parsed) IP6Header() IP6Header {
 	return IP6Header{
 		IPID:    ipid,
 		IPProto: q.IPProto,
-		Src:     q.Src.IP,
-		Dst:     q.Dst.IP,
+		Src:     q.Src.IP(),
+		Dst:     q.Dst.IP(),
 	}
 }
 
@@ -373,8 +355,8 @@ func (q *Parsed) UDP4Header() UDP4Header {
 	}
 	return UDP4Header{
 		IP4Header: q.IP4Header(),
-		SrcPort:   q.Src.Port,
-		DstPort:   q.Dst.Port,
+		SrcPort:   q.Src.Port(),
+		DstPort:   q.Dst.Port(),
 	}
 }
 

net/packet/packet_test.go

@@ -378,11 +378,9 @@ func TestParsedString(t *testing.T) {
 		})
 	}
 
-	var sink string
 	allocs := testing.AllocsPerRun(1000, func() {
-		sink = tests[0].qdecode.String()
+		sinkString = tests[0].qdecode.String()
 	})
-	_ = sink
 	if allocs != 1 {
 		t.Errorf("allocs = %v; want 1", allocs)
 	}
@@ -532,3 +530,33 @@ func TestMarshalResponse(t *testing.T) {
 		})
 	}
 }
+
+var sinkString string
+
+func BenchmarkString(b *testing.B) {
+	benches := []struct {
+		name string
+		buf  []byte
+	}{
+		{"tcp4", tcp4PacketBuffer},
+		{"tcp6", tcp6RequestBuffer},
+		{"udp4", udp4RequestBuffer},
+		{"udp6", udp6RequestBuffer},
+		{"icmp4", icmp4RequestBuffer},
+		{"icmp6", icmp6PacketBuffer},
+		{"igmp", igmpPacketBuffer},
+		{"unknown", unknownPacketBuffer},
+	}
+
+	for _, bench := range benches {
+		b.Run(bench.name, func(b *testing.B) {
+			b.ReportAllocs()
+			var p Parsed
+			p.Decode(bench.buf)
+			b.ResetTimer()
+			for i := 0; i < b.N; i++ {
+				sinkString = p.String()
+			}
+		})
+	}
+}

net/packet/tsmp.go

@@ -14,6 +14,7 @@ import (
 	"encoding/binary"
 	"errors"
 	"fmt"
+	"log"
 
 	"inet.af/netaddr"
 	"tailscale.com/net/flowtrack"
@@ -143,7 +144,7 @@ func (h TailscaleRejectedHeader) Marshal(buf []byte) error {
 	if len(buf) > maxPacketLength {
 		return errLargePacket
 	}
-	if h.Src.IP.Is4() {
+	if h.Src.IP().Is4() {
 		iph := IP4Header{
 			IPProto: ipproto.TSMP,
 			Src:     h.IPSrc,
@@ -151,7 +152,7 @@ func (h TailscaleRejectedHeader) Marshal(buf []byte) error {
 		}
 		iph.Marshal(buf)
 		buf = buf[ip4HeaderLength:]
-	} else if h.Src.IP.Is6() {
+	} else if h.Src.IP().Is6() {
 		iph := IP6Header{
 			IPProto: ipproto.TSMP,
 			Src:     h.IPSrc,
@@ -165,8 +166,8 @@ func (h TailscaleRejectedHeader) Marshal(buf []byte) error {
 	buf[0] = byte(TSMPTypeRejectedConn)
 	buf[1] = byte(h.Proto)
 	buf[2] = byte(h.Reason)
-	binary.BigEndian.PutUint16(buf[3:5], h.Src.Port)
-	binary.BigEndian.PutUint16(buf[5:7], h.Dst.Port)
+	binary.BigEndian.PutUint16(buf[3:5], h.Src.Port())
+	binary.BigEndian.PutUint16(buf[5:7], h.Dst.Port())
 
 	if h.hasFlags() {
 		var flags byte
@@ -190,10 +191,10 @@ func (pp *Parsed) AsTailscaleRejectedHeader() (h TailscaleRejectedHeader, ok boo
 	h = TailscaleRejectedHeader{
 		Proto:  ipproto.Proto(p[1]),
 		Reason: TailscaleRejectReason(p[2]),
-		IPSrc:  pp.Src.IP,
-		IPDst:  pp.Dst.IP,
-		Src:    netaddr.IPPort{IP: pp.Dst.IP, Port: binary.BigEndian.Uint16(p[3:5])},
-		Dst:    netaddr.IPPort{IP: pp.Src.IP, Port: binary.BigEndian.Uint16(p[5:7])},
+		IPSrc:  pp.Src.IP(),
+		IPDst:  pp.Dst.IP(),
+		Src:    netaddr.IPPortFrom(pp.Dst.IP(), binary.BigEndian.Uint16(p[3:5])),
+		Dst:    netaddr.IPPortFrom(pp.Src.IP(), binary.BigEndian.Uint16(p[5:7])),
 	}
 	if len(p) > 7 {
 		flags := p[7]
@@ -232,6 +233,7 @@ type TSMPPongReply struct {
 // AsTSMPPong returns pp as a TSMPPongReply and whether it is one.
 // The pong.IPHeader field is not populated.
 func (pp *Parsed) AsTSMPPong() (pong TSMPPongReply, ok bool) {
+	log.Println("TSMPPONG")
 	if pp.IPProto != ipproto.TSMP {
 		return
 	}

net/portmapper/portmapper.go

@@ -84,7 +84,7 @@ type pmpMapping struct {
 
 // externalValid reports whether m.external is valid, with both its IP and Port populated.
 func (m *pmpMapping) externalValid() bool {
-	return !m.external.IP.IsZero() && m.external.Port != 0
+	return !m.external.IP().IsZero() && m.external.Port() != 0
 }
 
 // release does a best effort fire-and-forget release of the PMP mapping m.
@@ -94,8 +94,8 @@ func (m *pmpMapping) release() {
 		return
 	}
 	defer uc.Close()
-	pkt := buildPMPRequestMappingPacket(m.internal.Port, m.external.Port, pmpMapLifetimeDelete)
-	uc.WriteTo(pkt, netaddr.IPPort{IP: m.gw, Port: pmpPort}.UDPAddr())
+	pkt := buildPMPRequestMappingPacket(m.internal.Port(), m.external.Port(), pmpMapLifetimeDelete)
+	uc.WriteTo(pkt, netaddr.IPPortFrom(m.gw, pmpPort).UDPAddr())
 }
 
 // NewClient returns a new portmapping client.
@@ -256,7 +256,7 @@ func (c *Client) CreateOrGetMapping(ctx context.Context) (external netaddr.IPPor
 	localPort := c.localPort
 	m := &pmpMapping{
 		gw:       gw,
-		internal: netaddr.IPPort{IP: myIP, Port: localPort},
+		internal: netaddr.IPPortFrom(myIP, localPort),
 	}
 
 	// prevPort is the port we had most previously, if any. We try
@@ -271,7 +271,7 @@ func (c *Client) CreateOrGetMapping(ctx context.Context) (external netaddr.IPPor
 			return m.external, nil
 		}
 		// The mapping might still be valid, so just try to renew it.
-		prevPort = m.external.Port
+		prevPort = m.external.Port()
 	}
 
 	// If we just did a Probe (e.g. via netchecker) but didn't
@@ -279,7 +279,7 @@ func (c *Client) CreateOrGetMapping(ctx context.Context) (external netaddr.IPPor
 	// again. Cuts down latency for most clients.
 	haveRecentPMP := c.sawPMPRecentlyLocked()
 	if haveRecentPMP {
-		m.external.IP = c.pmpPubIP
+		m.external = m.external.WithIP(c.pmpPubIP)
 	}
 	if c.lastProbe.After(now.Add(-5*time.Second)) && !haveRecentPMP {
 		c.mu.Unlock()
@@ -297,11 +297,11 @@ func (c *Client) CreateOrGetMapping(ctx context.Context) (external netaddr.IPPor
 	uc.SetReadDeadline(time.Now().Add(portMapServiceTimeout))
 	defer closeCloserOnContextDone(ctx, uc)()
 
-	pmpAddr := netaddr.IPPort{IP: gw, Port: pmpPort}
+	pmpAddr := netaddr.IPPortFrom(gw, pmpPort)
 	pmpAddru := pmpAddr.UDPAddr()
 
 	// Ask for our external address if needed.
-	if m.external.IP.IsZero() {
+	if m.external.IP().IsZero() {
 		if _, err := uc.WriteTo(pmpReqExternalAddrPacket, pmpAddru); err != nil {
 			return netaddr.IPPort{}, err
 		}
@@ -337,10 +337,10 @@ func (c *Client) CreateOrGetMapping(ctx context.Context) (external netaddr.IPPor
 				return netaddr.IPPort{}, NoMappingError{fmt.Errorf("PMP response Op=0x%x,Res=0x%x", pres.OpCode, pres.ResultCode)}
 			}
 			if pres.OpCode == pmpOpReply|pmpOpMapPublicAddr {
-				m.external.IP = pres.PublicAddr
+				m.external = m.external.WithIP(pres.PublicAddr)
 			}
 			if pres.OpCode == pmpOpReply|pmpOpMapUDP {
-				m.external.Port = pres.ExternalPort
+				m.external = m.external.WithPort(pres.ExternalPort)
 				d := time.Duration(pres.MappingValidSeconds) * time.Second
 				d /= 2 // renew in half the time
 				m.useUntil = time.Now().Add(d)
@@ -468,9 +468,9 @@ func (c *Client) Probe(ctx context.Context) (res ProbeResult, err error) {
 	defer cancel()
 	defer closeCloserOnContextDone(ctx, uc)()
 
-	pcpAddr := netaddr.IPPort{IP: gw, Port: pcpPort}.UDPAddr()
-	pmpAddr := netaddr.IPPort{IP: gw, Port: pmpPort}.UDPAddr()
-	upnpAddr := netaddr.IPPort{IP: gw, Port: upnpPort}.UDPAddr()
+	pcpAddr := netaddr.IPPortFrom(gw, pcpPort).UDPAddr()
+	pmpAddr := netaddr.IPPortFrom(gw, pmpPort).UDPAddr()
+	upnpAddr := netaddr.IPPortFrom(gw, upnpPort).UDPAddr()
 
 	// Don't send probes to services that we recently learned (for
 	// the same gw/myIP) are available. See

net/tsaddr/tsaddr.go

@@ -92,7 +92,7 @@ func TailscaleEphemeral6Range() netaddr.IPPrefix {
 // Currently used to work around a Windows limitation when programming
 // IPv6 routes in corner cases.
 func Tailscale4To6Placeholder() netaddr.IP {
-	return Tailscale4To6Range().IP
+	return Tailscale4To6Range().IP()
 }
 
 // Tailscale4To6 returns a Tailscale IPv6 address that maps 1:1 to the
@@ -102,7 +102,7 @@ func Tailscale4To6(ipv4 netaddr.IP) netaddr.IP {
 	if !ipv4.Is4() || !IsTailscaleIP(ipv4) {
 		return netaddr.IP{}
 	}
-	ret := Tailscale4To6Range().IP.As16()
+	ret := Tailscale4To6Range().IP().As16()
 	v4 := ipv4.As4()
 	copy(ret[13:], v4[1:])
 	return netaddr.IPFrom16(ret)
@@ -172,16 +172,16 @@ func NewContainsIPFunc(addrs []netaddr.IPPrefix) func(ip netaddr.IP) bool {
 	// Fast paths for 1 and 2 IPs:
 	if len(addrs) == 1 {
 		a := addrs[0]
-		return func(ip netaddr.IP) bool { return ip == a.IP }
+		return func(ip netaddr.IP) bool { return ip == a.IP() }
 	}
 	if len(addrs) == 2 {
 		a, b := addrs[0], addrs[1]
-		return func(ip netaddr.IP) bool { return ip == a.IP || ip == b.IP }
+		return func(ip netaddr.IP) bool { return ip == a.IP() || ip == b.IP() }
 	}
 	// General case:
 	m := map[netaddr.IP]bool{}
 	for _, a := range addrs {
-		m[a.IP] = true
+		m[a.IP()] = true
 	}
 	return func(ip netaddr.IP) bool { return m[ip] }
 }

net/tstun/fake.go

@@ -8,7 +8,7 @@ import (
 	"io"
 	"os"
 
-	"github.com/tailscale/wireguard-go/tun"
+	"golang.zx2c4.com/wireguard/tun"
 )
 
 type fakeTUN struct {

net/tstun/ifstatus_noop.go

@@ -9,7 +9,7 @@ package tstun
 import (
 	"time"
 
-	"github.com/tailscale/wireguard-go/tun"
+	"golang.zx2c4.com/wireguard/tun"
 	"tailscale.com/types/logger"
 )
 

net/tstun/ifstatus_windows.go

@@ -9,7 +9,7 @@ import (
 	"sync"
 	"time"
 
-	"github.com/tailscale/wireguard-go/tun"
+	"golang.zx2c4.com/wireguard/tun"
 	"golang.zx2c4.com/wireguard/windows/tunnel/winipcfg"
 	"tailscale.com/types/logger"
 )

net/tstun/tun.go

@@ -13,7 +13,7 @@ import (
 	"runtime"
 	"time"
 
-	"github.com/tailscale/wireguard-go/tun"
+	"golang.zx2c4.com/wireguard/tun"
 	"tailscale.com/types/logger"
 	"tailscale.com/version/distro"
 )

net/tstun/tun_notwindows.go

@@ -6,7 +6,7 @@
 
 package tstun
 
-import "github.com/tailscale/wireguard-go/tun"
+import "golang.zx2c4.com/wireguard/tun"
 
 func interfaceName(dev tun.Device) (string, error) {
 	return dev.Name()

net/tstun/tun_windows.go

@@ -5,9 +5,9 @@
 package tstun
 
 import (
-	"github.com/tailscale/wireguard-go/tun"
-	"github.com/tailscale/wireguard-go/tun/wintun"
 	"golang.org/x/sys/windows"
+	"golang.zx2c4.com/wireguard/tun"
+	"golang.zx2c4.com/wireguard/tun/wintun"
 	"golang.zx2c4.com/wireguard/windows/tunnel/winipcfg"
 )
 

net/tstun/wrap.go

@@ -9,13 +9,14 @@ package tstun
 import (
 	"errors"
 	"io"
+	"log"
 	"os"
 	"sync"
 	"sync/atomic"
 	"time"
 
-	"github.com/tailscale/wireguard-go/device"
-	"github.com/tailscale/wireguard-go/tun"
+	"golang.zx2c4.com/wireguard/device"
+	"golang.zx2c4.com/wireguard/tun"
 	"inet.af/netaddr"
 	"tailscale.com/net/packet"
 	"tailscale.com/types/ipproto"
@@ -276,6 +277,7 @@ func (t *Wrapper) poll() {
 var magicDNSIPPort = netaddr.MustParseIPPort("100.100.100.100:0")
 
 func (t *Wrapper) filterOut(p *packet.Parsed) filter.Response {
+	log.Println("FILTEROUT")
 	// Fake ICMP echo responses to MagicDNS (100.100.100.100).
 	if p.IsEchoRequest() && p.Dst == magicDNSIPPort {
 		header := p.ICMP4Header()
@@ -352,7 +354,7 @@ func (t *Wrapper) Read(buf []byte, offset int) (int, error) {
 	p.Decode(buf[offset : offset+n])
 
 	if m, ok := t.destIPActivity.Load().(map[netaddr.IP]func()); ok {
-		if fn := m[p.Dst.IP]; fn != nil {
+		if fn := m[p.Dst.IP()]; fn != nil {
 			fn()
 		}
 	}
@@ -376,6 +378,7 @@ func (t *Wrapper) Read(buf []byte, offset int) (int, error) {
 }
 
 func (t *Wrapper) filterIn(buf []byte) filter.Response {
+	log.Println("FILTERIN")
 	p := parsedPacketPool.Get().(*packet.Parsed)
 	defer parsedPacketPool.Put(p)
 	p.Decode(buf)
@@ -412,7 +415,7 @@ func (t *Wrapper) filterIn(buf []byte) filter.Response {
 		p.IPProto == ipproto.TCP &&
 		p.TCPFlags&packet.TCPSyn != 0 &&
 		t.PeerAPIPort != nil {
-		if port, ok := t.PeerAPIPort(p.Dst.IP); ok && port == p.Dst.Port {
+		if port, ok := t.PeerAPIPort(p.Dst.IP()); ok && port == p.Dst.Port() {
 			outcome = filter.Accept
 		}
 	}
@@ -425,8 +428,8 @@ func (t *Wrapper) filterIn(buf []byte) filter.Response {
 		// can show them a rejection history with reasons.
 		if p.IPVersion == 4 && p.IPProto == ipproto.TCP && p.TCPFlags&packet.TCPSyn != 0 && !t.disableTSMPRejected {
 			rj := packet.TailscaleRejectedHeader{
-				IPSrc:  p.Dst.IP,
-				IPDst:  p.Src.IP,
+				IPSrc:  p.Dst.IP(),
+				IPDst:  p.Src.IP(),
 				Src:    p.Src,
 				Dst:    p.Dst,
 				Proto:  p.IPProto,
@@ -532,11 +535,12 @@ func (t *Wrapper) InjectInboundCopy(packet []byte) error {
 }
 
 func (t *Wrapper) injectOutboundPong(pp *packet.Parsed, req packet.TSMPPingRequest) {
+	log.Println("INJECT OUTBOUND")
 	pong := packet.TSMPPongReply{
 		Data: req.Data,
 	}
 	if t.PeerAPIPort != nil {
-		pong.PeerAPIPort, _ = t.PeerAPIPort(pp.Dst.IP)
+		pong.PeerAPIPort, _ = t.PeerAPIPort(pp.Dst.IP())
 	}
 	switch pp.IPVersion {
 	case 4:
@@ -568,8 +572,10 @@ func (t *Wrapper) InjectOutbound(packet []byte) error {
 	}
 	select {
 	case <-t.closed:
+		log.Println("Closed")
 		return ErrClosed
 	case t.outbound <- packet:
+		log.Println("t.outbound <- packet")
 		return nil
 	}
 }

net/tstun/wrap_test.go

@@ -14,7 +14,7 @@ import (
 	"testing"
 	"unsafe"
 
-	"github.com/tailscale/wireguard-go/tun/tuntest"
+	"golang.zx2c4.com/wireguard/tun/tuntest"
 	"inet.af/netaddr"
 	"tailscale.com/net/packet"
 	"tailscale.com/types/ipproto"
@@ -82,7 +82,7 @@ func nets(nets ...string) (ret []netaddr.IPPrefix) {
 			if ip.Is6() {
 				bits = 128
 			}
-			ret = append(ret, netaddr.IPPrefix{IP: ip, Bits: bits})
+			ret = append(ret, netaddr.IPPrefixFrom(ip, bits))
 		} else {
 			pfx, err := netaddr.ParseIPPrefix(s)
 			if err != nil {

paths/paths.go

@@ -13,9 +13,9 @@ import (
 	"sync/atomic"
 )
 
-// IOSSharedDir is a string set by the iOS app on start
+// AppSharedDir is a string set by the iOS or Android app on start
 // containing a directory we can read/write in.
-var IOSSharedDir atomic.Value
+var AppSharedDir atomic.Value
 
 // DefaultTailscaledSocket returns the path to the tailscaled Unix socket
 // or the empty string if there's no reasonable default.

scripts/installer.sh

@@ -0,0 +1,399 @@
+#!/bin/sh
+# Copyright (c) 2021 Tailscale Inc & AUTHORS All rights reserved.
+# Use of this source code is governed by a BSD-style
+# license that can be found in the LICENSE file.
+#
+# This script detects the current operating system, and installs
+# Tailscale according to that OS's conventions.
+
+set -eu
+
+# All the code is wrapped in a main function that gets called at the
+# bottom of the file, so that a truncated partial download doesn't end
+# up executing half a script.
+main() {
+	# Step 1: detect the current linux distro, version, and packaging system.
+	#
+	# We rely on a combination of 'uname' and /etc/os-release to find
+	# an OS name and version, and from there work out what
+	# installation method we should be using.
+	#
+	# The end result of this step is that the following three
+	# variables are populated, if detection was successful.
+	OS=""
+	VERSION=""
+	PACKAGETYPE=""
+
+	if [ -f /etc/os-release ]; then
+		# /etc/os-release populates a number of shell variables. We care about the following:
+		#  - ID: the short name of the OS (e.g. "debian", "freebsd")
+		#  - VERSION_ID: the numeric release version for the OS, if any (e.g. "18.04")
+		#  - VERSION_CODENAME: the codename of the OS release, if any (e.g. "buster")
+		. /etc/os-release
+		case "$ID" in
+			ubuntu)
+				OS="$ID"
+				VERSION="$VERSION_CODENAME"
+				PACKAGETYPE="apt"
+				;;
+			debian)
+				OS="$ID"
+				VERSION="$VERSION_CODENAME"
+				PACKAGETYPE="apt"
+				;;
+			raspbian)
+				OS="$ID"
+				VERSION="$VERSION_CODENAME"
+				PACKAGETYPE="apt"
+				;;
+			centos)
+				OS="$ID"
+				VERSION="$VERSION_ID"
+				PACKAGETYPE="dnf"
+				if [ "$VERSION" = "7" ]; then
+					PACKAGETYPE="yum"
+				fi
+				;;
+			rhel)
+				OS="$ID"
+				VERSION="$(echo "$VERSION_ID" | cut -f1 -d.)"
+				PACKAGETYPE="dnf"
+				;;
+			fedora)
+				OS="$ID"
+				VERSION=""
+				PACKAGETYPE="dnf"
+				;;
+			amzn)
+				OS="amazon-linux"
+				VERSION="$VERSION_ID"
+				PACKAGETYPE="yum"
+				;;
+			opensuse-leap)
+				OS="opensuse"
+				VERSION="leap/$VERSION_ID"
+				PACKAGETYPE="zypper"
+				;;
+			opensuse-tumbleweed)
+				OS="opensuse"
+				VERSION="tumbleweed"
+				PACKAGETYPE="zypper"
+				;;
+ 			arch)
+				OS="$ID"
+				VERSION="" # rolling release
+				PACKAGETYPE="pacman"
+				;;
+			manjaro)
+				OS="$ID"
+				VERSION="" # rolling release
+				PACKAGETYPE="pacman"
+				;;
+			alpine)
+				OS="$ID"
+				VERSION="$VERSION_ID"
+				PACKAGETYPE="apk"
+				;;
+			nixos)
+				echo "Please add Tailscale to your NixOS configuration directly:"
+				echo
+				echo "services.tailscale.enable = true;"
+				exit 1
+				;;
+			void)
+				OS="$ID"
+				VERSION="" # rolling release
+				PACKAGETYPE="xbps"
+				;;
+			gentoo)
+				OS="$ID"
+				VERSION="" # rolling release
+				PACKAGETYPE="emerge"
+				;;
+			freebsd)
+				OS="$ID"
+				VERSION="$(echo "$VERSION_ID" | cut -f1 -d.)"
+				PACKAGETYPE="pkg"
+				;;
+			# TODO: wsl?
+			# TODO: synology? qnap?
+		esac
+	fi
+
+	# If we failed to detect something through os-release, consult
+	# uname and try to infer things from that.
+	if [ -z "$OS" ]; then
+		if type uname >/dev/null 2>&1; then
+			case "$(uname)" in
+				FreeBSD)
+					# FreeBSD before 12.2 doesn't have
+					# /etc/os-release, so we wouldn't have found it in
+					# the os-release probing above.
+					OS="freebsd"
+					VERSION="$(freebsd-version | cut -f1 -d.)"
+					PACKAGETYPE="pkg"
+					;;
+				OpenBSD)
+					OS="openbsd"
+					VERSION="$(uname -r)"
+					PACKAGETYPE=""
+					;;
+				Darwin)
+					OS="macos"
+					VERSION="$(sw_vers -productVersion | cut -f1-2 -d.)"
+					PACKAGETYPE="appstore"
+					;;
+				Linux)
+					OS="other-linux"
+					VERSION=""
+					PACKAGETYPE=""
+					;;
+			esac
+		fi
+	fi
+
+	# Step 2: having detected an OS we support, is it one of the
+	# versions we support?
+	OS_UNSUPPORTED=
+	case "$OS" in
+		ubuntu)
+			if [ "$VERSION" != "xenial" ] && \
+			   [ "$VERSION" != "bionic" ] && \
+			   [ "$VERSION" != "eoan" ] && \
+			   [ "$VERSION" != "focal" ] && \
+			   [ "$VERSION" != "groovy" ] && \
+			   [ "$VERSION" != "hirsute" ]
+			then
+				OS_UNSUPPORTED=1
+			fi
+		;;
+		debian)
+			if [ "$VERSION" != "stretch" ] && \
+			   [ "$VERSION" != "buster" ] && \
+			   [ "$VERSION" != "bullseye" ] && \
+			   [ "$VERSION" != "sid" ]
+			then
+				OS_UNSUPPORTED=1
+			fi
+		;;
+		raspbian)
+			if [ "$VERSION" != "buster" ]
+			then
+				OS_UNSUPPORTED=1
+			fi
+		;;
+		centos)
+			if [ "$VERSION" != "7" ] && \
+			   [ "$VERSION" != "8" ]
+			then
+				OS_UNSUPPORTED=1
+			fi
+		;;
+		rhel)
+			if [ "$VERSION" != "8" ]
+			then
+				OS_UNSUPPORTED=1
+			fi
+		;;
+		amazon-linux)
+			if [ "$VERSION" != "2" ]
+			then
+				OS_UNSUPPORTED=1
+			fi
+		;;
+		opensuse)
+			if [ "$VERSION" != "leap/15.1" ] && \
+			   [ "$VERSION" != "leap/15.2" ] && \
+			   [ "$VERSION" != "tumbleweed" ]
+			then
+				OS_UNSUPPORTED=1
+			fi
+			;;
+		arch)
+			# Rolling release, no version checking needed.
+			;;
+		manjaro)
+			# Rolling release, no version checking needed.
+			;;
+		alpine)
+			# All versions supported, no version checking needed.
+			# TODO: is that true? When was tailscale packaged?
+			;;
+		void)
+			# Rolling release, no version checking needed.
+			;;
+		gentoo)
+			# Rolling release, no version checking needed.
+			;;
+		freebsd)
+			if [ "$VERSION" != "12" ] && \
+			   [ "$VERSION" != "13" ]
+			then
+				OS_UNSUPPORTED=1
+			fi
+			;;
+		openbsd)
+			OS_UNSUPPORTED=1
+			;;
+		macos)
+			# We delegate macOS installation to the app store, it will
+			# perform version checks for us.
+			;;
+		other-linux)
+			OS_UNSUPPORTED=1
+			;;
+		*)
+			OS_UNSUPPORTED=1
+			;;
+	esac
+	if [ "$OS_UNSUPPORTED" = "1" ]; then
+		case "$OS" in
+			other-linux)
+				echo "Couldn't determine what kind of Linux is running."
+				echo "You could try the static binaries at:"
+				echo "https://pkgs.tailscale.com/stable/#static"
+				;;
+			"")
+				echo "Couldn't determine what operating system you're running."
+				;;
+			*)
+				echo "$OS $VERSION isn't supported by this script yet."
+				;;
+		esac
+		echo
+		echo "If you'd like us to support your system better, please email support@tailscale.com"
+		echo "and tell us what OS you're running."
+		echo
+		echo "Please include the following information we gathered from your system:"
+		echo
+		echo "OS=$OS"
+		echo "VERSION=$VERSION"
+		echo "PACKAGETYPE=$PACKAGETYPE"
+		if type uname >/dev/null 2>&1; then
+			echo "UNAME=$(uname -a)"
+		else
+			echo "UNAME="
+		fi
+		echo
+		if [ -f /etc/os-release ]; then
+			cat /etc/os-release
+		else
+			echo "No /etc/os-release"
+		fi
+		exit 1
+	fi
+
+	# Step 3: work out if we can run privileged commands, and if so,
+	# how.
+	CAN_ROOT=
+	SUDO=
+	if [ "$(id -u)" = 0 ]; then
+		CAN_ROOT=1
+		SUDO=""
+	elif type sudo >/dev/null; then
+		CAN_ROOT=1
+		SUDO="sudo"
+	elif type doas >/dev/null; then
+		CAN_ROOT=1
+		SUDO="doas"
+	fi
+	if [ "$CAN_ROOT" != "1" ]; then
+		echo "This installer needs to run commands as root."
+		echo "We tried looking for 'sudo' and 'doas', but couldn't find them."
+		echo "Either re-run this script as root, or set up sudo/doas."
+		exit 1
+	fi
+
+
+	# Step 4: run the installation.
+	echo "Installing Tailscale for $OS $VERSION, using method $PACKAGETYPE"
+	case "$PACKAGETYPE" in
+		apt)
+			# Ideally we want to use curl, but on some installs we
+			# only have wget. Detect and use what's available.
+			CURL=
+			if type curl >/dev/null; then
+				CURL="curl -fsSL"
+			elif type wget >/dev/null; then
+				CURL="wget -q -O-"
+			fi
+			if [ -z "$CURL" ]; then
+				echo "The installer needs either curl or wget to download files."
+				echo "Please install either curl or wget to proceed."
+				exit 1
+			fi
+
+			# TODO: use newfangled per-repo signature scheme
+			set -x
+			$CURL "https://pkgs.tailscale.com/stable/$OS/$VERSION.gpg" | $SUDO apt-key add -
+			$CURL "https://pkgs.tailscale.com/stable/$OS/$VERSION.list" | $SUDO tee /etc/apt/sources.list.d/tailscale.list
+			$SUDO apt-get update
+			$SUDO apt-get install tailscale
+			set +x
+		;;
+		yum)
+			set -x
+			$SUDO yum install yum-utils
+			$SUDO yum-config-manager --add-repo "https://pkgs.tailscale.com/stable/$OS/$VERSION/tailscale.repo"
+			$SUDO yum install tailscale
+			$SUDO systemctl enable --now tailscaled
+			set +x
+		;;
+		dnf)
+			set -x
+			$SUDO dnf config-manager --add-repo "https://pkgs.tailscale.com/stable/$OS/$VERSION/tailscale.repo"
+			$SUDO dnf install tailscale
+			$SUDO systemctl enable --now tailscaled
+			set +x
+		;;
+		zypper)
+			set -x
+			$SUDO zypper ar -g -r "https://pkgs.tailscale.com/stable/$OS/$VERSION/tailscale.repo"
+			$SUDO zypper ref
+			$SUDO zypper in tailscale
+			$SUDO systemctl enable --now tailscaled
+			set +x
+			;;
+		pacman)
+			set -x
+			$SUDO pacman -S tailscale
+			$SUDO systemctl enable --now tailscaled
+			set +x
+			;;
+		apk)
+			set -x
+			$SUDO apk add tailscale
+			$SUDO rc-update add tailscale
+			set +x
+			;;
+		xbps)
+			set -x
+			$SUDO xbps-install tailscale
+			set +x
+			;;
+		emerge)
+			set -x
+			$SUDO emerge net-vpn/tailscale
+			set +x
+			;;
+		appstore)
+			set -x
+			open "https://apps.apple.com/us/app/tailscale/id1475387142"
+			set +x
+			;;
+		*)
+			echo "unexpected: unknown package type $PACKAGETYPE"
+			exit 1
+			;;
+	esac
+
+	echo "Installation complete! Log in to start using Tailscale by running:"
+	echo
+	if [ -z "$SUDO" ]; then
+		echo "tailscale up"
+	else
+		echo "$SUDO tailscale up"
+	fi
+}
+
+main

tailcfg/tailcfg.go

@@ -7,7 +7,7 @@ package tailcfg
 //go:generate go run tailscale.com/cmd/cloner --type=User,Node,Hostinfo,NetInfo,Login,DNSConfig,DNSResolver,RegisterResponse --clonefunc=true --output=tailcfg_clone.go
 
 import (
-	"bytes"
+	"encoding/hex"
 	"errors"
 	"fmt"
 	"reflect"
@@ -679,7 +679,7 @@ func (et EndpointType) String() string {
 
 // Endpoint is an endpoint IPPort and an associated type.
 // It doesn't currently go over the wire as is but is instead
-// broken up into two parallel slices in MapReqeust, for compatibility
+// broken up into two parallel slices in MapRequest, for compatibility
 // reasons. But this type is used in the codebase.
 type Endpoint struct {
 	Addr netaddr.IPPort
@@ -749,6 +749,9 @@ type MapRequest struct {
 	//     * "minimize-netmap": have control minimize the netmap, removing
 	//       peers that are unreachable per ACLS.
 	DebugFlags []string `json:",omitempty"`
+
+	// Basic boolean field to determine if a Ping is being intitiated
+	Ping bool
 }
 
 // PortRange represents a range of UDP or TCP port numbers.
@@ -884,6 +887,27 @@ type PingRequest struct {
 	// Log is whether to log about this ping in the success case.
 	// For failure cases, the client will log regardless.
 	Log bool `json:",omitempty"`
+
+	Initiator        string // admin@email; "system" (for Tailscale)
+	TestIP           netaddr.IP
+	Types            string // empty means all: TSMP+ICMP+disco
+	StopAfterNDirect int    // 1 means stop on 1st direct ping; 4 means 4 direct pings; 0 means do MaxPings and stop
+	MaxPings         int    // MaxPings total, direct or DERPed
+	PayloadSize      int    // default: 0 extra bytes
+}
+
+// According to https://roamresearch.com/#/app/ts-corp/page/4Bn_Famn2
+// Client can stream responses back via HTTP
+// We will add a struct with the proper fields
+type StreamedPingResult struct {
+	IP      netaddr.IP
+	SeqNum  int     // somewhat redundant with TxID but for clarity
+	SentTo  NodeID  // for exit/subnet relays
+	TxID    string  // N hex bytes random
+	Dir     string  // "in"/"out"
+	Type    string  // ICMP, disco, TSMP, ...
+	Via     string  // "direct", "derp-nyc", ...
+	Seconds float64 // for Dir "in" only
 }
 
 type MapResponse struct {
@@ -1026,10 +1050,16 @@ func (k MachineKey) MarshalText() ([]byte, error)     { return keyMarshalText("m
 func (k MachineKey) HexString() string                { return fmt.Sprintf("%x", k[:]) }
 func (k *MachineKey) UnmarshalText(text []byte) error { return keyUnmarshalText(k[:], "mkey:", text) }
 
+func appendKey(base []byte, prefix string, k [32]byte) []byte {
+	ret := append(base, make([]byte, len(prefix)+64)...)
+	buf := ret[len(base):]
+	copy(buf, prefix)
+	hex.Encode(buf[len(prefix):], k[:])
+	return ret
+}
+
 func keyMarshalText(prefix string, k [32]byte) []byte {
-	buf := bytes.NewBuffer(make([]byte, 0, len(prefix)+64))
-	fmt.Fprintf(buf, "%s%x", prefix, k[:])
-	return buf.Bytes()
+	return appendKey(nil, prefix, k)
 }
 
 func keyUnmarshalText(dst []byte, prefix string, text []byte) error {
@@ -1060,6 +1090,7 @@ func (k DiscoKey) String() string                   { return fmt.Sprintf("discok
 func (k DiscoKey) MarshalText() ([]byte, error)     { return keyMarshalText("discokey:", k), nil }
 func (k *DiscoKey) UnmarshalText(text []byte) error { return keyUnmarshalText(k[:], "discokey:", text) }
 func (k DiscoKey) ShortString() string              { return fmt.Sprintf("d:%x", k[:8]) }
+func (k DiscoKey) AppendTo(b []byte) []byte         { return appendKey(b, "discokey:", k) }
 
 // IsZero reports whether k is the zero value.
 func (k DiscoKey) IsZero() bool { return k == DiscoKey{} }

tailcfg/tailcfg_test.go

@@ -14,6 +14,7 @@ import (
 
 	"inet.af/netaddr"
 	"tailscale.com/types/wgkey"
+	"tailscale.com/version"
 )
 
 func fieldsOf(t reflect.Type) (fields []string) {
@@ -518,3 +519,35 @@ func TestEndpointTypeMarshal(t *testing.T) {
 		t.Errorf("got %s; want %s", got, want)
 	}
 }
+
+var sinkBytes []byte
+
+func BenchmarkKeyMarshalText(b *testing.B) {
+	b.ReportAllocs()
+	var k [32]byte
+	for i := 0; i < b.N; i++ {
+		sinkBytes = keyMarshalText("prefix", k)
+	}
+}
+
+func TestAppendKeyAllocs(t *testing.T) {
+	if version.IsRace() {
+		t.Skip("skipping in race detector") // append(b, make([]byte, N)...) not optimized in compiler with race
+	}
+	var k [32]byte
+	n := int(testing.AllocsPerRun(1000, func() {
+		sinkBytes = keyMarshalText("prefix", k)
+	}))
+	if n != 1 {
+		t.Fatalf("allocs = %v; want 1", n)
+	}
+}
+
+func TestDiscoKeyAppend(t *testing.T) {
+	d := DiscoKey{1: 1, 2: 2}
+	got := string(d.AppendTo([]byte("foo")))
+	want := "foodiscokey:0001020000000000000000000000000000000000000000000000000000000000"
+	if got != want {
+		t.Errorf("got %q; want %q", got, want)
+	}
+}

tsnet/example/tshello/tshello.go

@@ -0,0 +1,43 @@
+// Copyright (c) 2021 Tailscale Inc & AUTHORS All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+// The tshello server demonstrates how to use Tailscale as a library.
+package main
+
+import (
+	"fmt"
+	"html"
+	"log"
+	"net/http"
+	"strings"
+
+	"tailscale.com/tsnet"
+)
+
+func main() {
+	s := new(tsnet.Server)
+	ln, err := s.Listen("tcp", ":80")
+	if err != nil {
+		log.Fatal(err)
+	}
+	log.Fatal(http.Serve(ln, http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		who, ok := s.WhoIs(r.RemoteAddr)
+		if !ok {
+			http.Error(w, "WhoIs failed", 500)
+			return
+		}
+		fmt.Fprintf(w, "<html><body><h1>Hello, world!</h1>\n")
+		fmt.Fprintf(w, "<p>You are <b>%s</b> from <b>%s</b> (%s)</p>",
+			html.EscapeString(who.UserProfile.LoginName),
+			html.EscapeString(firstLabel(who.Node.ComputedName)),
+			r.RemoteAddr)
+	})))
+}
+
+func firstLabel(s string) string {
+	if i := strings.Index(s, "."); i != -1 {
+		return s[:i]
+	}
+	return s
+}

tsnet/tsnet.go

@@ -0,0 +1,274 @@
+// Copyright (c) 2021 Tailscale Inc & AUTHORS All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+// Package tsnet provides Tailscale as a library.
+//
+// It is an experimental work in progress.
+package tsnet
+
+import (
+	"errors"
+	"fmt"
+	"log"
+	"net"
+	"os"
+	"path/filepath"
+	"strconv"
+	"strings"
+	"sync"
+	"time"
+
+	"inet.af/netaddr"
+	"tailscale.com/client/tailscale/apitype"
+	"tailscale.com/control/controlclient"
+	"tailscale.com/ipn"
+	"tailscale.com/ipn/ipnlocal"
+	"tailscale.com/smallzstd"
+	"tailscale.com/types/logger"
+	"tailscale.com/wgengine"
+	"tailscale.com/wgengine/monitor"
+	"tailscale.com/wgengine/netstack"
+)
+
+// Server is an embedded Tailscale server.
+//
+// Its exported fields may be changed until the first call to Listen.
+type Server struct {
+	// Dir specifies the name of the directory to use for
+	// state. If empty, a directory is selected automatically
+	// under os.UserConfigDir (https://golang.org/pkg/os/#UserConfigDir).
+	// based on the name of the binary.
+	Dir string
+
+	// Hostname is the hostname to present to the control server.
+	// If empty, the binary name is used.l
+	Hostname string
+
+	// Logf, if non-nil, specifies the logger to use. By default,
+	// log.Printf is used.
+	Logf logger.Logf
+
+	initOnce sync.Once
+	initErr  error
+	lb       *ipnlocal.LocalBackend
+	// the state directory
+	dir      string
+	hostname string
+
+	mu        sync.Mutex
+	listeners map[listenKey]*listener
+}
+
+// WhoIs reports the node and user who owns the node with the given
+// address. The addr may be an ip:port (as from an
+// http.Request.RemoteAddr) or just an IP address.
+func (s *Server) WhoIs(addr string) (w *apitype.WhoIsResponse, ok bool) {
+	ipp, err := netaddr.ParseIPPort(addr)
+	if err != nil {
+		ip, err := netaddr.ParseIP(addr)
+		if err != nil {
+			return nil, false
+		}
+		ipp = ipp.WithIP(ip)
+	}
+	n, up, ok := s.lb.WhoIs(ipp)
+	if !ok {
+		return nil, false
+	}
+	return &apitype.WhoIsResponse{
+		Node:        n,
+		UserProfile: &up,
+	}, true
+}
+
+func (s *Server) doInit() {
+	if err := s.start(); err != nil {
+		s.initErr = fmt.Errorf("tsnet: %w", err)
+	}
+}
+
+func (s *Server) start() error {
+	if v, _ := strconv.ParseBool(os.Getenv("TAILSCALE_USE_WIP_CODE")); !v {
+		return errors.New("code disabled without environment variable TAILSCALE_USE_WIP_CODE set true")
+	}
+
+	exe, err := os.Executable()
+	if err != nil {
+		return err
+	}
+	prog := strings.TrimSuffix(strings.ToLower(filepath.Base(exe)), ".exe")
+
+	s.hostname = s.Hostname
+	if s.hostname == "" {
+		s.hostname = prog
+	}
+
+	s.dir = s.Dir
+	if s.dir == "" {
+		confDir, err := os.UserConfigDir()
+		if err != nil {
+			return err
+		}
+		s.dir = filepath.Join(confDir, "tslib-"+prog)
+		if err := os.MkdirAll(s.dir, 0700); err != nil {
+			return err
+		}
+	}
+	if fi, err := os.Stat(s.dir); err != nil {
+		return err
+	} else if !fi.IsDir() {
+		return fmt.Errorf("%v is not a directory", s.dir)
+	}
+
+	logf := s.Logf
+	if logf == nil {
+		logf = log.Printf
+	}
+
+	// TODO(bradfitz): start logtail? don't use filch, perhaps?
+	// only upload plumbed Logf?
+
+	linkMon, err := monitor.New(logf)
+	if err != nil {
+		return err
+	}
+
+	eng, err := wgengine.NewUserspaceEngine(logf, wgengine.Config{
+		ListenPort:  0,
+		LinkMonitor: linkMon,
+	})
+	if err != nil {
+		return err
+	}
+
+	tunDev, magicConn, ok := eng.(wgengine.InternalsGetter).GetInternals()
+	if !ok {
+		return fmt.Errorf("%T is not a wgengine.InternalsGetter", eng)
+	}
+
+	ns, err := netstack.Create(logf, tunDev, eng, magicConn, false)
+	if err != nil {
+		return fmt.Errorf("netstack.Create: %w", err)
+	}
+	ns.ForwardTCPIn = s.forwardTCP
+	if err := ns.Start(); err != nil {
+		return fmt.Errorf("failed to start netstack: %w", err)
+	}
+
+	statePath := filepath.Join(s.dir, "tailscaled.state")
+	store, err := ipn.NewFileStore(statePath)
+	if err != nil {
+		return err
+	}
+	logid := "tslib-TODO"
+
+	lb, err := ipnlocal.NewLocalBackend(logf, logid, store, eng)
+	if err != nil {
+		return fmt.Errorf("NewLocalBackend: %v", err)
+	}
+	s.lb = lb
+	lb.SetDecompressor(func() (controlclient.Decompressor, error) {
+		return smallzstd.NewDecoder(nil)
+	})
+	prefs := ipn.NewPrefs()
+	prefs.Hostname = s.hostname
+	prefs.WantRunning = true
+	err = lb.Start(ipn.Options{
+		StateKey:    ipn.GlobalDaemonStateKey,
+		UpdatePrefs: prefs,
+	})
+	if err != nil {
+		return fmt.Errorf("starting backend: %w", err)
+	}
+	if os.Getenv("TS_LOGIN") == "1" {
+		s.lb.StartLoginInteractive()
+	}
+	return nil
+}
+
+func (s *Server) forwardTCP(c net.Conn, port uint16) {
+	s.mu.Lock()
+	ln, ok := s.listeners[listenKey{"tcp", "", fmt.Sprint(port)}]
+	s.mu.Unlock()
+	if !ok {
+		c.Close()
+		return
+	}
+	t := time.NewTimer(time.Second)
+	defer t.Stop()
+	select {
+	case ln.conn <- c:
+	case <-t.C:
+		c.Close()
+	}
+}
+
+func (s *Server) Listen(network, addr string) (net.Listener, error) {
+	host, port, err := net.SplitHostPort(addr)
+	if err != nil {
+		return nil, fmt.Errorf("tsnet: %w", err)
+	}
+
+	s.initOnce.Do(s.doInit)
+	if s.initErr != nil {
+		return nil, s.initErr
+	}
+
+	key := listenKey{network, host, port}
+	ln := &listener{
+		s:    s,
+		key:  key,
+		addr: addr,
+
+		conn: make(chan net.Conn),
+	}
+	s.mu.Lock()
+	if s.listeners == nil {
+		s.listeners = map[listenKey]*listener{}
+	}
+	if _, ok := s.listeners[key]; ok {
+		s.mu.Unlock()
+		return nil, fmt.Errorf("tsnet: listener already open for %s, %s", network, addr)
+	}
+	s.listeners[key] = ln
+	s.mu.Unlock()
+	return ln, nil
+}
+
+type listenKey struct {
+	network string
+	host    string
+	port    string
+}
+
+type listener struct {
+	s    *Server
+	key  listenKey
+	addr string
+	conn chan net.Conn
+}
+
+func (ln *listener) Accept() (net.Conn, error) {
+	c, ok := <-ln.conn
+	if !ok {
+		return nil, fmt.Errorf("tsnet: %w", net.ErrClosed)
+	}
+	return c, nil
+}
+
+func (ln *listener) Addr() net.Addr { return addr{ln} }
+func (ln *listener) Close() error {
+	ln.s.mu.Lock()
+	defer ln.s.mu.Unlock()
+	if v, ok := ln.s.listeners[ln.key]; ok && v == ln {
+		delete(ln.s.listeners, ln.key)
+		close(ln.conn)
+	}
+	return nil
+}
+
+type addr struct{ ln *listener }
+
+func (a addr) Network() string { return a.ln.key.network }
+func (a addr) String() string  { return a.ln.addr }

tstest/integration/integration_test.go

@@ -10,6 +10,7 @@ import (
 	crand "crypto/rand"
 	"crypto/tls"
 	"encoding/json"
+	"errors"
 	"fmt"
 	"io"
 	"io/ioutil"
@@ -21,6 +22,7 @@ import (
 	"os/exec"
 	"path"
 	"path/filepath"
+	"regexp"
 	"runtime"
 	"strings"
 	"sync"
@@ -41,8 +43,11 @@ import (
 	"tailscale.com/types/key"
 	"tailscale.com/types/logger"
 	"tailscale.com/types/nettype"
+	"tailscale.com/version"
 )
 
+// var verbose = flag.Bool("verbose", true, "verbose debug logs")
+
 var mainError atomic.Value // of error
 
 func TestMain(m *testing.M) {
@@ -54,14 +59,12 @@ func TestMain(m *testing.M) {
 		fmt.Fprintf(os.Stderr, "FAIL: %v\n", err)
 		os.Exit(1)
 	}
+	time.Sleep(5 * time.Second)
 	os.Exit(0)
 }
 
-func TestIntegration(t *testing.T) {
-	if runtime.GOOS == "windows" {
-		t.Skip("not tested/working on Windows yet")
-	}
-
+func TestOneNodeUp_NoAuth(t *testing.T) {
+	t.Parallel()
 	bins := buildTestBinaries(t)
 
 	env := newTestEnv(t, bins)
@@ -69,8 +72,8 @@ func TestIntegration(t *testing.T) {
 
 	n1 := newTestNode(t, env)
 
-	dcmd := n1.StartDaemon(t)
-	defer dcmd.Process.Kill()
+	d1 := n1.StartDaemon(t)
+	defer d1.Kill()
 
 	n1.AwaitListening(t)
 
@@ -87,44 +90,141 @@ func TestIntegration(t *testing.T) {
 		t.Error(err)
 	}
 
-	t.Logf("Running up --login-server=%s ...", env.ControlServer.URL)
-	if err := n1.Tailscale("up", "--login-server="+env.ControlServer.URL).Run(); err != nil {
-		t.Fatalf("up: %v", err)
-	}
+	n1.MustUp()
 
 	if d, _ := time.ParseDuration(os.Getenv("TS_POST_UP_SLEEP")); d > 0 {
 		t.Logf("Sleeping for %v to give 'up' time to misbehave (https://github.com/tailscale/tailscale/issues/1840) ...", d)
 		time.Sleep(d)
 	}
 
-	var ip string
-	if err := tstest.WaitFor(20*time.Second, func() error {
-		out, err := n1.Tailscale("ip").Output()
-		if err != nil {
-			return err
+	t.Logf("Got IP: %v", n1.AwaitIP(t))
+	n1.AwaitRunning(t)
+
+	d1.MustCleanShutdown(t)
+
+	t.Logf("number of HTTP logcatcher requests: %v", env.LogCatcher.numRequests())
+}
+
+func TestOneNodeUp_Auth(t *testing.T) {
+	t.Parallel()
+	bins := buildTestBinaries(t)
+
+	env := newTestEnv(t, bins)
+	defer env.Close()
+	env.Control.RequireAuth = true
+
+	n1 := newTestNode(t, env)
+	d1 := n1.StartDaemon(t)
+	defer d1.Kill()
+
+	n1.AwaitListening(t)
+
+	st := n1.MustStatus(t)
+	t.Logf("Status: %s", st.BackendState)
+
+	t.Logf("Running up --login-server=%s ...", env.ControlServer.URL)
+
+	cmd := n1.Tailscale("up", "--login-server="+env.ControlServer.URL)
+	var authCountAtomic int32
+	cmd.Stdout = &authURLParserWriter{fn: func(urlStr string) error {
+		if env.Control.CompleteAuth(urlStr) {
+			atomic.AddInt32(&authCountAtomic, 1)
+			t.Logf("completed auth path %s", urlStr)
+			return nil
+		}
+		err := fmt.Errorf("Failed to complete auth path to %q", urlStr)
+		t.Log(err)
+		return err
+	}}
+	cmd.Stderr = cmd.Stdout
+	if err := cmd.Run(); err != nil {
+		t.Fatalf("up: %v", err)
+	}
+	t.Logf("Got IP: %v", n1.AwaitIP(t))
+
+	n1.AwaitRunning(t)
+
+	if n := atomic.LoadInt32(&authCountAtomic); n != 1 {
+		t.Errorf("Auth URLs completed = %d; want 1", n)
+	}
+
+	d1.MustCleanShutdown(t)
+
+}
+
+func TestTwoNodes(t *testing.T) {
+	t.Parallel()
+	bins := buildTestBinaries(t)
+
+	env := newTestEnv(t, bins)
+	defer env.Close()
+
+	// Create two nodes:
+	n1 := newTestNode(t, env)
+	d1 := n1.StartDaemon(t)
+	defer d1.Kill()
+
+	n2 := newTestNode(t, env)
+	d2 := n2.StartDaemon(t)
+	defer d2.Kill()
+
+	n1.AwaitListening(t)
+	n2.AwaitListening(t)
+	n1.MustUp()
+	n2.MustUp()
+	n1.AwaitRunning(t)
+	n2.AwaitRunning(t)
+
+	if err := tstest.WaitFor(2*time.Second, func() error {
+		st := n1.MustStatus(t)
+		if len(st.Peer) == 0 {
+			return errors.New("no peers")
+		}
+		if len(st.Peer) > 1 {
+			return fmt.Errorf("got %d peers; want 1", len(st.Peer))
+		}
+		peer := st.Peer[st.Peers()[0]]
+		if peer.ID == st.Self.ID {
+			return errors.New("peer is self")
 		}
-		ip = string(out)
 		return nil
 	}); err != nil {
 		t.Error(err)
 	}
-	t.Logf("Got IP: %v", ip)
 
-	dcmd.Process.Signal(os.Interrupt)
+	d1.MustCleanShutdown(t)
+	d2.MustCleanShutdown(t)
+}
 
-	ps, err := dcmd.Process.Wait()
-	if err != nil {
-		t.Fatalf("tailscaled Wait: %v", err)
+func TestNodeAddressIPFields(t *testing.T) {
+	t.Parallel()
+	bins := buildTestBinaries(t)
+
+	env := newTestEnv(t, bins)
+	defer env.Close()
+
+	n1 := newTestNode(t, env)
+	d1 := n1.StartDaemon(t)
+	defer d1.Kill()
+
+	n1.AwaitListening(t)
+	n1.MustUp()
+	n1.AwaitRunning(t)
+
+	testNodes := env.Control.AllNodes()
+
+	if len(testNodes) != 1 {
+		t.Errorf("Expected %d nodes, got %d", 1, len(testNodes))
 	}
-	if ps.ExitCode() != 0 {
-		t.Errorf("tailscaled ExitCode = %d; want 0", ps.ExitCode())
+	node := testNodes[0]
+	if len(node.Addresses) == 0 {
+		t.Errorf("Empty Addresses field in node")
+	}
+	if len(node.AllowedIPs) == 0 {
+		t.Errorf("Empty AllowedIPs field in node")
 	}
 
-	t.Logf("number of HTTP logcatcher requests: %v", env.LogCatcher.numRequests())
-	if err := env.TrafficTrap.Err(); err != nil {
-		t.Errorf("traffic trap: %v", err)
-		t.Logf("logs: %s", env.LogCatcher.logsString())
-	}
+	d1.MustCleanShutdown(t)
 }
 
 // testBinaries are the paths to a tailscaled and tailscale binary.
@@ -139,16 +239,18 @@ type testBinaries struct {
 // if they fail to compile.
 func buildTestBinaries(t testing.TB) *testBinaries {
 	td := t.TempDir()
+	build(t, td, "tailscale.com/cmd/tailscaled", "tailscale.com/cmd/tailscale")
 	return &testBinaries{
 		dir:    td,
-		daemon: build(t, td, "tailscale.com/cmd/tailscaled"),
-		cli:    build(t, td, "tailscale.com/cmd/tailscale"),
+		daemon: filepath.Join(td, "tailscaled"+exe()),
+		cli:    filepath.Join(td, "tailscale"+exe()),
 	}
 }
 
 // testEnv contains the test environment (set of servers) used by one
 // or more nodes.
 type testEnv struct {
+	t        testing.TB
 	Binaries *testBinaries
 
 	LogCatcher       *logCatcher
@@ -168,13 +270,20 @@ type testEnv struct {
 //
 // Call Close to shut everything down.
 func newTestEnv(t testing.TB, bins *testBinaries) *testEnv {
+	if runtime.GOOS == "windows" {
+		t.Skip("not tested/working on Windows yet")
+	}
 	derpMap, derpShutdown := runDERPAndStun(t, logger.Discard)
 	logc := new(logCatcher)
 	control := &testcontrol.Server{
 		DERPMap: derpMap,
 	}
 	trafficTrap := new(trafficTrap)
+	log.Println("SERVER ATTACHED")
+	log.Println(len(control.PingRequestC))
+	// go func() { control.PingRequestC <- true }()
 	e := &testEnv{
+		t:                 t,
 		Binaries:          bins,
 		LogCatcher:        logc,
 		LogCatcherServer:  httptest.NewServer(logc),
@@ -184,10 +293,16 @@ func newTestEnv(t testing.TB, bins *testBinaries) *testEnv {
 		TrafficTrapServer: httptest.NewServer(trafficTrap),
 		derpShutdown:      derpShutdown,
 	}
+	e.Control.BaseURL = e.ControlServer.URL
 	return e
 }
 
 func (e *testEnv) Close() error {
+	if err := e.TrafficTrap.Err(); err != nil {
+		e.t.Errorf("traffic trap: %v", err)
+		e.t.Logf("logs: %s", e.LogCatcher.logsString())
+	}
+
 	e.LogCatcherServer.Close()
 	e.TrafficTrapServer.Close()
 	e.ControlServer.Close()
@@ -218,9 +333,28 @@ func newTestNode(t *testing.T, env *testEnv) *testNode {
 	}
 }
 
+type Daemon struct {
+	Process *os.Process
+}
+
+func (d *Daemon) Kill() {
+	d.Process.Kill()
+}
+
+func (d *Daemon) MustCleanShutdown(t testing.TB) {
+	d.Process.Signal(os.Interrupt)
+	ps, err := d.Process.Wait()
+	if err != nil {
+		t.Fatalf("tailscaled Wait: %v", err)
+	}
+	if ps.ExitCode() != 0 {
+		t.Errorf("tailscaled ExitCode = %d; want 0", ps.ExitCode())
+	}
+}
+
 // StartDaemon starts the node's tailscaled, failing if it fails to
 // start.
-func (n *testNode) StartDaemon(t testing.TB) *exec.Cmd {
+func (n *testNode) StartDaemon(t testing.TB) *Daemon {
 	cmd := exec.Command(n.env.Binaries.daemon,
 		"--tun=userspace-networking",
 		"--state="+n.stateFile,
@@ -234,7 +368,17 @@ func (n *testNode) StartDaemon(t testing.TB) *exec.Cmd {
 	if err := cmd.Start(); err != nil {
 		t.Fatalf("starting tailscaled: %v", err)
 	}
-	return cmd
+	return &Daemon{
+		Process: cmd.Process,
+	}
+}
+
+func (n *testNode) MustUp() {
+	t := n.env.t
+	t.Logf("Running up --login-server=%s ...", n.env.ControlServer.URL)
+	if err := n.Tailscale("up", "--login-server="+n.env.ControlServer.URL).Run(); err != nil {
+		t.Fatalf("up: %v", err)
+	}
 }
 
 // AwaitListening waits for the tailscaled to be serving local clients
@@ -252,6 +396,40 @@ func (n *testNode) AwaitListening(t testing.TB) {
 	}
 }
 
+func (n *testNode) AwaitIP(t testing.TB) (ips string) {
+	t.Helper()
+	if err := tstest.WaitFor(20*time.Second, func() error {
+		out, err := n.Tailscale("ip").Output()
+		if err != nil {
+			return err
+		}
+		ips = string(out)
+		return nil
+	}); err != nil {
+		t.Fatalf("awaiting an IP address: %v", err)
+	}
+	if ips == "" {
+		t.Fatalf("returned IP address was blank")
+	}
+	return ips
+}
+
+func (n *testNode) AwaitRunning(t testing.TB) {
+	t.Helper()
+	if err := tstest.WaitFor(20*time.Second, func() error {
+		st, err := n.Status()
+		if err != nil {
+			return err
+		}
+		if st.BackendState != "Running" {
+			return fmt.Errorf("in state %q", st.BackendState)
+		}
+		return nil
+	}); err != nil {
+		t.Fatalf("failure/timeout waiting for transition to Running status: %v", err)
+	}
+}
+
 // Tailscale returns a command that runs the tailscale CLI with the provided arguments.
 // It does not start the process.
 func (n *testNode) Tailscale(arg ...string) *exec.Cmd {
@@ -261,15 +439,23 @@ func (n *testNode) Tailscale(arg ...string) *exec.Cmd {
 	return cmd
 }
 
-func (n *testNode) MustStatus(tb testing.TB) *ipnstate.Status {
-	tb.Helper()
+func (n *testNode) Status() (*ipnstate.Status, error) {
 	out, err := n.Tailscale("status", "--json").CombinedOutput()
 	if err != nil {
-		tb.Fatalf("getting status: %v, %s", err, out)
+		return nil, fmt.Errorf("running tailscale status: %v, %s", err, out)
 	}
 	st := new(ipnstate.Status)
 	if err := json.Unmarshal(out, st); err != nil {
-		tb.Fatalf("parsing status json: %v, from: %s", err, out)
+		return nil, fmt.Errorf("decoding tailscale status JSON: %w", err)
+	}
+	return st, nil
+}
+
+func (n *testNode) MustStatus(tb testing.TB) *ipnstate.Status {
+	tb.Helper()
+	st, err := n.Status()
+	if err != nil {
+		tb.Fatal(err)
 	}
 	return st
 }
@@ -291,21 +477,44 @@ func findGo(t testing.TB) string {
 	} else if !fi.Mode().IsRegular() {
 		t.Fatalf("%v is unexpected %v", goBin, fi.Mode())
 	}
-	t.Logf("using go binary %v", goBin)
 	return goBin
 }
 
-func build(t testing.TB, outDir, target string) string {
-	exe := ""
-	if runtime.GOOS == "windows" {
-		exe = ".exe"
+// buildMu limits our use of "go build" to one at a time, so we don't
+// fight Go's built-in caching trying to do the same build concurrently.
+var buildMu sync.Mutex
+
+func build(t testing.TB, outDir string, targets ...string) {
+	buildMu.Lock()
+	defer buildMu.Unlock()
+
+	t0 := time.Now()
+	defer func() { t.Logf("built %s in %v", targets, time.Since(t0).Round(time.Millisecond)) }()
+
+	goBin := findGo(t)
+	cmd := exec.Command(goBin, "install")
+	if version.IsRace() {
+		cmd.Args = append(cmd.Args, "-race")
 	}
-	bin := filepath.Join(outDir, path.Base(target)) + exe
-	errOut, err := exec.Command(findGo(t), "build", "-o", bin, target).CombinedOutput()
-	if err != nil {
-		t.Fatalf("failed to build %v: %v, %s", target, err, errOut)
+	cmd.Args = append(cmd.Args, targets...)
+	cmd.Env = append(os.Environ(), "GOARCH="+runtime.GOARCH, "GOBIN="+outDir)
+	errOut, err := cmd.CombinedOutput()
+	if err == nil {
+		return
 	}
-	return bin
+	if strings.Contains(string(errOut), "when GOBIN is set") {
+		// Fallback slow path for cross-compiled binaries.
+		for _, target := range targets {
+			outFile := filepath.Join(outDir, path.Base(target)+exe())
+			cmd := exec.Command(goBin, "build", "-o", outFile, target)
+			cmd.Env = append(os.Environ(), "GOARCH="+runtime.GOARCH)
+			if errOut, err := cmd.CombinedOutput(); err != nil {
+				t.Fatalf("failed to build %v with %v: %v, %s", target, goBin, err, errOut)
+			}
+		}
+		return
+	}
+	t.Fatalf("failed to build %v with %v: %v, %s", targets, goBin, err, errOut)
 }
 
 // logCatcher is a minimal logcatcher for the logtail upload client.
@@ -378,6 +587,9 @@ func (lc *logCatcher) ServeHTTP(w http.ResponseWriter, r *http.Request) {
 	} else {
 		for _, ent := range jreq {
 			fmt.Fprintf(&lc.buf, "%s\n", strings.TrimSpace(ent.Text))
+			if testing.Verbose() {
+				fmt.Fprintf(os.Stderr, "%s\n", strings.TrimSpace(ent.Text))
+			}
 		}
 	}
 	w.WriteHeader(200) // must have no content, but not a 204
@@ -454,3 +666,152 @@ func runDERPAndStun(t testing.TB, logf logger.Logf) (derpMap *tailcfg.DERPMap, c
 
 	return m, cleanup
 }
+
+type authURLParserWriter struct {
+	buf bytes.Buffer
+	fn  func(urlStr string) error
+}
+
+var authURLRx = regexp.MustCompile(`(https?://\S+/auth/\S+)`)
+
+func (w *authURLParserWriter) Write(p []byte) (n int, err error) {
+	n, err = w.buf.Write(p)
+	m := authURLRx.FindSubmatch(w.buf.Bytes())
+	if m != nil {
+		urlStr := string(m[1])
+		w.buf.Reset() // so it's not matched again
+		if err := w.fn(urlStr); err != nil {
+			return 0, err
+		}
+	}
+	return n, err
+}
+
+type panicOnUseTransport struct{}
+
+func (panicOnUseTransport) RoundTrip(*http.Request) (*http.Response, error) {
+	panic("unexpected HTTP request")
+}
+func TestTwoNodePing(t *testing.T) {
+
+	// < --->
+	t.Parallel()
+	bins := buildTestBinaries(t)
+
+	env := newTestEnv(t, bins)
+	t.Log("Env :", env.ControlServer.URL)
+	res, err := http.Get(env.ControlServer.URL + "/ping")
+	t.Log("RESPONSE", res)
+	if err != nil {
+		t.Error(err)
+	}
+	defer env.Close()
+
+	// Create two nodes:
+	n1 := newTestNode(t, env)
+	d1 := n1.StartDaemon(t)
+	defer d1.Kill()
+
+	n2 := newTestNode(t, env)
+	d2 := n2.StartDaemon(t)
+	defer d2.Kill()
+
+	n1.AwaitListening(t)
+	n2.AwaitListening(t)
+	n1.MustUp()
+	n2.MustUp()
+	n1.AwaitRunning(t)
+	n2.AwaitRunning(t)
+	ip1 := n1.AwaitIP(t)
+	ip2 := n2.AwaitIP(t)
+	t.Logf("Node IPs : %s, %s\n", ip1, ip2)
+
+	if err := tstest.WaitFor(2*time.Second, func() error {
+		st := n1.MustStatus(t)
+		t.Log("CURPEER", len(st.Peer))
+		var peers []*ipnstate.PeerStatus
+		for _, peer := range st.Peers() {
+			ps := st.Peer[peer]
+			if ps.ShareeNode {
+				continue
+			}
+			peers = append(peers, ps)
+		}
+		jsonForm, _ := json.MarshalIndent(peers[0], "", " ")
+		t.Log("PeerStatus", string(jsonForm))
+		if len(st.Peer) == 0 {
+			return errors.New("no peers")
+		}
+		if len(st.Peer) > 1 {
+			return fmt.Errorf("got %d peers; want 1", len(st.Peer))
+		}
+		peer := st.Peer[st.Peers()[0]]
+		if peer.ID == st.Self.ID {
+			return errors.New("peer is self")
+		}
+		return nil
+	}); err != nil {
+		t.Error(err)
+	}
+
+	d1.MustCleanShutdown(t)
+	d2.MustCleanShutdown(t)
+}
+
+// Tests if our addPingRequest function works
+func TestAddPingRequest(t *testing.T) {
+
+}
+
+// Test such that we can simulate the ping instead of hardcoding in the map response
+func TestControlSelectivePing(t *testing.T) {
+	t.Parallel()
+	bins := buildTestBinaries(t)
+
+	env := newTestEnv(t, bins)
+	log.Println("POSTSTARTUP")
+	defer env.Close()
+
+	// Create two nodes:
+	n1 := newTestNode(t, env)
+	d1 := n1.StartDaemon(t)
+	defer d1.Kill()
+
+	n2 := newTestNode(t, env)
+	d2 := n2.StartDaemon(t)
+	defer d2.Kill()
+
+	n1.AwaitListening(t)
+	n2.AwaitListening(t)
+	n1.MustUp()
+	n2.MustUp()
+	n1.AwaitRunning(t)
+	n2.AwaitRunning(t)
+
+	// Wait for server to start serveMap
+	if err := tstest.WaitFor(2*time.Second, func() error {
+		t.Log("ENOUGHTIME")
+		env.Control.AddControlPingRequest()
+		if len(env.Control.PingRequestC) == 0 {
+			return errors.New("failed to add to PingRequestC")
+		}
+		log.Println("CHANNEL LENGTH", len(env.Control.PingRequestC))
+		return nil
+	}); err != nil {
+		t.Error(err)
+	}
+
+	// Wait for a MapResponse
+	if err := tstest.WaitFor(20*time.Second, func() error {
+		// Simulate the time needed for MapResponse method call.
+		time.Sleep(500 * time.Millisecond)
+		if len(env.Control.PingRequestC) == 1 {
+			t.Error("Expected PingRequestC to be empty")
+		}
+		return nil
+	}); err != nil {
+		t.Error(err)
+	}
+	d1.MustCleanShutdown(t)
+	d2.MustCleanShutdown(t)
+}

tstest/integration/testcontrol/testcontrol.go

@@ -17,6 +17,8 @@ import (
 	"log"
 	"math/rand"
 	"net/http"
+	"net/url"
+	"sort"
 	"strings"
 	"sync"
 	"time"
@@ -34,19 +36,56 @@ import (
 // Server is a control plane server. Its zero value is ready for use.
 // Everything is stored in-memory in one tailnet.
 type Server struct {
-	Logf    logger.Logf      // nil means to use the log package
-	DERPMap *tailcfg.DERPMap // nil means to use prod DERP map
+	Logf         logger.Logf      // nil means to use the log package
+	DERPMap      *tailcfg.DERPMap // nil means to use prod DERP map
+	RequireAuth  bool
+	BaseURL      string // must be set to e.g. "http://127.0.0.1:1234" with no trailing URL
+	Verbose      bool
+	PingRequestC chan bool
 
-	initMuxOnce sync.Once
-	mux         *http.ServeMux
+	initMuxOnce       sync.Once
+	mux               *http.ServeMux
+	initPRchannelOnce sync.Once
 
-	mu      sync.Mutex
-	pubKey  wgkey.Key
-	privKey wgkey.Private
-	nodes   map[tailcfg.NodeKey]*tailcfg.Node
-	users   map[tailcfg.NodeKey]*tailcfg.User
-	logins  map[tailcfg.NodeKey]*tailcfg.Login
-	updates map[tailcfg.NodeID]chan updateType
+	mu            sync.Mutex
+	pubKey        wgkey.Key
+	privKey       wgkey.Private
+	nodes         map[tailcfg.NodeKey]*tailcfg.Node
+	users         map[tailcfg.NodeKey]*tailcfg.User
+	logins        map[tailcfg.NodeKey]*tailcfg.Login
+	updates       map[tailcfg.NodeID]chan updateType
+	authPath      map[string]*AuthPath
+	nodeKeyAuthed map[tailcfg.NodeKey]bool // key => true once authenticated
+}
+
+// NumNodes returns the number of nodes in the testcontrol server.
+//
+// This is useful when connecting a bunch of virtual machines to a testcontrol
+// server to see how many of them connected successfully.
+func (s *Server) NumNodes() int {
+	s.mu.Lock()
+	defer s.mu.Unlock()
+
+	return len(s.nodes)
+}
+
+type AuthPath struct {
+	nodeKey tailcfg.NodeKey
+
+	closeOnce sync.Once
+	ch        chan struct{}
+	success   bool
+}
+
+func (ap *AuthPath) completeSuccessfully() {
+	ap.success = true
+	close(ap.ch)
+}
+
+// CompleteSuccessfully completes the login path successfully, as if
+// the user did the whole auth dance.
+func (ap *AuthPath) CompleteSuccessfully() {
+	ap.closeOnce.Do(ap.completeSuccessfully)
 }
 
 func (s *Server) logf(format string, a ...interface{}) {
@@ -58,14 +97,26 @@ func (s *Server) logf(format string, a ...interface{}) {
 }
 
 func (s *Server) initMux() {
+	log.Println("Mux inited")
 	s.mux = http.NewServeMux()
 	s.mux.HandleFunc("/", s.serveUnhandled)
 	s.mux.HandleFunc("/key", s.serveKey)
 	s.mux.HandleFunc("/machine/", s.serveMachine)
+	s.mux.HandleFunc("/ping", s.receivePingInfo)
+	s.mux.HandleFunc("/mockpingrequest", s.serveMockPing)
+}
+
+func (s *Server) initPingRequestC() {
+	log.Println("Channel created")
+	s.PingRequestC = make(chan bool, 1)
+	// s.AddControlPingRequest()
+	// log.Println("Channel length : ", len(s.PingRequestC))
 }
 
 func (s *Server) ServeHTTP(w http.ResponseWriter, r *http.Request) {
+	log.Println("HTTPSERVE")
 	s.initMuxOnce.Do(s.initMux)
+	s.initPRchannelOnce.Do(s.initPingRequestC)
 	s.mux.ServeHTTP(w, r)
 }
 
@@ -75,6 +126,12 @@ func (s *Server) serveUnhandled(w http.ResponseWriter, r *http.Request) {
 	go panic(fmt.Sprintf("testcontrol.Server received unhandled request: %s", got.Bytes()))
 }
 
+func (s *Server) serveMockPing(w http.ResponseWriter, r *http.Request) {
+	w.Header().Set("Content-Type", "text/plain")
+	s.AddControlPingRequest()
+	io.WriteString(w, "A ControlPingRequest has been queued for our next MapResponse.")
+}
+
 func (s *Server) publicKey() wgkey.Key {
 	pub, _ := s.keyPair()
 	return pub
@@ -142,6 +199,28 @@ func (s *Server) Node(nodeKey tailcfg.NodeKey) *tailcfg.Node {
 	return s.nodes[nodeKey].Clone()
 }
 
+func (s *Server) AllNodes() (nodes []*tailcfg.Node) {
+	s.mu.Lock()
+	defer s.mu.Unlock()
+	for _, n := range s.nodes {
+		nodes = append(nodes, n.Clone())
+	}
+	sort.Slice(nodes, func(i, j int) bool {
+		return nodes[i].StableID < nodes[j].StableID
+	})
+	return nodes
+}
+
+// AddControlPingRequest enqueues a bool to PingRequestC.
+// in serveMap this will result to a ControlPingRequest
+// added to the next MapResponse sent to the client
+func (s *Server) AddControlPingRequest() {
+	// Redundant check to avoid errors when called multiple times
+	if len(s.PingRequestC) == 0 {
+		s.PingRequestC <- true
+	}
+}
+
 func (s *Server) getUser(nodeKey tailcfg.NodeKey) (*tailcfg.User, *tailcfg.Login) {
 	s.mu.Lock()
 	defer s.mu.Unlock()
@@ -178,7 +257,58 @@ func (s *Server) getUser(nodeKey tailcfg.NodeKey) (*tailcfg.User, *tailcfg.Login
 	return user, login
 }
 
+// authPathDone returns a close-only struct that's closed when the
+// authPath ("/auth/XXXXXX") has authenticated.
+func (s *Server) authPathDone(authPath string) <-chan struct{} {
+	s.mu.Lock()
+	defer s.mu.Unlock()
+	if a, ok := s.authPath[authPath]; ok {
+		return a.ch
+	}
+	return nil
+}
+
+func (s *Server) addAuthPath(authPath string, nodeKey tailcfg.NodeKey) {
+	s.mu.Lock()
+	defer s.mu.Unlock()
+	if s.authPath == nil {
+		s.authPath = map[string]*AuthPath{}
+	}
+	s.authPath[authPath] = &AuthPath{
+		ch:      make(chan struct{}),
+		nodeKey: nodeKey,
+	}
+}
+
+// CompleteAuth marks the provided path or URL (containing
+// "/auth/...")  as successfully authenticated, unblocking any
+// requests blocked on that in serveRegister.
+func (s *Server) CompleteAuth(authPathOrURL string) bool {
+	i := strings.Index(authPathOrURL, "/auth/")
+	if i == -1 {
+		return false
+	}
+	authPath := authPathOrURL[i:]
+
+	s.mu.Lock()
+	defer s.mu.Unlock()
+	ap, ok := s.authPath[authPath]
+	if !ok {
+		return false
+	}
+	if ap.nodeKey.IsZero() {
+		panic("zero AuthPath.NodeKey")
+	}
+	if s.nodeKeyAuthed == nil {
+		s.nodeKeyAuthed = map[tailcfg.NodeKey]bool{}
+	}
+	s.nodeKeyAuthed[ap.nodeKey] = true
+	ap.CompleteSuccessfully()
+	return true
+}
+
 func (s *Server) serveRegister(w http.ResponseWriter, r *http.Request, mkey tailcfg.MachineKey) {
+	log.Println("SERVE REGISTER CALLED")
 	var req tailcfg.RegisterRequest
 	if err := s.decode(mkey, r.Body, &req); err != nil {
 		panic(fmt.Sprintf("serveRegister: decode: %v", err))
@@ -189,28 +319,71 @@ func (s *Server) serveRegister(w http.ResponseWriter, r *http.Request, mkey tail
 	if req.NodeKey.IsZero() {
 		panic("serveRegister: request has zero node key")
 	}
+	if s.Verbose {
+		j, _ := json.MarshalIndent(req, "", "\t")
+		log.Printf("Got %T: %s", req, j)
+	}
+
+	// If this is a followup request, wait until interactive followup URL visit complete.
+	if req.Followup != "" {
+		followupURL, err := url.Parse(req.Followup)
+		if err != nil {
+			panic(err)
+		}
+		doneCh := s.authPathDone(followupURL.Path)
+		select {
+		case <-r.Context().Done():
+			return
+		case <-doneCh:
+		}
+		// TODO(bradfitz): support a side test API to mark an
+		// auth as failued so we can send an error response in
+		// some follow-ups? For now all are successes.
+	}
 
 	user, login := s.getUser(req.NodeKey)
 	s.mu.Lock()
 	if s.nodes == nil {
 		s.nodes = map[tailcfg.NodeKey]*tailcfg.Node{}
 	}
+
+	machineAuthorized := true // TODO: add Server.RequireMachineAuth
+
+	allowedIPs := []netaddr.IPPrefix{
+		netaddr.MustParseIPPrefix(fmt.Sprintf("100.64.%d.%d/32", uint8(tailcfg.NodeID(user.ID)>>8), uint8(tailcfg.NodeID(user.ID)))),
+	}
+
 	s.nodes[req.NodeKey] = &tailcfg.Node{
 		ID:                tailcfg.NodeID(user.ID),
 		StableID:          tailcfg.StableNodeID(fmt.Sprintf("TESTCTRL%08x", int(user.ID))),
 		User:              user.ID,
 		Machine:           mkey,
 		Key:               req.NodeKey,
-		MachineAuthorized: true,
+		MachineAuthorized: machineAuthorized,
+		Addresses:         allowedIPs,
+		AllowedIPs:        allowedIPs,
+	}
+	requireAuth := s.RequireAuth
+	if requireAuth && s.nodeKeyAuthed[req.NodeKey] {
+		requireAuth = false
 	}
 	s.mu.Unlock()
 
+	authURL := ""
+	if requireAuth {
+		randHex := make([]byte, 10)
+		crand.Read(randHex)
+		authPath := fmt.Sprintf("/auth/%x", randHex)
+		s.addAuthPath(authPath, req.NodeKey)
+		authURL = s.BaseURL + authPath
+	}
+
 	res, err := s.encode(mkey, false, tailcfg.RegisterResponse{
 		User:              *user,
 		Login:             *login,
 		NodeKeyExpired:    false,
-		MachineAuthorized: true,
-		AuthURL:           "", // all good; TODO(bradfitz): add ways to not start all good.
+		MachineAuthorized: machineAuthorized,
+		AuthURL:           authURL,
 	})
 	if err != nil {
 		go panic(fmt.Sprintf("serveRegister: encode: %v", err))
@@ -239,6 +412,24 @@ func (s *Server) updateLocked(source string, peers []tailcfg.NodeID) {
 	}
 }
 
+// Adds a PingRequest to a MapResponse, we will ping the first peer.
+func (s *Server) addPingRequest(res *tailcfg.MapResponse) error {
+	if len(res.Peers) == 0 {
+		return errors.New("MapResponse has no peers to ping")
+	}
+
+	if len(res.Peers[0].Addresses) == 0 || len(res.Peers[0].AllowedIPs) == 0 {
+		return errors.New("peer has no Addresses or no AllowedIPs")
+	}
+	targetIP := res.Peers[0].AllowedIPs[0].IP()
+	res.PingRequest = &tailcfg.PingRequest{URL: s.BaseURL + "/ping", TestIP: targetIP, Types: "tsmp"}
+	// jsonRes, _ := json.MarshalIndent(res, "", " ")
+	// log.Println("jsonprint", string(jsonRes))
+	// log.Println("respeers", res.Peers)
+	// log.Println("allnodes", s.AllNodes(), res.Node.AllowedIPs)
+	return nil
+}
+
 // sendUpdate sends updateType to dst if dst is non-nil and
 // has capacity.
 func sendUpdate(dst chan<- updateType, updateType updateType) {
@@ -254,7 +445,23 @@ func sendUpdate(dst chan<- updateType, updateType updateType) {
 	}
 }
 
+func (s *Server) UpdateNode(n *tailcfg.Node) (peersToUpdate []tailcfg.NodeID) {
+	s.mu.Lock()
+	defer s.mu.Unlock()
+	if n.Key.IsZero() {
+		panic("zero nodekey")
+	}
+	s.nodes[n.Key] = n.Clone()
+	for _, n2 := range s.nodes {
+		if n.ID != n2.ID {
+			peersToUpdate = append(peersToUpdate, n2.ID)
+		}
+	}
+	return peersToUpdate
+}
+
 func (s *Server) serveMap(w http.ResponseWriter, r *http.Request, mkey tailcfg.MachineKey) {
+	log.Println("SERVEMAP CALLED")
 	ctx := r.Context()
 
 	req := new(tailcfg.MapRequest)
@@ -279,10 +486,8 @@ func (s *Server) serveMap(w http.ResponseWriter, r *http.Request, mkey tailcfg.M
 	if !req.ReadOnly {
 		endpoints := filterInvalidIPv6Endpoints(req.Endpoints)
 		node.Endpoints = endpoints
-		// TODO: more
-		// TODO: register node,
-		//s.UpdateEndpoint(mkey, req.NodeKey,
-		// XXX
+		node.DiscoKey = req.DiscoKey
+		peersToUpdate = s.UpdateNode(node)
 	}
 
 	nodeID := node.ID
@@ -309,9 +514,20 @@ func (s *Server) serveMap(w http.ResponseWriter, r *http.Request, mkey tailcfg.M
 	streaming := req.Stream && !req.ReadOnly
 	compress := req.Compress != ""
 
+	log.Println("CREATED MAPREQ", *req)
+	log.Println("REQUEST", r)
+	log.Println("REQBODY", r.Body)
 	w.WriteHeader(200)
 	for {
 		res, err := s.MapResponse(req)
+		log.Println("LENGTHER", len(s.PingRequestC))
+		select {
+		case <-s.PingRequestC:
+			log.Println("PINGADD", len(s.PingRequestC))
+			s.addPingRequest(res)
+		default:
+			log.Println("NOTEXIST")
+		}
 		if err != nil {
 			// TODO: log
 			return
@@ -372,6 +588,7 @@ var prodDERPMap = derpmap.Prod()
 //
 // No updates to s are done here.
 func (s *Server) MapResponse(req *tailcfg.MapRequest) (res *tailcfg.MapResponse, err error) {
+	log.Println("MAPREQUEST : ", string(JsonPrint(req)))
 	node := s.Node(req.NodeKey)
 	if node == nil {
 		// node key rotated away (once test server supports that)
@@ -389,10 +606,17 @@ func (s *Server) MapResponse(req *tailcfg.MapRequest) (res *tailcfg.MapResponse,
 		CollectServices: "true",
 		PacketFilter:    tailcfg.FilterAllowAll,
 	}
+	for _, p := range s.AllNodes() {
+		if p.StableID != node.StableID {
+			res.Peers = append(res.Peers, p)
+		}
+	}
+
 	res.Node.Addresses = []netaddr.IPPrefix{
 		netaddr.MustParseIPPrefix(fmt.Sprintf("100.64.%d.%d/32", uint8(node.ID>>8), uint8(node.ID))),
 	}
 	res.Node.AllowedIPs = res.Node.Addresses
+
 	return res, nil
 }
 
@@ -506,7 +730,7 @@ func keepClientEndpoint(ep string) bool {
 		// the incoming JSON response.
 		return false
 	}
-	ip := ipp.IP
+	ip := ipp.IP()
 	if ip.Zone() != "" {
 		return false
 	}
@@ -543,3 +767,24 @@ func breakSameNodeMapResponseStreams(req *tailcfg.MapRequest) bool {
 	}
 	return true
 }
+
+// This is where the PUT requests will go
+func (s *Server) receivePingInfo(w http.ResponseWriter, r *http.Request) {
+	if r.Method != "PUT" {
+		log.Println("Received NON PUT request, should panic if this happens after")
+		// panic("Only PUT requests are supported currently")
+	}
+	w.Header().Set("Content-Type", "text/plain")
+	reqBody, err := ioutil.ReadAll(r.Body)
+	if err != nil {
+		panic("Failed to read request body")
+	}
+	log.Println("Ping Info Received", string(reqBody))
+	w.WriteHeader(200)
+	io.WriteString(w, "Ping Streamed Back : "+string(reqBody))
+}
+
+func JsonPrint(item interface{}) []byte {
+	res, _ := json.MarshalIndent(item, "", " ")
+	return res
+}

tstest/integration/vms/doc.go

@@ -0,0 +1,7 @@
+// Copyright (c) 2021 Tailscale Inc & AUTHORS All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+// Package vms does VM-based integration/functional tests by using
+// qemu and a bank of pre-made VM images.
+package vms

tstest/integration/vms/vms_test.go

@@ -0,0 +1,532 @@
+// Copyright (c) 2021 Tailscale Inc & AUTHORS All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+// +build linux
+
+package vms
+
+import (
+	"crypto/sha256"
+	"encoding/hex"
+	"flag"
+	"fmt"
+	"io"
+	"net"
+	"net/http"
+	"os"
+	"os/exec"
+	"path"
+	"path/filepath"
+	"regexp"
+	"strings"
+	"sync"
+	"syscall"
+	"testing"
+	"text/template"
+	"time"
+
+	expect "github.com/google/goexpect"
+	"golang.org/x/crypto/ssh"
+	"inet.af/netaddr"
+	"tailscale.com/tstest/integration/testcontrol"
+)
+
+var runVMTests = flag.Bool("run-vm-tests", false, "if set, run expensive (10G+ ram) VM based integration tests")
+
+type Distro struct {
+	name           string // amazon-linux
+	url            string // URL to a qcow2 image
+	sha256sum      string // hex-encoded sha256 sum of contents of URL
+	mem            int    // VM memory in megabytes
+	packageManager string // yum/apt/dnf/zypper
+}
+
+func (d *Distro) InstallPre() string {
+	switch d.packageManager {
+	case "yum":
+		return ` - [ yum, update, gnupg2 ]
+`
+	case "apt":
+		return ` - [ apt-get, update ]
+ - [ apt-get, "-y", install, curl, "apt-transport-https", gnupg2 ]
+`
+	}
+
+	return ""
+}
+
+// fetchDistro fetches a distribution from the internet if it doesn't already exist locally. It
+// also validates the sha256 sum from a known good hash.
+func fetchDistro(t *testing.T, resultDistro Distro) {
+	t.Helper()
+
+	cdir, err := os.UserCacheDir()
+	if err != nil {
+		t.Fatalf("can't find cache dir: %v", err)
+	}
+	cdir = filepath.Join(cdir, "tailscale", "vm-test")
+
+	qcowPath := filepath.Join(cdir, "qcow2", resultDistro.sha256sum)
+
+	_, err = os.Stat(qcowPath)
+	if err != nil {
+		t.Logf("downloading distro image %s to %s", resultDistro.url, qcowPath)
+		fout, err := os.Create(qcowPath)
+		if err != nil {
+			t.Fatal(err)
+		}
+		resp, err := http.Get(resultDistro.url)
+		if err != nil {
+			t.Fatalf("can't fetch qcow2 for %s (%s): %v", resultDistro.name, resultDistro.url, err)
+		}
+
+		if resp.StatusCode != http.StatusOK {
+			resp.Body.Close()
+			t.Fatalf("%s replied %s", resultDistro.url, resp.Status)
+		}
+
+		_, err = io.Copy(fout, resp.Body)
+		resp.Body.Close()
+		if err != nil {
+			t.Fatalf("download of %s failed: %v", resultDistro.url, err)
+		}
+
+		err = fout.Close()
+		if err != nil {
+			t.Fatalf("can't close fout: %v", err)
+		}
+
+		fin, err := os.Open(qcowPath)
+		if err != nil {
+			t.Fatal(err)
+		}
+
+		hasher := sha256.New()
+		if _, err := io.Copy(hasher, fin); err != nil {
+			t.Fatal(err)
+		}
+		hash := hex.EncodeToString(hasher.Sum(nil))
+
+		if hash != resultDistro.sha256sum {
+			t.Logf("got:  %q", hash)
+			t.Logf("want: %q", resultDistro.sha256sum)
+			t.Fatal("hash mismatch, someone is doing something nasty")
+		}
+
+		t.Logf("hash check passed (%s)", resultDistro.sha256sum)
+	}
+}
+
+// run runs a command or fails the test.
+func run(t *testing.T, dir, prog string, args ...string) {
+	t.Helper()
+	t.Logf("running: %s %s", prog, strings.Join(args, " "))
+
+	cmd := exec.Command(prog, args...)
+	cmd.Stdout = os.Stdout
+	cmd.Stderr = os.Stderr
+	cmd.Dir = dir
+	if err := cmd.Run(); err != nil {
+		t.Fatal(err)
+	}
+}
+
+// mkLayeredQcow makes a layered qcow image that allows us to keep the upstream VM images
+// pristine and only do our changes on an overlay.
+func mkLayeredQcow(t *testing.T, tdir string, d Distro) {
+	t.Helper()
+
+	cdir, err := os.UserCacheDir()
+	if err != nil {
+		t.Fatalf("can't find cache dir: %v", err)
+	}
+	cdir = filepath.Join(cdir, "tailscale", "vm-test")
+
+	run(t, tdir, "qemu-img", "create",
+		"-f", "qcow2",
+		"-o", "backing_file="+filepath.Join(cdir, "qcow2", d.sha256sum),
+		filepath.Join(tdir, d.name+".qcow2"),
+	)
+}
+
+// mkSeed makes the cloud-init seed ISO that is used to configure a VM with tailscale.
+func mkSeed(t *testing.T, d Distro, sshKey, hostURL, tdir string, port int) {
+	t.Helper()
+
+	dir := filepath.Join(tdir, d.name, "seed")
+	os.MkdirAll(dir, 0700)
+
+	// make meta-data
+	{
+		fout, err := os.Create(filepath.Join(dir, "meta-data"))
+		if err != nil {
+			t.Fatal(err)
+		}
+
+		err = template.Must(template.New("meta-data.yaml").Parse(metaDataTemplate)).Execute(fout, struct {
+			ID       string
+			Hostname string
+		}{
+			ID:       "31337",
+			Hostname: d.name,
+		})
+		if err != nil {
+			t.Fatal(err)
+		}
+
+		err = fout.Close()
+		if err != nil {
+			t.Fatal(err)
+		}
+	}
+
+	// make user-data
+	{
+		fout, err := os.Create(filepath.Join(dir, "user-data"))
+		if err != nil {
+			t.Fatal(err)
+		}
+
+		err = template.Must(template.New("user-data.yaml").Parse(userDataTemplate)).Execute(fout, struct {
+			SSHKey     string
+			HostURL    string
+			Hostname   string
+			Port       int
+			InstallPre string
+		}{
+			SSHKey:     strings.TrimSpace(sshKey),
+			HostURL:    hostURL,
+			Hostname:   d.name,
+			Port:       port,
+			InstallPre: d.InstallPre(),
+		})
+		if err != nil {
+			t.Fatal(err)
+		}
+
+		err = fout.Close()
+		if err != nil {
+			t.Fatal(err)
+		}
+	}
+
+	run(t, tdir, "genisoimage",
+		"-output", filepath.Join(dir, "seed.iso"),
+		"-volid", "cidata", "-joliet", "-rock",
+		filepath.Join(dir, "meta-data"),
+		filepath.Join(dir, "user-data"),
+	)
+}
+
+// mkVM makes a KVM-accelerated virtual machine and prepares it for introduction to the
+// testcontrol server. The function it returns is for killing the virtual machine when it
+// is time for it to die.
+func mkVM(t *testing.T, n int, d Distro, sshKey, hostURL, tdir string) func() {
+	t.Helper()
+
+	cdir, err := os.UserCacheDir()
+	if err != nil {
+		t.Fatalf("can't find cache dir: %v", err)
+	}
+	cdir = filepath.Join(cdir, "within", "mkvm")
+	os.MkdirAll(filepath.Join(cdir, "qcow2"), 0755)
+	os.MkdirAll(filepath.Join(cdir, "seed"), 0755)
+
+	port := 23100 + n
+
+	fetchDistro(t, d)
+	mkLayeredQcow(t, tdir, d)
+	mkSeed(t, d, sshKey, hostURL, tdir, port)
+
+	driveArg := fmt.Sprintf("file=%s,if=virtio", filepath.Join(tdir, d.name+".qcow2"))
+
+	args := []string{
+		"-machine", "pc-q35-5.1,accel=kvm,usb=off,vmport=off,dump-guest-core=off",
+		"-netdev", fmt.Sprintf("user,hostfwd=::%d-:22,id=net0", port),
+		"-device", "virtio-net-pci,netdev=net0,id=net0,mac=8a:28:5c:30:1f:25",
+		"-m", fmt.Sprint(d.mem),
+		"-boot", "c",
+		"-drive", driveArg,
+		"-cdrom", filepath.Join(tdir, d.name, "seed", "seed.iso"),
+		"-vnc", fmt.Sprintf(":%d", n),
+	}
+
+	t.Logf("running: qemu-system-x86_64 %s", strings.Join(args, " "))
+
+	cmd := exec.Command("qemu-system-x86_64", args...)
+	err = cmd.Start()
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	time.Sleep(time.Second)
+
+	if err := cmd.Process.Signal(syscall.Signal(0)); err != nil {
+		t.Fatal("qemu is not running")
+	}
+
+	return func() {
+		err := cmd.Process.Kill()
+		if err != nil {
+			t.Errorf("can't kill %s (%d): %v", d.name, cmd.Process.Pid, err)
+		}
+	}
+}
+
+// TestVMIntegrationEndToEnd creates a virtual machine with mkvm(1X), installs tailscale on it and then ensures that it connects to the network successfully.
+func TestVMIntegrationEndToEnd(t *testing.T) {
+	if !*runVMTests {
+		t.Skip("not running integration tests (need -run-vm-tests)")
+	}
+
+	if _, err := exec.LookPath("qemu-system-x86_64"); err != nil {
+		t.Logf("hint: nix-shell -p go -p qemu -p cdrkit --run 'go test -v -timeout=60m -run-vm-tests'")
+		t.Fatalf("missing dependency: %v", err)
+	}
+
+	if _, err := exec.LookPath("genisoimage"); err != nil {
+		t.Logf("hint: nix-shell -p go -p qemu -p cdrkit --run 'go test -v -timeout=60m -run-vm-tests'")
+		t.Fatalf("missing dependency: %v", err)
+	}
+
+	distros := []Distro{
+		{"amazon-linux", "https://cdn.amazonlinux.com/os-images/2.0.20210427.0/kvm/amzn2-kvm-2.0.20210427.0-x86_64.xfs.gpt.qcow2", "6ef9daef32cec69b2d0088626ec96410cd24afc504d57278bbf2f2ba2b7e529b", 512, "yum"},
+		{"centos-7", "https://cloud.centos.org/centos/7/images/CentOS-7-x86_64-GenericCloud.qcow2", "1db30c9c272fb37b00111b93dcebff16c278384755bdbe158559e9c240b73b80", 512, "yum"},
+		{"centos-8", "https://cloud.centos.org/centos/8/x86_64/images/CentOS-8-GenericCloud-8.3.2011-20201204.2.x86_64.qcow2", "7ec97062618dc0a7ebf211864abf63629da1f325578868579ee70c495bed3ba0", 768, "dnf"},
+		{"debian-9", "https://cdimage.debian.org/cdimage/openstack/9.13.21-20210511/debian-9.13.21-20210511-openstack-amd64.qcow2", "0667a08e2d947b331aee068db4bbf3a703e03edaf5afa52e23d534adff44b62a", 512, "apt"},
+		{"debian-10", "https://cdimage.debian.org/images/cloud/buster/20210329-591/debian-10-generic-amd64-20210329-591.qcow2", "70c61956095870c4082103d1a7a1cb5925293f8405fc6cb348588ec97e8611b0", 768, "apt"},
+		{"fedora-34", "https://download.fedoraproject.org/pub/fedora/linux/releases/34/Cloud/x86_64/images/Fedora-Cloud-Base-34-1.2.x86_64.qcow2", "b9b621b26725ba95442d9a56cbaa054784e0779a9522ec6eafff07c6e6f717ea", 768, "dnf"},
+		{"opensuse-leap-15.1", "https://download.opensuse.org/repositories/Cloud:/Images:/Leap_15.1/images/openSUSE-Leap-15.1-OpenStack.x86_64.qcow2", "3203e256dab5981ca3301408574b63bc522a69972fbe9850b65b54ff44a96e0a", 512, "zypper"},
+		{"opensuse-leap-15.2", "https://download.opensuse.org/repositories/Cloud:/Images:/Leap_15.2/images/openSUSE-Leap-15.2-OpenStack.x86_64.qcow2", "4df9cee9281d1f57d20f79dc65d76e255592b904760e73c0dd44ac753a54330f", 512, "zypper"},
+		{"opensuse-tumbleweed", "https://download.opensuse.org/tumbleweed/appliances/openSUSE-Tumbleweed-JeOS.x86_64-OpenStack-Cloud.qcow2", "ba3ecd281045b5019f0fb11378329a644a41870b77631ea647b128cd07eb804b", 512, "zypper"},
+		{"ubuntu-16-04", "https://cloud-images.ubuntu.com/xenial/current/xenial-server-cloudimg-amd64-disk1.img", "50a21bc067c05e0c73bf5d8727ab61152340d93073b3dc32eff18b626f7d813b", 512, "apt"},
+		{"ubuntu-18-04", "https://cloud-images.ubuntu.com/bionic/current/bionic-server-cloudimg-amd64.img", "08396cf95c18534a2e3f88289bd92d18eee76f0e75813636b3ab9f1e603816d7", 512, "apt"},
+		{"ubuntu-20-04", "https://cloud-images.ubuntu.com/focal/current/focal-server-cloudimg-amd64.img", "513158b22ff0f08d0a078d8d60293bcddffdb17094a7809c76c52aba415ecc54", 512, "apt"},
+		{"ubuntu-20-10", "https://cloud-images.ubuntu.com/groovy/current/groovy-server-cloudimg-amd64.img", "e470df72fce4fb8d0ee4ef8af8eed740ee3bf51290515eb42e5c747725e98b6d", 512, "apt"},
+		{"ubuntu-21-04", "https://cloud-images.ubuntu.com/hirsute/current/hirsute-server-cloudimg-amd64.img", "7fab8eda0bcf6f8f6e63845ccf1e29de4706e3359c82d3888835093020fe6f05", 512, "apt"},
+	}
+
+	dir := t.TempDir()
+
+	ln, err := net.Listen("tcp", deriveBindhost(t)+":0")
+	if err != nil {
+		t.Fatalf("can't make TCP listener: %v", err)
+	}
+	defer ln.Close()
+	t.Logf("host:port: %s", ln.Addr())
+
+	cs := &testcontrol.Server{}
+
+	var (
+		ipMu  sync.Mutex
+		ipMap = map[string]string{} // SSH port => IP address
+	)
+
+	mux := http.NewServeMux()
+	mux.Handle("/", cs)
+
+	// This handler will let the virtual machines tell the host information about that VM.
+	// This is used to maintain a list of port->IP address mappings that are known to be
+	// working. This allows later steps to connect over SSH. This returns no response to
+	// clients because no response is needed.
+	mux.HandleFunc("/myip/", func(w http.ResponseWriter, r *http.Request) {
+		ipMu.Lock()
+		defer ipMu.Unlock()
+
+		name := path.Base(r.URL.Path)
+		host, _, _ := net.SplitHostPort(r.RemoteAddr)
+		ipMap[name] = host
+		t.Logf("%s: %v", name, host)
+	})
+
+	hs := &http.Server{Handler: mux}
+	go hs.Serve(ln)
+
+	run(t, dir, "ssh-keygen", "-t", "ed25519", "-f", "machinekey", "-N", ``)
+	pubkey, err := os.ReadFile(filepath.Join(dir, "machinekey.pub"))
+	if err != nil {
+		t.Fatalf("can't read ssh key: %v", err)
+	}
+
+	privateKey, err := os.ReadFile(filepath.Join(dir, "machinekey"))
+	if err != nil {
+		t.Fatalf("can't read ssh private key: %v", err)
+	}
+
+	signer, err := ssh.ParsePrivateKey(privateKey)
+	if err != nil {
+		t.Fatalf("can't parse private key: %v", err)
+	}
+
+	loginServer := fmt.Sprintf("http://%s", ln.Addr())
+	t.Logf("loginServer: %s", loginServer)
+
+	cancels := make(chan func(), len(distros))
+
+	t.Run("mkvm", func(t *testing.T) {
+		for n, distro := range distros {
+			n, distro := n, distro
+			t.Run(distro.name, func(t *testing.T) {
+				t.Parallel()
+
+				cancel := mkVM(t, n, distro, string(pubkey), loginServer, dir)
+				cancels <- cancel
+			})
+		}
+	})
+
+	close(cancels)
+	for cancel := range cancels {
+		//lint:ignore SA9001 They do actually get ran
+		defer cancel()
+
+		if len(cancels) == 0 {
+			t.Log("all VMs started")
+			break
+		}
+	}
+
+	t.Run("wait-for-vms", func(t *testing.T) {
+		t.Log("waiting for VMs to register")
+		waiter := time.NewTicker(time.Second)
+		defer waiter.Stop()
+		n := 0
+		for {
+			<-waiter.C
+			ipMu.Lock()
+			if len(ipMap) == len(distros) {
+				ipMu.Unlock()
+				break
+			} else {
+				if n%30 == 0 {
+					t.Logf("ipMap:   %d", len(ipMap))
+					t.Logf("distros: %d", len(distros))
+				}
+			}
+			n++
+			ipMu.Unlock()
+		}
+	})
+
+	ipMu.Lock()
+	defer ipMu.Unlock()
+	t.Run("join-net", func(t *testing.T) {
+		for port := range ipMap {
+			port := port
+			t.Run(port, func(t *testing.T) {
+				config := &ssh.ClientConfig{
+					User:            "ts",
+					Auth:            []ssh.AuthMethod{ssh.PublicKeys(signer), ssh.Password("hunter2")},
+					HostKeyCallback: ssh.InsecureIgnoreHostKey(),
+				}
+
+				cli, err := ssh.Dial("tcp", fmt.Sprintf("127.0.0.1:%s", port), config)
+				if err != nil {
+					t.Fatalf("can't dial 127.0.0.1:%s: %v", port, err)
+				}
+				defer cli.Close()
+
+				t.Parallel()
+				t.Logf("about to ssh into 127.0.0.1:%s", port)
+				timeout := 5 * time.Minute
+
+				e, _, err := expect.SpawnSSH(cli, timeout, expect.Verbose(true), expect.VerboseWriter(os.Stdout))
+				if err != nil {
+					t.Fatalf("%s: can't register a shell session: %v", port, err)
+				}
+				defer e.Close()
+
+				_, _, err = e.Expect(regexp.MustCompile(`(\$|\>)`), timeout)
+				if err != nil {
+					t.Fatalf("%s: can't get a shell: %v", port, err)
+				}
+				t.Logf("got shell for %s", port)
+				err = e.Send(fmt.Sprintf("sudo tailscale up --login-server %s\n", loginServer))
+				if err != nil {
+					t.Fatalf("%s: can't send tailscale up command: %v", port, err)
+				}
+				_, _, err = e.Expect(regexp.MustCompile(`Success.`), timeout)
+				if err != nil {
+					t.Fatalf("can't extract URL: %v", err)
+				}
+			})
+		}
+	})
+
+	if numNodes := cs.NumNodes(); numNodes != len(ipMap) {
+		t.Errorf("wanted %d nodes, got: %d", len(ipMap), numNodes)
+	}
+}
+
+func deriveBindhost(t *testing.T) string {
+	t.Helper()
+
+	ifaces, err := net.Interfaces()
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	rex := regexp.MustCompile(`^(eth|enp|wlp|wlan)`)
+
+	for _, iface := range ifaces {
+		t.Logf("found interface %s: %d", iface.Name, iface.Flags&net.FlagUp)
+		if (iface.Flags & net.FlagUp) == 0 {
+			continue
+		}
+
+		if rex.MatchString(iface.Name) {
+			addrs, err := iface.Addrs()
+			if err != nil {
+				t.Fatalf("can't get address for %s: %v", iface.Name, err)
+			}
+
+			for _, addr := range addrs {
+				return netaddr.MustParseIPPrefix(addr.String()).IP().String()
+			}
+		}
+	}
+
+	t.Fatal("can't find a bindhost")
+	return "invalid"
+}
+
+func TestDeriveBindhost(t *testing.T) {
+	t.Log(deriveBindhost(t))
+}
+
+const metaDataTemplate = `instance-id: {{.ID}}
+local-hostname: {{.Hostname}}`
+
+const userDataTemplate = `#cloud-config
+#vim:syntax=yaml
+
+cloud_config_modules:
+ - runcmd
+
+cloud_final_modules:
+ - [users-groups, always]
+ - [scripts-user, once-per-instance]
+
+users:
+  - name: ts
+    plain_text_passwd: hunter2
+    groups: [ wheel ]
+    sudo: [ "ALL=(ALL) NOPASSWD:ALL" ]
+    shell: /bin/sh
+    ssh-authorized-keys:
+    - {{.SSHKey}}
+
+write_files:
+  - path: /etc/cloud/cloud.cfg.d/80_disable_network_after_firstboot.cfg
+    content: |
+      # Disable network configuration after first boot
+      network:
+        config: disabled
+
+runcmd:
+{{.InstallPre}}
+ - [ "sh", "-c", "curl https://raw.githubusercontent.com/tailscale/tailscale/Xe/test-install-script-libvirtd/scripts/installer.sh | sh" ]
+ - [ systemctl, enable, --now, tailscaled.service ]
+ - [ curl, "{{.HostURL}}/myip/{{.Port}}", "-H", "User-Agent: {{.Hostname}}" ]
+`

tstest/natlab/firewall.go

@@ -52,7 +52,7 @@ func (s FirewallType) key(src, dst netaddr.IPPort) fwKey {
 	switch s {
 	case EndpointIndependentFirewall:
 	case AddressDependentFirewall:
-		k.dst.IP = dst.IP
+		k.dst = k.dst.WithIP(dst.IP())
 	case AddressAndPortDependentFirewall:
 		k.dst = dst
 	default:

tstest/natlab/nat.go

@@ -62,7 +62,7 @@ func (t NATType) key(src, dst netaddr.IPPort) natKey {
 	switch t {
 	case EndpointIndependentNAT:
 	case AddressDependentNAT:
-		k.dst.IP = dst.IP
+		k.dst = k.dst.WithIP(dst.IP())
 	case AddressAndPortDependentNAT:
 		k.dst = dst
 	default:
@@ -171,7 +171,7 @@ func (n *SNAT44) HandleIn(p *Packet, iif *Interface) *Packet {
 func (n *SNAT44) HandleForward(p *Packet, iif, oif *Interface) *Packet {
 	switch {
 	case oif == n.ExternalInterface:
-		if p.Src.IP == oif.V4() {
+		if p.Src.IP() == oif.V4() {
 			// Packet already NATed and is just retraversing Forward,
 			// don't touch it again.
 			return p
@@ -237,10 +237,7 @@ func (n *SNAT44) allocateMappedPort() (net.PacketConn, netaddr.IPPort) {
 	if err != nil {
 		panic(fmt.Sprintf("ran out of NAT ports: %v", err))
 	}
-	addr := netaddr.IPPort{
-		IP:   ip,
-		Port: uint16(pc.LocalAddr().(*net.UDPAddr).Port),
-	}
+	addr := netaddr.IPPortFrom(ip, uint16(pc.LocalAddr().(*net.UDPAddr).Port))
 	return pc, addr
 }
 

tstest/natlab/natlab.go

@@ -138,7 +138,7 @@ func (n *Network) allocIPv4(iface *Interface) netaddr.IP {
 		return netaddr.IP{}
 	}
 	if n.lastV4.IsZero() {
-		n.lastV4 = n.Prefix4.IP
+		n.lastV4 = n.Prefix4.IP()
 	}
 	a := n.lastV4.As16()
 	addOne(&a, 15)
@@ -157,7 +157,7 @@ func (n *Network) allocIPv6(iface *Interface) netaddr.IP {
 		return netaddr.IP{}
 	}
 	if n.lastV6.IsZero() {
-		n.lastV6 = n.Prefix6.IP
+		n.lastV6 = n.Prefix6.IP()
 	}
 	a := n.lastV6.As16()
 	addOne(&a, 15)
@@ -183,15 +183,15 @@ func (n *Network) write(p *Packet) (num int, err error) {
 
 	n.mu.Lock()
 	defer n.mu.Unlock()
-	iface, ok := n.machine[p.Dst.IP]
+	iface, ok := n.machine[p.Dst.IP()]
 	if !ok {
 		// If the destination is within the network's authoritative
 		// range, no route to host.
-		if p.Dst.IP.Is4() && n.Prefix4.Contains(p.Dst.IP) {
+		if p.Dst.IP().Is4() && n.Prefix4.Contains(p.Dst.IP()) {
 			p.Trace("no route to %v", p.Dst.IP)
 			return len(p.Payload), nil
 		}
-		if p.Dst.IP.Is6() && n.Prefix6.Contains(p.Dst.IP) {
+		if p.Dst.IP().Is6() && n.Prefix6.Contains(p.Dst.IP()) {
 			p.Trace("no route to %v", p.Dst.IP)
 			return len(p.Payload), nil
 		}
@@ -363,7 +363,7 @@ func (m *Machine) isLocalIP(ip netaddr.IP) bool {
 func (m *Machine) deliverIncomingPacket(p *Packet, iface *Interface) {
 	p.setLocator("mach=%s if=%s", m.Name, iface.name)
 
-	if m.isLocalIP(p.Dst.IP) {
+	if m.isLocalIP(p.Dst.IP()) {
 		m.deliverLocalPacket(p, iface)
 	} else {
 		m.forwardPacket(p, iface)
@@ -391,13 +391,13 @@ func (m *Machine) deliverLocalPacket(p *Packet, iface *Interface) {
 	defer m.mu.Unlock()
 
 	conns := m.conns4
-	if p.Dst.IP.Is6() {
+	if p.Dst.IP().Is6() {
 		conns = m.conns6
 	}
 	possibleDsts := []netaddr.IPPort{
 		p.Dst,
-		netaddr.IPPort{IP: v6unspec, Port: p.Dst.Port},
-		netaddr.IPPort{IP: v4unspec, Port: p.Dst.Port},
+		netaddr.IPPortFrom(v6unspec, p.Dst.Port()),
+		netaddr.IPPortFrom(v4unspec, p.Dst.Port()),
 	}
 	for _, dest := range possibleDsts {
 		c, ok := conns[dest]
@@ -417,7 +417,7 @@ func (m *Machine) deliverLocalPacket(p *Packet, iface *Interface) {
 }
 
 func (m *Machine) forwardPacket(p *Packet, iif *Interface) {
-	oif, err := m.interfaceForIP(p.Dst.IP)
+	oif, err := m.interfaceForIP(p.Dst.IP())
 	if err != nil {
 		p.Trace("%v", err)
 		return
@@ -501,7 +501,7 @@ func (m *Machine) Attach(interfaceName string, n *Network) *Interface {
 		}
 	}
 	sort.Slice(m.routes, func(i, j int) bool {
-		return m.routes[i].prefix.Bits > m.routes[j].prefix.Bits
+		return m.routes[i].prefix.Bits() > m.routes[j].prefix.Bits()
 	})
 
 	return f
@@ -515,33 +515,33 @@ var (
 func (m *Machine) writePacket(p *Packet) (n int, err error) {
 	p.setLocator("mach=%s", m.Name)
 
-	iface, err := m.interfaceForIP(p.Dst.IP)
+	iface, err := m.interfaceForIP(p.Dst.IP())
 	if err != nil {
 		p.Trace("%v", err)
 		return 0, err
 	}
-	origSrcIP := p.Src.IP
+	origSrcIP := p.Src.IP()
 	switch {
-	case p.Src.IP == v4unspec:
+	case p.Src.IP() == v4unspec:
 		p.Trace("assigning srcIP=%s", iface.V4())
-		p.Src.IP = iface.V4()
-	case p.Src.IP == v6unspec:
+		p.Src = p.Src.WithIP(iface.V4())
+	case p.Src.IP() == v6unspec:
 		// v6unspec in Go means "any src, but match address families"
-		if p.Dst.IP.Is6() {
+		if p.Dst.IP().Is6() {
 			p.Trace("assigning srcIP=%s", iface.V6())
-			p.Src.IP = iface.V6()
-		} else if p.Dst.IP.Is4() {
+			p.Src = p.Src.WithIP(iface.V6())
+		} else if p.Dst.IP().Is4() {
 			p.Trace("assigning srcIP=%s", iface.V4())
-			p.Src.IP = iface.V4()
+			p.Src = p.Src.WithIP(iface.V4())
 		}
 	default:
-		if !iface.Contains(p.Src.IP) {
-			err := fmt.Errorf("can't send to %v with src %v on interface %v", p.Dst.IP, p.Src.IP, iface)
+		if !iface.Contains(p.Src.IP()) {
+			err := fmt.Errorf("can't send to %v with src %v on interface %v", p.Dst.IP(), p.Src.IP(), iface)
 			p.Trace("%v", err)
 			return 0, err
 		}
 	}
-	if p.Src.IP.IsZero() {
+	if p.Src.IP().IsZero() {
 		err := fmt.Errorf("no matching address for address family for %v", origSrcIP)
 		p.Trace("%v", err)
 		return 0, err
@@ -602,12 +602,12 @@ func (m *Machine) pickEphemPort() (port uint16, err error) {
 
 func (m *Machine) portInUseLocked(port uint16) bool {
 	for ipp := range m.conns4 {
-		if ipp.Port == port {
+		if ipp.Port() == port {
 			return true
 		}
 	}
 	for ipp := range m.conns6 {
-		if ipp.Port == port {
+		if ipp.Port() == port {
 			return true
 		}
 	}
@@ -617,7 +617,7 @@ func (m *Machine) portInUseLocked(port uint16) bool {
 func (m *Machine) registerConn4(c *conn) error {
 	m.mu.Lock()
 	defer m.mu.Unlock()
-	if c.ipp.IP.Is6() && c.ipp.IP != v6unspec {
+	if c.ipp.IP().Is6() && c.ipp.IP() != v6unspec {
 		return fmt.Errorf("registerConn4 got IPv6 %s", c.ipp)
 	}
 	return registerConn(&m.conns4, c)
@@ -632,7 +632,7 @@ func (m *Machine) unregisterConn4(c *conn) {
 func (m *Machine) registerConn6(c *conn) error {
 	m.mu.Lock()
 	defer m.mu.Unlock()
-	if c.ipp.IP.Is4() {
+	if c.ipp.IP().Is4() {
 		return fmt.Errorf("registerConn6 got IPv4 %s", c.ipp)
 	}
 	return registerConn(&m.conns6, c)
@@ -707,7 +707,7 @@ func (m *Machine) ListenPacket(ctx context.Context, network, address string) (ne
 			return nil, nil
 		}
 	}
-	ipp := netaddr.IPPort{IP: ip, Port: port}
+	ipp := netaddr.IPPortFrom(ip, port)
 
 	c := &conn{
 		m:   m,

tstest/natlab/natlab_test.go

@@ -49,8 +49,8 @@ func TestSendPacket(t *testing.T) {
 	ifFoo := foo.Attach("eth0", internet)
 	ifBar := bar.Attach("enp0s1", internet)
 
-	fooAddr := netaddr.IPPort{IP: ifFoo.V4(), Port: 123}
-	barAddr := netaddr.IPPort{IP: ifBar.V4(), Port: 456}
+	fooAddr := netaddr.IPPortFrom(ifFoo.V4(), 123)
+	barAddr := netaddr.IPPortFrom(ifBar.V4(), 456)
 
 	ctx := context.Background()
 	fooPC, err := foo.ListenPacket(ctx, "udp4", fooAddr.String())
@@ -111,10 +111,10 @@ func TestMultiNetwork(t *testing.T) {
 		t.Fatal(err)
 	}
 
-	clientAddr := netaddr.IPPort{IP: ifClient.V4(), Port: 123}
-	natLANAddr := netaddr.IPPort{IP: ifNATLAN.V4(), Port: 456}
-	natWANAddr := netaddr.IPPort{IP: ifNATWAN.V4(), Port: 456}
-	serverAddr := netaddr.IPPort{IP: ifServer.V4(), Port: 789}
+	clientAddr := netaddr.IPPortFrom(ifClient.V4(), 123)
+	natLANAddr := netaddr.IPPortFrom(ifNATLAN.V4(), 456)
+	natWANAddr := netaddr.IPPortFrom(ifNATWAN.V4(), 456)
+	serverAddr := netaddr.IPPortFrom(ifServer.V4(), 789)
 
 	const msg1, msg2 = "hello", "world"
 	if _, err := natPC.WriteTo([]byte(msg1), clientAddr.UDPAddr()); err != nil {
@@ -154,8 +154,8 @@ type trivialNAT struct {
 }
 
 func (n *trivialNAT) HandleIn(p *Packet, iface *Interface) *Packet {
-	if iface == n.wanIf && p.Dst.IP == n.wanIf.V4() {
-		p.Dst.IP = n.clientIP
+	if iface == n.wanIf && p.Dst.IP() == n.wanIf.V4() {
+		p.Dst = p.Dst.WithIP(n.clientIP)
 	}
 	return p
 }
@@ -167,13 +167,13 @@ func (n trivialNAT) HandleOut(p *Packet, iface *Interface) *Packet {
 func (n *trivialNAT) HandleForward(p *Packet, iif, oif *Interface) *Packet {
 	// Outbound from LAN -> apply NAT, continue
 	if iif == n.lanIf && oif == n.wanIf {
-		if p.Src.IP == n.clientIP {
-			p.Src.IP = n.wanIf.V4()
+		if p.Src.IP() == n.clientIP {
+			p.Src = p.Src.WithIP(n.wanIf.V4())
 		}
 		return p
 	}
 	// Return traffic to LAN, allow if right dst.
-	if iif == n.wanIf && oif == n.lanIf && p.Dst.IP == n.clientIP {
+	if iif == n.wanIf && oif == n.lanIf && p.Dst.IP() == n.clientIP {
 		return p
 	}
 	// Else drop.
@@ -216,7 +216,7 @@ func TestPacketHandler(t *testing.T) {
 	}
 
 	const msg = "some message"
-	serverAddr := netaddr.IPPort{IP: ifServer.V4(), Port: 456}
+	serverAddr := netaddr.IPPortFrom(ifServer.V4(), 456)
 	if _, err := clientPC.WriteTo([]byte(msg), serverAddr.UDPAddr()); err != nil {
 		t.Fatal(err)
 	}
@@ -230,7 +230,7 @@ func TestPacketHandler(t *testing.T) {
 	if string(buf) != msg {
 		t.Errorf("read %q; want %q", buf, msg)
 	}
-	mappedAddr := netaddr.IPPort{IP: ifNATWAN.V4(), Port: 123}
+	mappedAddr := netaddr.IPPortFrom(ifNATWAN.V4(), 123)
 	if addr.String() != mappedAddr.String() {
 		t.Errorf("addr = %q; want %q", addr, mappedAddr)
 	}

tsweb/jsonhandler.go

@@ -89,3 +89,64 @@ func (fn JSONHandlerFunc) ServeHTTPReturn(w http.ResponseWriter, r *http.Request
 	w.Write(b)
 	return err
 }
+
+// TODO() Set this function such that chunk encoding works
+// Currently the same thing with chunking headers set.
+func (fn JSONHandlerFunc) ServeHTTPChunkEncodingReturn(w http.ResponseWriter, r *http.Request) error {
+	w.Header().Set("Connection", "Keep-Alive")
+	w.Header().Set("Transfer-Encoding", "chunked")
+	w.Header().Set("X-Content-Type-Options", "nosniff")
+	var resp *response
+	status, data, err := fn(r)
+	if err != nil {
+		if werr, ok := err.(HTTPError); ok {
+			resp = &response{
+				Status: "error",
+				Error:  werr.Msg,
+				Data:   data,
+			}
+			// Unwrap the HTTPError here because we are communicating with
+			// the client in this handler. We don't want the wrapping
+			// ReturnHandler to do it too.
+			err = werr.Err
+			if werr.Msg != "" {
+				err = fmt.Errorf("%s: %w", werr.Msg, err)
+			}
+			// take status from the HTTPError to encourage error handling in one location
+			if status != 0 && status != werr.Code {
+				err = fmt.Errorf("[unexpected] non-zero status that does not match HTTPError status, status: %d, HTTPError.code: %d: %w", status, werr.Code, err)
+			}
+			status = werr.Code
+		} else {
+			status = http.StatusInternalServerError
+			resp = &response{
+				Status: "error",
+				Error:  "internal server error",
+			}
+		}
+	} else if status == 0 {
+		status = http.StatusInternalServerError
+		resp = &response{
+			Status: "error",
+			Error:  "internal server error",
+		}
+	} else if err == nil {
+		resp = &response{
+			Status: "success",
+			Data:   data,
+		}
+	}
+	b, jerr := json.Marshal(resp)
+	if jerr != nil {
+		w.WriteHeader(http.StatusInternalServerError)
+		w.Write([]byte(`{"status":"error","error":"json marshal error"}`))
+		if err != nil {
+			return fmt.Errorf("%w, and then we could not respond: %v", err, jerr)
+		}
+		return jerr
+	}
+
+	w.WriteHeader(status)
+	w.Write(b)
+	return err
+}

types/netmap/netmap_test.go

@@ -250,7 +250,7 @@ func TestConciseDiffFrom(t *testing.T) {
 						DERP:       "127.3.3.40:2",
 						Endpoints:  []string{"192.168.0.100:41641", "1.1.1.1:41641"},
 						DiscoKey:   testDiscoKey("f00f00f00f"),
-						AllowedIPs: []netaddr.IPPrefix{{IP: netaddr.IPv4(100, 102, 103, 104), Bits: 32}},
+						AllowedIPs: []netaddr.IPPrefix{netaddr.IPPrefixFrom(netaddr.IPv4(100, 102, 103, 104), 32)},
 					},
 				},
 			},
@@ -263,7 +263,7 @@ func TestConciseDiffFrom(t *testing.T) {
 						DERP:       "127.3.3.40:2",
 						Endpoints:  []string{"192.168.0.100:41641", "1.1.1.1:41641"},
 						DiscoKey:   testDiscoKey("ba4ba4ba4b"),
-						AllowedIPs: []netaddr.IPPrefix{{IP: netaddr.IPv4(100, 102, 103, 104), Bits: 32}},
+						AllowedIPs: []netaddr.IPPrefix{netaddr.IPPrefixFrom(netaddr.IPv4(100, 102, 103, 104), 32)},
 					},
 				},
 			},

types/strbuilder/strbuilder.go

@@ -1,74 +0,0 @@
-// Copyright (c) 2020 Tailscale Inc & AUTHORS All rights reserved.
-// Use of this source code is governed by a BSD-style
-// license that can be found in the LICENSE file.
-
-// Package strbuilder defines a string builder type that allocates
-// less than the standard library's strings.Builder by using a
-// sync.Pool, so it doesn't matter if the compiler can't prove that
-// the builder doesn't escape into the fmt package, etc.
-package strbuilder
-
-import (
-	"bytes"
-	"strconv"
-	"sync"
-)
-
-var pool = sync.Pool{
-	New: func() interface{} { return new(Builder) },
-}
-
-type Builder struct {
-	bb      bytes.Buffer
-	scratch [20]byte // long enough for MinInt64, MaxUint64
-	locked  bool     // in pool, not for use
-}
-
-// Get returns a new or reused string Builder.
-func Get() *Builder {
-	b := pool.Get().(*Builder)
-	b.bb.Reset()
-	b.locked = false
-	return b
-}
-
-// String both returns the Builder's string, and returns the builder
-// to the pool.
-func (b *Builder) String() string {
-	if b.locked {
-		panic("String called twiced on Builder")
-	}
-	s := b.bb.String()
-	b.locked = true
-	pool.Put(b)
-	return s
-}
-
-func (b *Builder) WriteByte(v byte) error {
-	return b.bb.WriteByte(v)
-}
-
-func (b *Builder) WriteString(s string) (int, error) {
-	return b.bb.WriteString(s)
-}
-
-func (b *Builder) Write(p []byte) (int, error) {
-	return b.bb.Write(p)
-}
-
-func (b *Builder) WriteInt(v int64) {
-	b.Write(strconv.AppendInt(b.scratch[:0], v, 10))
-}
-
-func (b *Builder) WriteUint(v uint64) {
-	b.Write(strconv.AppendUint(b.scratch[:0], v, 10))
-}
-
-// Grow grows the buffer's capacity, if necessary, to guarantee space
-// for another n bytes. After Grow(n), at least n bytes can be written
-// to the buffer without another allocation. If n is negative, Grow
-// will panic. If the buffer can't grow it will panic with
-// ErrTooLarge.
-func (b *Builder) Grow(n int) {
-	b.bb.Grow(n)
-}

types/strbuilder/strbuilder_test.go

@@ -1,52 +0,0 @@
-// Copyright (c) 2020 Tailscale Inc & AUTHORS All rights reserved.
-// Use of this source code is governed by a BSD-style
-// license that can be found in the LICENSE file.
-
-package strbuilder
-
-import (
-	"math"
-	"testing"
-)
-
-func TestBuilder(t *testing.T) {
-	const want = "Hello, world 123 -456!"
-	bang := []byte("!")
-	var got string
-	allocs := testing.AllocsPerRun(1000, func() {
-		sb := Get()
-		sb.WriteString("Hello, world ")
-		sb.WriteUint(123)
-		sb.WriteByte(' ')
-		sb.WriteInt(-456)
-		sb.Write(bang)
-		got = sb.String()
-	})
-	if got != want {
-		t.Errorf("got %q; want %q", got, want)
-	}
-	if allocs != 1 {
-		t.Errorf("allocs = %v; want 1", allocs)
-	}
-}
-
-// Verifies scratch buf is large enough.
-func TestIntBounds(t *testing.T) {
-	const want = "-9223372036854775808 9223372036854775807 18446744073709551615"
-	var got string
-	allocs := testing.AllocsPerRun(1000, func() {
-		sb := Get()
-		sb.WriteInt(math.MinInt64)
-		sb.WriteByte(' ')
-		sb.WriteInt(math.MaxInt64)
-		sb.WriteByte(' ')
-		sb.WriteUint(math.MaxUint64)
-		got = sb.String()
-	})
-	if got != want {
-		t.Errorf("got %q; want %q", got, want)
-	}
-	if allocs != 1 {
-		t.Errorf("allocs = %v; want 1", allocs)
-	}
-}

types/wgkey/key.go

@@ -10,7 +10,6 @@
 package wgkey
 
 import (
-	"bytes"
 	"crypto/rand"
 	"crypto/subtle"
 	"encoding/base64"
@@ -72,14 +71,23 @@ func ParsePrivateHex(v string) (Private, error) {
 	return pk, nil
 }
 
-func (k Key) Base64() string    { return base64.StdEncoding.EncodeToString(k[:]) }
-func (k Key) String() string    { return k.ShortString() }
-func (k Key) HexString() string { return hex.EncodeToString(k[:]) }
-func (k Key) Equal(k2 Key) bool { return subtle.ConstantTimeCompare(k[:], k2[:]) == 1 }
+func (k Key) Base64() string           { return base64.StdEncoding.EncodeToString(k[:]) }
+func (k Key) String() string           { return k.ShortString() }
+func (k Key) HexString() string        { return hex.EncodeToString(k[:]) }
+func (k Key) Equal(k2 Key) bool        { return subtle.ConstantTimeCompare(k[:], k2[:]) == 1 }
+func (k Key) AppendTo(b []byte) []byte { return appendKey(b, "", k) }
 
 func (k *Key) ShortString() string {
-	long := k.Base64()
-	return "[" + long[0:5] + "]"
+	// The goal here is to generate "[" + base64.StdEncoding.EncodeToString(k[:])[:5] + "]".
+	// Since we only care about the first 5 characters, it suffices to encode the first 4 bytes of k.
+	// Encoding those 4 bytes requires 8 bytes.
+	// Make dst have size 9, to fit the leading '[' plus those 8 bytes.
+	// We slice the unused ones away at the end.
+	dst := make([]byte, 9)
+	dst[0] = '['
+	base64.StdEncoding.Encode(dst[1:], k[:4])
+	dst[6] = ']'
+	return string(dst[:7])
 }
 
 func (k *Key) IsZero() bool {
@@ -106,11 +114,10 @@ func (k *Key) UnmarshalJSON(b []byte) error {
 		return errors.New("wgkey.Key: UnmarshalJSON not given a string")
 	}
 	b = b[1 : len(b)-1]
-	key, err := ParseHex(string(b))
-	if err != nil {
-		return fmt.Errorf("wgkey.Key: UnmarshalJSON: %v", err)
+	if len(b) != 2*Size {
+		return fmt.Errorf("wgkey.Key: UnmarshalJSON input wrong size: %d", len(b))
 	}
-	copy(k[:], key[:])
+	hex.Decode(k[:], b)
 	return nil
 }
 
@@ -171,13 +178,17 @@ func (k *Private) Public() Key {
 	return (Key)(p)
 }
 
-func (k Private) MarshalText() ([]byte, error) {
-	// TODO(josharian): use encoding/hex instead?
-	buf := new(bytes.Buffer)
-	fmt.Fprintf(buf, `privkey:%x`, k[:])
-	return buf.Bytes(), nil
+func appendKey(base []byte, prefix string, k [32]byte) []byte {
+	ret := append(base, make([]byte, len(prefix)+64)...)
+	buf := ret[len(base):]
+	copy(buf, prefix)
+	hex.Encode(buf[len(prefix):], k[:])
+	return ret
 }
 
+func (k Private) MarshalText() ([]byte, error) { return appendKey(nil, "privkey:", k), nil }
+func (k Private) AppendTo(b []byte) []byte     { return appendKey(b, "privkey:", k) }
+
 func (k *Private) UnmarshalText(b []byte) error {
 	s := string(b)
 	if !strings.HasPrefix(s, `privkey:`) {

types/wgkey/key_test.go

@@ -156,3 +156,28 @@ func BenchmarkMarshalJSON(b *testing.B) {
 		}
 	}
 }
+
+func BenchmarkUnmarshalJSON(b *testing.B) {
+	b.ReportAllocs()
+	var k Key
+	buf, err := k.MarshalJSON()
+	if err != nil {
+		b.Fatal(err)
+	}
+	for i := 0; i < b.N; i++ {
+		err := k.UnmarshalJSON(buf)
+		if err != nil {
+			b.Fatal(err)
+		}
+	}
+}
+
+var sinkString string
+
+func BenchmarkShortString(b *testing.B) {
+	b.ReportAllocs()
+	var k Key
+	for i := 0; i < b.N; i++ {
+		sinkString = k.ShortString()
+	}
+}

util/dnsname/dnsname.go

@@ -24,13 +24,16 @@ func ToFQDN(s string) (FQDN, error) {
 	if isValidFQDN(s) {
 		return FQDN(s), nil
 	}
-	if len(s) == 0 {
+	if len(s) == 0 || s == "." {
 		return FQDN("."), nil
 	}
 
 	if s[len(s)-1] == '.' {
 		s = s[:len(s)-1]
 	}
+	if s[0] == '.' {
+		s = s[1:]
+	}
 	if len(s) > maxNameLength {
 		return "", fmt.Errorf("%q is too long to be a DNS name", s)
 	}

util/dnsname/dnsname_test.go

@@ -20,11 +20,12 @@ func TestFQDN(t *testing.T) {
 		{".", ".", false, 0},
 		{"foo.com", "foo.com.", false, 2},
 		{"foo.com.", "foo.com.", false, 2},
+		{".foo.com.", "foo.com.", false, 2},
+		{".foo.com", "foo.com.", false, 2},
 		{"com", "com.", false, 1},
 		{"www.tailscale.com", "www.tailscale.com.", false, 3},
 		{"aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa.com", "", true, 0},
 		{strings.Repeat("aaaaa.", 60) + "com", "", true, 0},
-		{".com", "", true, 0},
 		{"foo..com", "", true, 0},
 	}
 

version/race.go

@@ -0,0 +1,11 @@
+// Copyright (c) 2021 Tailscale Inc & AUTHORS All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+// +build race
+
+package version
+
+// IsRace reports whether the current binary was built with the Go
+// race detector enabled.
+func IsRace() bool { return true }

version/race_off.go

@@ -0,0 +1,11 @@
+// Copyright (c) 2021 Tailscale Inc & AUTHORS All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+// +build !race
+
+package version
+
+// IsRace reports whether the current binary was built with the Go
+// race detector enabled.
+func IsRace() bool { return false }

wf/firewall.go

@@ -0,0 +1,510 @@
+// Copyright (c) 2021 Tailscale Inc & AUTHORS All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+// +build windows
+
+package wf
+
+import (
+	"fmt"
+	"os"
+
+	"golang.org/x/sys/windows"
+	"inet.af/netaddr"
+	"inet.af/wf"
+)
+
+// Known addresses.
+var (
+	linkLocalRange           = netaddr.MustParseIPPrefix("ff80::/10")
+	linkLocalDHCPMulticast   = netaddr.MustParseIP("ff02::1:2")
+	siteLocalDHCPMulticast   = netaddr.MustParseIP("ff05::1:3")
+	linkLocalRouterMulticast = netaddr.MustParseIP("ff02::2")
+)
+
+type direction int
+
+const (
+	directionInbound direction = iota
+	directionOutbound
+	directionBoth
+)
+
+type protocol int
+
+const (
+	protocolV4 protocol = iota
+	protocolV6
+	protocolAll
+)
+
+// getLayers returns the wf.LayerIDs where the rules should be added based
+// on the protocol and direction.
+func (p protocol) getLayers(d direction) []wf.LayerID {
+	var layers []wf.LayerID
+	if p == protocolAll || p == protocolV4 {
+		if d == directionBoth || d == directionInbound {
+			layers = append(layers, wf.LayerALEAuthRecvAcceptV4)
+		}
+		if d == directionBoth || d == directionOutbound {
+			layers = append(layers, wf.LayerALEAuthConnectV4)
+		}
+	}
+	if p == protocolAll || p == protocolV6 {
+		if d == directionBoth || d == directionInbound {
+			layers = append(layers, wf.LayerALEAuthRecvAcceptV6)
+		}
+		if d == directionBoth || d == directionOutbound {
+			layers = append(layers, wf.LayerALEAuthConnectV6)
+		}
+	}
+	return layers
+}
+
+func ruleName(action wf.Action, l wf.LayerID, name string) string {
+	switch l {
+	case wf.LayerALEAuthConnectV4:
+		return fmt.Sprintf("%s outbound %s (IPv4)", action, name)
+	case wf.LayerALEAuthConnectV6:
+		return fmt.Sprintf("%s outbound %s (IPv6)", action, name)
+	case wf.LayerALEAuthRecvAcceptV4:
+		return fmt.Sprintf("%s inbound %s (IPv4)", action, name)
+	case wf.LayerALEAuthRecvAcceptV6:
+		return fmt.Sprintf("%s inbound %s (IPv6)", action, name)
+	}
+	return ""
+}
+
+// Firewall uses the Windows Filtering Platform to implement a network firewall.
+type Firewall struct {
+	luid       uint64
+	providerID wf.ProviderID
+	sublayerID wf.SublayerID
+	session    *wf.Session
+
+	permittedRoutes map[netaddr.IPPrefix][]*wf.Rule
+}
+
+// New returns a new Firewall for the provdied interface ID.
+func New(luid uint64) (*Firewall, error) {
+	session, err := wf.New(&wf.Options{
+		Name:    "Tailscale firewall",
+		Dynamic: true,
+	})
+	if err != nil {
+		return nil, err
+	}
+	wguid, err := windows.GenerateGUID()
+	if err != nil {
+		return nil, err
+	}
+	providerID := wf.ProviderID(wguid)
+	if err := session.AddProvider(&wf.Provider{
+		ID:   providerID,
+		Name: "Tailscale provider",
+	}); err != nil {
+		return nil, err
+	}
+	wguid, err = windows.GenerateGUID()
+	if err != nil {
+		return nil, err
+	}
+	sublayerID := wf.SublayerID(wguid)
+	if err := session.AddSublayer(&wf.Sublayer{
+		ID:     sublayerID,
+		Name:   "Tailscale permissive and blocking filters",
+		Weight: 0,
+	}); err != nil {
+		return nil, err
+	}
+	f := &Firewall{
+		luid:            luid,
+		session:         session,
+		providerID:      providerID,
+		sublayerID:      sublayerID,
+		permittedRoutes: make(map[netaddr.IPPrefix][]*wf.Rule),
+	}
+	if err := f.enable(); err != nil {
+		return nil, err
+	}
+	return f, nil
+}
+
+type weight uint64
+
+const (
+	weightTailscaleTraffic weight = 15
+	weightKnownTraffic     weight = 12
+	weightCatchAll         weight = 0
+)
+
+func (f *Firewall) enable() error {
+	if err := f.permitTailscaleService(weightTailscaleTraffic); err != nil {
+		return fmt.Errorf("permitTailscaleService failed: %w", err)
+	}
+
+	if err := f.permitTunInterface(weightTailscaleTraffic); err != nil {
+		return fmt.Errorf("permitTunInterface failed: %w", err)
+	}
+
+	if err := f.permitDNS(weightTailscaleTraffic); err != nil {
+		return fmt.Errorf("permitDNS failed: %w", err)
+	}
+
+	if err := f.permitLoopback(weightKnownTraffic); err != nil {
+		return fmt.Errorf("permitLoopback failed: %w", err)
+	}
+
+	if err := f.permitDHCPv4(weightKnownTraffic); err != nil {
+		return fmt.Errorf("permitDHCPv4 failed: %w", err)
+	}
+
+	if err := f.permitDHCPv6(weightKnownTraffic); err != nil {
+		return fmt.Errorf("permitDHCPv6 failed: %w", err)
+	}
+
+	if err := f.permitNDP(weightKnownTraffic); err != nil {
+		return fmt.Errorf("permitNDP failed: %w", err)
+	}
+
+	/* TODO: actually evaluate if this does anything and if we need this. It's layer 2; our other rules are layer 3.
+	 *  In other words, if somebody complains, try enabling it. For now, keep it off.
+	 * TODO(maisem): implement this.
+	err = permitHyperV(session, baseObjects, weightKnownTraffic)
+	if err != nil {
+		return wrapErr(err)
+	}
+	*/
+
+	if err := f.blockAll(weightCatchAll); err != nil {
+		return fmt.Errorf("blockAll failed: %w", err)
+	}
+	return nil
+}
+
+// UpdatedPermittedRoutes adds rules to allow incoming and outgoing connections
+// from the provided prefixes. It will also remove rules for routes that were
+// previously added but have been removed.
+func (f *Firewall) UpdatePermittedRoutes(newRoutes []netaddr.IPPrefix) error {
+	var routesToAdd []netaddr.IPPrefix
+	routeMap := make(map[netaddr.IPPrefix]bool)
+	for _, r := range newRoutes {
+		routeMap[r] = true
+		if _, ok := f.permittedRoutes[r]; !ok {
+			routesToAdd = append(routesToAdd, r)
+		}
+	}
+	var routesToRemove []netaddr.IPPrefix
+	for r := range f.permittedRoutes {
+		if !routeMap[r] {
+			routesToRemove = append(routesToRemove, r)
+		}
+	}
+	for _, r := range routesToRemove {
+		for _, rule := range f.permittedRoutes[r] {
+			if err := f.session.DeleteRule(rule.ID); err != nil {
+				return err
+			}
+		}
+		delete(f.permittedRoutes, r)
+	}
+	for _, r := range routesToAdd {
+		conditions := []*wf.Match{
+			{
+				Field: wf.FieldIPRemoteAddress,
+				Op:    wf.MatchTypeEqual,
+				Value: r,
+			},
+		}
+		var p protocol
+		if r.IP().Is4() {
+			p = protocolV4
+		} else {
+			p = protocolV6
+		}
+		rules, err := f.addRules("local route", weightKnownTraffic, conditions, wf.ActionPermit, p, directionBoth)
+		if err != nil {
+			return err
+		}
+		f.permittedRoutes[r] = rules
+	}
+	return nil
+}
+
+func (f *Firewall) newRule(name string, w weight, layer wf.LayerID, conditions []*wf.Match, action wf.Action) (*wf.Rule, error) {
+	id, err := windows.GenerateGUID()
+	if err != nil {
+		return nil, err
+	}
+	return &wf.Rule{
+		Name:       ruleName(action, layer, name),
+		ID:         wf.RuleID(id),
+		Provider:   f.providerID,
+		Sublayer:   f.sublayerID,
+		Layer:      layer,
+		Weight:     uint64(w),
+		Conditions: conditions,
+		Action:     action,
+	}, nil
+}
+
+func (f *Firewall) addRules(name string, w weight, conditions []*wf.Match, action wf.Action, p protocol, d direction) ([]*wf.Rule, error) {
+	var rules []*wf.Rule
+	for _, l := range p.getLayers(d) {
+		r, err := f.newRule(name, w, l, conditions, action)
+		if err != nil {
+			return nil, err
+		}
+		if err := f.session.AddRule(r); err != nil {
+			return nil, err
+		}
+		rules = append(rules, r)
+	}
+	return rules, nil
+}
+
+func (f *Firewall) blockAll(w weight) error {
+	_, err := f.addRules("all", w, nil, wf.ActionBlock, protocolAll, directionBoth)
+	return err
+}
+
+func (f *Firewall) permitNDP(w weight) error {
+	// These are aliased according to:
+	// https://social.msdn.microsoft.com/Forums/azure/en-US/eb2aa3cd-5f1c-4461-af86-61e7d43ccc23/filtering-icmp-by-type-code?forum=wfp
+	fieldICMPType := wf.FieldIPLocalPort
+	fieldICMPCode := wf.FieldIPRemotePort
+
+	var icmpConditions = func(t, c uint16, remoteAddress interface{}) []*wf.Match {
+		conditions := []*wf.Match{
+			{
+				Field: wf.FieldIPProtocol,
+				Op:    wf.MatchTypeEqual,
+				Value: wf.IPProtoICMPV6,
+			},
+			{
+				Field: fieldICMPType,
+				Op:    wf.MatchTypeEqual,
+				Value: t,
+			},
+			{
+				Field: fieldICMPCode,
+				Op:    wf.MatchTypeEqual,
+				Value: c,
+			},
+		}
+		if remoteAddress != nil {
+			conditions = append(conditions, &wf.Match{
+				Field: wf.FieldIPRemoteAddress,
+				Op:    wf.MatchTypeEqual,
+				Value: linkLocalRouterMulticast,
+			})
+		}
+		return conditions
+	}
+	/* TODO: actually handle the hop limit somehow! The rules should vaguely be:
+	 *  - icmpv6 133: must be outgoing, dst must be FF02::2/128, hop limit must be 255
+	 *  - icmpv6 134: must be incoming, src must be FE80::/10, hop limit must be 255
+	 *  - icmpv6 135: either incoming or outgoing, hop limit must be 255
+	 *  - icmpv6 136: either incoming or outgoing, hop limit must be 255
+	 *  - icmpv6 137: must be incoming, src must be FE80::/10, hop limit must be 255
+	 */
+
+	//
+	// Router Solicitation Message
+	// ICMP type 133, code 0. Outgoing.
+	//
+	conditions := icmpConditions(133, 0, linkLocalRouterMulticast)
+	if _, err := f.addRules("NDP type 133", w, conditions, wf.ActionPermit, protocolV6, directionOutbound); err != nil {
+		return err
+	}
+
+	//
+	// Router Advertisement Message
+	// ICMP type 134, code 0. Incoming.
+	//
+	conditions = icmpConditions(134, 0, linkLocalRange)
+	if _, err := f.addRules("NDP type 134", w, conditions, wf.ActionPermit, protocolV6, directionInbound); err != nil {
+		return err
+	}
+
+	//
+	// Neighbor Solicitation Message
+	// ICMP type 135, code 0. Bi-directional.
+	//
+	conditions = icmpConditions(135, 0, nil)
+	if _, err := f.addRules("NDP type 135", w, conditions, wf.ActionPermit, protocolV6, directionBoth); err != nil {
+		return err
+	}
+
+	//
+	// Neighbor Advertisement Message
+	// ICMP type 136, code 0. Bi-directional.
+	//
+	conditions = icmpConditions(136, 0, nil)
+	if _, err := f.addRules("NDP type 136", w, conditions, wf.ActionPermit, protocolV6, directionBoth); err != nil {
+		return err
+	}
+
+	//
+	// Redirect Message
+	// ICMP type 137, code 0. Incoming.
+	//
+	conditions = icmpConditions(137, 0, linkLocalRange)
+	if _, err := f.addRules("NDP type 137", w, conditions, wf.ActionPermit, protocolV6, directionInbound); err != nil {
+		return err
+	}
+	return nil
+}
+
+func (f *Firewall) permitDHCPv6(w weight) error {
+	var dhcpConditions = func(remoteAddrs ...interface{}) []*wf.Match {
+		conditions := []*wf.Match{
+			{
+				Field: wf.FieldIPProtocol,
+				Op:    wf.MatchTypeEqual,
+				Value: wf.IPProtoUDP,
+			},
+			{
+				Field: wf.FieldIPLocalAddress,
+				Op:    wf.MatchTypeEqual,
+				Value: linkLocalRange,
+			},
+			{
+				Field: wf.FieldIPLocalPort,
+				Op:    wf.MatchTypeEqual,
+				Value: uint16(546),
+			},
+			{
+				Field: wf.FieldIPRemotePort,
+				Op:    wf.MatchTypeEqual,
+				Value: uint16(547),
+			},
+		}
+		for _, a := range remoteAddrs {
+			conditions = append(conditions, &wf.Match{
+				Field: wf.FieldIPRemoteAddress,
+				Op:    wf.MatchTypeEqual,
+				Value: a,
+			})
+		}
+		return conditions
+	}
+	conditions := dhcpConditions(linkLocalDHCPMulticast, siteLocalDHCPMulticast)
+	if _, err := f.addRules("DHCP request", w, conditions, wf.ActionPermit, protocolV6, directionOutbound); err != nil {
+		return err
+	}
+	conditions = dhcpConditions(linkLocalRange)
+	if _, err := f.addRules("DHCP response", w, conditions, wf.ActionPermit, protocolV6, directionInbound); err != nil {
+		return err
+	}
+	return nil
+}
+
+func (f *Firewall) permitDHCPv4(w weight) error {
+	var dhcpConditions = func(remoteAddrs ...interface{}) []*wf.Match {
+		conditions := []*wf.Match{
+			{
+				Field: wf.FieldIPProtocol,
+				Op:    wf.MatchTypeEqual,
+				Value: wf.IPProtoUDP,
+			},
+			{
+				Field: wf.FieldIPLocalPort,
+				Op:    wf.MatchTypeEqual,
+				Value: uint16(68),
+			},
+			{
+				Field: wf.FieldIPRemotePort,
+				Op:    wf.MatchTypeEqual,
+				Value: uint16(67),
+			},
+		}
+		for _, a := range remoteAddrs {
+			conditions = append(conditions, &wf.Match{
+				Field: wf.FieldIPRemoteAddress,
+				Op:    wf.MatchTypeEqual,
+				Value: a,
+			})
+		}
+		return conditions
+	}
+	conditions := dhcpConditions(netaddr.IPv4(255, 255, 255, 255))
+	if _, err := f.addRules("DHCP request", w, conditions, wf.ActionPermit, protocolV4, directionOutbound); err != nil {
+		return err
+	}
+
+	conditions = dhcpConditions()
+	if _, err := f.addRules("DHCP response", w, conditions, wf.ActionPermit, protocolV4, directionInbound); err != nil {
+		return err
+	}
+	return nil
+}
+
+func (f *Firewall) permitTunInterface(w weight) error {
+	condition := []*wf.Match{
+		{
+			Field: wf.FieldIPLocalInterface,
+			Op:    wf.MatchTypeEqual,
+			Value: f.luid,
+		},
+	}
+	_, err := f.addRules("on TUN", w, condition, wf.ActionPermit, protocolAll, directionBoth)
+	return err
+}
+
+func (f *Firewall) permitLoopback(w weight) error {
+	condition := []*wf.Match{
+		{
+			Field: wf.FieldFlags,
+			Op:    wf.MatchTypeEqual,
+			Value: wf.ConditionFlagIsLoopback,
+		},
+	}
+	_, err := f.addRules("on loopback", w, condition, wf.ActionPermit, protocolAll, directionBoth)
+	return err
+}
+
+func (f *Firewall) permitDNS(w weight) error {
+	conditions := []*wf.Match{
+		{
+			Field: wf.FieldIPRemotePort,
+			Op:    wf.MatchTypeEqual,
+			Value: uint16(53),
+		},
+		// Repeat the condition type for logical OR.
+		{
+			Field: wf.FieldIPProtocol,
+			Op:    wf.MatchTypeEqual,
+			Value: wf.IPProtoUDP,
+		},
+		{
+			Field: wf.FieldIPProtocol,
+			Op:    wf.MatchTypeEqual,
+			Value: wf.IPProtoTCP,
+		},
+	}
+	_, err := f.addRules("DNS", w, conditions, wf.ActionPermit, protocolAll, directionBoth)
+	return err
+}
+
+func (f *Firewall) permitTailscaleService(w weight) error {
+	currentFile, err := os.Executable()
+	if err != nil {
+		return err
+	}
+
+	appID, err := wf.AppID(currentFile)
+	if err != nil {
+		return fmt.Errorf("could not get app id for %q: %w", currentFile, err)
+	}
+	conditions := []*wf.Match{
+		{
+			Field: wf.FieldALEAppID,
+			Op:    wf.MatchTypeEqual,
+			Value: appID,
+		},
+	}
+	_, err = f.addRules("unrestricted traffic for Tailscale service", w, conditions, wf.ActionPermit, protocolAll, directionBoth)
+	return err
+}

wgengine/bench/bench.go

@@ -80,14 +80,14 @@ func main() {
 
 	// tx=134236 rx=133166 (1070 = 0.80% loss) (1088.9 Mbits/sec)
 	case 101:
-		setupWGTest(logf, traf, Addr1, Addr2)
+		setupWGTest(nil, logf, traf, Addr1, Addr2)
 
 	default:
 		log.Fatalf("provide a valid test number (0..n)")
 	}
 
 	logf("initialized ok.")
-	traf.Start(Addr1.IP, Addr2.IP, PayloadSize+ICMPMinSize, 0)
+	traf.Start(Addr1.IP(), Addr2.IP(), PayloadSize+ICMPMinSize, 0)
 
 	var cur, prev Snapshot
 	var pps int64

wgengine/bench/bench_test.go

@@ -43,7 +43,7 @@ func BenchmarkBatchTCP(b *testing.B) {
 
 func BenchmarkWireGuardTest(b *testing.B) {
 	run(b, func(logf logger.Logf, traf *TrafficGen) {
-		setupWGTest(logf, traf, Addr1, Addr2)
+		setupWGTest(b, logf, traf, Addr1, Addr2)
 	})
 }
 
@@ -78,7 +78,7 @@ func runOnce(b *testing.B, setup SetupFunc, payload int) {
 	logf("initialized. (n=%v)", b.N)
 	b.SetBytes(int64(payload))
 
-	traf.Start(Addr1.IP, Addr2.IP, payload, int64(b.N))
+	traf.Start(Addr1.IP(), Addr2.IP(), payload, int64(b.N))
 
 	var cur, prev Snapshot
 	var pps int64

wgengine/bench/trafficgen.go

@@ -180,6 +180,7 @@ func (t *TrafficGen) Generate(b []byte, ofs int) int {
 // GotPacket processes a packet that came back on the receive side.
 func (t *TrafficGen) GotPacket(b []byte, ofs int) {
 	t.mu.Lock()
+	defer t.mu.Unlock()
 
 	s := &t.cur
 	seq := int64(binary.BigEndian.Uint64(
@@ -203,9 +204,6 @@ func (t *TrafficGen) GotPacket(b []byte, ofs int) {
 
 	f := t.onFirstPacket
 	t.onFirstPacket = nil
-
-	t.mu.Unlock()
-
 	if f != nil {
 		f()
 	}

wgengine/bench/wg.go

@@ -5,13 +5,14 @@
 package main
 
 import (
+	"errors"
 	"io"
 	"log"
 	"os"
-	"strings"
 	"sync"
+	"testing"
 
-	"github.com/tailscale/wireguard-go/tun"
+	"golang.zx2c4.com/wireguard/tun"
 	"inet.af/netaddr"
 
 	"tailscale.com/net/dns"
@@ -25,7 +26,7 @@ import (
 	"tailscale.com/wgengine/wgcfg"
 )
 
-func setupWGTest(logf logger.Logf, traf *TrafficGen, a1, a2 netaddr.IPPrefix) {
+func setupWGTest(b *testing.B, logf logger.Logf, traf *TrafficGen, a1, a2 netaddr.IPPrefix) {
 	l1 := logger.WithPrefix(logf, "e1: ")
 	k1, err := wgkey.NewPrivate()
 	if err != nil {
@@ -49,6 +50,9 @@ func setupWGTest(logf logger.Logf, traf *TrafficGen, a1, a2 netaddr.IPPrefix) {
 	if err != nil {
 		log.Fatalf("e1 init: %v", err)
 	}
+	if b != nil {
+		b.Cleanup(e1.Close)
+	}
 
 	l2 := logger.WithPrefix(logf, "e2: ")
 	k2, err := wgkey.NewPrivate()
@@ -73,6 +77,9 @@ func setupWGTest(logf logger.Logf, traf *TrafficGen, a1, a2 netaddr.IPPrefix) {
 	if err != nil {
 		log.Fatalf("e2 init: %v", err)
 	}
+	if b != nil {
+		b.Cleanup(e2.Close)
+	}
 
 	e1.SetFilter(filter.NewAllowAllForTest(l1))
 	e2.SetFilter(filter.NewAllowAllForTest(l2))
@@ -80,15 +87,25 @@ func setupWGTest(logf logger.Logf, traf *TrafficGen, a1, a2 netaddr.IPPrefix) {
 	var wait sync.WaitGroup
 	wait.Add(2)
 
+	var e1waitDoneOnce sync.Once
 	e1.SetStatusCallback(func(st *wgengine.Status, err error) {
+		if errors.Is(err, wgengine.ErrEngineClosing) {
+			return
+		}
 		if err != nil {
 			log.Fatalf("e1 status err: %v", err)
 		}
 		logf("e1 status: %v", *st)
 
 		var eps []string
+		var ipps []netaddr.IPPort
 		for _, ep := range st.LocalAddrs {
 			eps = append(eps, ep.Addr.String())
+			ipps = append(ipps, ep.Addr)
+		}
+		endpoint := wgcfg.Endpoints{
+			PublicKey: c1.PrivateKey.Public(),
+			IPPorts:   wgcfg.NewIPPortSet(ipps...),
 		}
 
 		n := tailcfg.Node{
@@ -107,22 +124,32 @@ func setupWGTest(logf logger.Logf, traf *TrafficGen, a1, a2 netaddr.IPPrefix) {
 		p := wgcfg.Peer{
 			PublicKey:  c1.PrivateKey.Public(),
 			AllowedIPs: []netaddr.IPPrefix{a1},
-			Endpoints:  strings.Join(eps, ","),
+			Endpoints:  endpoint,
 		}
 		c2.Peers = []wgcfg.Peer{p}
 		e2.Reconfig(&c2, &router.Config{}, new(dns.Config))
-		wait.Done()
+		e1waitDoneOnce.Do(wait.Done)
 	})
 
+	var e2waitDoneOnce sync.Once
 	e2.SetStatusCallback(func(st *wgengine.Status, err error) {
+		if errors.Is(err, wgengine.ErrEngineClosing) {
+			return
+		}
 		if err != nil {
 			log.Fatalf("e2 status err: %v", err)
 		}
 		logf("e2 status: %v", *st)
 
 		var eps []string
+		var ipps []netaddr.IPPort
 		for _, ep := range st.LocalAddrs {
 			eps = append(eps, ep.Addr.String())
+			ipps = append(ipps, ep.Addr)
+		}
+		endpoint := wgcfg.Endpoints{
+			PublicKey: c2.PrivateKey.Public(),
+			IPPorts:   wgcfg.NewIPPortSet(ipps...),
 		}
 
 		n := tailcfg.Node{
@@ -141,11 +168,11 @@ func setupWGTest(logf logger.Logf, traf *TrafficGen, a1, a2 netaddr.IPPrefix) {
 		p := wgcfg.Peer{
 			PublicKey:  c2.PrivateKey.Public(),
 			AllowedIPs: []netaddr.IPPrefix{a2},
-			Endpoints:  strings.Join(eps, ","),
+			Endpoints:  endpoint,
 		}
 		c1.Peers = []wgcfg.Peer{p}
 		e1.Reconfig(&c1, &router.Config{}, new(dns.Config))
-		wait.Done()
+		e2waitDoneOnce.Do(wait.Done)
 	})
 
 	// Not using DERP in this test (for now?).

wgengine/filter/filter.go

@@ -98,8 +98,8 @@ const (
 // everything. Use in tests only, as it permits some kinds of spoofing
 // attacks to reach the OS network stack.
 func NewAllowAllForTest(logf logger.Logf) *Filter {
-	any4 := netaddr.IPPrefix{IP: netaddr.IPv4(0, 0, 0, 0), Bits: 0}
-	any6 := netaddr.IPPrefix{IP: netaddr.IPFrom16([16]byte{}), Bits: 0}
+	any4 := netaddr.IPPrefixFrom(netaddr.IPv4(0, 0, 0, 0), 0)
+	any6 := netaddr.IPPrefixFrom(netaddr.IPFrom16([16]byte{}), 0)
 	ms := []Match{
 		{
 			Srcs: []netaddr.IPPrefix{any4},
@@ -185,12 +185,12 @@ func matchesFamily(ms matches, keep func(netaddr.IP) bool) matches {
 		var retm Match
 		retm.IPProto = m.IPProto
 		for _, src := range m.Srcs {
-			if keep(src.IP) {
+			if keep(src.IP()) {
 				retm.Srcs = append(retm.Srcs, src)
 			}
 		}
 		for _, dst := range m.Dsts {
-			if keep(dst.Net.IP) {
+			if keep(dst.Net.IP()) {
 				retm.Dsts = append(retm.Dsts, dst)
 			}
 		}
@@ -266,12 +266,10 @@ func (f *Filter) CheckTCP(srcIP, dstIP netaddr.IP, dstPort uint16) Response {
 	default:
 		panic("unreachable")
 	}
-	pkt.Src.IP = srcIP
-	pkt.Dst.IP = dstIP
+	pkt.Src = netaddr.IPPortFrom(srcIP, 0)
+	pkt.Dst = netaddr.IPPortFrom(dstIP, dstPort)
 	pkt.IPProto = ipproto.TCP
 	pkt.TCPFlags = packet.TCPSyn
-	pkt.Src.Port = 0
-	pkt.Dst.Port = dstPort
 
 	return f.RunIn(pkt, 0)
 }
@@ -321,7 +319,7 @@ func (f *Filter) runIn4(q *packet.Parsed) (r Response, why string) {
 	// A compromised peer could try to send us packets for
 	// destinations we didn't explicitly advertise. This check is to
 	// prevent that.
-	if !f.local.Contains(q.Dst.IP) {
+	if !f.local.Contains(q.Dst.IP()) {
 		return Drop, "destination not allowed"
 	}
 
@@ -378,7 +376,7 @@ func (f *Filter) runIn6(q *packet.Parsed) (r Response, why string) {
 	// A compromised peer could try to send us packets for
 	// destinations we didn't explicitly advertise. This check is to
 	// prevent that.
-	if !f.local.Contains(q.Dst.IP) {
+	if !f.local.Contains(q.Dst.IP()) {
 		return Drop, "destination not allowed"
 	}
 
@@ -480,11 +478,11 @@ func (f *Filter) pre(q *packet.Parsed, rf RunFlags, dir direction) Response {
 		return Drop
 	}
 
-	if q.Dst.IP.IsMulticast() {
+	if q.Dst.IP().IsMulticast() {
 		f.logRateLimit(rf, q, dir, Drop, "multicast")
 		return Drop
 	}
-	if q.Dst.IP.IsLinkLocalUnicast() && q.Dst.IP != gcpDNSAddr {
+	if q.Dst.IP().IsLinkLocalUnicast() && q.Dst.IP() != gcpDNSAddr {
 		f.logRateLimit(rf, q, dir, Drop, "link-local-unicast")
 		return Drop
 	}
@@ -506,7 +504,7 @@ func (f *Filter) pre(q *packet.Parsed, rf RunFlags, dir direction) Response {
 
 // loggingAllowed reports whether p can appear in logs at all.
 func (f *Filter) loggingAllowed(p *packet.Parsed) bool {
-	return f.logIPs.Contains(p.Src.IP) && f.logIPs.Contains(p.Dst.IP)
+	return f.logIPs.Contains(p.Src.IP()) && f.logIPs.Contains(p.Dst.IP())
 }
 
 // omitDropLogging reports whether packet p, which has already been
@@ -518,5 +516,5 @@ func omitDropLogging(p *packet.Parsed, dir direction) bool {
 		return false
 	}
 
-	return p.Dst.IP.IsMulticast() || (p.Dst.IP.IsLinkLocalUnicast() && p.Dst.IP != gcpDNSAddr) || p.IPProto == ipproto.IGMP
+	return p.Dst.IP().IsMulticast() || (p.Dst.IP().IsLinkLocalUnicast() && p.Dst.IP() != gcpDNSAddr) || p.IPProto == ipproto.IGMP
 }

wgengine/filter/filter_test.go

@@ -120,9 +120,9 @@ func TestFilter(t *testing.T) {
 		if test.p.IPProto == ipproto.TCP {
 			var got Response
 			if test.p.IPVersion == 4 {
-				got = acl.CheckTCP(test.p.Src.IP, test.p.Dst.IP, test.p.Dst.Port)
+				got = acl.CheckTCP(test.p.Src.IP(), test.p.Dst.IP(), test.p.Dst.Port())
 			} else {
-				got = acl.CheckTCP(test.p.Src.IP, test.p.Dst.IP, test.p.Dst.Port)
+				got = acl.CheckTCP(test.p.Src.IP(), test.p.Dst.IP(), test.p.Dst.Port())
 			}
 			if test.want != got {
 				t.Errorf("#%d CheckTCP got=%v want=%v packet:%v", i, got, test.want, test.p)
@@ -254,7 +254,9 @@ func TestParseIPSet(t *testing.T) {
 			}
 			t.Errorf("parseIPSet(%q, %v) error: %v; want error %q", tt.host, tt.bits, err, tt.wantErr)
 		}
-		if diff := cmp.Diff(got, tt.want, cmp.Comparer(func(a, b netaddr.IP) bool { return a == b })); diff != "" {
+		compareIP := cmp.Comparer(func(a, b netaddr.IP) bool { return a == b })
+		compareIPPrefix := cmp.Comparer(func(a, b netaddr.IPPrefix) bool { return a == b })
+		if diff := cmp.Diff(got, tt.want, compareIP, compareIPPrefix); diff != "" {
 			t.Errorf("parseIPSet(%q, %v) = %s; want %s", tt.host, tt.bits, got, tt.want)
 			continue
 		}
@@ -425,10 +427,10 @@ func TestLoggingPrivacy(t *testing.T) {
 	f.logIPs = logB.IPSet()
 
 	var (
-		ts4       = netaddr.IPPort{IP: tsaddr.CGNATRange().IP.Next(), Port: 1234}
-		internet4 = netaddr.IPPort{IP: netaddr.MustParseIP("8.8.8.8"), Port: 1234}
-		ts6       = netaddr.IPPort{IP: tsaddr.TailscaleULARange().IP.Next(), Port: 1234}
-		internet6 = netaddr.IPPort{IP: netaddr.MustParseIP("2001::1"), Port: 1234}
+		ts4       = netaddr.IPPortFrom(tsaddr.CGNATRange().IP().Next(), 1234)
+		internet4 = netaddr.IPPortFrom(netaddr.MustParseIP("8.8.8.8"), 1234)
+		ts6       = netaddr.IPPortFrom(tsaddr.TailscaleULARange().IP().Next(), 1234)
+		internet6 = netaddr.IPPortFrom(netaddr.MustParseIP("2001::1"), 1234)
 	)
 
 	tests := []struct {
@@ -545,10 +547,8 @@ func parsed(proto ipproto.Proto, src, dst string, sport, dport uint16) packet.Pa
 	var ret packet.Parsed
 	ret.Decode(dummyPacket)
 	ret.IPProto = proto
-	ret.Src.IP = sip
-	ret.Src.Port = sport
-	ret.Dst.IP = dip
-	ret.Dst.Port = dport
+	ret.Src = netaddr.IPPortFrom(sip, sport)
+	ret.Dst = netaddr.IPPortFrom(dip, dport)
 	ret.TCPFlags = packet.TCPSyn
 
 	if sip.Is4() {
@@ -674,7 +674,7 @@ func nets(nets ...string) (ret []netaddr.IPPrefix) {
 			if ip.Is6() {
 				bits = 128
 			}
-			ret = append(ret, netaddr.IPPrefix{IP: ip, Bits: bits})
+			ret = append(ret, netaddr.IPPrefixFrom(ip, bits))
 		} else {
 			pfx, err := netaddr.ParseIPPrefix(s)
 			if err != nil {

wgengine/filter/match.go

@@ -85,14 +85,14 @@ func (ms matches) match(q *packet.Parsed) bool {
 		if !protoInList(q.IPProto, m.IPProto) {
 			continue
 		}
-		if !ipInList(q.Src.IP, m.Srcs) {
+		if !ipInList(q.Src.IP(), m.Srcs) {
 			continue
 		}
 		for _, dst := range m.Dsts {
-			if !dst.Net.Contains(q.Dst.IP) {
+			if !dst.Net.Contains(q.Dst.IP()) {
 				continue
 			}
-			if !dst.Ports.contains(q.Dst.Port) {
+			if !dst.Ports.contains(q.Dst.Port()) {
 				continue
 			}
 			return true
@@ -103,11 +103,11 @@ func (ms matches) match(q *packet.Parsed) bool {
 
 func (ms matches) matchIPsOnly(q *packet.Parsed) bool {
 	for _, m := range ms {
-		if !ipInList(q.Src.IP, m.Srcs) {
+		if !ipInList(q.Src.IP(), m.Srcs) {
 			continue
 		}
 		for _, dst := range m.Dsts {
-			if dst.Net.Contains(q.Dst.IP) {
+			if dst.Net.Contains(q.Dst.IP()) {
 				return true
 			}
 		}

wgengine/filter/tailcfg.go

@@ -99,8 +99,8 @@ func parseIPSet(arg string, bits *int) ([]netaddr.IPPrefix, error) {
 	if arg == "*" {
 		// User explicitly requested wildcard.
 		return []netaddr.IPPrefix{
-			{IP: zeroIP4, Bits: 0},
-			{IP: zeroIP6, Bits: 0},
+			netaddr.IPPrefixFrom(zeroIP4, 0),
+			netaddr.IPPrefixFrom(zeroIP6, 0),
 		}, nil
 	}
 	if strings.Contains(arg, "/") {
@@ -124,7 +124,7 @@ func parseIPSet(arg string, bits *int) ([]netaddr.IPPrefix, error) {
 		if err != nil {
 			return nil, err
 		}
-		r := netaddr.IPRange{From: ip1, To: ip2}
+		r := netaddr.IPRangeFrom(ip1, ip2)
 		if !r.Valid() {
 			return nil, fmt.Errorf("invalid IP range %q", arg)
 		}
@@ -141,5 +141,5 @@ func parseIPSet(arg string, bits *int) ([]netaddr.IPPrefix, error) {
 		}
 		bits8 = uint8(*bits)
 	}
-	return []netaddr.IPPrefix{{IP: ip, Bits: bits8}}, nil
+	return []netaddr.IPPrefix{netaddr.IPPrefixFrom(ip, bits8)}, nil
 }

wgengine/magicsock/legacy.go

@@ -10,23 +10,23 @@ import (
 	"crypto/subtle"
 	"encoding/binary"
 	"errors"
-	"fmt"
 	"hash"
 	"net"
 	"strings"
 	"sync"
 	"time"
 
-	"github.com/tailscale/wireguard-go/conn"
-	"github.com/tailscale/wireguard-go/tai64n"
 	"golang.org/x/crypto/blake2s"
 	"golang.org/x/crypto/chacha20poly1305"
 	"golang.org/x/crypto/poly1305"
+	"golang.zx2c4.com/wireguard/conn"
+	"golang.zx2c4.com/wireguard/tai64n"
 	"inet.af/netaddr"
 	"tailscale.com/ipn/ipnstate"
 	"tailscale.com/types/key"
 	"tailscale.com/types/logger"
 	"tailscale.com/types/wgkey"
+	"tailscale.com/wgengine/wgcfg"
 )
 
 var (
@@ -34,7 +34,11 @@ var (
 	errDisabled       = errors.New("magicsock: legacy networking disabled")
 )
 
-func (c *Conn) createLegacyEndpointLocked(pk key.Public, addrs string) (conn.Endpoint, error) {
+// createLegacyEndpointLocked creates a new wireguard-go endpoint for a legacy connection.
+// pk is the public key of the remote peer. addrs is the ordered set of addresses for the remote peer.
+// rawDest is the encoded wireguard-go endpoint string. It should be treated as a black box.
+// It is provided so that addrSet.DstToString can return it when requested by wireguard-go.
+func (c *Conn) createLegacyEndpointLocked(pk key.Public, addrs wgcfg.IPPortSet, rawDest string) (conn.Endpoint, error) {
 	if c.disableLegacy {
 		return nil, errDisabled
 	}
@@ -43,17 +47,9 @@ func (c *Conn) createLegacyEndpointLocked(pk key.Public, addrs string) (conn.End
 		Logf:      c.logf,
 		publicKey: pk,
 		curAddr:   -1,
+		rawdst:    rawDest,
 	}
-
-	if addrs != "" {
-		for _, ep := range strings.Split(addrs, ",") {
-			ipp, err := netaddr.ParseIPPort(ep)
-			if err != nil {
-				return nil, fmt.Errorf("bogus address %q", ep)
-			}
-			a.ipPorts = append(a.ipPorts, ipp)
-		}
-	}
+	a.ipPorts = append(a.ipPorts, addrs.IPPorts()...)
 
 	// If this endpoint is being updated, remember its old set of
 	// endpoints so we can remove any (from c.addrsByUDP) that are
@@ -66,7 +62,7 @@ func (c *Conn) createLegacyEndpointLocked(pk key.Public, addrs string) (conn.End
 
 	// Add entries to c.addrsByUDP.
 	for _, ipp := range a.ipPorts {
-		if ipp.IP == derpMagicIPAddr {
+		if ipp.IP() == derpMagicIPAddr {
 			continue
 		}
 		c.addrsByUDP[ipp] = a
@@ -74,7 +70,7 @@ func (c *Conn) createLegacyEndpointLocked(pk key.Public, addrs string) (conn.End
 
 	// Remove previous c.addrsByUDP entries that are no longer in the new set.
 	for _, ipp := range oldIPP {
-		if ipp.IP != derpMagicIPAddr && c.addrsByUDP[ipp] != a {
+		if ipp.IP() != derpMagicIPAddr && c.addrsByUDP[ipp] != a {
 			delete(c.addrsByUDP, ipp)
 		}
 	}
@@ -384,13 +380,16 @@ type addrSet struct {
 	// set to a better one. This is only to suppress some
 	// redundant logs.
 	loggedLogPriMask uint32
+
+	// rawdst is the destination string from/for wireguard-go.
+	rawdst string
 }
 
 // derpID returns this addrSet's home DERP node, or 0 if none is found.
 func (as *addrSet) derpID() int {
 	for _, ua := range as.ipPorts {
-		if ua.IP == derpMagicIPAddr {
-			return int(ua.Port)
+		if ua.IP() == derpMagicIPAddr {
+			return int(ua.Port())
 		}
 	}
 	return 0
@@ -426,20 +425,10 @@ func (a *addrSet) DstToBytes() []byte {
 	return packIPPort(a.dst())
 }
 func (a *addrSet) DstToString() string {
-	var addrs []string
-	for _, addr := range a.ipPorts {
-		addrs = append(addrs, addr.String())
-	}
-
-	a.mu.Lock()
-	defer a.mu.Unlock()
-	if a.roamAddr != nil {
-		addrs = append(addrs, a.roamAddr.String())
-	}
-	return strings.Join(addrs, ",")
+	return a.rawdst
 }
 func (a *addrSet) DstIP() net.IP {
-	return a.dst().IP.IPAddr().IP // TODO: add netaddr accessor to cut an alloc here?
+	return a.dst().IP().IPAddr().IP // TODO: add netaddr accessor to cut an alloc here?
 }
 func (a *addrSet) SrcIP() net.IP       { return nil }
 func (a *addrSet) SrcToString() string { return "" }
@@ -448,7 +437,7 @@ func (a *addrSet) ClearSrc()           {}
 // updateDst records receipt of a packet from new. This is used to
 // potentially update the transmit address used for this addrSet.
 func (a *addrSet) updateDst(new netaddr.IPPort) error {
-	if new.IP == derpMagicIPAddr {
+	if new.IP() == derpMagicIPAddr {
 		// Never consider DERP addresses as a viable candidate for
 		// either curAddr or roamAddr. It's only ever a last resort
 		// choice, never a preferred choice.
@@ -550,7 +539,7 @@ func (as *addrSet) populatePeerStatus(ps *ipnstate.PeerStatus) {
 
 	ps.LastWrite = as.lastSend
 	for i, ua := range as.ipPorts {
-		if ua.IP == derpMagicIPAddr {
+		if ua.IP() == derpMagicIPAddr {
 			continue
 		}
 		uaStr := ua.String()

wgengine/magicsock/magicsock.go

@@ -11,9 +11,11 @@ import (
 	"context"
 	crand "crypto/rand"
 	"encoding/binary"
+	"encoding/json"
 	"errors"
 	"fmt"
 	"hash/fnv"
+	"log"
 	"math"
 	"math/rand"
 	"net"
@@ -26,10 +28,9 @@ import (
 	"sync/atomic"
 	"time"
 
-	"github.com/tailscale/wireguard-go/conn"
-	"go4.org/mem"
 	"golang.org/x/crypto/nacl/box"
 	"golang.org/x/time/rate"
+	"golang.zx2c4.com/wireguard/conn"
 	"inet.af/netaddr"
 	"tailscale.com/control/controlclient"
 	"tailscale.com/derp"
@@ -401,7 +402,7 @@ type Options struct {
 	// and 10 seconds seems like a good trade-off between often
 	// enough and not too often.) The provided func is called
 	// while holding userspaceEngine.wgLock and likely calls
-	// Conn.CreateEndpoint, which acquires Conn.mu. As such, you
+	// Conn.ParseEndpoint, which acquires Conn.mu. As such, you
 	// should not hold Conn.mu while calling it.
 	NoteRecvActivity func(tailcfg.DiscoKey)
 
@@ -824,6 +825,7 @@ func (c *Conn) LastRecvActivityOfDisco(dk tailcfg.DiscoKey) time.Time {
 
 // Ping handles a "tailscale ping" CLI query.
 func (c *Conn) Ping(peer *tailcfg.Node, res *ipnstate.PingResult, cb func(*ipnstate.PingResult)) {
+	log.Println("CLIPING")
 	c.mu.Lock()
 	defer c.mu.Unlock()
 	if c.privateKey.IsZero() {
@@ -832,7 +834,7 @@ func (c *Conn) Ping(peer *tailcfg.Node, res *ipnstate.PingResult, cb func(*ipnst
 		return
 	}
 	if len(peer.Addresses) > 0 {
-		res.NodeIP = peer.Addresses[0].IP.String()
+		res.NodeIP = peer.Addresses[0].IP().String()
 	}
 	res.NodeName = peer.Name // prefer DNS name
 	if res.NodeName == "" {
@@ -878,11 +880,11 @@ func (c *Conn) Ping(peer *tailcfg.Node, res *ipnstate.PingResult, cb func(*ipnst
 // c.mu must be held
 func (c *Conn) populateCLIPingResponseLocked(res *ipnstate.PingResult, latency time.Duration, ep netaddr.IPPort) {
 	res.LatencySeconds = latency.Seconds()
-	if ep.IP != derpMagicIPAddr {
+	if ep.IP() != derpMagicIPAddr {
 		res.Endpoint = ep.String()
 		return
 	}
-	regionID := int(ep.Port)
+	regionID := int(ep.Port())
 	res.DERPRegionID = regionID
 	if c.derpMap != nil {
 		if dr, ok := c.derpMap.Regions[regionID]; ok {
@@ -965,7 +967,7 @@ func (c *Conn) goDerpConnect(node int) {
 	if node == 0 {
 		return
 	}
-	go c.derpWriteChanOfAddr(netaddr.IPPort{IP: derpMagicIPAddr, Port: uint16(node)}, key.Public{})
+	go c.derpWriteChanOfAddr(netaddr.IPPortFrom(derpMagicIPAddr, uint16(node)), key.Public{})
 }
 
 // determineEndpoints returns the machine's endpoint addresses. It
@@ -1037,7 +1039,7 @@ func (c *Conn) determineEndpoints(ctx context.Context) ([]tailcfg.Endpoint, erro
 			ips = loopback
 		}
 		for _, ip := range ips {
-			addAddr(netaddr.IPPort{IP: ip, Port: uint16(localAddr.Port)}, tailcfg.EndpointLocal)
+			addAddr(netaddr.IPPortFrom(ip, uint16(localAddr.Port)), tailcfg.EndpointLocal)
 		}
 	} else {
 		// Our local endpoint is bound to a particular address.
@@ -1169,7 +1171,7 @@ func (c *Conn) sendUDPStd(addr *net.UDPAddr, b []byte) (sent bool, err error) {
 // IPv6 address when the local machine doesn't have IPv6 support
 // returns (false, nil); it's not an error, but nothing was sent.
 func (c *Conn) sendAddr(addr netaddr.IPPort, pubKey key.Public, b []byte) (sent bool, err error) {
-	if addr.IP != derpMagicIPAddr {
+	if addr.IP() != derpMagicIPAddr {
 		return c.sendUDP(addr, b)
 	}
 
@@ -1211,10 +1213,10 @@ const bufferedDerpWritesBeforeDrop = 32
 // If peer is non-zero, it can be used to find an active reverse
 // path, without using addr.
 func (c *Conn) derpWriteChanOfAddr(addr netaddr.IPPort, peer key.Public) chan<- derpWriteRequest {
-	if addr.IP != derpMagicIPAddr {
+	if addr.IP() != derpMagicIPAddr {
 		return nil
 	}
-	regionID := int(addr.Port)
+	regionID := int(addr.Port())
 
 	if c.networkDown() {
 		return nil
@@ -1402,7 +1404,7 @@ func (c *Conn) runDerpReader(ctx context.Context, derpFakeAddr netaddr.IPPort, d
 	}
 
 	didCopy := make(chan struct{}, 1)
-	regionID := int(derpFakeAddr.Port)
+	regionID := int(derpFakeAddr.Port())
 	res := derpReadResult{regionID: regionID}
 	var pkt derp.ReceivedPacket
 	res.copyBuf = func(dst []byte) int {
@@ -1676,7 +1678,7 @@ func (c *Conn) processDERPReadResult(dm derpReadResult, b []byte) (n int, ep con
 		return 0, nil
 	}
 
-	ipp := netaddr.IPPort{IP: derpMagicIPAddr, Port: uint16(regionID)}
+	ipp := netaddr.IPPortFrom(derpMagicIPAddr, uint16(regionID))
 	if c.handleDiscoMessage(b[:n], ipp) {
 		return 0, nil
 	}
@@ -1696,7 +1698,7 @@ func (c *Conn) processDERPReadResult(dm derpReadResult, b []byte) (n int, ep con
 		if discoEp == nil && c.noteRecvActivity != nil {
 			didNoteRecvActivity = true
 			c.mu.Unlock()          // release lock before calling noteRecvActivity
-			c.noteRecvActivity(dk) // (calls back into CreateEndpoint)
+			c.noteRecvActivity(dk) // (calls back into ParseEndpoint)
 			// Now require the lock. No invariants need to be rechecked; just
 			// 1-2 map lookups follow that are harmless if, say, the peer has
 			// been deleted during this time.
@@ -1837,7 +1839,7 @@ func (c *Conn) handleDiscoMessage(msg []byte, src netaddr.IPPort) (isDiscoMsg bo
 		// We don't have an active endpoint for this sender but we knew about the node, so
 		// it's an idle endpoint that doesn't yet exist in the wireguard config. We now have
 		// to notify the userspace engine (via noteRecvActivity) so wireguard-go can create
-		// an Endpoint (ultimately calling our CreateEndpoint).
+		// an Endpoint (ultimately calling our ParseEndpoint).
 		c.logf("magicsock: got disco message from idle peer, starting lazy conf for %v, %v", peerNode.Key.ShortString(), sender.ShortString())
 		if c.noteRecvActivity == nil {
 			c.logf("magicsock: [unexpected] have node without endpoint, without c.noteRecvActivity hook")
@@ -1851,7 +1853,7 @@ func (c *Conn) handleDiscoMessage(msg []byte, src netaddr.IPPort) (isDiscoMsg bo
 		// We can't hold Conn.mu while calling noteRecvActivity.
 		// noteRecvActivity acquires userspaceEngine.wgLock (and per our
 		// lock ordering rules: wgLock must come first), and also calls
-		// back into our Conn.CreateEndpoint, which would double-acquire
+		// back into our Conn.ParseEndpoint, which would double-acquire
 		// Conn.mu.
 		c.mu.Unlock()
 		c.noteRecvActivity(sender)
@@ -1922,7 +1924,7 @@ func (c *Conn) handleDiscoMessage(msg []byte, src netaddr.IPPort) (isDiscoMsg bo
 		}
 		de.handlePongConnLocked(dm, src)
 	case *disco.CallMeMaybe:
-		if src.IP != derpMagicIPAddr {
+		if src.IP() != derpMagicIPAddr {
 			// CallMeMaybe messages should only come via DERP.
 			c.logf("[unexpected] CallMeMaybe packets should only come via DERP")
 			return
@@ -2212,6 +2214,7 @@ func nodesEqual(x, y []*tailcfg.Node) bool {
 // conditionally sent to SetDERPMap instead.
 func (c *Conn) SetNetworkMap(nm *netmap.NetworkMap) {
 	c.mu.Lock()
+	log.Println("NETMAP being set")
 	defer c.mu.Unlock()
 
 	if c.netMap != nil && nodesEqual(c.netMap.Peers, nm.Peers) {
@@ -2722,7 +2725,7 @@ func (c *Conn) resetEndpointStates() {
 
 // packIPPort packs an IPPort into the form wanted by WireGuard.
 func packIPPort(ua netaddr.IPPort) []byte {
-	ip := ua.IP.Unmap()
+	ip := ua.IP().Unmap()
 	a := ip.As16()
 	ipb := a[:]
 	if ip.Is4() {
@@ -2730,50 +2733,38 @@ func packIPPort(ua netaddr.IPPort) []byte {
 	}
 	b := make([]byte, 0, len(ipb)+2)
 	b = append(b, ipb...)
-	b = append(b, byte(ua.Port))
-	b = append(b, byte(ua.Port>>8))
+	b = append(b, byte(ua.Port()))
+	b = append(b, byte(ua.Port()>>8))
 	return b
 }
 
 // ParseEndpoint is called by WireGuard to connect to an endpoint.
-//
-// keyAddrs is the 32 byte public key of the peer followed by addrs.
-// Addrs is either:
-//
-//  1) a comma-separated list of UDP ip:ports (the peer doesn't have a discovery key)
-//  2) "<hex-discovery-key>.disco.tailscale:12345", a magic value that means the peer
-//     is running code that supports active discovery, so CreateEndpoint returns
-//     a discoEndpoint.
-func (c *Conn) ParseEndpoint(keyAddrs string) (conn.Endpoint, error) {
-	if len(keyAddrs) < 32 {
-		c.logf("[unexpected] ParseEndpoint keyAddrs too short: %q", keyAddrs)
-		return nil, errors.New("endpoint string too short")
+// endpointStr is a json-serialized wgcfg.Endpoints struct.
+// If those Endpoints contain an active discovery key, ParseEndpoint returns a discoEndpoint.
+// Otherwise it returns a legacy endpoint.
+func (c *Conn) ParseEndpoint(endpointStr string) (conn.Endpoint, error) {
+	var endpoints wgcfg.Endpoints
+	err := json.Unmarshal([]byte(endpointStr), &endpoints)
+	if err != nil {
+		return nil, fmt.Errorf("magicsock: ParseEndpoint: json.Unmarshal failed on %q: %w", endpointStr, err)
 	}
-	var pk key.Public
-	copy(pk[:], keyAddrs)
-	addrs := keyAddrs[len(pk):]
+	pk := key.Public(endpoints.PublicKey)
+	discoKey := endpoints.DiscoKey
+	c.logf("magicsock: ParseEndpoint: key=%s: disco=%s ipps=%s", pk.ShortString(), discoKey.ShortString(), derpStr(endpoints.IPPorts.String()))
+
 	c.mu.Lock()
 	defer c.mu.Unlock()
-
-	c.logf("magicsock: ParseEndpoint: key=%s: %s", pk.ShortString(), derpStr(addrs))
-
-	if !strings.HasSuffix(addrs, wgcfg.EndpointDiscoSuffix) {
-		return c.createLegacyEndpointLocked(pk, addrs)
-	}
-
-	discoHex := strings.TrimSuffix(addrs, wgcfg.EndpointDiscoSuffix)
-	discoKey, err := key.NewPublicFromHexMem(mem.S(discoHex))
-	if err != nil {
-		return nil, fmt.Errorf("magicsock: invalid discokey endpoint %q for %v: %w", addrs, pk.ShortString(), err)
+	if discoKey.IsZero() {
+		return c.createLegacyEndpointLocked(pk, endpoints.IPPorts, endpointStr)
 	}
 	de := &discoEndpoint{
-		c:                  c,
-		publicKey:          tailcfg.NodeKey(pk),        // peer public key (for WireGuard + DERP)
-		discoKey:           tailcfg.DiscoKey(discoKey), // for discovery mesages
-		discoShort:         tailcfg.DiscoKey(discoKey).ShortString(),
-		wgEndpointHostPort: addrs,
-		sentPing:           map[stun.TxID]sentPing{},
-		endpointState:      map[netaddr.IPPort]*endpointState{},
+		c:             c,
+		publicKey:     tailcfg.NodeKey(pk),        // peer public key (for WireGuard + DERP)
+		discoKey:      tailcfg.DiscoKey(discoKey), // for discovery mesages
+		discoShort:    tailcfg.DiscoKey(discoKey).ShortString(),
+		wgEndpoint:    endpointStr,
+		sentPing:      map[stun.TxID]sentPing{},
+		endpointState: map[netaddr.IPPort]*endpointState{},
 	}
 	de.initFakeUDPAddr()
 	de.updateFromNode(c.nodeOfDisco[de.discoKey])
@@ -2984,15 +2975,15 @@ func peerShort(k key.Public) string {
 }
 
 func sbPrintAddr(sb *strings.Builder, a netaddr.IPPort) {
-	is6 := a.IP.Is6()
+	is6 := a.IP().Is6()
 	if is6 {
 		sb.WriteByte('[')
 	}
-	fmt.Fprintf(sb, "%s", a.IP)
+	fmt.Fprintf(sb, "%s", a.IP())
 	if is6 {
 		sb.WriteByte(']')
 	}
-	fmt.Fprintf(sb, ":%d", a.Port)
+	fmt.Fprintf(sb, ":%d", a.Port())
 }
 
 func (c *Conn) derpRegionCodeOfAddrLocked(ipPort string) string {
@@ -3029,15 +3020,15 @@ func (c *Conn) UpdateStatus(sb *ipnstate.StatusBuilder) {
 			if !addr.IsSingleIP() {
 				continue
 			}
-			sb.AddTailscaleIP(addr.IP)
+			sb.AddTailscaleIP(addr.IP())
 			// TailAddr previously only allowed for a
 			// single Tailscale IP. For compatibility for
 			// a couple releases starting with 1.8, keep
 			// that field pulled out separately.
-			if addr.IP.Is4() {
-				tailAddr4 = addr.IP.String()
+			if addr.IP().Is4() {
+				tailAddr4 = addr.IP().String()
 			}
-			tailscaleIPs = append(tailscaleIPs, addr.IP)
+			tailscaleIPs = append(tailscaleIPs, addr.IP())
 		}
 	}
 
@@ -3096,8 +3087,8 @@ func (c *Conn) UpdateStatus(sb *ipnstate.StatusBuilder) {
 }
 
 func ippDebugString(ua netaddr.IPPort) string {
-	if ua.IP == derpMagicIPAddr {
-		return fmt.Sprintf("derp-%d", ua.Port)
+	if ua.IP() == derpMagicIPAddr {
+		return fmt.Sprintf("derp-%d", ua.Port())
 	}
 	return ua.String()
 }
@@ -3110,12 +3101,12 @@ type discoEndpoint struct {
 	numStopAndResetAtomic int64
 
 	// These fields are initialized once and never modified.
-	c                  *Conn
-	publicKey          tailcfg.NodeKey  // peer public key (for WireGuard + DERP)
-	discoKey           tailcfg.DiscoKey // for discovery mesages
-	discoShort         string           // ShortString of discoKey
-	fakeWGAddr         netaddr.IPPort   // the UDP address we tell wireguard-go we're using
-	wgEndpointHostPort string           // string from CreateEndpoint: "<hex-discovery-key>.disco.tailscale:12345"
+	c          *Conn
+	publicKey  tailcfg.NodeKey  // peer public key (for WireGuard + DERP)
+	discoKey   tailcfg.DiscoKey // for discovery mesages
+	discoShort string           // ShortString of discoKey
+	fakeWGAddr netaddr.IPPort   // the UDP address we tell wireguard-go we're using
+	wgEndpoint string           // string from ParseEndpoint, holds a JSON-serialized wgcfg.Endpoints
 
 	// Owned by Conn.mu:
 	lastPingFrom netaddr.IPPort
@@ -3266,10 +3257,7 @@ func (de *discoEndpoint) initFakeUDPAddr() {
 	addr[0] = 0xfd
 	addr[1] = 0x00
 	binary.BigEndian.PutUint64(addr[2:], uint64(reflect.ValueOf(de).Pointer()))
-	de.fakeWGAddr = netaddr.IPPort{
-		IP:   netaddr.IPFrom16(addr),
-		Port: 12345,
-	}
+	de.fakeWGAddr = netaddr.IPPortFrom(netaddr.IPFrom16(addr), 12345)
 }
 
 // isFirstRecvActivityInAwhile notes that receive activity has occured for this
@@ -3295,7 +3283,7 @@ func (de *discoEndpoint) String() string {
 func (de *discoEndpoint) ClearSrc()           {}
 func (de *discoEndpoint) SrcToString() string { panic("unused") } // unused by wireguard-go
 func (de *discoEndpoint) SrcIP() net.IP       { panic("unused") } // unused by wireguard-go
-func (de *discoEndpoint) DstToString() string { return de.wgEndpointHostPort }
+func (de *discoEndpoint) DstToString() string { return de.wgEndpoint }
 func (de *discoEndpoint) DstIP() net.IP       { panic("unused") }
 func (de *discoEndpoint) DstToBytes() []byte  { return packIPPort(de.fakeWGAddr) }
 
@@ -3405,6 +3393,7 @@ func (de *discoEndpoint) send(b []byte) error {
 	now := time.Now()
 
 	de.mu.Lock()
+	log.Println("Discosend")
 	udpAddr, derpAddr := de.addrForSendLocked(now)
 	if udpAddr.IsZero() || now.After(de.trustBestAddrUntil) {
 		de.sendPingsLocked(now, true)
@@ -3463,7 +3452,9 @@ func (de *discoEndpoint) removeSentPingLocked(txid stun.TxID, sp sentPing) {
 // The caller (startPingLocked) should've already been recorded the ping in
 // sentPing and set up the timer.
 func (de *discoEndpoint) sendDiscoPing(ep netaddr.IPPort, txid stun.TxID, logLevel discoLogLevel) {
+	log.Println("sendDiscoPing")
 	sent, _ := de.sendDiscoMessage(ep, &disco.Ping{TxID: [12]byte(txid)}, logLevel)
+	log.Println(sent)
 	if !sent {
 		de.forgetPing(txid)
 	}
@@ -3642,9 +3633,10 @@ func (de *discoEndpoint) noteConnectivityChange() {
 // It should be called with the Conn.mu held.
 func (de *discoEndpoint) handlePongConnLocked(m *disco.Pong, src netaddr.IPPort) {
 	de.mu.Lock()
+	log.Println("Disco Reached")
 	defer de.mu.Unlock()
 
-	isDerp := src.IP == derpMagicIPAddr
+	isDerp := src.IP() == derpMagicIPAddr
 
 	sp, ok := de.sentPing[m.TxID]
 	if !ok {
@@ -3720,13 +3712,13 @@ func betterAddr(a, b addrLatency) bool {
 	if a.IsZero() {
 		return false
 	}
-	if a.IP.Is6() && b.IP.Is4() {
+	if a.IP().Is6() && b.IP().Is4() {
 		// Prefer IPv6 for being a bit more robust, as long as
 		// the latencies are roughly equivalent.
 		if a.latency/10*9 < b.latency {
 			return true
 		}
-	} else if a.IP.Is4() && b.IP.Is6() {
+	} else if a.IP().Is4() && b.IP().Is6() {
 		if betterAddr(b, a) {
 			return false
 		}
@@ -3766,7 +3758,7 @@ func (de *discoEndpoint) handleCallMeMaybe(m *disco.CallMeMaybe) {
 	}
 	var newEPs []netaddr.IPPort
 	for _, ep := range m.MyNumber {
-		if ep.IP.Is6() && ep.IP.IsLinkLocalUnicast() {
+		if ep.IP().Is6() && ep.IP().IsLinkLocalUnicast() {
 			// We send these out, but ignore them for now.
 			// TODO: teach the ping code to ping on all interfaces
 			// for these.

wgengine/magicsock/magicsock_test.go

@@ -10,6 +10,7 @@ import (
 	crand "crypto/rand"
 	"crypto/tls"
 	"encoding/binary"
+	"encoding/json"
 	"errors"
 	"fmt"
 	"io/ioutil"
@@ -25,9 +26,9 @@ import (
 	"time"
 	"unsafe"
 
-	"github.com/tailscale/wireguard-go/device"
-	"github.com/tailscale/wireguard-go/tun/tuntest"
 	"golang.org/x/crypto/nacl/box"
+	"golang.zx2c4.com/wireguard/device"
+	"golang.zx2c4.com/wireguard/tun/tuntest"
 	"inet.af/netaddr"
 	"tailscale.com/derp"
 	"tailscale.com/derp/derphttp"
@@ -167,7 +168,7 @@ func newMagicStack(t testing.TB, logf logger.Logf, l nettype.PacketListener, der
 	tsTun.SetFilter(filter.NewAllowAllForTest(logf))
 
 	wgLogger := wglog.NewLogger(logf)
-	dev := device.NewDevice(tsTun, conn.Bind(), wgLogger.DeviceLogger, new(device.DeviceOptions))
+	dev := device.NewDevice(tsTun, conn.Bind(), wgLogger.DeviceLogger)
 	dev.Up()
 
 	// Wait for magicsock to connect up to DERP.
@@ -251,13 +252,13 @@ func meshStacks(logf logger.Logf, ms []*magicStack) (cleanup func()) {
 		nm := &netmap.NetworkMap{
 			PrivateKey: me.privateKey,
 			NodeKey:    tailcfg.NodeKey(me.privateKey.Public()),
-			Addresses:  []netaddr.IPPrefix{{IP: netaddr.IPv4(1, 0, 0, byte(myIdx+1)), Bits: 32}},
+			Addresses:  []netaddr.IPPrefix{netaddr.IPPrefixFrom(netaddr.IPv4(1, 0, 0, byte(myIdx+1)), 32)},
 		}
 		for i, peer := range ms {
 			if i == myIdx {
 				continue
 			}
-			addrs := []netaddr.IPPrefix{{IP: netaddr.IPv4(1, 0, 0, byte(i+1)), Bits: 32}}
+			addrs := []netaddr.IPPrefix{netaddr.IPPrefixFrom(netaddr.IPv4(1, 0, 0, byte(i+1)), 32)}
 			peer := &tailcfg.Node{
 				ID:         tailcfg.NodeID(i + 1),
 				Name:       fmt.Sprintf("node%d", i+1),
@@ -432,7 +433,7 @@ func TestPickDERPFallback(t *testing.T) {
 	// But move if peers are elsewhere.
 	const otherNode = 789
 	c.addrsByKey = map[key.Public]*addrSet{
-		{1}: {ipPorts: []netaddr.IPPort{{IP: derpMagicIPAddr, Port: otherNode}}},
+		{1}: {ipPorts: []netaddr.IPPort{netaddr.IPPortFrom(derpMagicIPAddr, otherNode)}},
 	}
 	if got := c.pickDERPFallback(); got != otherNode {
 		t.Errorf("didn't join peers: got %v; want %v", got, someNode)
@@ -453,7 +454,7 @@ func makeConfigs(t *testing.T, addrs []netaddr.IPPort) []wgcfg.Config {
 		privKeys = append(privKeys, wgkey.Private(privKey))
 
 		addresses = append(addresses, []netaddr.IPPrefix{
-			parseCIDR(t, fmt.Sprintf("1.0.0.%d/32", i+1)),
+			netaddr.MustParseIPPrefix(fmt.Sprintf("1.0.0.%d/32", i+1)),
 		})
 	}
 
@@ -468,10 +469,14 @@ func makeConfigs(t *testing.T, addrs []netaddr.IPPort) []wgcfg.Config {
 			if peerNum == i {
 				continue
 			}
+			publicKey := privKeys[peerNum].Public()
 			peer := wgcfg.Peer{
-				PublicKey:           privKeys[peerNum].Public(),
-				AllowedIPs:          addresses[peerNum],
-				Endpoints:           addr.String(),
+				PublicKey:  publicKey,
+				AllowedIPs: addresses[peerNum],
+				Endpoints: wgcfg.Endpoints{
+					PublicKey: publicKey,
+					IPPorts:   wgcfg.NewIPPortSet(addr),
+				},
 				PersistentKeepalive: 25,
 			}
 			cfg.Peers = append(cfg.Peers, peer)
@@ -481,15 +486,6 @@ func makeConfigs(t *testing.T, addrs []netaddr.IPPort) []wgcfg.Config {
 	return cfgs
 }
 
-func parseCIDR(t *testing.T, addr string) netaddr.IPPrefix {
-	t.Helper()
-	cidr, err := netaddr.ParseIPPrefix(addr)
-	if err != nil {
-		t.Fatal(err)
-	}
-	return cidr
-}
-
 // TestDeviceStartStop exercises the startup and shutdown logic of
 // wireguard-go, which is intimately intertwined with magicsock's own
 // lifecycle. We seem to be good at generating deadlocks here, so if
@@ -513,7 +509,7 @@ func TestDeviceStartStop(t *testing.T) {
 
 	tun := tuntest.NewChannelTUN()
 	wgLogger := wglog.NewLogger(t.Logf)
-	dev := device.NewDevice(tun.TUN(), conn.Bind(), wgLogger.DeviceLogger, new(device.DeviceOptions))
+	dev := device.NewDevice(tun.TUN(), conn.Bind(), wgLogger.DeviceLogger)
 	dev.Up()
 	dev.Close()
 }
@@ -891,8 +887,8 @@ func testTwoDevicePing(t *testing.T, d *devices) {
 	defer m2.Close()
 
 	addrs := []netaddr.IPPort{
-		{IP: d.m1IP, Port: m1.conn.LocalPort()},
-		{IP: d.m2IP, Port: m2.conn.LocalPort()},
+		netaddr.IPPortFrom(d.m1IP, m1.conn.LocalPort()),
+		netaddr.IPPortFrom(d.m2IP, m2.conn.LocalPort()),
 	}
 	cfgs := makeConfigs(t, addrs)
 
@@ -1251,6 +1247,19 @@ func newNonLegacyTestConn(t testing.TB) *Conn {
 	return conn
 }
 
+func makeEndpoint(tb testing.TB, public tailcfg.NodeKey, disco tailcfg.DiscoKey) string {
+	tb.Helper()
+	ep := wgcfg.Endpoints{
+		PublicKey: wgkey.Key(public),
+		DiscoKey:  disco,
+	}
+	buf, err := json.Marshal(ep)
+	if err != nil {
+		tb.Fatal(err)
+	}
+	return string(buf)
+}
+
 // addTestEndpoint sets conn's network map to a single peer expected
 // to receive packets from sendConn (or DERP), and returns that peer's
 // nodekey and discokey.
@@ -1270,7 +1279,7 @@ func addTestEndpoint(tb testing.TB, conn *Conn, sendConn net.PacketConn) (tailcf
 		},
 	})
 	conn.SetPrivateKey(wgkey.Private{0: 1})
-	_, err := conn.ParseEndpoint(string(nodeKey[:]) + "0000000000000000000000000000000000000000000000000000000000000001.disco.tailscale:12345")
+	_, err := conn.ParseEndpoint(makeEndpoint(tb, nodeKey, discoKey))
 	if err != nil {
 		tb.Fatal(err)
 	}
@@ -1444,7 +1453,7 @@ func TestSetNetworkMapChangingNodeKey(t *testing.T) {
 			},
 		},
 	})
-	_, err := conn.ParseEndpoint(string(nodeKey1[:]) + "0000000000000000000000000000000000000000000000000000000000000001.disco.tailscale:12345")
+	_, err := conn.ParseEndpoint(makeEndpoint(t, nodeKey1, discoKey))
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -1546,7 +1555,7 @@ func TestEndpointSetsEqual(t *testing.T) {
 	s := func(ports ...uint16) (ret []tailcfg.Endpoint) {
 		for _, port := range ports {
 			ret = append(ret, tailcfg.Endpoint{
-				Addr: netaddr.IPPort{Port: port},
+				Addr: netaddr.IPPortFrom(netaddr.IP{}, port),
 			})
 		}
 		return

wgengine/mem_ios.go

@@ -0,0 +1,21 @@
+// Copyright (c) 2021 Tailscale Inc & AUTHORS All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package wgengine
+
+import (
+	"golang.zx2c4.com/wireguard/device"
+)
+
+// iOS has a very restrictive memory limit on network extensions.
+// Reduce the maximum amount of memory that wireguard-go can allocate
+// to avoid getting killed.
+
+func init() {
+	device.QueueStagedSize = 64
+	device.QueueOutboundSize = 64
+	device.QueueInboundSize = 64
+	device.QueueHandshakeSize = 64
+	device.PreallocatedBuffersPerPool = 64
+}

wgengine/monitor/monitor_linux.go

@@ -130,11 +130,11 @@ func netaddrIP(std net.IP) netaddr.IP {
 
 func netaddrIPPrefix(std net.IP, bits uint8) netaddr.IPPrefix {
 	ip, _ := netaddr.FromStdIP(std)
-	return netaddr.IPPrefix{IP: ip, Bits: bits}
+	return netaddr.IPPrefixFrom(ip, bits)
 }
 
 func condNetAddrPrefix(ipp netaddr.IPPrefix) string {
-	if ipp.IP.IsZero() {
+	if ipp.IP().IsZero() {
 		return ""
 	}
 	return ipp.String()
@@ -157,7 +157,7 @@ type newRouteMessage struct {
 const tsTable = 52
 
 func (m *newRouteMessage) ignore() bool {
-	return m.Table == tsTable || tsaddr.IsTailscaleIP(m.Dst.IP)
+	return m.Table == tsTable || tsaddr.IsTailscaleIP(m.Dst.IP())
 }
 
 // newAddrMessage is a message for a new address being added.

wgengine/netstack/netstack.go

@@ -48,6 +48,12 @@ const debugNetstack = false
 // and implements wgengine.FakeImpl to act as a userspace network
 // stack when Tailscale is running in fake mode.
 type Impl struct {
+	// ForwardTCPIn, if non-nil, handles forwarding an inbound TCP
+	// connection.
+	// TODO(bradfitz): provide mechanism for tsnet to reject a
+	// port other than accepting it and closing it.
+	ForwardTCPIn func(c net.Conn, port uint16)
+
 	ipstack     *stack.Stack
 	linkEP      *channel.Endpoint
 	tundev      *tstun.Wrapper
@@ -173,7 +179,7 @@ func DNSMapFromNetworkMap(nm *netmap.NetworkMap) DNSMap {
 	suffix := nm.MagicDNSSuffix()
 
 	if nm.Name != "" && len(nm.Addresses) > 0 {
-		ip := nm.Addresses[0].IP
+		ip := nm.Addresses[0].IP()
 		ret[strings.TrimRight(nm.Name, ".")] = ip
 		if dnsname.HasSuffix(nm.Name, suffix) {
 			ret[dnsname.TrimSuffix(nm.Name, suffix)] = ip
@@ -181,7 +187,7 @@ func DNSMapFromNetworkMap(nm *netmap.NetworkMap) DNSMap {
 	}
 	for _, p := range nm.Peers {
 		if p.Name != "" && len(p.Addresses) > 0 {
-			ip := p.Addresses[0].IP
+			ip := p.Addresses[0].IP()
 			ret[strings.TrimRight(p.Name, ".")] = ip
 			if dnsname.HasSuffix(p.Name, suffix) {
 				ret[dnsname.TrimSuffix(p.Name, suffix)] = ip
@@ -221,8 +227,8 @@ func (ns *Impl) removeSubnetAddress(ip netaddr.IP) {
 
 func ipPrefixToAddressWithPrefix(ipp netaddr.IPPrefix) tcpip.AddressWithPrefix {
 	return tcpip.AddressWithPrefix{
-		Address:   tcpip.Address(ipp.IP.IPAddr().IP),
-		PrefixLen: int(ipp.Bits),
+		Address:   tcpip.Address(ipp.IP().IPAddr().IP),
+		PrefixLen: int(ipp.Bits()),
 	}
 }
 
@@ -316,7 +322,7 @@ func (m DNSMap) Resolve(ctx context.Context, addr string) (netaddr.IPPort, error
 	// Try MagicDNS first, else otherwise a real DNS lookup.
 	ip := m[host]
 	if !ip.IsZero() {
-		return netaddr.IPPort{IP: ip, Port: uint16(port16)}, nil
+		return netaddr.IPPortFrom(ip, uint16(port16)), nil
 	}
 
 	// No MagicDNS name so try real DNS.
@@ -329,7 +335,7 @@ func (m DNSMap) Resolve(ctx context.Context, addr string) (netaddr.IPPort, error
 		return netaddr.IPPort{}, fmt.Errorf("DNS lookup returned no results for %q", host)
 	}
 	ip, _ = netaddr.FromStdIP(ips[0])
-	return netaddr.IPPort{IP: ip, Port: uint16(port16)}, nil
+	return netaddr.IPPortFrom(ip, uint16(port16)), nil
 }
 
 func (ns *Impl) DialContextTCP(ctx context.Context, addr string) (*gonet.TCPConn, error) {
@@ -343,11 +349,11 @@ func (ns *Impl) DialContextTCP(ctx context.Context, addr string) (*gonet.TCPConn
 	}
 	remoteAddress := tcpip.FullAddress{
 		NIC:  nicID,
-		Addr: tcpip.Address(remoteIPPort.IP.IPAddr().IP),
-		Port: remoteIPPort.Port,
+		Addr: tcpip.Address(remoteIPPort.IP().IPAddr().IP),
+		Port: remoteIPPort.Port(),
 	}
 	var ipType tcpip.NetworkProtocolNumber
-	if remoteIPPort.IP.Is4() {
+	if remoteIPPort.IP().Is4() {
 		ipType = ipv4.ProtocolNumber
 	} else {
 		ipType = ipv6.ProtocolNumber
@@ -389,7 +395,7 @@ func (ns *Impl) isLocalIP(ip netaddr.IP) bool {
 }
 
 func (ns *Impl) injectInbound(p *packet.Parsed, t *tstun.Wrapper) filter.Response {
-	if ns.onlySubnets && ns.isLocalIP(p.Dst.IP) {
+	if ns.onlySubnets && ns.isLocalIP(p.Dst.IP()) {
 		// In hybrid ("only subnets") mode, bail out early if
 		// the traffic is destined for an actual Tailscale
 		// address. The real host OS interface will handle it.
@@ -441,11 +447,15 @@ func (ns *Impl) acceptTCP(r *tcp.ForwarderRequest) {
 		r.Complete(true)
 		return
 	}
+	r.Complete(false)
+	c := gonet.NewTCPConn(&wq, ep)
+	if ns.ForwardTCPIn != nil {
+		ns.ForwardTCPIn(c, reqDetails.LocalPort)
+		return
+	}
 	if isTailscaleIP {
 		dialAddr = tcpip.Address(net.ParseIP("127.0.0.1")).To4()
 	}
-	r.Complete(false)
-	c := gonet.NewTCPConn(&wq, ep)
 	ns.forwardTCP(c, &wq, dialAddr, reqDetails.LocalPort)
 }
 

wgengine/pendopen.go

@@ -115,8 +115,8 @@ func (e *userspaceEngine) trackOpenPostFilterOut(pp *packet.Parsed, t *tstun.Wra
 	// Don't start timers tracking those. They won't succeed anyway. Avoids log spam
 	// like:
 	//    open-conn-track: timeout opening (100.115.73.60:52501 => 17.125.252.5:443); no associated peer node
-	if runtime.GOOS == "ios" && flow.Dst.Port == 443 && !tsaddr.IsTailscaleIP(flow.Dst.IP) {
-		if _, err := e.peerForIP(flow.Dst.IP); err != nil {
+	if runtime.GOOS == "ios" && flow.Dst.Port() == 443 && !tsaddr.IsTailscaleIP(flow.Dst.IP()) {
+		if _, err := e.peerForIP(flow.Dst.IP()); err != nil {
 			return
 		}
 	}
@@ -156,7 +156,7 @@ func (e *userspaceEngine) onOpenTimeout(flow flowtrack.Tuple) {
 	}
 
 	// Diagnose why it might've timed out.
-	n, err := e.peerForIP(flow.Dst.IP)
+	n, err := e.peerForIP(flow.Dst.IP())
 	if err != nil {
 		e.logf("open-conn-track: timeout opening %v; peerForIP: %v", flow, err)
 		return
@@ -193,7 +193,7 @@ func (e *userspaceEngine) onOpenTimeout(flow flowtrack.Tuple) {
 	if ps == nil {
 		onlyZeroRoute := true // whether peerForIP returned n only because its /0 route matched
 		for _, r := range n.AllowedIPs {
-			if r.Bits != 0 && r.Contains(flow.Dst.IP) {
+			if r.Bits() != 0 && r.Contains(flow.Dst.IP()) {
 				onlyZeroRoute = false
 				break
 			}