util/deephash: move funcs to methods

Signed-off-by: Brad Fitzpatrick <bradfitz@tailscale.com>

.github/workflows/xe-experimental-vm-test.yml

@@ -3,14 +3,11 @@ name: "integration-vms"
 on:
   # # NOTE(Xe): uncomment this region when testing the test
   # pull_request:
-  #   branches:
-  #     - 'main'
+  #   branches: [ main ]
+  push:
+    branches: [ main ]
   release:
     types: [ created ]
-  schedule:
-    # At minute 0 past hour 6 and 18
-    # https://crontab.guru/#00_6,18_*_*_*
-    - cron: '00 6,18 * * *'
 
 jobs:
   experimental-linux-vm-test:
@@ -34,3 +31,17 @@ jobs:
           TMPDIR: "/tmp"
           XDG_CACHE_HOME: "/var/lib/ghrunner/cache"
 
+      - uses: k0kubun/action-slack@v2.0.0
+        with:
+          payload: |
+            {
+              "attachments": [{
+                "text": "${{ job.status }}: ${{ github.workflow }} <https://github.com/${{ github.repository }}/commit/${{ github.sha }}/checks|${{ env.COMMIT_DATE }} #${{ env.COMMIT_NUMBER_OF_DAY }}> " +
+                        "(<https://github.com/${{ github.repository }}/commit/${{ github.sha }}|" + "${{ github.sha }}".substring(0, 10) + ">) " +
+                        "of ${{ github.repository }}@" + "${{ github.ref }}".split('/').reverse()[0] + " by ${{ github.event.head_commit.committer.name }}",
+                "color": "danger"
+              }]
+            }
+        env:
+          SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK_URL }}
+        if: failure() && github.event_name == 'push'

VERSION.txt

@@ -1 +1 @@
-1.9.0
+1.11.0

api.md

@@ -471,15 +471,16 @@ Determines what rules match for a user on an ACL without saving the ACL to the s
 ##### Parameters
 
 ###### Query Parameters
-`user` - A user's email. The provided ACL is queried with this user to determine which rules match.
+`type` - can be 'user' or 'ipport'
+`previewFor` - if type=user, a user's email. If type=ipport, a IP address + port like "10.0.0.1:80".
+The provided ACL is queried with this paramater to determine which rules match.
 
 ###### POST Body
 ACL JSON or HuJSON (see https://tailscale.com/kb/1018/acls)
 
 ##### Example
 ```
-POST /api/v2/tailnet/example.com/acl/preiew
-curl 'https://api.tailscale.com/api/v2/tailnet/example.com/acl?user=user1@example.com' \
+curl 'https://api.tailscale.com/api/v2/tailnet/example.com/acl/preview?previewFor=user1@example.com&type=user' \
   -u "tskey-yourapikey123:" \
   --data-binary '// Example/default ACLs for unrestricted connections.
 {

client/tailscale/tailscale.go

@@ -24,6 +24,7 @@ import (
 	"tailscale.com/ipn/ipnstate"
 	"tailscale.com/paths"
 	"tailscale.com/safesocket"
+	"tailscale.com/tailcfg"
 )
 
 // TailscaledSocket is the tailscaled Unix socket.
@@ -278,3 +279,17 @@ func SetDNS(ctx context.Context, name, value string) error {
 	_, err := send(ctx, "POST", "/localapi/v0/set-dns?"+v.Encode(), 200, nil)
 	return err
 }
+
+// CurrentDERPMap returns the current DERPMap that is being used by the local tailscaled.
+// It is intended to be used with netcheck to see availability of DERPs.
+func CurrentDERPMap(ctx context.Context) (*tailcfg.DERPMap, error) {
+	var derpMap tailcfg.DERPMap
+	res, err := send(ctx, "GET", "/localapi/v0/derpmap", 200, nil)
+	if err != nil {
+		return nil, err
+	}
+	if err = json.Unmarshal(res, &derpMap); err != nil {
+		return nil, fmt.Errorf("invalid derp map json: %w", err)
+	}
+	return &derpMap, nil
+}

cmd/cloner/cloner.go

@@ -246,7 +246,9 @@ func gen(buf *bytes.Buffer, imports map[string]struct{}, name string, typ *types
 					writef("\t\tdst.%s[k] = append([]%s{}, src.%s[k]...)", fname, n, fname)
 					writef("\t}")
 				} else if containsPointers(ft.Elem()) {
-					writef("\t\t" + `panic("TODO map value pointers")`)
+					writef("\tfor k, v := range src.%s {", fname)
+					writef("\t\tdst.%s[k] = v.Clone()", fname)
+					writef("\t}")
 				} else {
 					writef("\tfor k, v := range src.%s {", fname)
 					writef("\t\tdst.%s[k] = v", fname)

cmd/derper/derper.go

@@ -12,8 +12,6 @@ import (
 	"errors"
 	"expvar"
 	"flag"
-	"fmt"
-	"html"
 	"io"
 	"io/ioutil"
 	"log"
@@ -35,7 +33,6 @@ import (
 	"tailscale.com/tsweb"
 	"tailscale.com/types/key"
 	"tailscale.com/types/wgkey"
-	"tailscale.com/version"
 )
 
 var (
@@ -49,6 +46,7 @@ var (
 	meshPSKFile   = flag.String("mesh-psk-file", defaultMeshPSKFile(), "if non-empty, path to file containing the mesh pre-shared key file. It should contain some hex string; whitespace is trimmed.")
 	meshWith      = flag.String("mesh-with", "", "optional comma-separated list of hostnames to mesh with; the server's own hostname can be in the list")
 	bootstrapDNS  = flag.String("bootstrap-dns-names", "", "optional comma-separated list of hostnames to make available at /bootstrap-dns")
+	verifyClients = flag.Bool("verify-clients", false, "verify clients to this DERP server through a local tailscaled instance.")
 )
 
 type config struct {
@@ -125,6 +123,7 @@ func main() {
 	letsEncrypt := tsweb.IsProd443(*addr)
 
 	s := derp.NewServer(key.Private(cfg.PrivateKey), log.Printf)
+	s.SetVerifyClient(*verifyClients)
 
 	if *meshPSKFile != "" {
 		b, err := ioutil.ReadFile(*meshPSKFile)
@@ -143,8 +142,7 @@ func main() {
 	}
 	expvar.Publish("derp", s.ExpVar())
 
-	// Create our own mux so we don't expose /debug/ stuff to the world.
-	mux := tsweb.NewMux(debugHandler(s))
+	mux := http.NewServeMux()
 	mux.Handle("/derp", derphttp.Handler(s))
 	go refreshBootstrapDNSLoop()
 	mux.HandleFunc("/bootstrap-dns", handleBootstrapDNS)
@@ -164,6 +162,18 @@ func main() {
 			io.WriteString(w, "<p>Debug info at <a href='/debug/'>/debug/</a>.</p>\n")
 		}
 	}))
+	debug := tsweb.Debugger(mux)
+	debug.KV("TLS hostname", *hostname)
+	debug.KV("Mesh key", s.HasMeshKey())
+	debug.Handle("check", "Consistency check", http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		err := s.ConsistencyCheck()
+		if err != nil {
+			http.Error(w, err.Error(), 500)
+		} else {
+			io.WriteString(w, "derp.Server ConsistencyCheck okay")
+		}
+	}))
+	debug.Handle("traffic", "Traffic check", http.HandlerFunc(s.ServeDebugTraffic))
 
 	if *runSTUN {
 		go serveSTUN()
@@ -217,39 +227,6 @@ func main() {
 	}
 }
 
-func debugHandler(s *derp.Server) http.Handler {
-	return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
-		if r.RequestURI == "/debug/check" {
-			err := s.ConsistencyCheck()
-			if err != nil {
-				http.Error(w, err.Error(), 500)
-			} else {
-				io.WriteString(w, "derp.Server ConsistencyCheck okay")
-			}
-			return
-		}
-		f := func(format string, args ...interface{}) { fmt.Fprintf(w, format, args...) }
-		f(`<html><body>
-<h1>DERP debug</h1>
-<ul>
-`)
-		f("<li><b>Hostname:</b> %v</li>\n", html.EscapeString(*hostname))
-		f("<li><b>Uptime:</b> %v</li>\n", tsweb.Uptime())
-		f("<li><b>Mesh Key:</b> %v</li>\n", s.HasMeshKey())
-		f("<li><b>Version:</b> %v</li>\n", html.EscapeString(version.Long))
-
-		f(`<li><a href="/debug/vars">/debug/vars</a> (Go)</li>
-   <li><a href="/debug/varz">/debug/varz</a> (Prometheus)</li>
-   <li><a href="/debug/pprof/">/debug/pprof/</a></li>
-   <li><a href="/debug/pprof/goroutine?debug=1">/debug/pprof/goroutine</a> (collapsed)</li>
-   <li><a href="/debug/pprof/goroutine?debug=2">/debug/pprof/goroutine</a> (full)</li>
-   <li><a href="/debug/check">/debug/check</a> internal consistency check</li>
-<ul>
-</html>
-`)
-	})
-}
-
 func serveSTUN() {
 	pc, err := net.ListenPacket("udp", ":3478")
 	if err != nil {

cmd/microproxy/microproxy.go

@@ -55,9 +55,13 @@ func main() {
 		log.Fatalf("Couldn't parse URL %q: %v", *goVarsURL, err)
 	}
 
-	mux := tsweb.NewMux(http.HandlerFunc(debugHandler))
+	mux := http.NewServeMux()
+	tsweb.Debugger(mux) // registers /debug/*
 	mux.Handle("/metrics", tsweb.Protected(proxy))
-	mux.Handle("/varz", tsweb.Protected(tsweb.StdHandler(&goVarsHandler{*goVarsURL}, log.Printf)))
+	mux.Handle("/varz", tsweb.Protected(tsweb.StdHandler(&goVarsHandler{*goVarsURL}, tsweb.HandlerOptions{
+		Quiet200s: true,
+		Logf:      log.Printf,
+	})))
 
 	ch := &certHolder{
 		hostname: *hostname,
@@ -167,23 +171,3 @@ func (c *certHolder) loadLocked() error {
 	c.loaded = time.Now()
 	return nil
 }
-
-// debugHandler serves a page with links to tsweb-managed debug URLs
-// at /debug/.
-func debugHandler(w http.ResponseWriter, r *http.Request) {
-	f := func(format string, args ...interface{}) { fmt.Fprintf(w, format, args...) }
-	f(`<html><body>
-<h1>microproxy debug</h1>
-<ul>
-`)
-	f("<li><b>Hostname:</b> %v</li>\n", *hostname)
-	f("<li><b>Uptime:</b> %v</li>\n", tsweb.Uptime())
-	f(`<li><a href="/debug/vars">/debug/vars</a> (Go)</li>
-   <li><a href="/debug/varz">/debug/varz</a> (Prometheus)</li>
-   <li><a href="/debug/pprof/">/debug/pprof/</a></li>
-   <li><a href="/debug/pprof/goroutine?debug=1">/debug/pprof/goroutine</a> (collapsed)</li>
-   <li><a href="/debug/pprof/goroutine?debug=2">/debug/pprof/goroutine</a> (full)</li>
-<ul>
-</html>
-`)
-}

cmd/tailscale/cli/auth-redirect.html

@@ -0,0 +1,57 @@
+<html>
+<head>
+	<title>Redirecting...</title>
+	<style>
+		html,
+		body {
+			height: 100%;
+		}
+
+		html {
+			background-color: rgb(249, 247, 246);
+			font-family: ui-sans-serif, system-ui, -apple-system, BlinkMacSystemFont, "Segoe UI", Roboto, "Helvetica Neue", Arial, "Noto Sans", sans-serif, "Apple Color Emoji", "Segoe UI Emoji", "Segoe UI Symbol", "Noto Color Emoji";
+			line-height: 1.5;
+			-webkit-text-size-adjust: 100%;
+			-webkit-font-smoothing: antialiased;
+			-moz-osx-font-smoothing: grayscale;
+		}
+
+		body {
+			display: flex;
+			flex-direction: column;
+			align-items: center;
+			justify-content: center;
+		}
+
+		.spinner {
+			margin-bottom: 2rem;
+			border: 4px rgba(112, 110, 109, 0.5) solid;
+			border-left-color: transparent;
+			border-radius: 9999px;
+			width: 4rem;
+			height: 4rem;
+			-webkit-animation: spin 700ms linear infinite;
+      animation: spin 800ms linear infinite;
+		}
+
+		.label {
+			color: rgb(112, 110, 109);
+			padding-left: 0.4rem;
+		}
+
+		@-webkit-keyframes spin {
+			to {
+				transform: rotate(360deg);
+			}
+		}
+
+		@keyframes spin {
+			to {
+				transform: rotate(360deg);
+			}
+		}
+	</style>
+</head> <body>
+	<div class="spinner"></div>
+	<div class="label">Redirecting...</div>
+</body>

cmd/tailscale/cli/cli_test.go

@@ -402,6 +402,28 @@ func TestCheckForAccidentalSettingReverts(t *testing.T) {
 			},
 			want: accidentalUpPrefix + " --hostname=foo --exit-node=100.64.5.7",
 		},
+		{
+			name:  "ignore_login_server_synonym",
+			flags: []string{"--login-server=https://controlplane.tailscale.com"},
+			curPrefs: &ipn.Prefs{
+				ControlURL:       "https://login.tailscale.com",
+				AllowSingleHosts: true,
+				CorpDNS:          true,
+				NetfilterMode:    preftype.NetfilterOn,
+			},
+			want: "", // not an error
+		},
+		{
+			name:  "ignore_login_server_synonym_on_other_change",
+			flags: []string{"--netfilter-mode=off"},
+			curPrefs: &ipn.Prefs{
+				ControlURL:       "https://login.tailscale.com",
+				AllowSingleHosts: true,
+				CorpDNS:          false,
+				NetfilterMode:    preftype.NetfilterOn,
+			},
+			want: accidentalUpPrefix + " --netfilter-mode=off --accept-dns=false",
+		},
 	}
 	for _, tt := range tests {
 		t.Run(tt.name, func(t *testing.T) {

cmd/tailscale/cli/debug.go

@@ -31,6 +31,7 @@ var debugCmd = &ffcli.Command{
 		fs.BoolVar(&debugArgs.goroutines, "daemon-goroutines", false, "If true, dump the tailscaled daemon's goroutines")
 		fs.BoolVar(&debugArgs.ipn, "ipn", false, "If true, subscribe to IPN notifications")
 		fs.BoolVar(&debugArgs.prefs, "prefs", false, "If true, dump active prefs")
+		fs.BoolVar(&debugArgs.derpMap, "derp", false, "If true, dump DERP map")
 		fs.BoolVar(&debugArgs.pretty, "pretty", false, "If true, pretty-print output (for --prefs)")
 		fs.BoolVar(&debugArgs.netMap, "netmap", true, "whether to include netmap in --ipn mode")
 		fs.BoolVar(&debugArgs.localCreds, "local-creds", false, "print how to connect to local tailscaled")
@@ -44,6 +45,7 @@ var debugArgs struct {
 	goroutines bool
 	ipn        bool
 	netMap     bool
+	derpMap    bool
 	file       string
 	prefs      bool
 	pretty     bool
@@ -87,6 +89,18 @@ func runDebug(ctx context.Context, args []string) error {
 		os.Stdout.Write(goroutines)
 		return nil
 	}
+	if debugArgs.derpMap {
+		dm, err := tailscale.CurrentDERPMap(ctx)
+		if err != nil {
+			return fmt.Errorf(
+				"failed to get local derp map, instead `curl %s/derpmap/default`: %w", ipn.DefaultControlURL, err,
+			)
+		}
+		enc := json.NewEncoder(os.Stdout)
+		enc.SetIndent("", "\t")
+		enc.Encode(dm)
+		return nil
+	}
 	if debugArgs.ipn {
 		c, bc, ctx, cancel := connect(ctx)
 		defer cancel()

cmd/tailscale/cli/file.go

@@ -74,7 +74,6 @@ func runCp(ctx context.Context, args []string) error {
 		return runCpTargets(ctx, args)
 	}
 	if len(args) < 2 {
-		//lint:ignore ST1005 no sorry need that colon at the end
 		return errors.New("usage: tailscale file cp <files...> <target>:")
 	}
 	files, target := args[:len(args)-1], args[len(args)-1]
@@ -97,14 +96,12 @@ func runCp(ctx context.Context, args []string) error {
 		return err
 	}
 
-	peerAPIBase, lastSeen, isOffline, err := discoverPeerAPIBase(ctx, ip)
+	peerAPIBase, isOffline, err := discoverPeerAPIBase(ctx, ip)
 	if err != nil {
 		return fmt.Errorf("can't send to %s: %v", target, err)
 	}
 	if isOffline {
 		fmt.Fprintf(os.Stderr, "# warning: %s is offline\n", target)
-	} else if !lastSeen.IsZero() && time.Since(lastSeen) > lastSeenOld {
-		fmt.Fprintf(os.Stderr, "# warning: %s last seen %v ago\n", target, time.Since(lastSeen).Round(time.Minute))
 	}
 
 	if len(files) > 1 {
@@ -182,14 +179,14 @@ func runCp(ctx context.Context, args []string) error {
 	return nil
 }
 
-func discoverPeerAPIBase(ctx context.Context, ipStr string) (base string, lastSeen time.Time, isOffline bool, err error) {
+func discoverPeerAPIBase(ctx context.Context, ipStr string) (base string, isOffline bool, err error) {
 	ip, err := netaddr.ParseIP(ipStr)
 	if err != nil {
-		return "", time.Time{}, false, err
+		return "", false, err
 	}
 	fts, err := tailscale.FileTargets(ctx)
 	if err != nil {
-		return "", time.Time{}, false, err
+		return "", false, err
 	}
 	for _, ft := range fts {
 		n := ft.Node
@@ -197,14 +194,11 @@ func discoverPeerAPIBase(ctx context.Context, ipStr string) (base string, lastSe
 			if a.IP() != ip {
 				continue
 			}
-			if n.LastSeen != nil {
-				lastSeen = *n.LastSeen
-			}
 			isOffline = n.Online != nil && !*n.Online
-			return ft.PeerAPIURL, lastSeen, isOffline, nil
+			return ft.PeerAPIURL, isOffline, nil
 		}
 	}
-	return "", time.Time{}, false, fileTargetErrorDetail(ctx, ip)
+	return "", false, fileTargetErrorDetail(ctx, ip)
 }
 
 // fileTargetErrorDetail returns a non-nil error saying why ip is an
@@ -274,8 +268,6 @@ func (r *slowReader) Read(p []byte) (n int, err error) {
 	return
 }
 
-const lastSeenOld = 20 * time.Minute
-
 func runCpTargets(ctx context.Context, args []string) error {
 	if len(args) > 0 {
 		return errors.New("invalid arguments with --targets")

cmd/tailscale/cli/netcheck.go

@@ -9,14 +9,18 @@ import (
 	"encoding/json"
 	"flag"
 	"fmt"
+	"io"
+	"io/ioutil"
 	"log"
+	"net/http"
 	"os"
 	"sort"
 	"strings"
 	"time"
 
 	"github.com/peterbourgon/ff/v2/ffcli"
-	"tailscale.com/derp/derpmap"
+	"tailscale.com/client/tailscale"
+	"tailscale.com/ipn"
 	"tailscale.com/net/netcheck"
 	"tailscale.com/net/portmapper"
 	"tailscale.com/tailcfg"
@@ -59,7 +63,13 @@ func runNetcheck(ctx context.Context, args []string) error {
 		fmt.Fprintln(os.Stderr, "# Warning: this JSON format is not yet considered a stable interface")
 	}
 
-	dm := derpmap.Prod()
+	dm, err := tailscale.CurrentDERPMap(ctx)
+	if err != nil {
+		dm, err = prodDERPMap(ctx, http.DefaultClient)
+		if err != nil {
+			return err
+		}
+	}
 	for {
 		t0 := time.Now()
 		report, err := c.GetReport(ctx, dm)
@@ -176,3 +186,27 @@ func portMapping(r *netcheck.Report) string {
 	}
 	return strings.Join(got, ", ")
 }
+
+func prodDERPMap(ctx context.Context, httpc *http.Client) (*tailcfg.DERPMap, error) {
+	req, err := http.NewRequestWithContext(ctx, "GET", ipn.DefaultControlURL+"/derpmap/default", nil)
+	if err != nil {
+		return nil, fmt.Errorf("create prodDERPMap request: %w", err)
+	}
+	res, err := httpc.Do(req)
+	if err != nil {
+		return nil, fmt.Errorf("fetch prodDERPMap failed: %w", err)
+	}
+	defer res.Body.Close()
+	b, err := ioutil.ReadAll(io.LimitReader(res.Body, 1<<20))
+	if err != nil {
+		return nil, fmt.Errorf("fetch prodDERPMap failed: %w", err)
+	}
+	if res.StatusCode != 200 {
+		return nil, fmt.Errorf("fetch prodDERPMap: %v: %s", res.Status, b)
+	}
+	var derpMap tailcfg.DERPMap
+	if err = json.Unmarshal(b, &derpMap); err != nil {
+		return nil, fmt.Errorf("fetch prodDERPMap: %w", err)
+	}
+	return &derpMap, nil
+}

cmd/tailscale/cli/up.go

@@ -586,6 +586,9 @@ func checkForAccidentalSettingReverts(flagSet *flag.FlagSet, curPrefs, newPrefs
 		if reflect.DeepEqual(valCur, valNew) {
 			continue
 		}
+		if flagName == "login-server" && isLoginServerSynonym(valCur) && isLoginServerSynonym(valNew) {
+			continue
+		}
 		missing = append(missing, fmtFlagValueArg(flagName, valCur))
 	}
 	if len(missing) == 0 {
@@ -632,6 +635,10 @@ func applyImplicitPrefs(prefs, oldPrefs *ipn.Prefs, curUser string) {
 	}
 }
 
+func isLoginServerSynonym(val interface{}) bool {
+	return val == "https://login.tailscale.com" || val == "https://controlplane.tailscale.com"
+}
+
 func flagAppliesToOS(flag, goos string) bool {
 	switch flag {
 	case "netfilter-mode", "snat-subnet-routes":

cmd/tailscale/cli/web.go

@@ -5,28 +5,29 @@
 package cli
 
 import (
-	"bufio"
 	"bytes"
 	"context"
 	_ "embed"
 	"encoding/json"
+	"encoding/xml"
 	"flag"
 	"fmt"
 	"html/template"
+	"io/ioutil"
 	"log"
 	"net/http"
 	"net/http/cgi"
-	"os"
+	"net/url"
 	"os/exec"
 	"runtime"
 	"strings"
 
 	"github.com/peterbourgon/ff/v2/ffcli"
-	"go4.org/mem"
 	"tailscale.com/client/tailscale"
 	"tailscale.com/ipn"
 	"tailscale.com/tailcfg"
 	"tailscale.com/types/preftype"
+	"tailscale.com/util/groupmember"
 	"tailscale.com/version/distro"
 )
 
@@ -36,6 +37,9 @@ var webHTML string
 //go:embed web.css
 var webCSS string
 
+//go:embed auth-redirect.html
+var authenticationRedirectHTML string
+
 var tmpl *template.Template
 
 func init() {
@@ -85,57 +89,98 @@ func runWeb(ctx context.Context, args []string) error {
 	return http.ListenAndServe(webArgs.listen, http.HandlerFunc(webHandler))
 }
 
-// authorize checks whether the provided user has access to the web UI.
-func authorize(name string) error {
-	if distro.Get() == distro.Synology {
-		return authorizeSynology(name)
+// authorize returns the name of the user accessing the web UI after verifying
+// whether the user has access to the web UI. The function will write the
+// error to the provided http.ResponseWriter.
+// Note: This is different from a tailscale user, and is typically the local
+// user on the node.
+func authorize(w http.ResponseWriter, r *http.Request) (string, error) {
+	switch distro.Get() {
+	case distro.Synology:
+		user, err := synoAuthn()
+		if err != nil {
+			http.Error(w, err.Error(), http.StatusUnauthorized)
+			return "", err
+		}
+		if err := authorizeSynology(user); err != nil {
+			http.Error(w, err.Error(), http.StatusForbidden)
+			return "", err
+		}
+		return user, nil
+	case distro.QNAP:
+		user, resp, err := qnapAuthn(r)
+		if err != nil {
+			http.Error(w, err.Error(), http.StatusUnauthorized)
+			return "", err
+		}
+		if resp.IsAdmin == 0 {
+			http.Error(w, err.Error(), http.StatusForbidden)
+			return "", err
+		}
+		return user, nil
 	}
-	return nil
+	return "", nil
 }
 
 // authorizeSynology checks whether the provided user has access to the web UI
 // by consulting the membership of the "administrators" group.
 func authorizeSynology(name string) error {
-	f, err := os.Open("/etc/group")
+	yes, err := groupmember.IsMemberOfGroup("administrators", name)
 	if err != nil {
 		return err
 	}
-	defer f.Close()
-	s := bufio.NewScanner(f)
-	var agLine string
-	for s.Scan() {
-		if !mem.HasPrefix(mem.B(s.Bytes()), mem.S("administrators:")) {
-			continue
-		}
-		agLine = s.Text()
-		break
+	if !yes {
+		return fmt.Errorf("not a member of administrators group")
 	}
-	if err := s.Err(); err != nil {
-		return err
-	}
-	if agLine == "" {
-		return fmt.Errorf("admin group not defined")
-	}
-	agEntry := strings.Split(agLine, ":")
-	if len(agEntry) < 4 {
-		return fmt.Errorf("malformed admin group entry")
-	}
-	agMembers := agEntry[3]
-	for _, m := range strings.Split(agMembers, ",") {
-		if m == name {
-			return nil
-		}
-	}
-	return fmt.Errorf("not a member of administrators group")
+	return nil
 }
 
-// authenticate returns the name of the user accessing the web UI.
-// Note: This is different from a tailscale user, and is typically the local
-// user on the node.
-func authenticate() (string, error) {
-	if distro.Get() != distro.Synology {
-		return "", nil
+type qnapAuthResponse struct {
+	AuthPassed int    `xml:"authPassed"`
+	IsAdmin    int    `xml:"isAdmin"`
+	AuthSID    string `xml:"authSid"`
+	ErrorValue int    `xml:"errorValue"`
+}
+
+func qnapAuthn(r *http.Request) (string, *qnapAuthResponse, error) {
+	user, err := r.Cookie("NAS_USER")
+	if err != nil {
+		return "", nil, err
 	}
+	token, err := r.Cookie("qtoken")
+	if err != nil {
+		return "", nil, err
+	}
+	query := url.Values{
+		"qtoken": []string{token.Value},
+		"user":   []string{user.Value},
+	}
+	u := url.URL{
+		Scheme:   r.URL.Scheme,
+		Host:     r.URL.Host,
+		Path:     "/cgi-bin/authLogin.cgi",
+		RawQuery: query.Encode(),
+	}
+	resp, err := http.Get(u.String())
+	if err != nil {
+		return "", nil, err
+	}
+	defer resp.Body.Close()
+	out, err := ioutil.ReadAll(resp.Body)
+	if err != nil {
+		return "", nil, err
+	}
+	authResp := &qnapAuthResponse{}
+	if err := xml.Unmarshal(out, authResp); err != nil {
+		return "", nil, err
+	}
+	if authResp.AuthPassed == 0 {
+		return "", nil, fmt.Errorf("not authenticated")
+	}
+	return user.Value, authResp, nil
+}
+
+func synoAuthn() (string, error) {
 	cmd := exec.Command("/usr/syno/synoman/webman/modules/authenticate.cgi")
 	out, err := cmd.CombinedOutput()
 	if err != nil {
@@ -144,10 +189,14 @@ func authenticate() (string, error) {
 	return strings.TrimSpace(string(out)), nil
 }
 
-func synoTokenRedirect(w http.ResponseWriter, r *http.Request) bool {
-	if distro.Get() != distro.Synology {
-		return false
+func authRedirect(w http.ResponseWriter, r *http.Request) bool {
+	if distro.Get() == distro.Synology {
+		return synoTokenRedirect(w, r)
 	}
+	return false
+}
+
+func synoTokenRedirect(w http.ResponseWriter, r *http.Request) bool {
 	if r.Header.Get("X-Syno-Token") != "" {
 		return false
 	}
@@ -181,80 +230,13 @@ req.send(null);
 </body></html>
 `
 
-const authenticationRedirectHTML = `
-<html>
-<head>
-	<title>Redirecting...</title>
-	<style>
-		html,
-		body {
-			height: 100%;
-		}
-
-		html {
-			background-color: rgb(249, 247, 246);
-			font-family: ui-sans-serif, system-ui, -apple-system, BlinkMacSystemFont, "Segoe UI", Roboto, "Helvetica Neue", Arial, "Noto Sans", sans-serif, "Apple Color Emoji", "Segoe UI Emoji", "Segoe UI Symbol", "Noto Color Emoji";
-			line-height: 1.5;
-			-webkit-text-size-adjust: 100%;
-			-webkit-font-smoothing: antialiased;
-			-moz-osx-font-smoothing: grayscale;
-		}
-
-		body {
-			display: flex;
-			flex-direction: column;
-			align-items: center;
-			justify-content: center;
-		}
-
-		.spinner {
-			margin-bottom: 2rem;
-			border: 4px rgba(112, 110, 109, 0.5) solid;
-			border-left-color: transparent;
-			border-radius: 9999px;
-			width: 4rem;
-			height: 4rem;
-			-webkit-animation: spin 700ms linear infinite;
-      animation: spin 800ms linear infinite;
-		}
-
-		.label {
-			color: rgb(112, 110, 109);
-			padding-left: 0.4rem;
-		}
-
-		@-webkit-keyframes spin {
-			to {
-				transform: rotate(360deg);
-			}
-		}
-
-		@keyframes spin {
-			to {
-				transform: rotate(360deg);
-			}
-		}
-	</style>
-</head>
-<body>
-	<div class="spinner"></div>
-	<div class="label">Redirecting...</div>
-</body>
-`
-
 func webHandler(w http.ResponseWriter, r *http.Request) {
-	if synoTokenRedirect(w, r) {
+	if authRedirect(w, r) {
 		return
 	}
 
-	user, err := authenticate()
+	user, err := authorize(w, r)
 	if err != nil {
-		http.Error(w, err.Error(), http.StatusUnauthorized)
-		return
-	}
-
-	if err := authorize(user); err != nil {
-		http.Error(w, err.Error(), http.StatusForbidden)
 		return
 	}
 
@@ -268,7 +250,7 @@ func webHandler(w http.ResponseWriter, r *http.Request) {
 		w.Header().Set("Content-Type", "application/json")
 		url, err := tailscaleUpForceReauth(r.Context())
 		if err != nil {
-			w.WriteHeader(500)
+			w.WriteHeader(http.StatusInternalServerError)
 			json.NewEncoder(w).Encode(mi{"error": err.Error()})
 			return
 		}
@@ -278,7 +260,7 @@ func webHandler(w http.ResponseWriter, r *http.Request) {
 
 	st, err := tailscale.Status(r.Context())
 	if err != nil {
-		http.Error(w, err.Error(), 500)
+		http.Error(w, err.Error(), http.StatusInternalServerError)
 		return
 	}
 
@@ -296,7 +278,7 @@ func webHandler(w http.ResponseWriter, r *http.Request) {
 
 	buf := new(bytes.Buffer)
 	if err := tmpl.Execute(buf, data); err != nil {
-		http.Error(w, err.Error(), 500)
+		http.Error(w, err.Error(), http.StatusInternalServerError)
 		return
 	}
 	w.Write(buf.Bytes())

cmd/tailscale/depaware.txt

@@ -15,13 +15,13 @@ tailscale.com/cmd/tailscale dependencies: (generated by github.com/tailscale/dep
         inet.af/netaddr                                              from tailscale.com/cmd/tailscale/cli+
         rsc.io/goversion/version                                     from tailscale.com/version
         tailscale.com/atomicfile                                     from tailscale.com/ipn
-        tailscale.com/client/tailscale                               from tailscale.com/cmd/tailscale/cli
+        tailscale.com/client/tailscale                               from tailscale.com/cmd/tailscale/cli+
         tailscale.com/client/tailscale/apitype                       from tailscale.com/client/tailscale+
         tailscale.com/cmd/tailscale/cli                              from tailscale.com/cmd/tailscale
         tailscale.com/derp                                           from tailscale.com/derp/derphttp
         tailscale.com/derp/derphttp                                  from tailscale.com/net/netcheck
-        tailscale.com/derp/derpmap                                   from tailscale.com/cmd/tailscale/cli
         tailscale.com/disco                                          from tailscale.com/derp
+        tailscale.com/hostinfo                                       from tailscale.com/net/interfaces
         tailscale.com/ipn                                            from tailscale.com/cmd/tailscale/cli+
         tailscale.com/ipn/ipnstate                                   from tailscale.com/cmd/tailscale/cli+
         tailscale.com/metrics                                        from tailscale.com/derp
@@ -53,7 +53,8 @@ tailscale.com/cmd/tailscale dependencies: (generated by github.com/tailscale/dep
         tailscale.com/types/wgkey                                    from tailscale.com/types/netmap+
         tailscale.com/util/dnsname                                   from tailscale.com/cmd/tailscale/cli+
    W    tailscale.com/util/endian                                    from tailscale.com/net/netns
-   L    tailscale.com/util/lineread                                  from tailscale.com/net/interfaces
+        tailscale.com/util/groupmember                               from tailscale.com/cmd/tailscale/cli
+        tailscale.com/util/lineread                                  from tailscale.com/net/interfaces+
         tailscale.com/version                                        from tailscale.com/cmd/tailscale/cli+
         tailscale.com/version/distro                                 from tailscale.com/cmd/tailscale/cli+
         tailscale.com/wgengine/filter                                from tailscale.com/types/netmap
@@ -118,13 +119,14 @@ tailscale.com/cmd/tailscale dependencies: (generated by github.com/tailscale/dep
         debug/macho                                                  from rsc.io/goversion/version
         debug/pe                                                     from rsc.io/goversion/version
         embed                                                        from tailscale.com/cmd/tailscale/cli
-        encoding                                                     from encoding/json
+        encoding                                                     from encoding/json+
         encoding/asn1                                                from crypto/x509+
         encoding/base64                                              from encoding/json+
         encoding/binary                                              from compress/gzip+
         encoding/hex                                                 from crypto/x509+
         encoding/json                                                from expvar+
         encoding/pem                                                 from crypto/tls+
+        encoding/xml                                                 from tailscale.com/cmd/tailscale/cli
         errors                                                       from bufio+
         expvar                                                       from tailscale.com/derp+
         flag                                                         from github.com/peterbourgon/ff/v2+
@@ -156,6 +158,7 @@ tailscale.com/cmd/tailscale dependencies: (generated by github.com/tailscale/dep
         os                                                           from crypto/rand+
         os/exec                                                      from github.com/toqueteos/webbrowser+
         os/signal                                                    from tailscale.com/cmd/tailscale/cli
+        os/user                                                      from tailscale.com/util/groupmember
         path                                                         from debug/dwarf+
         path/filepath                                                from crypto/x509+
         reflect                                                      from crypto/x509+

cmd/tailscaled/debug.go

@@ -11,6 +11,8 @@ import (
 	"errors"
 	"flag"
 	"fmt"
+	"io"
+	"io/ioutil"
 	"log"
 	"net/http"
 	"net/http/httptrace"
@@ -19,7 +21,7 @@ import (
 	"time"
 
 	"tailscale.com/derp/derphttp"
-	"tailscale.com/derp/derpmap"
+	"tailscale.com/ipn"
 	"tailscale.com/net/interfaces"
 	"tailscale.com/net/tshttpproxy"
 	"tailscale.com/tailcfg"
@@ -131,7 +133,26 @@ func getURL(ctx context.Context, urlStr string) error {
 }
 
 func checkDerp(ctx context.Context, derpRegion string) error {
-	dmap := derpmap.Prod()
+	req, err := http.NewRequestWithContext(ctx, "GET", ipn.DefaultControlURL+"/derpmap/default", nil)
+	if err != nil {
+		return fmt.Errorf("create derp map request: %w", err)
+	}
+	res, err := http.DefaultClient.Do(req)
+	if err != nil {
+		return fmt.Errorf("fetch derp map failed: %w", err)
+	}
+	defer res.Body.Close()
+	b, err := ioutil.ReadAll(io.LimitReader(res.Body, 1<<20))
+	if err != nil {
+		return fmt.Errorf("fetch derp map failed: %w", err)
+	}
+	if res.StatusCode != 200 {
+		return fmt.Errorf("fetch derp map: %v: %s", res.Status, b)
+	}
+	var dmap tailcfg.DERPMap
+	if err = json.Unmarshal(b, &dmap); err != nil {
+		return fmt.Errorf("fetch DERP map: %w", err)
+	}
 	getRegion := func() *tailcfg.DERPRegion {
 		for _, r := range dmap.Regions {
 			if r.RegionCode == derpRegion {

cmd/tailscaled/depaware.txt

@@ -25,7 +25,7 @@ tailscale.com/cmd/tailscaled dependencies: (generated by github.com/tailscale/de
    W 💣 github.com/tailscale/certstore                               from tailscale.com/control/controlclient
         github.com/tcnksm/go-httpstat                                from tailscale.com/net/netcheck
      💣 go4.org/intern                                               from inet.af/netaddr
-     💣 go4.org/mem                                                  from tailscale.com/control/controlclient+
+     💣 go4.org/mem                                                  from tailscale.com/derp+
         go4.org/unsafe/assume-no-moving-gc                           from go4.org/intern
      💣 golang.zx2c4.com/wireguard/conn                              from golang.zx2c4.com/wireguard/device+
    W 💣 golang.zx2c4.com/wireguard/conn/winrio                       from golang.zx2c4.com/wireguard/conn
@@ -40,6 +40,8 @@ tailscale.com/cmd/tailscaled dependencies: (generated by github.com/tailscale/de
    W 💣 golang.zx2c4.com/wireguard/tun/wintun                        from golang.zx2c4.com/wireguard/tun+
    W 💣 golang.zx2c4.com/wireguard/windows/tunnel/winipcfg           from tailscale.com/net/interfaces+
         inet.af/netaddr                                              from tailscale.com/control/controlclient+
+        inet.af/netstack/atomicbitops                                from inet.af/netstack/tcpip+
+     💣 inet.af/netstack/buffer                                      from inet.af/netstack/tcpip/stack
      💣 inet.af/netstack/gohacks                                     from inet.af/netstack/state/wire+
         inet.af/netstack/linewriter                                  from inet.af/netstack/log
         inet.af/netstack/log                                         from inet.af/netstack/state+
@@ -48,7 +50,7 @@ tailscale.com/cmd/tailscaled dependencies: (generated by github.com/tailscale/de
      💣 inet.af/netstack/state                                       from inet.af/netstack/tcpip+
         inet.af/netstack/state/wire                                  from inet.af/netstack/state
      💣 inet.af/netstack/sync                                        from inet.af/netstack/linewriter+
-     💣 inet.af/netstack/tcpip                                       from inet.af/netstack/tcpip/adapters/gonet+
+        inet.af/netstack/tcpip                                       from inet.af/netstack/tcpip/adapters/gonet+
         inet.af/netstack/tcpip/adapters/gonet                        from tailscale.com/wgengine/netstack
      💣 inet.af/netstack/tcpip/buffer                                from inet.af/netstack/tcpip/adapters/gonet+
         inet.af/netstack/tcpip/hash/jenkins                          from inet.af/netstack/tcpip/stack+
@@ -74,14 +76,14 @@ tailscale.com/cmd/tailscaled dependencies: (generated by github.com/tailscale/de
    W 💣 inet.af/wf                                                   from tailscale.com/wf
         rsc.io/goversion/version                                     from tailscale.com/version
         tailscale.com/atomicfile                                     from tailscale.com/ipn+
+        tailscale.com/client/tailscale                               from tailscale.com/derp
         tailscale.com/client/tailscale/apitype                       from tailscale.com/ipn/ipnlocal+
         tailscale.com/control/controlclient                          from tailscale.com/ipn/ipnlocal+
         tailscale.com/derp                                           from tailscale.com/derp/derphttp+
         tailscale.com/derp/derphttp                                  from tailscale.com/net/netcheck+
-        tailscale.com/derp/derpmap                                   from tailscale.com/cmd/tailscaled+
         tailscale.com/disco                                          from tailscale.com/derp+
         tailscale.com/health                                         from tailscale.com/control/controlclient+
-        tailscale.com/internal/deephash                              from tailscale.com/ipn/ipnlocal+
+        tailscale.com/hostinfo                                       from tailscale.com/control/controlclient+
         tailscale.com/ipn                                            from tailscale.com/ipn/ipnserver+
         tailscale.com/ipn/ipnlocal                                   from tailscale.com/ipn/ipnserver+
         tailscale.com/ipn/ipnserver                                  from tailscale.com/cmd/tailscaled
@@ -106,7 +108,8 @@ tailscale.com/cmd/tailscaled dependencies: (generated by github.com/tailscale/de
      💣 tailscale.com/net/netstat                                    from tailscale.com/ipn/ipnserver
         tailscale.com/net/packet                                     from tailscale.com/wgengine+
         tailscale.com/net/portmapper                                 from tailscale.com/net/netcheck+
-        tailscale.com/net/socks5                                     from tailscale.com/cmd/tailscaled
+        tailscale.com/net/socks5                                     from tailscale.com/net/socks5/tssocks
+        tailscale.com/net/socks5/tssocks                             from tailscale.com/cmd/tailscaled
         tailscale.com/net/stun                                       from tailscale.com/net/netcheck+
         tailscale.com/net/tlsdial                                    from tailscale.com/control/controlclient+
         tailscale.com/net/tsaddr                                     from tailscale.com/ipn/ipnlocal+
@@ -114,7 +117,7 @@ tailscale.com/cmd/tailscaled dependencies: (generated by github.com/tailscale/de
         tailscale.com/net/tstun                                      from tailscale.com/cmd/tailscaled+
         tailscale.com/paths                                          from tailscale.com/cmd/tailscaled+
         tailscale.com/portlist                                       from tailscale.com/ipn/ipnlocal
-        tailscale.com/safesocket                                     from tailscale.com/ipn/ipnserver
+        tailscale.com/safesocket                                     from tailscale.com/ipn/ipnserver+
         tailscale.com/smallzstd                                      from tailscale.com/ipn/ipnserver+
         tailscale.com/syncs                                          from tailscale.com/net/interfaces+
         tailscale.com/tailcfg                                        from tailscale.com/control/controlclient+
@@ -133,9 +136,11 @@ tailscale.com/cmd/tailscaled dependencies: (generated by github.com/tailscale/de
         tailscale.com/types/structs                                  from tailscale.com/control/controlclient+
         tailscale.com/types/wgkey                                    from tailscale.com/control/controlclient+
    L    tailscale.com/util/cmpver                                    from tailscale.com/net/dns
+        tailscale.com/util/deephash                                  from tailscale.com/ipn/ipnlocal+
         tailscale.com/util/dnsname                                   from tailscale.com/ipn/ipnstate+
   LW    tailscale.com/util/endian                                    from tailscale.com/net/netns+
-   L    tailscale.com/util/lineread                                  from tailscale.com/control/controlclient+
+        tailscale.com/util/groupmember                               from tailscale.com/ipn/ipnserver
+        tailscale.com/util/lineread                                  from tailscale.com/control/controlclient+
         tailscale.com/util/osshare                                   from tailscale.com/cmd/tailscaled+
         tailscale.com/util/pidowner                                  from tailscale.com/ipn/ipnserver
         tailscale.com/util/racebuild                                 from tailscale.com/logpolicy
@@ -149,7 +154,7 @@ tailscale.com/cmd/tailscaled dependencies: (generated by github.com/tailscale/de
         tailscale.com/wgengine/filter                                from tailscale.com/control/controlclient+
         tailscale.com/wgengine/magicsock                             from tailscale.com/wgengine+
         tailscale.com/wgengine/monitor                               from tailscale.com/wgengine+
-        tailscale.com/wgengine/netstack                              from tailscale.com/cmd/tailscaled
+        tailscale.com/wgengine/netstack                              from tailscale.com/cmd/tailscaled+
         tailscale.com/wgengine/router                                from tailscale.com/cmd/tailscaled+
         tailscale.com/wgengine/wgcfg                                 from tailscale.com/ipn/ipnlocal+
         tailscale.com/wgengine/wgcfg/nmcfg                           from tailscale.com/ipn/ipnlocal
@@ -223,7 +228,7 @@ tailscale.com/cmd/tailscaled dependencies: (generated by github.com/tailscale/de
         debug/elf                                                    from rsc.io/goversion/version
         debug/macho                                                  from rsc.io/goversion/version
         debug/pe                                                     from rsc.io/goversion/version
-   L    embed                                                        from tailscale.com/net/dns
+        embed                                                        from tailscale.com/net/dns+
         encoding                                                     from encoding/json+
         encoding/asn1                                                from crypto/x509+
         encoding/base64                                              from encoding/json+

cmd/tailscaled/tailscaled.go

@@ -24,22 +24,18 @@ import (
 	"runtime/debug"
 	"strconv"
 	"strings"
-	"sync"
 	"syscall"
 	"time"
 
 	"github.com/go-multierror/multierror"
-	"inet.af/netaddr"
 	"tailscale.com/ipn/ipnserver"
 	"tailscale.com/logpolicy"
 	"tailscale.com/net/dns"
-	"tailscale.com/net/socks5"
-	"tailscale.com/net/tsaddr"
+	"tailscale.com/net/socks5/tssocks"
 	"tailscale.com/net/tstun"
 	"tailscale.com/paths"
 	"tailscale.com/types/flagtype"
 	"tailscale.com/types/logger"
-	"tailscale.com/types/netmap"
 	"tailscale.com/util/osshare"
 	"tailscale.com/version"
 	"tailscale.com/version/distro"
@@ -228,6 +224,11 @@ func run() error {
 		if err != nil {
 			log.Fatalf("SOCKS5 listener: %v", err)
 		}
+		if strings.HasSuffix(args.socksAddr, ":0") {
+			// Log kernel-selected port number so integration tests
+			// can find it portably.
+			log.Printf("SOCKS5 listening on %v", socksListener.Addr())
+		}
 	}
 
 	e, useNetstack, err := createEngine(logf, linkMon)
@@ -243,35 +244,7 @@ func run() error {
 	}
 
 	if socksListener != nil {
-		srv := &socks5.Server{
-			Logf: logger.WithPrefix(logf, "socks5: "),
-		}
-		var (
-			mu  sync.Mutex // guards the following field
-			dns netstack.DNSMap
-		)
-		e.AddNetworkMapCallback(func(nm *netmap.NetworkMap) {
-			mu.Lock()
-			defer mu.Unlock()
-			dns = netstack.DNSMapFromNetworkMap(nm)
-		})
-		useNetstackForIP := func(ip netaddr.IP) bool {
-			// TODO(bradfitz): this isn't exactly right.
-			// We should also support subnets when the
-			// prefs are configured as such.
-			return tsaddr.IsTailscaleIP(ip)
-		}
-		srv.Dialer = func(ctx context.Context, network, addr string) (net.Conn, error) {
-			ipp, err := dns.Resolve(ctx, addr)
-			if err != nil {
-				return nil, err
-			}
-			if ns != nil && useNetstackForIP(ipp.IP()) {
-				return ns.DialContextTCP(ctx, addr)
-			}
-			var d net.Dialer
-			return d.DialContext(ctx, network, ipp.String())
-		}
+		srv := tssocks.NewServer(logger.WithPrefix(logf, "socks5: "), e, ns)
 		go func() {
 			log.Fatalf("SOCKS5 server exited: %v", srv.Serve(socksListener))
 		}()

cmd/tailscaled/tailscaled.openrc

@@ -0,0 +1,23 @@
+#!/sbin/openrc-run
+
+source /etc/default/tailscaled
+
+command="/usr/sbin/tailscaled"
+command_args="--state=/var/lib/tailscale/tailscaled.state --port=$PORT --socket=/var/run/tailscale/tailscaled.sock $FLAGS"
+command_background=true
+pidfile="/run/tailscaled.pid"
+start_stop_daemon_args="-1 /var/log/tailscaled.log -2 /var/log/tailscaled.log"
+
+depend() {
+    need net
+}
+
+start_pre() {
+    mkdir -p /var/run/tailscale
+    mkdir -p /var/lib/tailscale
+    $command --cleanup
+}
+
+stop_post() {
+    $command --cleanup
+}

cmd/tailscaled/tailscaled.service

@@ -2,7 +2,7 @@
 Description=Tailscale node agent
 Documentation=https://tailscale.com/kb/
 Wants=network-pre.target
-After=network-pre.target
+After=network-pre.target NetworkManager.service systemd-resolved.service
 
 [Service]
 EnvironmentFile=/etc/default/tailscaled

cmd/tailscaled/tailscaled_windows.go

@@ -19,6 +19,7 @@ package main // import "tailscale.com/cmd/tailscaled"
 
 import (
 	"context"
+	"encoding/json"
 	"fmt"
 	"log"
 	"os"
@@ -27,6 +28,7 @@ import (
 	"golang.org/x/sys/windows"
 	"golang.org/x/sys/windows/svc"
 	"golang.zx2c4.com/wireguard/windows/tunnel/winipcfg"
+	"inet.af/netaddr"
 	"tailscale.com/ipn/ipnserver"
 	"tailscale.com/logpolicy"
 	"tailscale.com/net/dns"
@@ -126,16 +128,6 @@ func beFirewallKillswitch() bool {
 	log.SetFlags(0)
 	log.Printf("killswitch subprocess starting, tailscale GUID is %s", os.Args[2])
 
-	go func() {
-		b := make([]byte, 16)
-		for {
-			_, err := os.Stdin.Read(b)
-			if err != nil {
-				log.Fatalf("parent process died or requested exit, exiting (%v)", err)
-			}
-		}
-	}()
-
 	guid, err := windows.GUIDFromString(os.Args[2])
 	if err != nil {
 		log.Fatalf("invalid GUID %q: %v", os.Args[2], err)
@@ -147,13 +139,25 @@ func beFirewallKillswitch() bool {
 	}
 
 	start := time.Now()
-	if _, err := wf.New(uint64(luid)); err != nil {
-		log.Fatalf("filewall creation failed: %v", err)
+	fw, err := wf.New(uint64(luid))
+	if err != nil {
+		log.Fatalf("failed to enable firewall: %v", err)
 	}
 	log.Printf("killswitch enabled, took %s", time.Since(start))
 
-	// Block until the monitor goroutine shuts us down.
-	select {}
+	// Note(maisem): when local lan access toggled, tailscaled needs to
+	// inform the firewall to let local routes through. The set of routes
+	// is passed in via stdin encoded in json.
+	dcd := json.NewDecoder(os.Stdin)
+	for {
+		var routes []netaddr.IPPrefix
+		if err := dcd.Decode(&routes); err != nil {
+			log.Fatalf("parent process died or requested exit, exiting (%v)", err)
+		}
+		if err := fw.UpdatePermittedRoutes(routes); err != nil {
+			log.Fatalf("failed to update routes (%v)", err)
+		}
+	}
 }
 
 func startIPNServer(ctx context.Context, logid string) error {

control/controlclient/direct.go

@@ -80,6 +80,7 @@ type Direct struct {
 	endpoints     []tailcfg.Endpoint
 	everEndpoints bool   // whether we've ever had non-empty endpoints
 	localPort     uint16 // or zero to mean auto
+	lastPingURL   string // last PingRequest.URL received, for dup suppresion
 }
 
 type Options struct {
@@ -353,6 +354,7 @@ func (c *Direct) doLogin(ctx context.Context, opt loginOpt) (mustRegen bool, new
 		if err != nil {
 			return regen, opt.URL, err
 		}
+		c.logf("control server key %s from %s", serverKey.HexString(), c.serverURL)
 
 		c.mu.Lock()
 		c.serverKey = serverKey
@@ -775,7 +777,7 @@ func (c *Direct) sendMapRequest(ctx context.Context, maxPolls int, cb func(*netm
 			health.GotStreamedMapResponse()
 		}
 
-		if pr := resp.PingRequest; pr != nil {
+		if pr := resp.PingRequest; pr != nil && c.isUniquePingRequest(pr) {
 			go answerPing(c.logf, c.httpc, pr)
 		}
 
@@ -1170,6 +1172,23 @@ func ipForwardingBroken(routes []netaddr.IPPrefix, state *interfaces.State) bool
 	return false
 }
 
+// isUniquePingRequest reports whether pr contains a new PingRequest.URL
+// not already handled, noting its value when returning true.
+func (c *Direct) isUniquePingRequest(pr *tailcfg.PingRequest) bool {
+	if pr == nil || pr.URL == "" {
+		// Bogus.
+		return false
+	}
+
+	c.mu.Lock()
+	defer c.mu.Unlock()
+	if pr.URL == c.lastPingURL {
+		return false
+	}
+	c.lastPingURL = pr.URL
+	return true
+}
+
 func answerPing(logf logger.Logf, c *http.Client, pr *tailcfg.PingRequest) {
 	if pr.URL == "" {
 		logf("invalid PingRequest with no URL")
@@ -1263,7 +1282,7 @@ func (c *Direct) SetDNS(ctx context.Context, req *tailcfg.SetDNSRequest) error {
 	defer res.Body.Close()
 	if res.StatusCode != 200 {
 		msg, _ := ioutil.ReadAll(res.Body)
-		return fmt.Errorf("sign-dns response: %v, %.200s", res.Status, strings.TrimSpace(string(msg)))
+		return fmt.Errorf("set-dns response: %v, %.200s", res.Status, strings.TrimSpace(string(msg)))
 	}
 	var setDNSRes struct{} // no fields yet
 	if err := decode(res, &setDNSRes, &serverKey, &machinePrivKey); err != nil {

control/controlclient/hostinfo_linux.go

@@ -9,13 +9,11 @@ package controlclient
 import (
 	"bytes"
 	"fmt"
-	"io"
 	"io/ioutil"
-	"os"
 	"strings"
 	"syscall"
 
-	"go4.org/mem"
+	"tailscale.com/hostinfo"
 	"tailscale.com/util/lineread"
 	"tailscale.com/version/distro"
 )
@@ -56,11 +54,11 @@ func osVersionLinux() string {
 		}
 		attrBuf.WriteByte(byte(b))
 	}
-	if inContainer() {
+	if hostinfo.InContainer() {
 		attrBuf.WriteString("; container")
 	}
-	if inKnative() {
-		attrBuf.WriteString("; env=kn")
+	if env := hostinfo.GetEnvType(); env != "" {
+		fmt.Fprintf(&attrBuf, "; env=%s", env)
 	}
 	attr := attrBuf.String()
 
@@ -93,31 +91,3 @@ func osVersionLinux() string {
 	}
 	return fmt.Sprintf("Other%s", attr)
 }
-
-func inContainer() (ret bool) {
-	lineread.File("/proc/1/cgroup", func(line []byte) error {
-		if mem.Contains(mem.B(line), mem.S("/docker/")) ||
-			mem.Contains(mem.B(line), mem.S("/lxc/")) {
-			ret = true
-			return io.EOF // arbitrary non-nil error to stop loop
-		}
-		return nil
-	})
-	lineread.File("/proc/mounts", func(line []byte) error {
-		if mem.Contains(mem.B(line), mem.S("fuse.lxcfs")) {
-			ret = true
-			return io.EOF
-		}
-		return nil
-	})
-	return
-}
-
-func inKnative() bool {
-	// https://cloud.google.com/run/docs/reference/container-contract#env-vars
-	if os.Getenv("K_REVISION") != "" && os.Getenv("K_CONFIGURATION") != "" &&
-		os.Getenv("K_SERVICE") != "" && os.Getenv("PORT") != "" {
-		return true
-	}
-	return false
-}

control/controlclient/map.go

@@ -6,8 +6,11 @@ package controlclient
 
 import (
 	"log"
+	"os"
 	"sort"
+	"strconv"
 
+	"inet.af/netaddr"
 	"tailscale.com/tailcfg"
 	"tailscale.com/types/logger"
 	"tailscale.com/types/netmap"
@@ -124,7 +127,7 @@ func (ms *mapSession) netmapForResponse(resp *tailcfg.MapResponse) *netmap.Netwo
 		nm.SelfNode = node
 		nm.Expiry = node.KeyExpiry
 		nm.Name = node.Name
-		nm.Addresses = node.Addresses
+		nm.Addresses = filterSelfAddresses(node.Addresses)
 		nm.User = node.User
 		nm.Hostinfo = node.Hostinfo
 		if node.MachineAuthorized {
@@ -280,3 +283,19 @@ func cloneNodes(v1 []*tailcfg.Node) []*tailcfg.Node {
 	}
 	return v2
 }
+
+var debugSelfIPv6Only, _ = strconv.ParseBool(os.Getenv("TS_DEBUG_SELF_V6_ONLY"))
+
+func filterSelfAddresses(in []netaddr.IPPrefix) (ret []netaddr.IPPrefix) {
+	switch {
+	default:
+		return in
+	case debugSelfIPv6Only:
+		for _, a := range in {
+			if a.IP().Is6() {
+				ret = append(ret, a)
+			}
+		}
+		return ret
+	}
+}

derp/derp_server.go

@@ -20,18 +20,24 @@ import (
 	"io"
 	"io/ioutil"
 	"log"
+	"math"
 	"math/big"
 	"math/rand"
+	"net/http"
 	"os"
+	"os/exec"
 	"runtime"
 	"strconv"
 	"strings"
 	"sync"
+	"sync/atomic"
 	"time"
 
 	"go4.org/mem"
 	"golang.org/x/crypto/nacl/box"
 	"golang.org/x/sync/errgroup"
+	"inet.af/netaddr"
+	"tailscale.com/client/tailscale"
 	"tailscale.com/disco"
 	"tailscale.com/metrics"
 	"tailscale.com/types/key"
@@ -120,6 +126,11 @@ type Server struct {
 	multiForwarderCreated    expvar.Int
 	multiForwarderDeleted    expvar.Int
 	removePktForwardOther    expvar.Int
+	avgQueueDuration         *uint64 // In milliseconds; accessed atomically
+
+	// verifyClients only accepts client connections to the DERP server if the clientKey is a
+	// known peer in the network, as specified by a running tailscaled's client's local api.
+	verifyClients bool
 
 	mu          sync.Mutex
 	closed      bool
@@ -138,6 +149,9 @@ type Server struct {
 	// because it includes intra-region forwarded packets as the
 	// src.
 	sentTo map[key.Public]map[key.Public]int64 // src => dst => dst's latest sclient.connNum
+
+	// maps from netaddr.IPPort to a client's public key
+	keyOfAddr map[netaddr.IPPort]key.Public
 }
 
 // PacketForwarder is something that can forward packets.
@@ -182,6 +196,8 @@ func NewServer(privateKey key.Private, logf logger.Logf) *Server {
 		memSys0:              ms.Sys,
 		watchers:             map[*sclient]bool{},
 		sentTo:               map[key.Public]map[key.Public]int64{},
+		avgQueueDuration:     new(uint64),
+		keyOfAddr:            map[netaddr.IPPort]key.Public{},
 	}
 	s.initMetacert()
 	s.packetsRecvDisco = s.packetsRecvByKind.Get("disco")
@@ -203,6 +219,13 @@ func (s *Server) SetMeshKey(v string) {
 	s.meshKey = v
 }
 
+// SetVerifyClients sets whether this DERP server verifies clients through tailscaled.
+//
+// It must be called before serving begins.
+func (s *Server) SetVerifyClient(v bool) {
+	s.verifyClients = v
+}
+
 // HasMeshKey reports whether the server is configured with a mesh key.
 func (s *Server) HasMeshKey() bool { return s.meshKey != "" }
 
@@ -339,6 +362,7 @@ func (s *Server) registerClient(c *sclient) {
 	if _, ok := s.clientsMesh[c.key]; !ok {
 		s.clientsMesh[c.key] = nil // just for varz of total users in cluster
 	}
+	s.keyOfAddr[c.remoteIPPort] = c.key
 	s.curClients.Add(1)
 	s.broadcastPeerStateChangeLocked(c.key, true)
 }
@@ -373,6 +397,8 @@ func (s *Server) unregisterClient(c *sclient) {
 		delete(s.watchers, c)
 	}
 
+	delete(s.keyOfAddr, c.remoteIPPort)
+
 	s.curClients.Add(-1)
 	if c.preferred {
 		s.curHomeClients.Add(-1)
@@ -446,20 +472,23 @@ func (s *Server) accept(nc Conn, brw *bufio.ReadWriter, remoteAddr string, connN
 	ctx, cancel := context.WithCancel(context.Background())
 	defer cancel()
 
+	remoteIPPort, _ := netaddr.ParseIPPort(remoteAddr)
+
 	c := &sclient{
-		connNum:     connNum,
-		s:           s,
-		key:         clientKey,
-		nc:          nc,
-		br:          br,
-		bw:          bw,
-		logf:        logger.WithPrefix(s.logf, fmt.Sprintf("derp client %v/%x: ", remoteAddr, clientKey)),
-		done:        ctx.Done(),
-		remoteAddr:  remoteAddr,
-		connectedAt: time.Now(),
-		sendQueue:   make(chan pkt, perClientSendQueueDepth),
-		peerGone:    make(chan key.Public),
-		canMesh:     clientInfo.MeshKey != "" && clientInfo.MeshKey == s.meshKey,
+		connNum:      connNum,
+		s:            s,
+		key:          clientKey,
+		nc:           nc,
+		br:           br,
+		bw:           bw,
+		logf:         logger.WithPrefix(s.logf, fmt.Sprintf("derp client %v/%x: ", remoteAddr, clientKey)),
+		done:         ctx.Done(),
+		remoteAddr:   remoteAddr,
+		remoteIPPort: remoteIPPort,
+		connectedAt:  time.Now(),
+		sendQueue:    make(chan pkt, perClientSendQueueDepth),
+		peerGone:     make(chan key.Public),
+		canMesh:      clientInfo.MeshKey != "" && clientInfo.MeshKey == s.meshKey,
 	}
 	if c.canMesh {
 		c.meshUpdate = make(chan struct{})
@@ -611,8 +640,9 @@ func (c *sclient) handleFrameForwardPacket(ft frameType, fl uint32) error {
 	}
 
 	return c.sendPkt(dst, pkt{
-		bs:  contents,
-		src: srcKey,
+		bs:         contents,
+		enqueuedAt: time.Now(),
+		src:        srcKey,
 	})
 }
 
@@ -665,8 +695,9 @@ func (c *sclient) handleFrameSendPacket(ft frameType, fl uint32) error {
 	}
 
 	p := pkt{
-		bs:  contents,
-		src: c.key,
+		bs:         contents,
+		enqueuedAt: time.Now(),
+		src:        c.key,
 	}
 	return c.sendPkt(dst, p)
 }
@@ -696,7 +727,7 @@ func (c *sclient) sendPkt(dst *sclient, p pkt) error {
 		}
 
 		select {
-		case <-dst.sendQueue:
+		case pkt := <-dst.sendQueue:
 			s.packetsDropped.Add(1)
 			s.packetsDroppedQueueHead.Add(1)
 			if verboseDropKeys[dstKey] {
@@ -705,6 +736,7 @@ func (c *sclient) sendPkt(dst *sclient, p pkt) error {
 				msg := fmt.Sprintf("tail drop %s -> %s", p.src.ShortString(), dstKey.ShortString())
 				c.s.limitedLogf(msg)
 			}
+			c.recordQueueTime(pkt.enqueuedAt)
 			if debug {
 				c.logf("dropping packet from client %x queue head", dstKey)
 			}
@@ -750,8 +782,17 @@ func (c *sclient) requestMeshUpdate() {
 }
 
 func (s *Server) verifyClient(clientKey key.Public, info *clientInfo) error {
-	// TODO(crawshaw): implement policy constraints on who can use the DERP server
-	// TODO(bradfitz): ... and at what rate.
+	if !s.verifyClients {
+		return nil
+	}
+	status, err := tailscale.Status(context.TODO())
+	if err != nil {
+		return fmt.Errorf("failed to query local tailscaled status: %w", err)
+	}
+	if _, exists := status.Peer[clientKey]; !exists {
+		return fmt.Errorf("client %v not in set of peers", clientKey)
+	}
+	// TODO(bradfitz): add policy for configurable bandwidth rate per client?
 	return nil
 }
 
@@ -885,18 +926,19 @@ func (s *Server) recvForwardPacket(br *bufio.Reader, frameLen uint32) (srcKey, d
 // (The "s" prefix is to more explicitly distinguish it from Client in derp_client.go)
 type sclient struct {
 	// Static after construction.
-	connNum    int64 // process-wide unique counter, incremented each Accept
-	s          *Server
-	nc         Conn
-	key        key.Public
-	info       clientInfo
-	logf       logger.Logf
-	done       <-chan struct{} // closed when connection closes
-	remoteAddr string          // usually ip:port from net.Conn.RemoteAddr().String()
-	sendQueue  chan pkt        // packets queued to this client; never closed
-	peerGone   chan key.Public // write request that a previous sender has disconnected (not used by mesh peers)
-	meshUpdate chan struct{}   // write request to write peerStateChange
-	canMesh    bool            // clientInfo had correct mesh token for inter-region routing
+	connNum      int64 // process-wide unique counter, incremented each Accept
+	s            *Server
+	nc           Conn
+	key          key.Public
+	info         clientInfo
+	logf         logger.Logf
+	done         <-chan struct{} // closed when connection closes
+	remoteAddr   string          // usually ip:port from net.Conn.RemoteAddr().String()
+	remoteIPPort netaddr.IPPort  // zero if remoteAddr is not ip:port.
+	sendQueue    chan pkt        // packets queued to this client; never closed
+	peerGone     chan key.Public // write request that a previous sender has disconnected (not used by mesh peers)
+	meshUpdate   chan struct{}   // write request to write peerStateChange
+	canMesh      bool            // clientInfo had correct mesh token for inter-region routing
 
 	// Owned by run, not thread-safe.
 	br          *bufio.Reader
@@ -927,11 +969,13 @@ type pkt struct {
 	// src is the who's the sender of the packet.
 	src key.Public
 
+	// enqueuedAt is when a packet was put onto a queue before it was sent,
+	// and is used for reporting metrics on the duration of packets in the queue.
+	enqueuedAt time.Time
+
 	// bs is the data packet bytes.
 	// The memory is owned by pkt.
 	bs []byte
-
-	// TODO(danderson): enqueue time, to measure queue latency?
 }
 
 func (c *sclient) setPreferred(v bool) {
@@ -959,6 +1003,25 @@ func (c *sclient) setPreferred(v bool) {
 	}
 }
 
+// expMovingAverage returns the new moving average given the previous average,
+// a new value, and an alpha decay factor.
+// https://en.wikipedia.org/wiki/Moving_average#Exponential_moving_average
+func expMovingAverage(prev, newValue, alpha float64) float64 {
+	return alpha*newValue + (1-alpha)*prev
+}
+
+// recordQueueTime updates the average queue duration metric after a packet has been sent.
+func (c *sclient) recordQueueTime(enqueuedAt time.Time) {
+	elapsed := float64(time.Since(enqueuedAt).Milliseconds())
+	for {
+		old := atomic.LoadUint64(c.s.avgQueueDuration)
+		newAvg := expMovingAverage(math.Float64frombits(old), elapsed, 0.1)
+		if atomic.CompareAndSwapUint64(c.s.avgQueueDuration, old, math.Float64bits(newAvg)) {
+			break
+		}
+	}
+}
+
 func (c *sclient) sendLoop(ctx context.Context) error {
 	defer func() {
 		// If the sender shuts down unilaterally due to an error, close so
@@ -1002,6 +1065,7 @@ func (c *sclient) sendLoop(ctx context.Context) error {
 			continue
 		case msg := <-c.sendQueue:
 			werr = c.sendPacket(msg.src, msg.bs)
+			c.recordQueueTime(msg.enqueuedAt)
 			continue
 		case <-keepAliveTick.C:
 			werr = c.sendKeepAlive()
@@ -1025,6 +1089,7 @@ func (c *sclient) sendLoop(ctx context.Context) error {
 			continue
 		case msg := <-c.sendQueue:
 			werr = c.sendPacket(msg.src, msg.bs)
+			c.recordQueueTime(msg.enqueuedAt)
 		case <-keepAliveTick.C:
 			werr = c.sendKeepAlive()
 		}
@@ -1290,6 +1355,9 @@ func (s *Server) ExpVar() expvar.Var {
 	m.Set("multiforwarder_created", &s.multiForwarderCreated)
 	m.Set("multiforwarder_deleted", &s.multiForwarderDeleted)
 	m.Set("packet_forwarder_delete_other_value", &s.removePktForwardOther)
+	m.Set("average_queue_duration_ms", expvar.Func(func() interface{} {
+		return math.Float64frombits(atomic.LoadUint64(s.avgQueueDuration))
+	}))
 	var expvarVersion expvar.String
 	expvarVersion.Set(version.Long)
 	m.Set("version", &expvarVersion)
@@ -1365,3 +1433,84 @@ func writePublicKey(bw *bufio.Writer, key *key.Public) error {
 	}
 	return nil
 }
+
+const minTimeBetweenLogs = 2 * time.Second
+
+// BytesSentRecv records the number of bytes that have been sent since the last traffic check
+// for a given process, as well as the public key of the process sending those bytes.
+type BytesSentRecv struct {
+	Sent uint64
+	Recv uint64
+	// Key is the public key of the client which sent/received these bytes.
+	Key key.Public
+}
+
+// parseSSOutput parses the output from the specific call to ss in ServeDebugTraffic.
+// Separated out for ease of testing.
+func parseSSOutput(raw string) map[netaddr.IPPort]BytesSentRecv {
+	newState := map[netaddr.IPPort]BytesSentRecv{}
+	// parse every 2 lines and get src and dst ips, and kv pairs
+	lines := strings.Split(raw, "\n")
+	for i := 0; i < len(lines); i += 2 {
+		ipInfo := strings.Fields(strings.TrimSpace(lines[i]))
+		if len(ipInfo) < 5 {
+			continue
+		}
+		src, err := netaddr.ParseIPPort(ipInfo[4])
+		if err != nil {
+			continue
+		}
+		stats := strings.Fields(strings.TrimSpace(lines[i+1]))
+		stat := BytesSentRecv{}
+		for _, s := range stats {
+			if strings.Contains(s, "bytes_sent") {
+				sent, err := strconv.Atoi(s[strings.Index(s, ":")+1:])
+				if err == nil {
+					stat.Sent = uint64(sent)
+				}
+			} else if strings.Contains(s, "bytes_received") {
+				recv, err := strconv.Atoi(s[strings.Index(s, ":")+1:])
+				if err == nil {
+					stat.Recv = uint64(recv)
+				}
+			}
+		}
+		newState[src] = stat
+	}
+	return newState
+}
+
+func (s *Server) ServeDebugTraffic(w http.ResponseWriter, r *http.Request) {
+	prevState := map[netaddr.IPPort]BytesSentRecv{}
+	enc := json.NewEncoder(w)
+	for r.Context().Err() == nil {
+		output, err := exec.Command("ss", "-i", "-H", "-t").Output()
+		if err != nil {
+			fmt.Fprintf(w, "ss failed: %v", err)
+			return
+		}
+		newState := parseSSOutput(string(output))
+		s.mu.Lock()
+		for k, next := range newState {
+			prev := prevState[k]
+			if prev.Sent < next.Sent || prev.Recv < next.Recv {
+				if pkey, ok := s.keyOfAddr[k]; ok {
+					next.Key = pkey
+					if err := enc.Encode(next); err != nil {
+						s.mu.Unlock()
+						return
+					}
+				}
+			}
+		}
+		s.mu.Unlock()
+		prevState = newState
+		if _, err := fmt.Fprintln(w); err != nil {
+			return
+		}
+		if f, ok := w.(http.Flusher); ok {
+			f.Flush()
+		}
+		time.Sleep(minTimeBetweenLogs)
+	}
+}

derp/derp_test.go

@@ -948,3 +948,14 @@ func waitConnect(t testing.TB, c *Client) {
 		t.Fatalf("client first Recv was unexpected type %T", v)
 	}
 }
+
+func TestParseSSOutput(t *testing.T) {
+	contents, err := ioutil.ReadFile("testdata/example_ss.txt")
+	if err != nil {
+		t.Errorf("ioutil.Readfile(example_ss.txt) failed: %v", err)
+	}
+	seen := parseSSOutput(string(contents))
+	if len(seen) == 0 {
+		t.Errorf("parseSSOutput expected non-empty map")
+	}
+}

derp/derpmap/derpmap.go

@@ -1,91 +0,0 @@
-// Copyright (c) 2020 Tailscale Inc & AUTHORS All rights reserved.
-// Use of this source code is governed by a BSD-style
-// license that can be found in the LICENSE file.
-
-// Package derpmap contains information about Tailscale.com's production DERP nodes.
-//
-// This package is only used by the "tailscale netcheck" command for debugging.
-// In normal operation the Tailscale nodes get this sent to them from the control
-// server.
-//
-// TODO: remove this package and make "tailscale netcheck" get the
-// list from the control server too.
-package derpmap
-
-import (
-	"fmt"
-	"strings"
-
-	"tailscale.com/tailcfg"
-)
-
-func derpNode(suffix, v4, v6 string) *tailcfg.DERPNode {
-	return &tailcfg.DERPNode{
-		Name:     suffix, // updated later
-		RegionID: 0,      // updated later
-		IPv4:     v4,
-		IPv6:     v6,
-	}
-}
-
-func derpRegion(id int, code, name string, nodes ...*tailcfg.DERPNode) *tailcfg.DERPRegion {
-	region := &tailcfg.DERPRegion{
-		RegionID:   id,
-		RegionName: name,
-		RegionCode: code,
-		Nodes:      nodes,
-	}
-	for _, n := range nodes {
-		n.Name = fmt.Sprintf("%d%s", id, n.Name)
-		n.RegionID = id
-		n.HostName = fmt.Sprintf("derp%s.tailscale.com", strings.TrimSuffix(n.Name, "a"))
-	}
-	return region
-}
-
-// Prod returns Tailscale's map of relay servers.
-//
-// This list is only used by cmd/tailscale's netcheck subcommand. In
-// normal operation the Tailscale nodes get this sent to them from the
-// control server.
-//
-// This list is subject to change and should not be relied on.
-func Prod() *tailcfg.DERPMap {
-	return &tailcfg.DERPMap{
-		Regions: map[int]*tailcfg.DERPRegion{
-			1: derpRegion(1, "nyc", "New York City",
-				derpNode("a", "159.89.225.99", "2604:a880:400:d1::828:b001"),
-			),
-			2: derpRegion(2, "sfo", "San Francisco",
-				derpNode("a", "167.172.206.31", "2604:a880:2:d1::c5:7001"),
-			),
-			3: derpRegion(3, "sin", "Singapore",
-				derpNode("a", "68.183.179.66", "2400:6180:0:d1::67d:8001"),
-			),
-			4: derpRegion(4, "fra", "Frankfurt",
-				derpNode("a", "167.172.182.26", "2a03:b0c0:3:e0::36e:9001"),
-			),
-			5: derpRegion(5, "syd", "Sydney",
-				derpNode("a", "103.43.75.49", "2001:19f0:5801:10b7:5400:2ff:feaa:284c"),
-			),
-			6: derpRegion(6, "blr", "Bangalore",
-				derpNode("a", "68.183.90.120", "2400:6180:100:d0::982:d001"),
-			),
-			7: derpRegion(7, "tok", "Tokyo",
-				derpNode("a", "167.179.89.145", "2401:c080:1000:467f:5400:2ff:feee:22aa"),
-			),
-			8: derpRegion(8, "lhr", "London",
-				derpNode("a", "167.71.139.179", "2a03:b0c0:1:e0::3cc:e001"),
-			),
-			9: derpRegion(9, "dfw", "Dallas",
-				derpNode("a", "207.148.3.137", "2001:19f0:6401:1d9c:5400:2ff:feef:bb82"),
-			),
-			10: derpRegion(10, "sea", "Seattle",
-				derpNode("a", "137.220.36.168", "2001:19f0:8001:2d9:5400:2ff:feef:bbb1"),
-			),
-			11: derpRegion(11, "sao", "São Paulo",
-				derpNode("a", "18.230.97.74", "2600:1f1e:ee4:5611:ec5c:1736:d43b:a454"),
-			),
-		},
-	}
-}

derp/testdata/example_ss.txt

@@ -0,0 +1,8 @@
+ESTAB 0      0       10.255.1.11:35238  34.210.105.16:https
+         cubic wscale:7,7 rto:236 rtt:34.14/3.432 ato:40 mss:1448 pmtu:1500 rcvmss:1448 advmss:1448 cwnd:8 ssthresh:6 bytes_sent:38056577 bytes_retrans:2918 bytes_acked:38053660 bytes_received:6973211 segs_out:165090 segs_in:124227 data_segs_out:78018 data_segs_in:71645 send 2.71Mbps lastsnd:1156 lastrcv:1120 lastack:1120 pacing_rate 3.26Mbps delivery_rate 2.35Mbps delivered:78017 app_limited busy:2586132ms retrans:0/6 dsack_dups:4 reordering:5 reord_seen:15 rcv_rtt:126355 rcv_space:65780 rcv_ssthresh:541928 minrtt:26.632
+ESTAB 0      80     100.79.58.14:ssh    100.95.73.104:58145
+         cubic wscale:6,7 rto:224 rtt:23.051/2.03 ato:172 mss:1228 pmtu:1280 rcvmss:1228 advmss:1228 cwnd:10 ssthresh:94 bytes_sent:1591815 bytes_retrans:944 bytes_acked:1590791 bytes_received:158925 segs_out:8070 segs_in:8858 data_segs_out:7452 data_segs_in:3789 send 4.26Mbps lastsnd:4 lastrcv:4 lastack:4 pacing_rate 8.52Mbps delivery_rate 10.9Mbps delivered:7451 app_limited busy:61656ms unacked:2 retrans:0/10 dsack_dups:10 rcv_rtt:174712 rcv_space:65025 rcv_ssthresh:64296 minrtt:16.186                                   
+ESTAB 0      374     10.255.1.11:43254 167.172.206.31:https
+         cubic wscale:7,7 rto:224 rtt:22.55/1.941 ato:40 mss:1448 pmtu:1500 rcvmss:1448 advmss:1448 cwnd:6 ssthresh:4 bytes_sent:14594668 bytes_retrans:173314 bytes_acked:14420981 bytes_received:4207111 segs_out:80566 segs_in:70310 data_segs_out:24317 data_segs_in:20365 send 3.08Mbps lastsnd:4 lastrcv:4 lastack:4 pacing_rate 3.7Mbps delivery_rate 3.05Mbps delivered:24111 app_limited busy:184820ms unacked:2 retrans:0/185 dsack_dups:1 reord_seen:3 rcv_rtt:651.262 rcv_space:226657 rcv_ssthresh:1557136 minrtt:10.18           
+ESTAB 0      0       10.255.1.11:33036    3.121.18.47:https
+         cubic wscale:7,7 rto:372 rtt:168.408/2.044 ato:40 mss:1448 pmtu:1500 rcvmss:1448 advmss:1448 cwnd:10 bytes_sent:27500 bytes_acked:27501 bytes_received:1386524 segs_out:10990 segs_in:11037 data_segs_out:303 data_segs_in:3414 send 688kbps lastsnd:125776 lastrcv:9640 lastack:22760 pacing_rate 1.38Mbps delivery_rate 482kbps delivered:304 app_limited busy:43024ms rcv_rtt:3345.12 rcv_space:62431 rcv_ssthresh:760472 minrtt:168.867

go.mod

@@ -34,15 +34,15 @@ require (
 	golang.org/x/crypto v0.0.0-20210513164829-c07d793c2f9a
 	golang.org/x/net v0.0.0-20210525063256-abc453219eb5
 	golang.org/x/sync v0.0.0-20210220032951-036812b2e83c
-	golang.org/x/sys v0.0.0-20210608053332-aa57babbf139
+	golang.org/x/sys v0.0.0-20210616094352-59db8d763f22
 	golang.org/x/term v0.0.0-20210503060354-a79de5458b56
-	golang.org/x/time v0.0.0-20210220033141-f8bda1e9f3ba
+	golang.org/x/time v0.0.0-20210611083556-38a9dc6acbc6
 	golang.org/x/tools v0.1.2
 	golang.zx2c4.com/wireguard v0.0.0-20210525143454-64cb82f2b3f5
 	golang.zx2c4.com/wireguard/windows v0.3.15-0.20210525143335-94c0476d63e3
 	honnef.co/go/tools v0.1.4
 	inet.af/netaddr v0.0.0-20210602152128-50f8686885e3
-	inet.af/netstack v0.0.0-20210317161235-a1bf4e56ef22
+	inet.af/netstack v0.0.0-20210622165351-29b14ebc044e
 	inet.af/peercred v0.0.0-20210318190834-4259e17bb763
 	inet.af/wf v0.0.0-20210516214145-a5343001b756
 	rsc.io/goversion v1.2.0

go.sum

@@ -784,14 +784,13 @@ golang.org/x/sys v0.0.0-20210301091718-77cc2087c03b/go.mod h1:h1NjWce9XRLGQEsW7w
 golang.org/x/sys v0.0.0-20210303074136-134d130e1a04/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20210305230114-8fe3ee5dd75b/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20210309040221-94ec62e08169/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20210316164454-77fc1eacc6aa/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20210330210617-4fbd30eecc44/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20210403161142-5e06dd20ab57/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20210525143221-35b2ab0089ea/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
-golang.org/x/sys v0.0.0-20210608053332-aa57babbf139 h1:C+AwYEtBp/VQwoLntUmQ/yx3MS9vmZaKNdw5eOpoQe8=
-golang.org/x/sys v0.0.0-20210608053332-aa57babbf139/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.0.0-20210616094352-59db8d763f22 h1:RqytpXGR1iVNX7psjB3ff8y7sNFinVFvkx1c8SjBkio=
+golang.org/x/sys v0.0.0-20210616094352-59db8d763f22/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw=
 golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
 golang.org/x/term v0.0.0-20210503060354-a79de5458b56 h1:b8jxX3zqjpqb2LklXPzKSGJhzyxCOZSz8ncv8Nv+y7w=
@@ -806,8 +805,8 @@ golang.org/x/text v0.3.7-0.20210524175448-3115f89c4b99 h1:ZEXtoJu1S0ie/EmdYnjY3C
 golang.org/x/text v0.3.7-0.20210524175448-3115f89c4b99/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ=
 golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
 golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
-golang.org/x/time v0.0.0-20210220033141-f8bda1e9f3ba h1:O8mE0/t419eoIwhTFpKVkHiTs/Igowgfkj25AcZrtiE=
-golang.org/x/time v0.0.0-20210220033141-f8bda1e9f3ba/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
+golang.org/x/time v0.0.0-20210611083556-38a9dc6acbc6 h1:Vv0JUPWTyeqUq42B2WJ1FeIDjjvGKoA2Ss+Ts0lAVbs=
+golang.org/x/time v0.0.0-20210611083556-38a9dc6acbc6/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
 golang.org/x/tools v0.0.0-20180221164845-07fd8470d635/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
 golang.org/x/tools v0.0.0-20180525024113-a5b4c53f6e8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
 golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
@@ -952,8 +951,8 @@ honnef.co/go/tools v0.1.4/go.mod h1:NgwopIslSNH47DimFoV78dnkksY2EFtX0ajyb3K/las=
 inet.af/netaddr v0.0.0-20210515010201-ad03edc7c841/go.mod h1:z0nx+Dh+7N7CC8V5ayHtHGpZpxLQZZxkIaaz6HN65Ls=
 inet.af/netaddr v0.0.0-20210602152128-50f8686885e3 h1:RlarOdsmOUCCvy7Xm1JchJIGuQsuKwD/Lo1bjYmfuQI=
 inet.af/netaddr v0.0.0-20210602152128-50f8686885e3/go.mod h1:z0nx+Dh+7N7CC8V5ayHtHGpZpxLQZZxkIaaz6HN65Ls=
-inet.af/netstack v0.0.0-20210317161235-a1bf4e56ef22 h1:DNtszwGa6w76qlIr+PbPEnlBJdiRV8SaxeigOy0q1gg=
-inet.af/netstack v0.0.0-20210317161235-a1bf4e56ef22/go.mod h1:GVx+5OZtbG4TVOW5ilmyRZAZXr1cNwfqUEkTOtWK0PM=
+inet.af/netstack v0.0.0-20210622165351-29b14ebc044e h1:z11NK94NQcI3DA+a3pUC/2dRYTph1kPX6B0FnCaMDzk=
+inet.af/netstack v0.0.0-20210622165351-29b14ebc044e/go.mod h1:fG3G1dekmK8oDX3iVzt8c0zICLMLSN8SjdxbXVt0WjU=
 inet.af/peercred v0.0.0-20210318190834-4259e17bb763 h1:gPSJmmVzmdy4kHhlCMx912GdiUz3k/RzJGg0ADqy1dg=
 inet.af/peercred v0.0.0-20210318190834-4259e17bb763/go.mod h1:FjawnflS/udxX+SvpsMgZfdqx2aykOlkISeAsADi5IU=
 inet.af/wf v0.0.0-20210516214145-a5343001b756 h1:muIT3C1rH3/xpvIH8blKkMvhctV7F+OtZqs7kcwHDBQ=

hostinfo/hostinfo.go

@@ -0,0 +1,117 @@
+// Copyright (c) 2020 Tailscale Inc & AUTHORS All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+// Package hostinfo answers questions about the host environment that Tailscale is
+// running on.
+//
+// TODO(bradfitz): move more of control/controlclient/hostinfo_* into this package.
+package hostinfo
+
+import (
+	"io"
+	"os"
+	"runtime"
+	"sync/atomic"
+
+	"go4.org/mem"
+	"tailscale.com/util/lineread"
+)
+
+// EnvType represents a known environment type.
+// The empty string, the default, means unknown.
+type EnvType string
+
+const (
+	KNative         = EnvType("kn")
+	AWSLambda       = EnvType("lm")
+	Heroku          = EnvType("hr")
+	AzureAppService = EnvType("az")
+)
+
+var envType atomic.Value // of EnvType
+
+func GetEnvType() EnvType {
+	if e, ok := envType.Load().(EnvType); ok {
+		return e
+	}
+	e := getEnvType()
+	envType.Store(e)
+	return e
+}
+
+func getEnvType() EnvType {
+	if inKnative() {
+		return KNative
+	}
+	if inAWSLambda() {
+		return AWSLambda
+	}
+	if inHerokuDyno() {
+		return Heroku
+	}
+	if inAzureAppService() {
+		return AzureAppService
+	}
+	return ""
+}
+
+// InContainer reports whether we're running in a container.
+func InContainer() bool {
+	if runtime.GOOS != "linux" {
+		return false
+	}
+	var ret bool
+	lineread.File("/proc/1/cgroup", func(line []byte) error {
+		if mem.Contains(mem.B(line), mem.S("/docker/")) ||
+			mem.Contains(mem.B(line), mem.S("/lxc/")) {
+			ret = true
+			return io.EOF // arbitrary non-nil error to stop loop
+		}
+		return nil
+	})
+	lineread.File("/proc/mounts", func(line []byte) error {
+		if mem.Contains(mem.B(line), mem.S("fuse.lxcfs")) {
+			ret = true
+			return io.EOF
+		}
+		return nil
+	})
+	return ret
+}
+
+func inKnative() bool {
+	// https://cloud.google.com/run/docs/reference/container-contract#env-vars
+	if os.Getenv("K_REVISION") != "" && os.Getenv("K_CONFIGURATION") != "" &&
+		os.Getenv("K_SERVICE") != "" && os.Getenv("PORT") != "" {
+		return true
+	}
+	return false
+}
+
+func inAWSLambda() bool {
+	// https://docs.aws.amazon.com/lambda/latest/dg/configuration-envvars.html
+	if os.Getenv("AWS_LAMBDA_FUNCTION_NAME") != "" &&
+		os.Getenv("AWS_LAMBDA_FUNCTION_VERSION") != "" &&
+		os.Getenv("AWS_LAMBDA_INITIALIZATION_TYPE") != "" &&
+		os.Getenv("AWS_LAMBDA_RUNTIME_API") != "" {
+		return true
+	}
+	return false
+}
+
+func inHerokuDyno() bool {
+	// https://devcenter.heroku.com/articles/dynos#local-environment-variables
+	if os.Getenv("PORT") != "" && os.Getenv("DYNO") != "" {
+		return true
+	}
+	return false
+}
+
+func inAzureAppService() bool {
+	if os.Getenv("APPSVC_RUN_ZIP") != "" && os.Getenv("WEBSITE_STACK") != "" &&
+		os.Getenv("WEBSITE_AUTH_AUTO_AAD") != "" {
+		return true
+	}
+	return false
+}

ipn/ipnlocal/local.go

@@ -29,7 +29,6 @@ import (
 	"tailscale.com/client/tailscale/apitype"
 	"tailscale.com/control/controlclient"
 	"tailscale.com/health"
-	"tailscale.com/internal/deephash"
 	"tailscale.com/ipn"
 	"tailscale.com/ipn/ipnstate"
 	"tailscale.com/ipn/policy"
@@ -46,6 +45,7 @@ import (
 	"tailscale.com/types/persist"
 	"tailscale.com/types/preftype"
 	"tailscale.com/types/wgkey"
+	"tailscale.com/util/deephash"
 	"tailscale.com/util/dnsname"
 	"tailscale.com/util/osshare"
 	"tailscale.com/util/systemd"
@@ -244,10 +244,10 @@ func (b *LocalBackend) linkChange(major bool, ifst *interfaces.State) {
 	// need updating to tweak default routes.
 	b.updateFilter(b.netMap, b.prefs)
 
-	if runtime.GOOS == "windows" && b.netMap != nil && b.state == ipn.Running {
+	if peerAPIListenAsync && b.netMap != nil && b.state == ipn.Running {
 		want := len(b.netMap.Addresses)
-		b.logf("linkChange: peerAPIListeners too low; trying again")
 		if len(b.peerAPIListeners) < want {
+			b.logf("linkChange: peerAPIListeners too low; trying again")
 			go b.initPeerAPIListener()
 		}
 	}
@@ -325,6 +325,7 @@ func (b *LocalBackend) updateStatus(sb *ipnstate.StatusBuilder, extraLocked func
 		s.AuthURL = b.authURLSticky
 		if b.netMap != nil {
 			s.MagicDNSSuffix = b.netMap.MagicDNSSuffix()
+			s.CertDomains = append([]string(nil), b.netMap.DNS.CertDomains...)
 		}
 	})
 	sb.MutateSelfStatus(func(ss *ipnstate.PeerStatus) {
@@ -1749,6 +1750,25 @@ func (b *LocalBackend) authReconfig() {
 	for _, peer := range nm.Peers {
 		set(peer.Name, peer.Addresses)
 	}
+	for _, rec := range nm.DNS.ExtraRecords {
+		switch rec.Type {
+		case "", "A", "AAAA":
+			// Treat these all the same for now: infer from the value
+		default:
+			// TODO: more
+			continue
+		}
+		ip, err := netaddr.ParseIP(rec.Value)
+		if err != nil {
+			// Ignore.
+			continue
+		}
+		fqdn, err := dnsname.ToFQDN(rec.Name)
+		if err != nil {
+			continue
+		}
+		dcfg.Hosts[fqdn] = append(dcfg.Hosts[fqdn], ip)
+	}
 
 	if uc.CorpDNS {
 		addDefault := func(resolvers []tailcfg.DNSResolver) {
@@ -1818,7 +1838,7 @@ func (b *LocalBackend) authReconfig() {
 		}
 	}
 
-	err = b.e.Reconfig(cfg, rcfg, &dcfg)
+	err = b.e.Reconfig(cfg, rcfg, &dcfg, nm.Debug)
 	if err == wgengine.ErrNoChanges {
 		return
 	}
@@ -1884,6 +1904,13 @@ func (b *LocalBackend) closePeerAPIListenersLocked() {
 	b.peerAPIListeners = nil
 }
 
+// peerAPIListenAsync is whether the operating system requires that we
+// retry listening on the peerAPI ip/port for whatever reason.
+//
+// On Windows, see Issue 1620.
+// On Android, see Issue 1960.
+const peerAPIListenAsync = runtime.GOOS == "windows" || runtime.GOOS == "android"
+
 func (b *LocalBackend) initPeerAPIListener() {
 	b.mu.Lock()
 	defer b.mu.Unlock()
@@ -1947,9 +1974,8 @@ func (b *LocalBackend) initPeerAPIListener() {
 		if !skipListen {
 			ln, err = ps.listen(a.IP(), b.prevIfState)
 			if err != nil {
-				if runtime.GOOS == "windows" {
-					// Expected for now. See Issue 1620.
-					// But we fix it later in linkChange
+				if peerAPIListenAsync {
+					// Expected. But we fix it later in linkChange
 					// ("peerAPIListeners too low").
 					continue
 				}
@@ -2082,7 +2108,7 @@ func (b *LocalBackend) routerConfig(cfg *wgcfg.Config, prefs *ipn.Prefs) *router
 		if !default6 {
 			rs.Routes = append(rs.Routes, ipv6Default)
 		}
-		if runtime.GOOS == "linux" {
+		if runtime.GOOS == "linux" || runtime.GOOS == "darwin" || runtime.GOOS == "windows" {
 			// Only allow local lan access on linux machines for now.
 			ips, _, err := interfaceRoutes()
 			if err != nil {
@@ -2173,7 +2199,7 @@ func (b *LocalBackend) enterState(newState ipn.State) {
 		b.blockEngineUpdates(true)
 		fallthrough
 	case ipn.Stopped:
-		err := b.e.Reconfig(&wgcfg.Config{}, &router.Config{}, &dns.Config{})
+		err := b.e.Reconfig(&wgcfg.Config{}, &router.Config{}, &dns.Config{}, nil)
 		if err != nil {
 			b.logf("Reconfig(down): %v", err)
 		}
@@ -2187,7 +2213,7 @@ func (b *LocalBackend) enterState(newState ipn.State) {
 		b.e.RequestStatus()
 	case ipn.Running:
 		var addrs []string
-		for _, addr := range b.netMap.Addresses {
+		for _, addr := range netMap.Addresses {
 			addrs = append(addrs, addr.IP().String())
 		}
 		systemd.Status("Connected; %s; %s", activeLogin, strings.Join(addrs, " "))
@@ -2293,7 +2319,7 @@ func (b *LocalBackend) stateMachine() {
 // a status update that predates the "I've shut down" update.
 func (b *LocalBackend) stopEngineAndWait() {
 	b.logf("stopEngineAndWait...")
-	b.e.Reconfig(&wgcfg.Config{}, &router.Config{}, &dns.Config{})
+	b.e.Reconfig(&wgcfg.Config{}, &router.Config{}, &dns.Config{}, nil)
 	b.requestEngineStatusAndWait()
 	b.logf("stopEngineAndWait: done.")
 }
@@ -2685,7 +2711,6 @@ func (b *LocalBackend) CheckIPForwarding() error {
 		return nil
 	}
 	if isBSD(runtime.GOOS) {
-		//lint:ignore ST1005 output to users as is
 		return fmt.Errorf("Subnet routing and exit nodes only work with additional manual configuration on %v, and is not currently officially supported.", runtime.GOOS)
 	}
 
@@ -2702,16 +2727,13 @@ func (b *LocalBackend) CheckIPForwarding() error {
 	for _, key := range keys {
 		bs, err := exec.Command("sysctl", "-n", key).Output()
 		if err != nil {
-			//lint:ignore ST1005 output to users as is
 			return fmt.Errorf("couldn't check %s (%v).\nSubnet routes won't work without IP forwarding.", key, err)
 		}
 		on, err := strconv.ParseBool(string(bytes.TrimSpace(bs)))
 		if err != nil {
-			//lint:ignore ST1005 output to users as is
 			return fmt.Errorf("couldn't parse %s (%v).\nSubnet routes won't work without IP forwarding.", key, err)
 		}
 		if !on {
-			//lint:ignore ST1005 output to users as is
 			return fmt.Errorf("%s is disabled. Subnet routes won't work.", key)
 		}
 	}
@@ -2730,3 +2752,13 @@ func (b *LocalBackend) PeerDialControlFunc() func(network, address string, c sys
 	}
 	return nil
 }
+
+// DERPMap returns the current DERPMap in use, or nil if not connected.
+func (b *LocalBackend) DERPMap() *tailcfg.DERPMap {
+	b.mu.Lock()
+	defer b.mu.Unlock()
+	if b.netMap == nil {
+		return nil
+	}
+	return b.netMap.DERPMap
+}

ipn/ipnlocal/peerapi_macios_ext.go

@@ -9,14 +9,12 @@ package ipnlocal
 import (
 	"errors"
 	"fmt"
-	"log"
 	"net"
-	"strings"
 	"syscall"
 
-	"golang.org/x/sys/unix"
 	"inet.af/netaddr"
 	"tailscale.com/net/interfaces"
+	"tailscale.com/net/netns"
 )
 
 func init() {
@@ -32,29 +30,7 @@ func initListenConfigNetworkExtension(nc *net.ListenConfig, ip netaddr.IP, st *i
 	if !ok {
 		return fmt.Errorf("no interface with name %q", tunIfName)
 	}
-	nc.Control = func(network, address string, c syscall.RawConn) error {
-		var sockErr error
-		err := c.Control(func(fd uintptr) {
-			sockErr = bindIf(fd, network, address, tunIf.Index)
-			log.Printf("peerapi: bind(%q, %q) on index %v = %v", network, address, tunIf.Index, sockErr)
-		})
-		if err != nil {
-			return err
-		}
-		return sockErr
-	}
-	return nil
-}
-
-func bindIf(fd uintptr, network, address string, ifIndex int) error {
-	v6 := strings.Contains(address, "]:") || strings.HasSuffix(network, "6") // hacky test for v6
-	proto := unix.IPPROTO_IP
-	opt := unix.IP_BOUND_IF
-	if v6 {
-		proto = unix.IPPROTO_IPV6
-		opt = unix.IPV6_BOUND_IF
-	}
-	return unix.SetsockoptInt(int(fd), proto, opt, ifIndex)
+	return netns.SetListenConfigInterfaceIndex(nc, tunIf.Index)
 }
 
 func peerDialControlFuncNetworkExtension(b *LocalBackend) func(network, address string, c syscall.RawConn) error {
@@ -68,17 +44,12 @@ func peerDialControlFuncNetworkExtension(b *LocalBackend) func(network, address
 			index = tunIf.Index
 		}
 	}
+	var lc net.ListenConfig
+	netns.SetListenConfigInterfaceIndex(&lc, index)
 	return func(network, address string, c syscall.RawConn) error {
 		if index == -1 {
 			return errors.New("failed to find TUN interface to bind to")
 		}
-		var sockErr error
-		err := c.Control(func(fd uintptr) {
-			sockErr = bindIf(fd, network, address, index)
-		})
-		if err != nil {
-			return err
-		}
-		return sockErr
+		return lc.Control(network, address, c)
 	}
 }

ipn/ipnserver/server.go

@@ -24,7 +24,6 @@ import (
 	"strconv"
 	"strings"
 	"sync"
-	"sync/atomic"
 	"syscall"
 	"time"
 
@@ -41,9 +40,11 @@ import (
 	"tailscale.com/safesocket"
 	"tailscale.com/smallzstd"
 	"tailscale.com/types/logger"
+	"tailscale.com/util/groupmember"
 	"tailscale.com/util/pidowner"
 	"tailscale.com/util/systemd"
 	"tailscale.com/version"
+	"tailscale.com/version/distro"
 	"tailscale.com/wgengine"
 )
 
@@ -347,51 +348,32 @@ func isReadonlyConn(ci connIdentity, operatorUID string, logf logger.Logf) bool
 		logf("connection from userid %v; is configured operator", uid)
 		return rw
 	}
-	var adminGroupID string
-	switch runtime.GOOS {
-	case "darwin":
-		adminGroupID = darwinAdminGroupID()
-	default:
-		logf("connection from userid %v; read-only", uid)
+	if yes, err := isLocalAdmin(uid); err != nil {
+		logf("connection from userid %v; read-only; %v", uid, err)
 		return ro
-	}
-	if adminGroupID == "" {
-		logf("connection from userid %v; no system admin group found, read-only", uid)
-		return ro
-	}
-	u, err := user.LookupId(uid)
-	if err != nil {
-		logf("connection from userid %v; failed to look up user; read-only", uid)
-		return ro
-	}
-	gids, err := u.GroupIds()
-	if err != nil {
-		logf("connection from userid %v; failed to look up groups; read-only", uid)
-		return ro
-	}
-	for _, gid := range gids {
-		if gid == adminGroupID {
-			logf("connection from userid %v; is local admin, has access", uid)
-			return rw
-		}
+	} else if yes {
+		logf("connection from userid %v; is local admin, has access", uid)
+		return rw
 	}
 	logf("connection from userid %v; read-only", uid)
 	return ro
 }
 
-var darwinAdminGroupIDCache atomic.Value // of string
-
-func darwinAdminGroupID() string {
-	s, _ := darwinAdminGroupIDCache.Load().(string)
-	if s != "" {
-		return s
-	}
-	g, err := user.LookupGroup("admin")
+func isLocalAdmin(uid string) (bool, error) {
+	u, err := user.LookupId(uid)
 	if err != nil {
-		return ""
+		return false, err
 	}
-	darwinAdminGroupIDCache.Store(g.Gid)
-	return g.Gid
+	var adminGroup string
+	switch {
+	case runtime.GOOS == "darwin":
+		adminGroup = "admin"
+	case distro.Get() == distro.QNAP:
+		adminGroup = "administrators"
+	default:
+		return false, fmt.Errorf("no system admin group found")
+	}
+	return groupmember.IsMemberOfGroup(adminGroup, u.Username)
 }
 
 // inUseOtherUserError is the error type for when the server is in use
@@ -415,12 +397,10 @@ func (s *server) checkConnIdentityLocked(ci connIdentity) error {
 			break
 		}
 		if ci.UserID != active.UserID {
-			//lint:ignore ST1005 we want to capitalize Tailscale here
 			return inUseOtherUserError{fmt.Errorf("Tailscale already in use by %s, pid %d", active.User.Username, active.Pid)}
 		}
 	}
 	if su := s.serverModeUser; su != nil && ci.UserID != su.Uid {
-		//lint:ignore ST1005 we want to capitalize Tailscale here
 		return inUseOtherUserError{fmt.Errorf("Tailscale already in use by %s", su.Username)}
 	}
 	return nil

ipn/ipnstate/ipnstate.go

@@ -45,6 +45,13 @@ type Status struct {
 	// has MagicDNS enabled.
 	MagicDNSSuffix string
 
+	// CertDomains are the set of DNS names for which the control
+	// plane server will assist with provisioning TLS
+	// certificates. See SetDNSRequest for dns-01 ACME challenges
+	// for e.g. LetsEncrypt. These names are FQDNs without
+	// trailing periods, and without any "_acme-challenge." prefix.
+	CertDomains []string
+
 	Peer map[key.Public]*PeerStatus
 	User map[tailcfg.UserID]tailcfg.UserProfile
 }

ipn/localapi/localapi.go

@@ -102,6 +102,8 @@ func (h *Handler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
 		h.serveFileTargets(w, r)
 	case "/localapi/v0/set-dns":
 		h.serveSetDNS(w, r)
+	case "/localapi/v0/derpmap":
+		h.serveDERPMap(w, r)
 	case "/":
 		io.WriteString(w, "tailscaled\n")
 	default:
@@ -403,6 +405,17 @@ func (h *Handler) serveSetDNS(w http.ResponseWriter, r *http.Request) {
 	json.NewEncoder(w).Encode(struct{}{})
 }
 
+func (h *Handler) serveDERPMap(w http.ResponseWriter, r *http.Request) {
+	if r.Method != "GET" {
+		http.Error(w, "want GET", 400)
+		return
+	}
+	w.Header().Set("Content-Type", "application/json")
+	e := json.NewEncoder(w)
+	e.SetIndent("", "\t")
+	e.Encode(h.b.DERPMap())
+}
+
 var dialPeerTransportOnce struct {
 	sync.Once
 	v *http.Transport
@@ -411,7 +424,7 @@ var dialPeerTransportOnce struct {
 func getDialPeerTransport(b *ipnlocal.LocalBackend) *http.Transport {
 	dialPeerTransportOnce.Do(func() {
 		t := http.DefaultTransport.(*http.Transport).Clone()
-		t.Dial = nil //lint:ignore SA1019 yes I know I'm setting it to nil defensively
+		t.Dial = nil
 		dialer := net.Dialer{
 			Timeout:   30 * time.Second,
 			KeepAlive: 30 * time.Second,

ipn/prefs.go

@@ -28,7 +28,7 @@ import (
 // DefaultControlURL returns the URL base of the control plane
 // ("coordination server") for use when no explicit one is configured.
 // The default control plane is the hosted version run by Tailscale.com.
-const DefaultControlURL = "https://login.tailscale.com"
+const DefaultControlURL = "https://controlplane.tailscale.com"
 
 // Prefs are the user modifiable settings of the Tailscale node agent.
 type Prefs struct {

ipn/prefs_test.go

@@ -92,13 +92,13 @@ func TestPrefsEqual(t *testing.T) {
 		},
 
 		{
-			&Prefs{ControlURL: "https://login.tailscale.com"},
+			&Prefs{ControlURL: "https://controlplane.tailscale.com"},
 			&Prefs{ControlURL: "https://login.private.co"},
 			false,
 		},
 		{
-			&Prefs{ControlURL: "https://login.tailscale.com"},
-			&Prefs{ControlURL: "https://login.tailscale.com"},
+			&Prefs{ControlURL: "https://controlplane.tailscale.com"},
+			&Prefs{ControlURL: "https://controlplane.tailscale.com"},
 			true,
 		},
 
@@ -324,7 +324,7 @@ func TestBasicPrefs(t *testing.T) {
 	tstest.PanicOnLog()
 
 	p := Prefs{
-		ControlURL: "https://login.tailscale.com",
+		ControlURL: "https://controlplane.tailscale.com",
 	}
 	checkPrefs(t, p)
 }
@@ -336,7 +336,7 @@ func TestPrefsPersist(t *testing.T) {
 		LoginName: "test@example.com",
 	}
 	p := Prefs{
-		ControlURL: "https://login.tailscale.com",
+		ControlURL: "https://controlplane.tailscale.com",
 		CorpDNS:    true,
 		Persist:    &c,
 	}

logtail/filch/filch.go

@@ -15,7 +15,6 @@ import (
 	"sync"
 )
 
-//lint:ignore U1000 work around false positive: https://github.com/dominikh/go-tools/issues/983
 var stderrFD = 2 // a variable for testing
 
 type Options struct {

net/dns/direct.go

@@ -2,23 +2,22 @@
 // Use of this source code is governed by a BSD-style
 // license that can be found in the LICENSE file.
 
-// +build linux freebsd openbsd
-
 package dns
 
 import (
 	"bufio"
 	"bytes"
+	"crypto/rand"
 	"fmt"
 	"io"
 	"io/ioutil"
 	"os"
 	"os/exec"
+	"path/filepath"
 	"runtime"
 	"strings"
 
 	"inet.af/netaddr"
-	"tailscale.com/atomicfile"
 	"tailscale.com/util/dnsname"
 )
 
@@ -77,21 +76,17 @@ func readResolv(r io.Reader) (config OSConfig, err error) {
 	return config, nil
 }
 
-func readResolvFile(path string) (OSConfig, error) {
-	var config OSConfig
-
-	f, err := os.Open(path)
+func (m directManager) readResolvFile(path string) (OSConfig, error) {
+	b, err := m.fs.ReadFile(path)
 	if err != nil {
-		return config, err
+		return OSConfig{}, err
 	}
-	defer f.Close()
-
-	return readResolv(f)
+	return readResolv(bytes.NewReader(b))
 }
 
 // readResolvConf reads DNS configuration from /etc/resolv.conf.
-func readResolvConf() (OSConfig, error) {
-	return readResolvFile(resolvConf)
+func (m directManager) readResolvConf() (OSConfig, error) {
+	return m.readResolvFile(resolvConf)
 }
 
 // resolvOwner returns the apparent owner of the resolv.conf
@@ -143,33 +138,39 @@ func isResolvedRunning() bool {
 	return err == nil
 }
 
-// directManager is a managerImpl which replaces /etc/resolv.conf with a file
+// directManager is an OSConfigurator which replaces /etc/resolv.conf with a file
 // generated from the given configuration, creating a backup of its old state.
 //
 // This way of configuring DNS is precarious, since it does not react
 // to the disappearance of the Tailscale interface.
 // The caller must call Down before program shutdown
 // or as cleanup if the program terminates unexpectedly.
-type directManager struct{}
+type directManager struct {
+	fs wholeFileFS
+}
 
-func newDirectManager() (directManager, error) {
-	return directManager{}, nil
+func newDirectManager() directManager {
+	return directManager{fs: directFS{}}
+}
+
+func newDirectManagerOnFS(fs wholeFileFS) directManager {
+	return directManager{fs: fs}
 }
 
 // ownedByTailscale reports whether /etc/resolv.conf seems to be a
 // tailscale-managed file.
 func (m directManager) ownedByTailscale() (bool, error) {
-	st, err := os.Stat(resolvConf)
+	isRegular, err := m.fs.Stat(resolvConf)
 	if err != nil {
 		if os.IsNotExist(err) {
 			return false, nil
 		}
 		return false, err
 	}
-	if !st.Mode().IsRegular() {
+	if !isRegular {
 		return false, nil
 	}
-	bs, err := ioutil.ReadFile(resolvConf)
+	bs, err := m.fs.ReadFile(resolvConf)
 	if err != nil {
 		return false, err
 	}
@@ -182,11 +183,11 @@ func (m directManager) ownedByTailscale() (bool, error) {
 // backupConfig creates or updates a backup of /etc/resolv.conf, if
 // resolv.conf does not currently contain a Tailscale-managed config.
 func (m directManager) backupConfig() error {
-	if _, err := os.Stat(resolvConf); err != nil {
+	if _, err := m.fs.Stat(resolvConf); err != nil {
 		if os.IsNotExist(err) {
 			// No resolv.conf, nothing to back up. Also get rid of any
 			// existing backup file, to avoid restoring something old.
-			os.Remove(backupConf)
+			m.fs.Remove(backupConf)
 			return nil
 		}
 		return err
@@ -200,11 +201,11 @@ func (m directManager) backupConfig() error {
 		return nil
 	}
 
-	return os.Rename(resolvConf, backupConf)
+	return m.fs.Rename(resolvConf, backupConf)
 }
 
 func (m directManager) restoreBackup() error {
-	if _, err := os.Stat(backupConf); err != nil {
+	if _, err := m.fs.Stat(backupConf); err != nil {
 		if os.IsNotExist(err) {
 			// No backup, nothing we can do.
 			return nil
@@ -215,7 +216,7 @@ func (m directManager) restoreBackup() error {
 	if err != nil {
 		return err
 	}
-	if _, err := os.Stat(resolvConf); err != nil && !os.IsNotExist(err) {
+	if _, err := m.fs.Stat(resolvConf); err != nil && !os.IsNotExist(err) {
 		return err
 	}
 	resolvConfExists := !os.IsNotExist(err)
@@ -223,12 +224,12 @@ func (m directManager) restoreBackup() error {
 	if resolvConfExists && !owned {
 		// There's already a non-tailscale config in place, get rid of
 		// our backup.
-		os.Remove(backupConf)
+		m.fs.Remove(backupConf)
 		return nil
 	}
 
 	// We own resolv.conf, and a backup exists.
-	if err := os.Rename(backupConf, resolvConf); err != nil {
+	if err := m.fs.Rename(backupConf, resolvConf); err != nil {
 		return err
 	}
 
@@ -247,7 +248,7 @@ func (m directManager) SetDNS(config OSConfig) error {
 
 		buf := new(bytes.Buffer)
 		writeResolvConf(buf, config.Nameservers, config.SearchDomains)
-		if err := atomicfile.WriteFile(resolvConf, buf.Bytes(), 0644); err != nil {
+		if err := atomicWriteFile(m.fs, resolvConf, buf.Bytes(), 0644); err != nil {
 			return err
 		}
 	}
@@ -279,7 +280,7 @@ func (m directManager) GetBaseConfig() (OSConfig, error) {
 		fileToRead = backupConf
 	}
 
-	return readResolvFile(fileToRead)
+	return m.readResolvFile(fileToRead)
 }
 
 func (m directManager) Close() error {
@@ -287,9 +288,9 @@ func (m directManager) Close() error {
 	// to it, but then we stopped because /etc/resolv.conf being a
 	// symlink to surprising places breaks snaps and other sandboxing
 	// things. Clean it up if it's still there.
-	os.Remove("/etc/resolv.tailscale.conf")
+	m.fs.Remove("/etc/resolv.tailscale.conf")
 
-	if _, err := os.Stat(backupConf); err != nil {
+	if _, err := m.fs.Stat(backupConf); err != nil {
 		if os.IsNotExist(err) {
 			// No backup, nothing we can do.
 			return nil
@@ -300,7 +301,7 @@ func (m directManager) Close() error {
 	if err != nil {
 		return err
 	}
-	_, err = os.Stat(resolvConf)
+	_, err = m.fs.Stat(resolvConf)
 	if err != nil && !os.IsNotExist(err) {
 		return err
 	}
@@ -309,12 +310,12 @@ func (m directManager) Close() error {
 	if resolvConfExists && !owned {
 		// There's already a non-tailscale config in place, get rid of
 		// our backup.
-		os.Remove(backupConf)
+		m.fs.Remove(backupConf)
 		return nil
 	}
 
 	// We own resolv.conf, and a backup exists.
-	if err := os.Rename(backupConf, resolvConf); err != nil {
+	if err := m.fs.Rename(backupConf, resolvConf); err != nil {
 		return err
 	}
 
@@ -324,3 +325,63 @@ func (m directManager) Close() error {
 
 	return nil
 }
+
+func atomicWriteFile(fs wholeFileFS, filename string, data []byte, perm os.FileMode) error {
+	var randBytes [12]byte
+	if _, err := rand.Read(randBytes[:]); err != nil {
+		return fmt.Errorf("atomicWriteFile: %w", err)
+	}
+
+	tmpName := fmt.Sprintf("%s.%x.tmp", filename, randBytes[:])
+	defer fs.Remove(tmpName)
+
+	if err := fs.WriteFile(tmpName, data, perm); err != nil {
+		return fmt.Errorf("atomicWriteFile: %w", err)
+	}
+	return fs.Rename(tmpName, filename)
+}
+
+// wholeFileFS is a high-level file system abstraction designed just for use
+// by directManager, with the goal that it is easy to implement over wsl.exe.
+//
+// All name parameters are absolute paths.
+type wholeFileFS interface {
+	Stat(name string) (isRegular bool, err error)
+	Rename(oldName, newName string) error
+	Remove(name string) error
+	ReadFile(name string) ([]byte, error)
+	WriteFile(name string, contents []byte, perm os.FileMode) error
+}
+
+// directFS is a wholeFileFS implemented directly on the OS.
+type directFS struct {
+	// prefix is file path prefix.
+	//
+	// All name parameters are absolute paths so this is typically a
+	// testing temporary directory like "/tmp".
+	prefix string
+}
+
+func (fs directFS) path(name string) string { return filepath.Join(fs.prefix, name) }
+
+func (fs directFS) Stat(name string) (isRegular bool, err error) {
+	fi, err := os.Stat(fs.path(name))
+	if err != nil {
+		return false, err
+	}
+	return fi.Mode().IsRegular(), nil
+}
+
+func (fs directFS) Rename(oldName, newName string) error {
+	return os.Rename(fs.path(oldName), fs.path(newName))
+}
+
+func (fs directFS) Remove(name string) error { return os.Remove(fs.path(name)) }
+
+func (fs directFS) ReadFile(name string) ([]byte, error) {
+	return ioutil.ReadFile(fs.path(name))
+}
+
+func (fs directFS) WriteFile(name string, contents []byte, perm os.FileMode) error {
+	return ioutil.WriteFile(fs.path(name), contents, perm)
+}

net/dns/direct_test.go

@@ -0,0 +1,83 @@
+// Copyright (c) 2021 Tailscale Inc & AUTHORS All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package dns
+
+import (
+	"io/ioutil"
+	"os"
+	"path/filepath"
+	"testing"
+
+	"inet.af/netaddr"
+	"tailscale.com/util/dnsname"
+)
+
+func TestSetDNS(t *testing.T) {
+	const orig = "nameserver 9.9.9.9 # orig"
+	tmp := t.TempDir()
+	resolvPath := filepath.Join(tmp, "etc", "resolv.conf")
+	backupPath := filepath.Join(tmp, "etc", "resolv.pre-tailscale-backup.conf")
+
+	if err := os.MkdirAll(filepath.Dir(resolvPath), 0777); err != nil {
+		t.Fatal(err)
+	}
+	if err := ioutil.WriteFile(resolvPath, []byte(orig), 0644); err != nil {
+		t.Fatal(err)
+	}
+
+	readFile := func(t *testing.T, path string) string {
+		t.Helper()
+		b, err := ioutil.ReadFile(path)
+		if err != nil {
+			t.Fatal(err)
+		}
+		return string(b)
+	}
+	assertBaseState := func(t *testing.T) {
+		if got := readFile(t, resolvPath); got != orig {
+			t.Fatalf("resolv.conf:\n%s, want:\n%s", got, orig)
+		}
+		if _, err := os.Stat(backupPath); !os.IsNotExist(err) {
+			t.Fatalf("resolv.conf backup: want it to be gone but: %v", err)
+		}
+	}
+
+	m := directManager{fs: directFS{prefix: tmp}}
+	if err := m.SetDNS(OSConfig{
+		Nameservers:   []netaddr.IP{netaddr.MustParseIP("8.8.8.8"), netaddr.MustParseIP("8.8.4.4")},
+		SearchDomains: []dnsname.FQDN{"ts.net.", "ts-dns.test."},
+		MatchDomains:  []dnsname.FQDN{"ignored."},
+	}); err != nil {
+		t.Fatal(err)
+	}
+	want := `# resolv.conf(5) file generated by tailscale
+# DO NOT EDIT THIS FILE BY HAND -- CHANGES WILL BE OVERWRITTEN
+
+nameserver 8.8.8.8
+nameserver 8.8.4.4
+search ts.net ts-dns.test
+`
+	if got := readFile(t, resolvPath); got != want {
+		t.Fatalf("resolv.conf:\n%s, want:\n%s", got, want)
+	}
+	if got := readFile(t, backupPath); got != orig {
+		t.Fatalf("resolv.conf backup:\n%s, want:\n%s", got, orig)
+	}
+
+	// Test that a nil OSConfig cleans up resolv.conf.
+	if err := m.SetDNS(OSConfig{}); err != nil {
+		t.Fatal(err)
+	}
+	assertBaseState(t)
+
+	// Test that Close cleans up resolv.conf.
+	if err := m.SetDNS(OSConfig{Nameservers: []netaddr.IP{netaddr.MustParseIP("8.8.8.8")}}); err != nil {
+		t.Fatal(err)
+	}
+	if err := m.Close(); err != nil {
+		t.Fatal(err)
+	}
+	assertBaseState(t)
+}

net/dns/ini.go

@@ -0,0 +1,29 @@
+// Copyright (c) 2021 Tailscale Inc & AUTHORS All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package dns
+
+import (
+	"regexp"
+	"strings"
+)
+
+// parseIni parses a basic .ini file, used for wsl.conf.
+func parseIni(data string) map[string]map[string]string {
+	sectionRE := regexp.MustCompile(`^\[([^]]+)\]`)
+	kvRE := regexp.MustCompile(`^\s*(\w+)\s*=\s*([^#]*)`)
+
+	ini := map[string]map[string]string{}
+	var section string
+	for _, line := range strings.Split(data, "\n") {
+		if res := sectionRE.FindStringSubmatch(line); len(res) > 1 {
+			section = res[1]
+			ini[section] = map[string]string{}
+		} else if res := kvRE.FindStringSubmatch(line); len(res) > 2 {
+			k, v := strings.TrimSpace(res[1]), strings.TrimSpace(res[2])
+			ini[section][k] = v
+		}
+	}
+	return ini
+}

net/dns/ini_test.go

@@ -0,0 +1,37 @@
+// Copyright (c) 2021 Tailscale Inc & AUTHORS All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package dns
+
+import (
+	"reflect"
+	"testing"
+)
+
+func TestParseIni(t *testing.T) {
+	var tests = []struct {
+		src  string
+		want map[string]map[string]string
+	}{
+		{
+			src: `# appended wsl.conf file
+[automount]
+	enabled = true
+	root=/mnt/
+# added by tailscale
+[network] # trailing comment
+generateResolvConf = false  # trailing comment`,
+			want: map[string]map[string]string{
+				"automount": map[string]string{"enabled": "true", "root": "/mnt/"},
+				"network":   map[string]string{"generateResolvConf": "false"},
+			},
+		},
+	}
+	for _, test := range tests {
+		got := parseIni(test.src)
+		if !reflect.DeepEqual(got, test.want) {
+			t.Errorf("for:\n%s\ngot:  %v\nwant: %v", test.src, got, test.want)
+		}
+	}
+}

net/dns/manager.go

@@ -20,8 +20,6 @@ import (
 // the lint exception is necessary and on others it is not,
 // and plain ignore complains if the exception is unnecessary.
 
-//lint:file-ignore U1000 reconfigTimeout is used on some platforms but not others
-
 // reconfigTimeout is the time interval within which Manager.{Up,Down} should complete.
 //
 // This is particularly useful because certain conditions can cause indefinite hangs
@@ -40,11 +38,11 @@ type Manager struct {
 }
 
 // NewManagers created a new manager from the given config.
-func NewManager(logf logger.Logf, oscfg OSConfigurator, linkMon *monitor.Mon) *Manager {
+func NewManager(logf logger.Logf, oscfg OSConfigurator, linkMon *monitor.Mon, linkSel resolver.ForwardLinkSelector) *Manager {
 	logf = logger.WithPrefix(logf, "dns: ")
 	m := &Manager{
 		logf:     logf,
-		resolver: resolver.New(logf, linkMon),
+		resolver: resolver.New(logf, linkMon, linkSel),
 		os:       oscfg,
 	}
 	m.logf("using %T", m.os)
@@ -209,7 +207,7 @@ func Cleanup(logf logger.Logf, interfaceName string) {
 		logf("creating dns cleanup: %v", err)
 		return
 	}
-	dns := NewManager(logf, oscfg, nil)
+	dns := NewManager(logf, oscfg, nil, nil)
 	if err := dns.Down(); err != nil {
 		logf("dns down: %v", err)
 	}

net/dns/manager_freebsd.go

@@ -15,7 +15,7 @@ import (
 func NewOSConfigurator(logf logger.Logf, _ string) (OSConfigurator, error) {
 	bs, err := ioutil.ReadFile("/etc/resolv.conf")
 	if os.IsNotExist(err) {
-		return newDirectManager()
+		return newDirectManager(), nil
 	}
 	if err != nil {
 		return nil, fmt.Errorf("reading /etc/resolv.conf: %w", err)
@@ -25,6 +25,6 @@ func NewOSConfigurator(logf logger.Logf, _ string) (OSConfigurator, error) {
 	case "resolvconf":
 		return newResolvconfManager(logf)
 	default:
-		return newDirectManager()
+		return newDirectManager(), nil
 	}
 }

net/dns/manager_linux.go

@@ -5,7 +5,6 @@
 package dns
 
 import (
-	"bytes"
 	"context"
 	"errors"
 	"fmt"
@@ -15,6 +14,7 @@ import (
 	"time"
 
 	"github.com/godbus/dbus/v5"
+	"inet.af/netaddr"
 	"tailscale.com/types/logger"
 	"tailscale.com/util/cmpver"
 )
@@ -42,7 +42,7 @@ func NewOSConfigurator(logf logger.Logf, interfaceName string) (ret OSConfigurat
 	bs, err := ioutil.ReadFile("/etc/resolv.conf")
 	if os.IsNotExist(err) {
 		dbg("rc", "missing")
-		return newDirectManager()
+		return newDirectManager(), nil
 	}
 	if err != nil {
 		return nil, fmt.Errorf("reading /etc/resolv.conf: %w", err)
@@ -51,9 +51,18 @@ func NewOSConfigurator(logf logger.Logf, interfaceName string) (ret OSConfigurat
 	switch resolvOwner(bs) {
 	case "systemd-resolved":
 		dbg("rc", "resolved")
+		// Some systems, for reasons known only to them, have a
+		// resolv.conf that has the word "systemd-resolved" in its
+		// header, but doesn't actually point to resolved. We mustn't
+		// try to program resolved in that case.
+		// https://github.com/tailscale/tailscale/issues/2136
+		if err := resolvedIsActuallyResolver(); err != nil {
+			dbg("resolved", "not-in-use")
+			return newDirectManager(), nil
+		}
 		if err := dbusPing("org.freedesktop.resolve1", "/org/freedesktop/resolve1"); err != nil {
 			dbg("resolved", "no")
-			return newDirectManager()
+			return newDirectManager(), nil
 		}
 		if err := dbusPing("org.freedesktop.NetworkManager", "/org/freedesktop/NetworkManager/DnsManager"); err != nil {
 			dbg("nm", "no")
@@ -79,109 +88,69 @@ func NewOSConfigurator(logf logger.Logf, interfaceName string) (ret OSConfigurat
 		// "unmanaged" interfaces - meaning NM 1.26.6 and later
 		// actively ignore DNS configuration we give it. So, for those
 		// NM versions, we can and must use resolved directly.
-		old, err := nmVersionOlderThan("1.26.6")
+		//
+		// Even more fun, even-older versions of NM won't let us set
+		// DNS settings if the interface isn't managed by NM, with a
+		// hard failure on DBus requests. Empirically, NM 1.22 does
+		// this. Based on the versions popular distros shipped, we
+		// conservatively decree that only 1.26.0 through 1.26.5 are
+		// "safe" to use for our purposes. This roughly matches
+		// distros released in the latter half of 2020.
+		//
+		// In a perfect world, we'd avoid this by replacing
+		// configuration out from under NM entirely (e.g. using
+		// directManager to overwrite resolv.conf), but in a world
+		// where resolved runs, we need to get correct configuration
+		// into resolved regardless of what's in resolv.conf (because
+		// resolved can also be queried over dbus, or via an NSS
+		// module that bypasses /etc/resolv.conf). Given that we must
+		// get correct configuration into resolved, we have no choice
+		// but to use NM, and accept the loss of IPv6 configuration
+		// that comes with it (see
+		// https://github.com/tailscale/tailscale/issues/1699,
+		// https://github.com/tailscale/tailscale/pull/1945)
+		safe, err := nmVersionBetween("1.26.0", "1.26.5")
 		if err != nil {
 			// Failed to figure out NM's version, can't make a correct
 			// decision.
 			return nil, fmt.Errorf("checking NetworkManager version: %v", err)
 		}
-		if old {
-			dbg("nm-old", "yes")
+		if safe {
+			dbg("nm-safe", "yes")
 			return newNMManager(interfaceName)
 		}
-		dbg("nm-old", "no")
+		dbg("nm-safe", "no")
 		return newResolvedManager(logf, interfaceName)
 	case "resolvconf":
 		dbg("rc", "resolvconf")
-		if err := resolvconfSourceIsNM(bs); err == nil {
-			dbg("src-is-nm", "yes")
-			if err := dbusPing("org.freedesktop.NetworkManager", "/org/freedesktop/NetworkManager/DnsManager"); err == nil {
-				dbg("nm", "yes")
-				old, err := nmVersionOlderThan("1.26.6")
-				if err != nil {
-					return nil, fmt.Errorf("checking NetworkManager version: %v", err)
-				}
-				if old {
-					dbg("nm-old", "yes")
-					return newNMManager(interfaceName)
-				} else {
-					dbg("nm-old", "no")
-				}
-			} else {
-				dbg("nm", "no")
-			}
-		} else {
-			dbg("src-is-nm", "no")
-		}
 		if _, err := exec.LookPath("resolvconf"); err != nil {
 			dbg("resolvconf", "no")
-			return newDirectManager()
+			return newDirectManager(), nil
 		}
 		dbg("resolvconf", "yes")
 		return newResolvconfManager(logf)
 	case "NetworkManager":
+		// You'd think we would use newNMManager somewhere in
+		// here. However, as explained in
+		// https://github.com/tailscale/tailscale/issues/1699 , using
+		// NetworkManager for DNS configuration carries with it the
+		// cost of losing IPv6 configuration on the Tailscale network
+		// interface. So, when we can avoid it, we bypass
+		// NetworkManager by replacing resolv.conf directly.
+		//
+		// If you ever try to put NMManager back here, keep in mind
+		// that versions >=1.26.6 will ignore DNS configuration
+		// anyway, so you still need a fallback path that uses
+		// directManager.
 		dbg("rc", "nm")
-		if err := dbusPing("org.freedesktop.NetworkManager", "/org/freedesktop/NetworkManager/DnsManager"); err != nil {
-			dbg("nm", "no")
-			return newDirectManager()
-		}
-		dbg("nm", "yes")
-		old, err := nmVersionOlderThan("1.26.6")
-		if err != nil {
-			return nil, fmt.Errorf("checking NetworkManager version: %v", err)
-		}
-		if old {
-			dbg("nm-old", "yes")
-			return newNMManager(interfaceName)
-		}
-		dbg("nm-old", "no")
-		return newDirectManager()
+		return newDirectManager(), nil
 	default:
 		dbg("rc", "unknown")
-		return newDirectManager()
+		return newDirectManager(), nil
 	}
 }
 
-func resolvconfSourceIsNM(resolvDotConf []byte) error {
-	b := bytes.NewBuffer(resolvDotConf)
-	cfg, err := readResolv(b)
-	if err != nil {
-		return fmt.Errorf("parsing /etc/resolv.conf: %w", err)
-	}
-
-	var (
-		paths = []string{
-			"/etc/resolvconf/run/interface/NetworkManager",
-			"/run/resolvconf/interface/NetworkManager",
-			"/var/run/resolvconf/interface/NetworkManager",
-			"/run/resolvconf/interfaces/NetworkManager",
-			"/var/run/resolvconf/interfaces/NetworkManager",
-		}
-		nmCfg OSConfig
-		found bool
-	)
-	for _, path := range paths {
-		nmCfg, err = readResolvFile(path)
-		if os.IsNotExist(err) {
-			continue
-		} else if err != nil {
-			return err
-		}
-		found = true
-		break
-	}
-	if !found {
-		return errors.New("NetworkManager resolvconf snippet not found")
-	}
-
-	if !nmCfg.Equal(cfg) {
-		return errors.New("NetworkManager config not applied by resolvconf")
-	}
-
-	return nil
-}
-
-func nmVersionOlderThan(want string) (bool, error) {
+func nmVersionBetween(first, last string) (bool, error) {
 	conn, err := dbus.SystemBus()
 	if err != nil {
 		// DBus probably not running.
@@ -199,7 +168,8 @@ func nmVersionOlderThan(want string) (bool, error) {
 		return false, fmt.Errorf("unexpected type %T for NM version", v.Value())
 	}
 
-	return cmpver.Compare(version, want) < 0, nil
+	outside := cmpver.Compare(version, first) < 0 || cmpver.Compare(version, last) > 0
+	return !outside, nil
 }
 
 func nmIsUsingResolved() error {
@@ -224,6 +194,17 @@ func nmIsUsingResolved() error {
 	return nil
 }
 
+func resolvedIsActuallyResolver() error {
+	cfg, err := newDirectManager().readResolvConf()
+	if err != nil {
+		return err
+	}
+	if len(cfg.Nameservers) != 1 || cfg.Nameservers[0] != netaddr.IPv4(127, 0, 0, 53) {
+		return errors.New("resolv.conf doesn't point to systemd-resolved")
+	}
+	return nil
+}
+
 func dbusPing(name, objectPath string) error {
 	ctx, cancel := context.WithTimeout(context.Background(), time.Second)
 	defer cancel()

net/dns/manager_openbsd.go

@@ -7,5 +7,5 @@ package dns
 import "tailscale.com/types/logger"
 
 func NewOSConfigurator(logger.Logf, string) (OSConfigurator, error) {
-	return newDirectManager()
+	return newDirectManager(), nil
 }

net/dns/manager_test.go

@@ -376,7 +376,7 @@ func TestManager(t *testing.T) {
 				SplitDNS:   test.split,
 				BaseConfig: test.bs,
 			}
-			m := NewManager(t.Logf, &f, nil)
+			m := NewManager(t.Logf, &f, nil, nil)
 			m.resolver.TestOnlySetHook(f.SetResolver)
 
 			if err := m.Set(test.in); err != nil {

net/dns/manager_windows.go

@@ -35,16 +35,18 @@ const (
 )
 
 type windowsManager struct {
-	logf      logger.Logf
-	guid      string
-	nrptWorks bool
+	logf       logger.Logf
+	guid       string
+	nrptWorks  bool
+	wslManager *wslManager
 }
 
 func NewOSConfigurator(logf logger.Logf, interfaceName string) (OSConfigurator, error) {
 	ret := windowsManager{
-		logf:      logf,
-		guid:      interfaceName,
-		nrptWorks: !isWindows7(),
+		logf:       logf,
+		guid:       interfaceName,
+		nrptWorks:  isWindows10OrBetter(),
+		wslManager: newWSLManager(logf),
 	}
 
 	// Best-effort: if our NRPT rule exists, try to delete it. Unlike
@@ -57,6 +59,13 @@ func NewOSConfigurator(logf logger.Logf, interfaceName string) (OSConfigurator,
 		ret.delKey(nrptBase)
 	}
 
+	// Log WSL status once at startup.
+	if distros, err := wslDistros(); err != nil {
+		logf("WSL: could not list distributions: %v", err)
+	} else {
+		logf("WSL: found %d distributions", len(distros))
+	}
+
 	return ret, nil
 }
 
@@ -296,6 +305,16 @@ func (m windowsManager) SetDNS(cfg OSConfig) error {
 		}
 	}()
 
+	// On initial setup of WSL, the restart caused by --shutdown is slow,
+	// so we do it out-of-line.
+	go func() {
+		if err := m.wslManager.SetDNS(cfg); err != nil {
+			m.logf("WSL SetDNS: %v", err) // continue
+		} else {
+			m.logf("WSL SetDNS: success")
+		}
+	}()
+
 	return nil
 }
 
@@ -407,22 +426,16 @@ var siteLocalResolvers = []netaddr.IP{
 	netaddr.MustParseIP("fec0:0:0:ffff::3"),
 }
 
-func isWindows7() bool {
+func isWindows10OrBetter() bool {
 	key, err := registry.OpenKey(registry.LOCAL_MACHINE, versionKey, registry.READ)
 	if err != nil {
-		// Fail safe, assume Windows 7.
-		return true
+		// Fail safe, assume old Windows.
+		return false
 	}
-	ver, _, err := key.GetStringValue("CurrentVersion")
-	if err != nil {
-		return true
+	// This key above only exists in Windows 10 and above. Its mere
+	// presence is good enough.
+	if _, _, err := key.GetIntegerValue("CurrentMajorVersionNumber"); err != nil {
+		return false
 	}
-	// Careful to not assume anything about version numbers beyond
-	// 6.3, Microsoft deprecated this registry key and locked its
-	// value to what it was in Windows 8.1. We can only use this to
-	// probe for versions before that. Good thing we only need Windows
-	// 7 (so far).
-	//
-	// And yes, Windows 7 is version 6.1. Don't ask.
-	return ver == "6.1"
+	return true
 }

net/dns/nm.go

@@ -4,8 +4,6 @@
 
 // +build linux
 
-//lint:file-ignore U1000 refactoring, temporarily unused code.
-
 package dns
 
 import (

net/dns/resolved.go

@@ -4,8 +4,6 @@
 
 // +build linux
 
-//lint:file-ignore U1000 refactoring, temporarily unused code.
-
 package dns
 
 import (
@@ -69,7 +67,7 @@ func isResolvedActive() bool {
 		return false
 	}
 
-	config, err := readResolvConf()
+	config, err := newDirectManager().readResolvConf()
 	if err != nil {
 		return false
 	}
@@ -82,7 +80,7 @@ func isResolvedActive() bool {
 	return false
 }
 
-// resolvedManager uses the systemd-resolved DBus API.
+// resolvedManager is an OSConfigurator which uses the systemd-resolved DBus API.
 type resolvedManager struct {
 	logf     logger.Logf
 	ifidx    int
@@ -107,7 +105,6 @@ func newResolvedManager(logf logger.Logf, interfaceName string) (*resolvedManage
 	}, nil
 }
 
-// Up implements managerImpl.
 func (m *resolvedManager) SetDNS(config OSConfig) error {
 	ctx, cancel := context.WithTimeout(context.Background(), reconfigTimeout)
 	defer cancel()

net/dns/resolver/forwarder.go

@@ -9,41 +9,30 @@ import (
 	"context"
 	"encoding/binary"
 	"errors"
-	"fmt"
 	"hash/crc32"
+	"io"
 	"math/rand"
 	"net"
 	"sync"
-	"syscall"
 	"time"
 
 	dns "golang.org/x/net/dns/dnsmessage"
 	"inet.af/netaddr"
-	"tailscale.com/logtail/backoff"
 	"tailscale.com/types/logger"
 	"tailscale.com/util/dnsname"
+	"tailscale.com/wgengine/monitor"
 )
 
 // headerBytes is the number of bytes in a DNS message header.
 const headerBytes = 12
 
-// connCount is the number of UDP connections to use for forwarding.
-const connCount = 32
-
 const (
-	// cleanupInterval is the interval between purged of timed-out entries from txMap.
-	cleanupInterval = 30 * time.Second
 	// responseTimeout is the maximal amount of time to wait for a DNS response.
 	responseTimeout = 5 * time.Second
 )
 
 var errNoUpstreams = errors.New("upstream nameservers not set")
 
-type forwardingRecord struct {
-	src       netaddr.IPPort
-	createdAt time.Time
-}
-
 // txid identifies a DNS transaction.
 //
 // As the standard DNS Request ID is only 16 bits, we extend it:
@@ -99,179 +88,218 @@ func getTxID(packet []byte) txid {
 	return (txid(hash) << 32) | txid(dnsid)
 }
 
+// clampEDNSSize attempts to limit the maximum EDNS response size. This is not
+// an exhaustive solution, instead only easy cases are currently handled in the
+// interest of speed and reduced complexity. Only OPT records at the very end of
+// the message with no option codes are addressed.
+// TODO: handle more situations if we discover that they happen often
+func clampEDNSSize(packet []byte, maxSize uint16) {
+	// optFixedBytes is the size of an OPT record with no option codes.
+	const optFixedBytes = 11
+	const edns0Version = 0
+
+	if len(packet) < headerBytes+optFixedBytes {
+		return
+	}
+
+	arCount := binary.BigEndian.Uint16(packet[10:12])
+	if arCount == 0 {
+		// OPT shows up in an AR, so there must be no OPT
+		return
+	}
+
+	opt := packet[len(packet)-optFixedBytes:]
+
+	if opt[0] != 0 {
+		// OPT NAME must be 0 (root domain)
+		return
+	}
+	if dns.Type(binary.BigEndian.Uint16(opt[1:3])) != dns.TypeOPT {
+		// Not an OPT record
+		return
+	}
+	requestedSize := binary.BigEndian.Uint16(opt[3:5])
+	// Ignore extended RCODE in opt[5]
+	if opt[6] != edns0Version {
+		// Be conservative and don't touch unknown versions.
+		return
+	}
+	// Ignore flags in opt[7:9]
+	if binary.BigEndian.Uint16(opt[10:12]) != 0 {
+		// RDLEN must be 0 (no variable length data). We're at the end of the
+		// packet so this should be 0 anyway)..
+		return
+	}
+
+	if requestedSize <= maxSize {
+		return
+	}
+
+	// Clamp the maximum size
+	binary.BigEndian.PutUint16(opt[3:5], maxSize)
+}
+
 type route struct {
-	suffix    dnsname.FQDN
-	resolvers []netaddr.IPPort
+	Suffix    dnsname.FQDN
+	Resolvers []netaddr.IPPort
 }
 
 // forwarder forwards DNS packets to a number of upstream nameservers.
 type forwarder struct {
-	logf logger.Logf
+	logf    logger.Logf
+	linkMon *monitor.Mon
+	linkSel ForwardLinkSelector
+
+	ctx       context.Context    // good until Close
+	ctxCancel context.CancelFunc // closes ctx
 
 	// responses is a channel by which responses are returned.
 	responses chan packet
-	// closed signals all goroutines to stop.
-	closed chan struct{}
-	// wg signals when all goroutines have stopped.
-	wg sync.WaitGroup
 
-	// conns are the UDP connections used for forwarding.
-	// A random one is selected for each request, regardless of the target upstream.
-	conns []*fwdConn
+	mu sync.Mutex // guards following
 
-	mu sync.Mutex
-	// routes are per-suffix resolvers to use.
-	routes []route                   // most specific routes first
-	txMap  map[txid]forwardingRecord // txids to in-flight requests
+	// routes are per-suffix resolvers to use, with
+	// the most specific routes first.
+	routes []route
 }
 
 func init() {
 	rand.Seed(time.Now().UnixNano())
 }
 
-func newForwarder(logf logger.Logf, responses chan packet) *forwarder {
-	ret := &forwarder{
+func newForwarder(logf logger.Logf, responses chan packet, linkMon *monitor.Mon, linkSel ForwardLinkSelector) *forwarder {
+	f := &forwarder{
 		logf:      logger.WithPrefix(logf, "forward: "),
+		linkMon:   linkMon,
+		linkSel:   linkSel,
 		responses: responses,
-		closed:    make(chan struct{}),
-		conns:     make([]*fwdConn, connCount),
-		txMap:     make(map[txid]forwardingRecord),
 	}
-
-	ret.wg.Add(connCount + 1)
-	for idx := range ret.conns {
-		ret.conns[idx] = newFwdConn(ret.logf, idx)
-		go ret.recv(ret.conns[idx])
-	}
-	go ret.cleanMap()
-
-	return ret
+	f.ctx, f.ctxCancel = context.WithCancel(context.Background())
+	return f
 }
 
-func (f *forwarder) Close() {
-	select {
-	case <-f.closed:
-		return
-	default:
-		// continue
-	}
-	close(f.closed)
-
-	for _, conn := range f.conns {
-		conn.close()
-	}
-
-	f.wg.Wait()
-}
-
-func (f *forwarder) rebindFromNetworkChange() {
-	for _, c := range f.conns {
-		c.mu.Lock()
-		c.reconnectLocked()
-		c.mu.Unlock()
-	}
+func (f *forwarder) Close() error {
+	f.ctxCancel()
+	return nil
 }
 
 func (f *forwarder) setRoutes(routes []route) {
 	f.mu.Lock()
+	defer f.mu.Unlock()
 	f.routes = routes
-	f.mu.Unlock()
+}
+
+var stdNetPacketListener packetListener = new(net.ListenConfig)
+
+type packetListener interface {
+	ListenPacket(ctx context.Context, network, address string) (net.PacketConn, error)
+}
+
+func (f *forwarder) packetListener(ip netaddr.IP) (packetListener, error) {
+	if f.linkSel == nil || initListenConfig == nil {
+		return stdNetPacketListener, nil
+	}
+	linkName := f.linkSel.PickLink(ip)
+	if linkName == "" {
+		return stdNetPacketListener, nil
+	}
+	lc := new(net.ListenConfig)
+	if err := initListenConfig(lc, f.linkMon, linkName); err != nil {
+		return nil, err
+	}
+	return lc, nil
 }
 
 // send sends packet to dst. It is best effort.
-func (f *forwarder) send(packet []byte, dst netaddr.IPPort) {
-	connIdx := rand.Intn(connCount)
-	conn := f.conns[connIdx]
-	conn.send(packet, dst)
-}
+//
+// send expects the reply to have the same txid as txidOut.
+//
+// The provided closeOnCtxDone lets send register values to Close if
+// the caller's ctx expires. This avoids send from allocating its own
+// waiting goroutine to interrupt the ReadFrom, as memory is tight on
+// iOS and we want the number of pending DNS lookups to be bursty
+// without too much associated goroutine/memory cost.
+func (f *forwarder) send(ctx context.Context, txidOut txid, closeOnCtxDone *closePool, packet []byte, dst netaddr.IPPort) ([]byte, error) {
+	// TODO(bradfitz): if dst.IP is 8.8.8.8 or 8.8.4.4 or 1.1.1.1, etc, or
+	// something dynamically probed earlier to support DoH or DoT,
+	// do that here instead.
 
-func (f *forwarder) recv(conn *fwdConn) {
-	defer f.wg.Done()
+	ln, err := f.packetListener(dst.IP())
+	if err != nil {
+		return nil, err
+	}
+	conn, err := ln.ListenPacket(ctx, "udp", ":0")
+	if err != nil {
+		f.logf("ListenPacket failed: %v", err)
+		return nil, err
+	}
+	defer conn.Close()
 
-	for {
-		select {
-		case <-f.closed:
-			return
-		default:
+	closeOnCtxDone.Add(conn)
+	defer closeOnCtxDone.Remove(conn)
+
+	if _, err := conn.WriteTo(packet, dst.UDPAddr()); err != nil {
+		if err := ctx.Err(); err != nil {
+			return nil, err
 		}
-		// The 1 extra byte is to detect packet truncation.
-		out := make([]byte, maxResponseBytes+1)
-		n := conn.read(out)
-		var truncated bool
-		if n > maxResponseBytes {
-			n = maxResponseBytes
-			truncated = true
+		return nil, err
+	}
+
+	// The 1 extra byte is to detect packet truncation.
+	out := make([]byte, maxResponseBytes+1)
+	n, _, err := conn.ReadFrom(out)
+	if err != nil {
+		if err := ctx.Err(); err != nil {
+			return nil, err
 		}
-		if n == 0 {
-			continue
-		}
-		if n < headerBytes {
-			f.logf("recv: packet too small (%d bytes)", n)
-		}
-
-		out = out[:n]
-		txid := getTxID(out)
-
-		if truncated {
-			const dnsFlagTruncated = 0x200
-			flags := binary.BigEndian.Uint16(out[2:4])
-			flags |= dnsFlagTruncated
-			binary.BigEndian.PutUint16(out[2:4], flags)
-
-			// TODO(#2067): Remove any incomplete records? RFC 1035 section 6.2
-			// states that truncation should head drop so that the authority
-			// section can be preserved if possible. However, the UDP read with
-			// a too-small buffer has already dropped the end, so that's the
-			// best we can do.
-		}
-
-		f.mu.Lock()
-
-		record, found := f.txMap[txid]
-		// At most one nameserver will return a response:
-		// the first one to do so will delete txid from the map.
-		if !found {
-			f.mu.Unlock()
-			continue
-		}
-		delete(f.txMap, txid)
-
-		f.mu.Unlock()
-
-		pkt := packet{out, record.src}
-		select {
-		case <-f.closed:
-			return
-		case f.responses <- pkt:
-			// continue
+		if packetWasTruncated(err) {
+			err = nil
+		} else {
+			return nil, err
 		}
 	}
+	truncated := n > maxResponseBytes
+	if truncated {
+		n = maxResponseBytes
+	}
+	if n < headerBytes {
+		f.logf("recv: packet too small (%d bytes)", n)
+	}
+	out = out[:n]
+	txid := getTxID(out)
+	if txid != txidOut {
+		return nil, errors.New("txid doesn't match")
+	}
+
+	if truncated {
+		const dnsFlagTruncated = 0x200
+		flags := binary.BigEndian.Uint16(out[2:4])
+		flags |= dnsFlagTruncated
+		binary.BigEndian.PutUint16(out[2:4], flags)
+
+		// TODO(#2067): Remove any incomplete records? RFC 1035 section 6.2
+		// states that truncation should head drop so that the authority
+		// section can be preserved if possible. However, the UDP read with
+		// a too-small buffer has already dropped the end, so that's the
+		// best we can do.
+	}
+
+	clampEDNSSize(out, maxResponseBytes)
+
+	return out, nil
 }
 
-// cleanMap periodically deletes timed-out forwarding records from f.txMap to bound growth.
-func (f *forwarder) cleanMap() {
-	defer f.wg.Done()
-
-	t := time.NewTicker(cleanupInterval)
-	defer t.Stop()
-
-	var now time.Time
-	for {
-		select {
-		case <-f.closed:
-			return
-		case now = <-t.C:
-			// continue
+// resolvers returns the resolvers to use for domain.
+func (f *forwarder) resolvers(domain dnsname.FQDN) []netaddr.IPPort {
+	f.mu.Lock()
+	routes := f.routes
+	f.mu.Unlock()
+	for _, route := range routes {
+		if route.Suffix == "." || route.Suffix.Contains(domain) {
+			return route.Resolvers
 		}
-
-		f.mu.Lock()
-		for k, v := range f.txMap {
-			if now.Sub(v.createdAt) > responseTimeout {
-				delete(f.txMap, k)
-			}
-		}
-		f.mu.Unlock()
 	}
+	return nil
 }
 
 // forward forwards the query to all upstream nameservers and returns the first response.
@@ -282,226 +310,62 @@ func (f *forwarder) forward(query packet) error {
 	}
 
 	txid := getTxID(query.bs)
+	clampEDNSSize(query.bs, maxResponseBytes)
 
-	f.mu.Lock()
-	routes := f.routes
-	f.mu.Unlock()
-
-	var resolvers []netaddr.IPPort
-	for _, route := range routes {
-		if route.suffix != "." && !route.suffix.Contains(domain) {
-			continue
-		}
-		resolvers = route.resolvers
-		break
-	}
+	resolvers := f.resolvers(domain)
 	if len(resolvers) == 0 {
 		return errNoUpstreams
 	}
 
-	f.mu.Lock()
-	f.txMap[txid] = forwardingRecord{
-		src:       query.addr,
-		createdAt: time.Now(),
-	}
-	f.mu.Unlock()
+	closeOnCtxDone := new(closePool)
+	defer closeOnCtxDone.Close()
 
-	// TODO(#2066): EDNS size clamping
+	ctx, cancel := context.WithTimeout(f.ctx, responseTimeout)
+	defer cancel()
 
-	for _, resolver := range resolvers {
-		f.send(query.bs, resolver)
-	}
+	resc := make(chan []byte, 1)
+	var (
+		mu       sync.Mutex
+		firstErr error
+	)
 
-	return nil
-}
-
-// A fwdConn manages a single connection used to forward DNS requests.
-// Net link changes can cause a *net.UDPConn to become permanently unusable, particularly on macOS.
-// fwdConn detects such situations and transparently creates new connections.
-type fwdConn struct {
-	// logf allows a fwdConn to log.
-	logf logger.Logf
-
-	// change allows calls to read to block until a the network connection has been replaced.
-	change *sync.Cond
-
-	// mu protects fields that follow it; it is also change's Locker.
-	mu sync.Mutex
-	// closed tracks whether fwdConn has been permanently closed.
-	closed bool
-	// conn is the current active connection.
-	conn net.PacketConn
-}
-
-func newFwdConn(logf logger.Logf, idx int) *fwdConn {
-	c := new(fwdConn)
-	c.logf = logger.WithPrefix(logf, fmt.Sprintf("fwdConn %d: ", idx))
-	c.change = sync.NewCond(&c.mu)
-	// c.conn is created lazily in send
-	return c
-}
-
-// send sends packet to dst using c's connection.
-// It is best effort. It is UDP, after all. Failures are logged.
-func (c *fwdConn) send(packet []byte, dst netaddr.IPPort) {
-	var b *backoff.Backoff // lazily initialized, since it is not needed in the common case
-	backOff := func(err error) {
-		if b == nil {
-			b = backoff.NewBackoff("dns-fwdConn-send", c.logf, 30*time.Second)
-		}
-		b.BackOff(context.Background(), err)
-	}
-
-	for {
-		// Gather the current connection.
-		// We can't hold the lock while we call WriteTo.
-		c.mu.Lock()
-		conn := c.conn
-		closed := c.closed
-		if closed {
-			c.mu.Unlock()
-			return
-		}
-		if conn == nil {
-			c.reconnectLocked()
-			c.mu.Unlock()
-			continue
-		}
-		c.mu.Unlock()
-
-		_, err := conn.WriteTo(packet, dst.UDPAddr())
-		if err == nil {
-			// Success
-			return
-		}
-		if errors.Is(err, net.ErrClosed) {
-			// We intentionally closed this connection.
-			// It has been replaced by a new connection. Try again.
-			continue
-		}
-		// Something else went wrong.
-		// We have three choices here: try again, give up, or create a new connection.
-		var opErr *net.OpError
-		if !errors.As(err, &opErr) {
-			// Weird. All errors from the net package should be *net.OpError. Bail.
-			c.logf("send: non-*net.OpErr %v (%T)", err, err)
-			return
-		}
-		if opErr.Temporary() || opErr.Timeout() {
-			// I doubt that either of these can happen (this is UDP),
-			// but go ahead and try again.
-			backOff(err)
-			continue
-		}
-		if errors.Is(err, syscall.EHOSTUNREACH) {
-			// "No route to host." The network stack is fine, but
-			// can't talk to this destination. Not much we can do
-			// about that, don't spam logs.
-			return
-		}
-		if networkIsDown(err) {
-			// Fail.
-			c.logf("send: network is down")
-			return
-		}
-		if networkIsUnreachable(err) {
-			// This can be caused by a link change.
-			// Replace the existing connection with a new one.
-			c.mu.Lock()
-			// It's possible that multiple senders discovered simultaneously
-			// that the network is unreachable. Avoid reconnecting multiple times:
-			// Only reconnect if the current connection is the one that we
-			// discovered to be problematic.
-			if c.conn == conn {
-				backOff(err)
-				c.reconnectLocked()
+	for _, ipp := range resolvers {
+		go func(ipp netaddr.IPPort) {
+			resb, err := f.send(ctx, txid, closeOnCtxDone, query.bs, ipp)
+			if err != nil {
+				mu.Lock()
+				defer mu.Unlock()
+				if firstErr == nil {
+					firstErr = err
+				}
+				return
 			}
-			c.mu.Unlock()
-			// Try again with our new network connection.
-			continue
+			select {
+			case resc <- resb:
+			default:
+			}
+		}(ipp)
+	}
+
+	select {
+	case v := <-resc:
+		select {
+		case <-ctx.Done():
+			return ctx.Err()
+		case f.responses <- packet{v, query.addr}:
+			return nil
 		}
-		// Unrecognized error. Fail.
-		c.logf("send: unrecognized error: %v", err)
-		return
+	case <-ctx.Done():
+		mu.Lock()
+		defer mu.Unlock()
+		if firstErr != nil {
+			return firstErr
+		}
+		return ctx.Err()
 	}
 }
 
-// read waits for a response from c's connection.
-// It returns the number of bytes read, which may be 0
-// in case of an error or a closed connection.
-func (c *fwdConn) read(out []byte) int {
-	for {
-		// Gather the current connection.
-		// We can't hold the lock while we call ReadFrom.
-		c.mu.Lock()
-		conn := c.conn
-		closed := c.closed
-		if closed {
-			c.mu.Unlock()
-			return 0
-		}
-		if conn == nil {
-			// There is no current connection.
-			// Wait for the connection to change, then try again.
-			c.change.Wait()
-			c.mu.Unlock()
-			continue
-		}
-		c.mu.Unlock()
-
-		n, _, err := conn.ReadFrom(out)
-		if err == nil || packetWasTruncated(err) {
-			// Success.
-			return n
-		}
-		if errors.Is(err, net.ErrClosed) {
-			// We intentionally closed this connection.
-			// It has been replaced by a new connection. Try again.
-			continue
-		}
-
-		c.logf("read: unrecognized error: %v", err)
-		return 0
-	}
-}
-
-// reconnectLocked replaces the current connection with a new one.
-// c.mu must be locked.
-func (c *fwdConn) reconnectLocked() {
-	c.closeConnLocked()
-	// Make a new connection.
-	conn, err := net.ListenPacket("udp", "")
-	if err != nil {
-		c.logf("ListenPacket failed: %v", err)
-	} else {
-		c.conn = conn
-	}
-	// Broadcast that a new connection is available.
-	c.change.Broadcast()
-}
-
-// closeCurrentConn closes the current connection.
-// c.mu must be locked.
-func (c *fwdConn) closeConnLocked() {
-	if c.conn == nil {
-		return
-	}
-	c.conn.Close() // unblocks all readers/writers, they'll pick up the next connection.
-	c.conn = nil
-}
-
-// close permanently closes c.
-func (c *fwdConn) close() {
-	c.mu.Lock()
-	defer c.mu.Unlock()
-	if c.closed {
-		return
-	}
-	c.closed = true
-	c.closeConnLocked()
-	// Unblock any remaining readers.
-	c.change.Broadcast()
-}
+var initListenConfig func(_ *net.ListenConfig, _ *monitor.Mon, tunName string) error
 
 // nameFromQuery extracts the normalized query name from bs.
 func nameFromQuery(bs []byte) (dnsname.FQDN, error) {
@@ -523,3 +387,48 @@ func nameFromQuery(bs []byte) (dnsname.FQDN, error) {
 	n := q.Name.Data[:q.Name.Length]
 	return dnsname.ToFQDN(rawNameToLower(n))
 }
+
+// closePool is a dynamic set of io.Closers to close as a group.
+// It's intended to be Closed at most once.
+//
+// The zero value is ready for use.
+type closePool struct {
+	mu     sync.Mutex
+	m      map[io.Closer]bool
+	closed bool
+}
+
+func (p *closePool) Add(c io.Closer) {
+	p.mu.Lock()
+	defer p.mu.Unlock()
+	if p.closed {
+		c.Close()
+		return
+	}
+	if p.m == nil {
+		p.m = map[io.Closer]bool{}
+	}
+	p.m[c] = true
+}
+
+func (p *closePool) Remove(c io.Closer) {
+	p.mu.Lock()
+	defer p.mu.Unlock()
+	if p.closed {
+		return
+	}
+	delete(p.m, c)
+}
+
+func (p *closePool) Close() error {
+	p.mu.Lock()
+	defer p.mu.Unlock()
+	if p.closed {
+		return nil
+	}
+	p.closed = true
+	for c := range p.m {
+		c.Close()
+	}
+	return nil
+}

net/dns/resolver/macios_ext.go

@@ -0,0 +1,27 @@
+// Copyright (c) 2021 Tailscale Inc & AUTHORS All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+// +build darwin,ts_macext ios,ts_macext
+
+package resolver
+
+import (
+	"errors"
+	"net"
+
+	"tailscale.com/net/netns"
+	"tailscale.com/wgengine/monitor"
+)
+
+func init() {
+	initListenConfig = initListenConfigNetworkExtension
+}
+
+func initListenConfigNetworkExtension(nc *net.ListenConfig, mon *monitor.Mon, tunName string) error {
+	nif, ok := mon.InterfaceState().Interface[tunName]
+	if !ok {
+		return errors.New("utun not found")
+	}
+	return netns.SetListenConfigInterfaceIndex(nc, nif.Interface.Index)
+}

net/dns/resolver/neterr_other.go

@@ -12,4 +12,4 @@ func networkIsUnreachable(err error) bool { return false }
 // packetWasTruncated returns true if err indicates truncation but the RecvFrom
 // that generated err was otherwise successful. It always returns false on this
 // platform.
-func packetWasTruncated(err error) bool   { return false }
+func packetWasTruncated(err error) bool { return false }

net/dns/resolver/tsdns.go

@@ -9,14 +9,15 @@ package resolver
 import (
 	"encoding/hex"
 	"errors"
+	"runtime"
 	"sort"
 	"strings"
 	"sync"
+	"sync/atomic"
 	"time"
 
 	dns "golang.org/x/net/dns/dnsmessage"
 	"inet.af/netaddr"
-	"tailscale.com/net/interfaces"
 	"tailscale.com/types/logger"
 	"tailscale.com/util/dnsname"
 	"tailscale.com/wgengine/monitor"
@@ -27,10 +28,20 @@ import (
 // truncation in a platform-agnostic way.
 const maxResponseBytes = 4095
 
-// queueSize is the maximal number of DNS requests that can await polling.
+// maxActiveQueries returns the maximal number of DNS requests that be
+// can running.
 // If EnqueueRequest is called when this many requests are already pending,
 // the request will be dropped to avoid blocking the caller.
-const queueSize = 64
+func maxActiveQueries() int32 {
+	if runtime.GOOS == "ios" {
+		// For memory paranoia reasons on iOS, match the
+		// historical Tailscale 1.x..1.8 behavior for now
+		// (just before the 1.10 release).
+		return 64
+	}
+	// But for other platforms, allow more burstiness:
+	return 256
+}
 
 // defaultTTL is the TTL of all responses from Resolver.
 const defaultTTL = 600 * time.Second
@@ -75,13 +86,12 @@ type Config struct {
 type Resolver struct {
 	logf               logger.Logf
 	linkMon            *monitor.Mon     // or nil
-	unregLinkMon       func()           // or nil
 	saveConfigForTests func(cfg Config) // used in tests to capture resolver config
 	// forwarder forwards requests to upstream nameservers.
 	forwarder *forwarder
 
-	// queue is a buffered channel holding DNS requests queued for resolution.
-	queue chan packet
+	activeQueriesAtomic int32 // number of DNS queries in flight
+
 	// responses is an unbuffered channel to which responses are returned.
 	responses chan packet
 	// errors is an unbuffered channel to which errors are returned.
@@ -98,27 +108,26 @@ type Resolver struct {
 	ipToHost     map[netaddr.IP]dnsname.FQDN
 }
 
+type ForwardLinkSelector interface {
+	// PickLink returns which network device should be used to query
+	// the DNS server at the given IP.
+	// The empty string means to use an unspecified default.
+	PickLink(netaddr.IP) (linkName string)
+}
+
 // New returns a new resolver.
 // linkMon optionally specifies a link monitor to use for socket rebinding.
-func New(logf logger.Logf, linkMon *monitor.Mon) *Resolver {
+func New(logf logger.Logf, linkMon *monitor.Mon, linkSel ForwardLinkSelector) *Resolver {
 	r := &Resolver{
 		logf:      logger.WithPrefix(logf, "dns: "),
 		linkMon:   linkMon,
-		queue:     make(chan packet, queueSize),
 		responses: make(chan packet),
 		errors:    make(chan error),
 		closed:    make(chan struct{}),
 		hostToIP:  map[dnsname.FQDN][]netaddr.IP{},
 		ipToHost:  map[netaddr.IP]dnsname.FQDN{},
 	}
-	r.forwarder = newForwarder(r.logf, r.responses)
-	if r.linkMon != nil {
-		r.unregLinkMon = r.linkMon.RegisterChangeCallback(r.onLinkMonitorChange)
-	}
-
-	r.wg.Add(1)
-	go r.poll()
-
+	r.forwarder = newForwarder(r.logf, r.responses, linkMon, linkSel)
 	return r
 }
 
@@ -140,13 +149,13 @@ func (r *Resolver) SetConfig(cfg Config) error {
 
 	for suffix, ips := range cfg.Routes {
 		routes = append(routes, route{
-			suffix:    suffix,
-			resolvers: ips,
+			Suffix:    suffix,
+			Resolvers: ips,
 		})
 	}
 	// Sort from longest prefix to shortest.
 	sort.Slice(routes, func(i, j int) bool {
-		return routes[i].suffix.NumLabels() > routes[j].suffix.NumLabels()
+		return routes[i].Suffix.NumLabels() > routes[j].Suffix.NumLabels()
 	})
 
 	r.forwarder.setRoutes(routes)
@@ -170,19 +179,7 @@ func (r *Resolver) Close() {
 	}
 	close(r.closed)
 
-	if r.unregLinkMon != nil {
-		r.unregLinkMon()
-	}
-
 	r.forwarder.Close()
-	r.wg.Wait()
-}
-
-func (r *Resolver) onLinkMonitorChange(changed bool, state *interfaces.State) {
-	if !changed {
-		return
-	}
-	r.forwarder.rebindFromNetworkChange()
 }
 
 // EnqueueRequest places the given DNS request in the resolver's queue.
@@ -192,11 +189,14 @@ func (r *Resolver) EnqueueRequest(bs []byte, from netaddr.IPPort) error {
 	select {
 	case <-r.closed:
 		return ErrClosed
-	case r.queue <- packet{bs, from}:
-		return nil
 	default:
+	}
+	if n := atomic.AddInt32(&r.activeQueriesAtomic, 1); n > maxActiveQueries() {
+		atomic.AddInt32(&r.activeQueriesAtomic, -1)
 		return errFullQueue
 	}
+	go r.handleQuery(packet{bs, from})
+	return nil
 }
 
 // NextResponse returns a DNS response to a previously enqueued request.
@@ -291,53 +291,34 @@ func (r *Resolver) resolveLocal(domain dnsname.FQDN, typ dns.Type) (netaddr.IP,
 // resolveReverse returns the unique domain name that maps to the given address.
 func (r *Resolver) resolveLocalReverse(ip netaddr.IP) (dnsname.FQDN, dns.RCode) {
 	r.mu.Lock()
-	ips := r.ipToHost
-	r.mu.Unlock()
-
-	name, found := ips[ip]
-	if !found {
+	defer r.mu.Unlock()
+	name, ok := r.ipToHost[ip]
+	if !ok {
 		return "", dns.RCodeNameError
 	}
 	return name, dns.RCodeSuccess
 }
 
-func (r *Resolver) poll() {
-	defer r.wg.Done()
+func (r *Resolver) handleQuery(pkt packet) {
+	defer atomic.AddInt32(&r.activeQueriesAtomic, -1)
 
-	var pkt packet
-	for {
+	out, err := r.respond(pkt.bs)
+	if err == errNotOurName {
+		err = r.forwarder.forward(pkt)
+		if err == nil {
+			// forward will send response into r.responses, nothing to do.
+			return
+		}
+	}
+	if err != nil {
 		select {
 		case <-r.closed:
-			return
-		case pkt = <-r.queue:
-			// continue
+		case r.errors <- err:
 		}
-
-		out, err := r.respond(pkt.bs)
-
-		if err == errNotOurName {
-			err = r.forwarder.forward(pkt)
-			if err == nil {
-				// forward will send response into r.responses, nothing to do.
-				continue
-			}
-		}
-
-		if err != nil {
-			select {
-			case <-r.closed:
-				return
-			case r.errors <- err:
-				// continue
-			}
-		} else {
-			pkt.bs = out
-			select {
-			case <-r.closed:
-				return
-			case r.responses <- pkt:
-				// continue
-			}
+	} else {
+		select {
+		case <-r.closed:
+		case r.responses <- packet{out, pkt.addr}:
 		}
 	}
 }
@@ -351,28 +332,44 @@ type response struct {
 	IP netaddr.IP
 }
 
-// parseQuery parses the query in given packet into a response struct.
-// if the parse is successful, resp.Name contains the normalized name being queried.
-// TODO: stuffing the query name in resp.Name temporarily is a hack. Clean it up.
-func parseQuery(query []byte, resp *response) error {
-	var parser dns.Parser
-	var err error
+var dnsParserPool = &sync.Pool{
+	New: func() interface{} {
+		return new(dnsParser)
+	},
+}
 
-	resp.Header, err = parser.Start(query)
+// dnsParser parses DNS queries using x/net/dns/dnsmessage.
+// These structs are pooled with dnsParserPool.
+type dnsParser struct {
+	Header   dns.Header
+	Question dns.Question
+
+	parser dns.Parser
+}
+
+func (p *dnsParser) response() *response {
+	return &response{Header: p.Header, Question: p.Question}
+}
+
+// zeroParser clears parser so it doesn't retain its most recently
+// parsed DNS query's []byte while it's sitting in a sync.Pool.
+// It's not useful to keep anyway: the next Start will do the same.
+func (p *dnsParser) zeroParser() { p.parser = dns.Parser{} }
+
+// parseQuery parses the query in given packet into p.Header and
+// p.Question.
+func (p *dnsParser) parseQuery(query []byte) error {
+	defer p.zeroParser()
+	var err error
+	p.Header, err = p.parser.Start(query)
 	if err != nil {
 		return err
 	}
-
-	if resp.Header.Response {
+	if p.Header.Response {
 		return errNotQuery
 	}
-
-	resp.Question, err = parser.Question()
-	if err != nil {
-		return err
-	}
-
-	return nil
+	p.Question, err = p.parser.Question()
+	return err
 }
 
 // marshalARecord serializes an A record into an active builder.
@@ -624,12 +621,13 @@ func (r *Resolver) respondReverse(query []byte, name dnsname.FQDN, resp *respons
 // respond returns a DNS response to query if it can be resolved locally.
 // Otherwise, it returns errNotOurName.
 func (r *Resolver) respond(query []byte) ([]byte, error) {
-	resp := new(response)
+	parser := dnsParserPool.Get().(*dnsParser)
+	defer dnsParserPool.Put(parser)
 
 	// ParseQuery is sufficiently fast to run on every DNS packet.
 	// This is considerably simpler than extracting the name by hand
 	// to shave off microseconds in case of delegation.
-	err := parseQuery(query, resp)
+	err := parser.parseQuery(query)
 	// We will not return this error: it is the sender's fault.
 	if err != nil {
 		if errors.Is(err, dns.ErrSectionDone) {
@@ -637,13 +635,15 @@ func (r *Resolver) respond(query []byte) ([]byte, error) {
 		} else {
 			r.logf("parseQuery(%02x): %v", query, err)
 		}
+		resp := parser.response()
 		resp.Header.RCode = dns.RCodeFormatError
 		return marshalResponse(resp)
 	}
-	rawName := resp.Question.Name.Data[:resp.Question.Name.Length]
+	rawName := parser.Question.Name.Data[:parser.Question.Name.Length]
 	name, err := dnsname.ToFQDN(rawNameToLower(rawName))
 	if err != nil {
 		// DNS packet unexpectedly contains an invalid FQDN.
+		resp := parser.response()
 		resp.Header.RCode = dns.RCodeFormatError
 		return marshalResponse(resp)
 	}
@@ -651,15 +651,17 @@ func (r *Resolver) respond(query []byte) ([]byte, error) {
 	// Always try to handle reverse lookups; delegate inside when not found.
 	// This way, queries for existent nodes do not leak,
 	// but we behave gracefully if non-Tailscale nodes exist in CGNATRange.
-	if resp.Question.Type == dns.TypePTR {
-		return r.respondReverse(query, name, resp)
+	if parser.Question.Type == dns.TypePTR {
+		return r.respondReverse(query, name, parser.response())
 	}
 
-	resp.IP, resp.Header.RCode = r.resolveLocal(name, resp.Question.Type)
-	// This return code is special: it requests forwarding.
-	if resp.Header.RCode == dns.RCodeRefused {
-		return nil, errNotOurName
+	ip, rcode := r.resolveLocal(name, parser.Question.Type)
+	if rcode == dns.RCodeRefused {
+		return nil, errNotOurName // sentinel error return value: it requests forwarding
 	}
 
+	resp := parser.response()
+	resp.Header.RCode = rcode
+	resp.IP = ip
 	return marshalResponse(resp)
 }

net/dns/resolver/tsdns_server_test.go

@@ -68,7 +68,7 @@ func resolveToIP(ipv4, ipv6 netaddr.IP, ns string) dns.HandlerFunc {
 
 // resolveToTXT returns a handler function which responds to queries of type TXT
 // it receives with the strings in txts.
-func resolveToTXT(txts []string) dns.HandlerFunc {
+func resolveToTXT(txts []string, ednsMaxSize uint16) dns.HandlerFunc {
 	return func(w dns.ResponseWriter, req *dns.Msg) {
 		m := new(dns.Msg)
 		m.SetReply(req)
@@ -93,6 +93,27 @@ func resolveToTXT(txts []string) dns.HandlerFunc {
 		}
 
 		m.Answer = append(m.Answer, ans)
+
+		queryInfo := &dns.TXT{
+			Hdr: dns.RR_Header{
+				Name:   "query-info.test.",
+				Rrtype: dns.TypeTXT,
+				Class:  dns.ClassINET,
+			},
+		}
+
+		if edns := req.IsEdns0(); edns == nil {
+			queryInfo.Txt = []string{"EDNS=false"}
+		} else {
+			queryInfo.Txt = []string{"EDNS=true", fmt.Sprintf("maxSize=%v", edns.UDPSize())}
+		}
+
+		m.Extra = append(m.Extra, queryInfo)
+
+		if ednsMaxSize > 0 {
+			m.SetEdns0(ednsMaxSize, false)
+		}
+
 		if err := w.WriteMsg(m); err != nil {
 			panic(err)
 		}

net/dns/resolver/tsdns_test.go

@@ -8,14 +8,19 @@ import (
 	"bytes"
 	"encoding/hex"
 	"errors"
+	"fmt"
 	"math/rand"
 	"net"
+	"runtime"
+	"strconv"
+	"strings"
 	"testing"
 
 	dns "golang.org/x/net/dns/dnsmessage"
 	"inet.af/netaddr"
 	"tailscale.com/tstest"
 	"tailscale.com/util/dnsname"
+	"tailscale.com/wgengine/monitor"
 )
 
 var testipv4 = netaddr.MustParseIP("1.2.3.4")
@@ -29,7 +34,9 @@ var dnsCfg = Config{
 	LocalDomains: []dnsname.FQDN{"ipn.dev."},
 }
 
-func dnspacket(domain dnsname.FQDN, tp dns.Type) []byte {
+const noEdns = 0
+
+func dnspacket(domain dnsname.FQDN, tp dns.Type, ednsSize uint16) []byte {
 	var dnsHeader dns.Header
 	question := dns.Question{
 		Name:  dns.MustNewName(domain.WithTrailingDot()),
@@ -38,19 +45,44 @@ func dnspacket(domain dnsname.FQDN, tp dns.Type) []byte {
 	}
 
 	builder := dns.NewBuilder(nil, dnsHeader)
-	builder.StartQuestions()
-	builder.Question(question)
+	if err := builder.StartQuestions(); err != nil {
+		panic(err)
+	}
+	if err := builder.Question(question); err != nil {
+		panic(err)
+	}
+
+	if ednsSize != noEdns {
+		if err := builder.StartAdditionals(); err != nil {
+			panic(err)
+		}
+
+		ednsHeader := dns.ResourceHeader{
+			Name:  dns.MustNewName("."),
+			Type:  dns.TypeOPT,
+			Class: dns.Class(ednsSize),
+		}
+
+		if err := builder.OPTResource(ednsHeader, dns.OPTResource{}); err != nil {
+			panic(err)
+		}
+	}
+
 	payload, _ := builder.Finish()
 
 	return payload
 }
 
 type dnsResponse struct {
-	ip        netaddr.IP
-	txt       []string
-	name      dnsname.FQDN
-	rcode     dns.RCode
-	truncated bool
+	ip               netaddr.IP
+	txt              []string
+	name             dnsname.FQDN
+	rcode            dns.RCode
+	truncated        bool
+	requestEdns      bool
+	requestEdnsSize  uint16
+	responseEdns     bool
+	responseEdnsSize uint16
 }
 
 func unpackResponse(payload []byte) (dnsResponse, error) {
@@ -86,48 +118,107 @@ func unpackResponse(payload []byte) (dnsResponse, error) {
 		return response, err
 	}
 
-	ah, err := parser.AnswerHeader()
+	for {
+		ah, err := parser.AnswerHeader()
+		if err == dns.ErrSectionDone {
+			break
+		}
+		if err != nil {
+			return response, err
+		}
+
+		switch ah.Type {
+		case dns.TypeA:
+			res, err := parser.AResource()
+			if err != nil {
+				return response, err
+			}
+			response.ip = netaddr.IPv4(res.A[0], res.A[1], res.A[2], res.A[3])
+		case dns.TypeAAAA:
+			res, err := parser.AAAAResource()
+			if err != nil {
+				return response, err
+			}
+			response.ip = netaddr.IPv6Raw(res.AAAA)
+		case dns.TypeTXT:
+			res, err := parser.TXTResource()
+			if err != nil {
+				return response, err
+			}
+			response.txt = res.TXT
+		case dns.TypeNS:
+			res, err := parser.NSResource()
+			if err != nil {
+				return response, err
+			}
+			response.name, err = dnsname.ToFQDN(res.NS.String())
+			if err != nil {
+				return response, err
+			}
+		default:
+			return response, errors.New("type not in {A, AAAA, NS}")
+		}
+	}
+
+	err = parser.SkipAllAuthorities()
 	if err != nil {
 		return response, err
 	}
 
-	switch ah.Type {
-	case dns.TypeA:
-		res, err := parser.AResource()
+	for {
+		ah, err := parser.AdditionalHeader()
+		if err == dns.ErrSectionDone {
+			break
+		}
 		if err != nil {
 			return response, err
 		}
-		response.ip = netaddr.IPv4(res.A[0], res.A[1], res.A[2], res.A[3])
-	case dns.TypeAAAA:
-		res, err := parser.AAAAResource()
-		if err != nil {
-			return response, err
+
+		switch ah.Type {
+		case dns.TypeOPT:
+			_, err := parser.OPTResource()
+			if err != nil {
+				return response, err
+			}
+			response.responseEdns = true
+			response.responseEdnsSize = uint16(ah.Class)
+		case dns.TypeTXT:
+			res, err := parser.TXTResource()
+			if err != nil {
+				return response, err
+			}
+			switch ah.Name.String() {
+			case "query-info.test.":
+				for _, msg := range res.TXT {
+					s := strings.SplitN(msg, "=", 2)
+					if len(s) != 2 {
+						continue
+					}
+					switch s[0] {
+					case "EDNS":
+						response.requestEdns, err = strconv.ParseBool(s[1])
+						if err != nil {
+							return response, err
+						}
+					case "maxSize":
+						sz, err := strconv.ParseUint(s[1], 10, 16)
+						if err != nil {
+							return response, err
+						}
+						response.requestEdnsSize = uint16(sz)
+					}
+				}
+			}
 		}
-		response.ip = netaddr.IPv6Raw(res.AAAA)
-	case dns.TypeTXT:
-		res, err := parser.TXTResource()
-		if err != nil {
-			return response, err
-		}
-		response.txt = res.TXT
-	case dns.TypeNS:
-		res, err := parser.NSResource()
-		if err != nil {
-			return response, err
-		}
-		response.name, err = dnsname.ToFQDN(res.NS.String())
-		if err != nil {
-			return response, err
-		}
-	default:
-		return response, errors.New("type not in {A, AAAA, NS}")
 	}
 
 	return response, nil
 }
 
 func syncRespond(r *Resolver, query []byte) ([]byte, error) {
-	r.EnqueueRequest(query, netaddr.IPPort{})
+	if err := r.EnqueueRequest(query, netaddr.IPPort{}); err != nil {
+		return nil, fmt.Errorf("EnqueueRequest: %w", err)
+	}
 	payload, _, err := r.NextResponse()
 	return payload, err
 }
@@ -210,8 +301,12 @@ func TestRDNSNameToIPv6(t *testing.T) {
 	}
 }
 
+func newResolver(t testing.TB) *Resolver {
+	return New(t.Logf, nil /* no link monitor */, nil /* no link selector */)
+}
+
 func TestResolveLocal(t *testing.T) {
-	r := New(t.Logf, nil)
+	r := newResolver(t)
 	defer r.Close()
 
 	r.SetConfig(dnsCfg)
@@ -251,7 +346,7 @@ func TestResolveLocal(t *testing.T) {
 }
 
 func TestResolveLocalReverse(t *testing.T) {
-	r := New(t.Logf, nil)
+	r := newResolver(t)
 	defer r.Close()
 
 	r.SetConfig(dnsCfg)
@@ -331,7 +426,7 @@ func TestDelegate(t *testing.T) {
 	// support these sizes of response without truncation because they are
 	// moderately common.
 	medTXT := generateTXT(1200, randSource)
-	largeTXT := generateTXT(4000, randSource)
+	largeTXT := generateTXT(3900, randSource)
 
 	// xlargeTXT is slightly above the maximum response size that we support,
 	// so there should be truncation.
@@ -342,26 +437,23 @@ func TestDelegate(t *testing.T) {
 	// intend to handle responses this large, so there should be truncation.
 	hugeTXT := generateTXT(64000, randSource)
 
-	v4server := serveDNS(t, "127.0.0.1:0",
-		"test.site.", resolveToIP(testipv4, testipv6, "dns.test.site."),
+	records := []interface{}{
+		"test.site.",
+		resolveToIP(testipv4, testipv6, "dns.test.site."),
 		"nxdomain.site.", resolveToNXDOMAIN,
-		"small.txt.", resolveToTXT(smallTXT),
-		"med.txt.", resolveToTXT(medTXT),
-		"large.txt.", resolveToTXT(largeTXT),
-		"xlarge.txt.", resolveToTXT(xlargeTXT),
-		"huge.txt.", resolveToTXT(hugeTXT))
+		"small.txt.", resolveToTXT(smallTXT, noEdns),
+		"smalledns.txt.", resolveToTXT(smallTXT, 512),
+		"med.txt.", resolveToTXT(medTXT, 1500),
+		"large.txt.", resolveToTXT(largeTXT, maxResponseBytes),
+		"xlarge.txt.", resolveToTXT(xlargeTXT, 8000),
+		"huge.txt.", resolveToTXT(hugeTXT, 65527),
+	}
+	v4server := serveDNS(t, "127.0.0.1:0", records...)
 	defer v4server.Shutdown()
-	v6server := serveDNS(t, "[::1]:0",
-		"test.site.", resolveToIP(testipv4, testipv6, "dns.test.site."),
-		"nxdomain.site.", resolveToNXDOMAIN,
-		"small.txt.", resolveToTXT(smallTXT),
-		"med.txt.", resolveToTXT(medTXT),
-		"large.txt.", resolveToTXT(largeTXT),
-		"xlarge.txt.", resolveToTXT(xlargeTXT),
-		"huge.txt.", resolveToTXT(hugeTXT))
+	v6server := serveDNS(t, "[::1]:0", records...)
 	defer v6server.Shutdown()
 
-	r := New(t.Logf, nil)
+	r := newResolver(t)
 	defer r.Close()
 
 	cfg := dnsCfg
@@ -380,53 +472,92 @@ func TestDelegate(t *testing.T) {
 	}{
 		{
 			"ipv4",
-			dnspacket("test.site.", dns.TypeA),
+			dnspacket("test.site.", dns.TypeA, noEdns),
 			dnsResponse{ip: testipv4, rcode: dns.RCodeSuccess},
 		},
 		{
 			"ipv6",
-			dnspacket("test.site.", dns.TypeAAAA),
+			dnspacket("test.site.", dns.TypeAAAA, noEdns),
 			dnsResponse{ip: testipv6, rcode: dns.RCodeSuccess},
 		},
 		{
 			"ns",
-			dnspacket("test.site.", dns.TypeNS),
+			dnspacket("test.site.", dns.TypeNS, noEdns),
 			dnsResponse{name: "dns.test.site.", rcode: dns.RCodeSuccess},
 		},
 		{
 			"nxdomain",
-			dnspacket("nxdomain.site.", dns.TypeA),
+			dnspacket("nxdomain.site.", dns.TypeA, noEdns),
 			dnsResponse{rcode: dns.RCodeNameError},
 		},
 		{
 			"smalltxt",
-			dnspacket("small.txt.", dns.TypeTXT),
-			dnsResponse{txt: smallTXT, rcode: dns.RCodeSuccess},
+			dnspacket("small.txt.", dns.TypeTXT, 8000),
+			dnsResponse{txt: smallTXT, rcode: dns.RCodeSuccess, requestEdns: true, requestEdnsSize: maxResponseBytes},
+		},
+		{
+			"smalltxtedns",
+			dnspacket("smalledns.txt.", dns.TypeTXT, 512),
+			dnsResponse{
+				txt:              smallTXT,
+				rcode:            dns.RCodeSuccess,
+				requestEdns:      true,
+				requestEdnsSize:  512,
+				responseEdns:     true,
+				responseEdnsSize: 512,
+			},
 		},
 		{
 			"medtxt",
-			dnspacket("med.txt.", dns.TypeTXT),
-			dnsResponse{txt: medTXT, rcode: dns.RCodeSuccess},
+			dnspacket("med.txt.", dns.TypeTXT, 2000),
+			dnsResponse{
+				txt:              medTXT,
+				rcode:            dns.RCodeSuccess,
+				requestEdns:      true,
+				requestEdnsSize:  2000,
+				responseEdns:     true,
+				responseEdnsSize: 1500,
+			},
 		},
 		{
 			"largetxt",
-			dnspacket("large.txt.", dns.TypeTXT),
-			dnsResponse{txt: largeTXT, rcode: dns.RCodeSuccess},
+			dnspacket("large.txt.", dns.TypeTXT, maxResponseBytes),
+			dnsResponse{
+				txt:              largeTXT,
+				rcode:            dns.RCodeSuccess,
+				requestEdns:      true,
+				requestEdnsSize:  maxResponseBytes,
+				responseEdns:     true,
+				responseEdnsSize: maxResponseBytes,
+			},
 		},
 		{
 			"xlargetxt",
-			dnspacket("xlarge.txt.", dns.TypeTXT),
-			dnsResponse{rcode: dns.RCodeSuccess, truncated: true},
+			dnspacket("xlarge.txt.", dns.TypeTXT, 8000),
+			dnsResponse{
+				rcode:     dns.RCodeSuccess,
+				truncated: true,
+				// request/response EDNS fields will be unset because of
+				// they were truncated away
+			},
 		},
 		{
 			"hugetxt",
-			dnspacket("huge.txt.", dns.TypeTXT),
-			dnsResponse{rcode: dns.RCodeSuccess, truncated: true},
+			dnspacket("huge.txt.", dns.TypeTXT, 8000),
+			dnsResponse{
+				rcode:     dns.RCodeSuccess,
+				truncated: true,
+				// request/response EDNS fields will be unset because of
+				// they were truncated away
+			},
 		},
 	}
 
 	for _, tt := range tests {
 		t.Run(tt.title, func(t *testing.T) {
+			if tt.title == "hugetxt" && runtime.GOOS == "darwin" {
+				t.Skip("known to not work on macOS: https://github.com/tailscale/tailscale/issues/2229")
+			}
 			payload, err := syncRespond(r, tt.query)
 			if err != nil {
 				t.Errorf("err = %v; want nil", err)
@@ -455,6 +586,18 @@ func TestDelegate(t *testing.T) {
 					}
 				}
 			}
+			if response.requestEdns != tt.response.requestEdns {
+				t.Errorf("requestEdns = %v; want %v", response.requestEdns, tt.response.requestEdns)
+			}
+			if response.requestEdnsSize != tt.response.requestEdnsSize {
+				t.Errorf("requestEdnsSize = %v; want %v", response.requestEdnsSize, tt.response.requestEdnsSize)
+			}
+			if response.responseEdns != tt.response.responseEdns {
+				t.Errorf("responseEdns = %v; want %v", response.requestEdns, tt.response.requestEdns)
+			}
+			if response.responseEdnsSize != tt.response.responseEdnsSize {
+				t.Errorf("responseEdnsSize = %v; want %v", response.responseEdnsSize, tt.response.responseEdnsSize)
+			}
 		})
 	}
 }
@@ -470,7 +613,7 @@ func TestDelegateSplitRoute(t *testing.T) {
 		"test.other.", resolveToIP(test4, test6, "dns.other."))
 	defer server2.Shutdown()
 
-	r := New(t.Logf, nil)
+	r := newResolver(t)
 	defer r.Close()
 
 	cfg := dnsCfg
@@ -487,12 +630,12 @@ func TestDelegateSplitRoute(t *testing.T) {
 	}{
 		{
 			"general",
-			dnspacket("test.site.", dns.TypeA),
+			dnspacket("test.site.", dns.TypeA, noEdns),
 			dnsResponse{ip: testipv4, rcode: dns.RCodeSuccess},
 		},
 		{
 			"override",
-			dnspacket("test.other.", dns.TypeA),
+			dnspacket("test.other.", dns.TypeA, noEdns),
 			dnsResponse{ip: test4, rcode: dns.RCodeSuccess},
 		},
 	}
@@ -527,7 +670,7 @@ func TestDelegateCollision(t *testing.T) {
 		"test.site.", resolveToIP(testipv4, testipv6, "dns.test.site."))
 	defer server.Shutdown()
 
-	r := New(t.Logf, nil)
+	r := newResolver(t)
 	defer r.Close()
 
 	cfg := dnsCfg
@@ -549,7 +692,7 @@ func TestDelegateCollision(t *testing.T) {
 
 	// packets will have the same dns txid.
 	for _, p := range packets {
-		payload := dnspacket(p.qname, p.qtype)
+		payload := dnspacket(p.qname, p.qtype, noEdns)
 		err := r.EnqueueRequest(payload, p.addr)
 		if err != nil {
 			t.Error(err)
@@ -741,7 +884,7 @@ var emptyResponse = []byte{
 }
 
 func TestFull(t *testing.T) {
-	r := New(t.Logf, nil)
+	r := newResolver(t)
 	defer r.Close()
 
 	r.SetConfig(dnsCfg)
@@ -752,15 +895,15 @@ func TestFull(t *testing.T) {
 		request  []byte
 		response []byte
 	}{
-		{"all", dnspacket("test1.ipn.dev.", dns.TypeALL), allResponse},
-		{"ipv4", dnspacket("test1.ipn.dev.", dns.TypeA), ipv4Response},
-		{"ipv6", dnspacket("test2.ipn.dev.", dns.TypeAAAA), ipv6Response},
-		{"no-ipv6", dnspacket("test1.ipn.dev.", dns.TypeAAAA), emptyResponse},
-		{"upper", dnspacket("TEST1.IPN.DEV.", dns.TypeA), ipv4UppercaseResponse},
-		{"ptr4", dnspacket("4.3.2.1.in-addr.arpa.", dns.TypePTR), ptrResponse},
+		{"all", dnspacket("test1.ipn.dev.", dns.TypeALL, noEdns), allResponse},
+		{"ipv4", dnspacket("test1.ipn.dev.", dns.TypeA, noEdns), ipv4Response},
+		{"ipv6", dnspacket("test2.ipn.dev.", dns.TypeAAAA, noEdns), ipv6Response},
+		{"no-ipv6", dnspacket("test1.ipn.dev.", dns.TypeAAAA, noEdns), emptyResponse},
+		{"upper", dnspacket("TEST1.IPN.DEV.", dns.TypeA, noEdns), ipv4UppercaseResponse},
+		{"ptr4", dnspacket("4.3.2.1.in-addr.arpa.", dns.TypePTR, noEdns), ptrResponse},
 		{"ptr6", dnspacket("f.0.e.0.d.0.c.0.b.0.a.0.9.0.8.0.7.0.6.0.5.0.4.0.3.0.2.0.1.0.0.0.ip6.arpa.",
-			dns.TypePTR), ptrResponse6},
-		{"nxdomain", dnspacket("test3.ipn.dev.", dns.TypeA), nxdomainResponse},
+			dns.TypePTR, noEdns), ptrResponse6},
+		{"nxdomain", dnspacket("test3.ipn.dev.", dns.TypeA, noEdns), nxdomainResponse},
 	}
 
 	for _, tt := range tests {
@@ -777,7 +920,7 @@ func TestFull(t *testing.T) {
 }
 
 func TestAllocs(t *testing.T) {
-	r := New(t.Logf, nil)
+	r := newResolver(t)
 	defer r.Close()
 	r.SetConfig(dnsCfg)
 
@@ -789,9 +932,9 @@ func TestAllocs(t *testing.T) {
 		want  int
 	}{
 		// Name lowercasing and response slice created by dns.NewBuilder.
-		{"forward", dnspacket("test1.ipn.dev.", dns.TypeA), 2},
+		{"forward", dnspacket("test1.ipn.dev.", dns.TypeA, noEdns), 2},
 		// 3 extra allocs in rdnsNameToIPv4 and one in marshalPTRRecord (dns.NewName).
-		{"reverse", dnspacket("4.3.2.1.in-addr.arpa.", dns.TypePTR), 5},
+		{"reverse", dnspacket("4.3.2.1.in-addr.arpa.", dns.TypePTR, noEdns), 5},
 	}
 
 	for _, tt := range tests {
@@ -831,7 +974,7 @@ func BenchmarkFull(b *testing.B) {
 		"test.site.", resolveToIP(testipv4, testipv6, "dns.test.site."))
 	defer server.Shutdown()
 
-	r := New(b.Logf, nil)
+	r := newResolver(b)
 	defer r.Close()
 
 	cfg := dnsCfg
@@ -845,9 +988,9 @@ func BenchmarkFull(b *testing.B) {
 		name    string
 		request []byte
 	}{
-		{"forward", dnspacket("test1.ipn.dev.", dns.TypeA)},
-		{"reverse", dnspacket("4.3.2.1.in-addr.arpa.", dns.TypePTR)},
-		{"delegated", dnspacket("test.site.", dns.TypeA)},
+		{"forward", dnspacket("test1.ipn.dev.", dns.TypeA, noEdns)},
+		{"reverse", dnspacket("4.3.2.1.in-addr.arpa.", dns.TypePTR, noEdns)},
+		{"delegated", dnspacket("test.site.", dns.TypeA, noEdns)},
 	}
 
 	for _, tt := range tests {
@@ -868,3 +1011,58 @@ func TestMarshalResponseFormatError(t *testing.T) {
 	}
 	t.Logf("response: %q", v)
 }
+
+func TestForwardLinkSelection(t *testing.T) {
+	old := initListenConfig
+	defer func() { initListenConfig = old }()
+
+	configCall := make(chan string, 1)
+	initListenConfig = func(nc *net.ListenConfig, mon *monitor.Mon, tunName string) error {
+		select {
+		case configCall <- tunName:
+			return nil
+		default:
+			t.Error("buffer full")
+			return errors.New("buffer full")
+		}
+	}
+
+	// specialIP is some IP we pretend that our link selector
+	// routes differently.
+	specialIP := netaddr.IPv4(1, 2, 3, 4)
+
+	fwd := newForwarder(t.Logf, nil, nil, linkSelFunc(func(ip netaddr.IP) string {
+		if ip == netaddr.IPv4(1, 2, 3, 4) {
+			return "special"
+		}
+		return ""
+	}))
+
+	// Test non-special IP.
+	if got, err := fwd.packetListener(netaddr.IP{}); err != nil {
+		t.Fatal(err)
+	} else if got != stdNetPacketListener {
+		t.Errorf("for IP zero value, didn't get expected packet listener")
+	}
+	select {
+	case v := <-configCall:
+		t.Errorf("unexpected ListenConfig call, with tunName %q", v)
+	default:
+	}
+
+	// Test that our special IP generates a call to initListenConfig.
+	if got, err := fwd.packetListener(specialIP); err != nil {
+		t.Fatal(err)
+	} else if got == stdNetPacketListener {
+		t.Errorf("special IP returned std packet listener; expected unique one")
+	}
+	if v, ok := <-configCall; !ok {
+		t.Errorf("didn't get ListenConfig call")
+	} else if v != "special" {
+		t.Errorf("got tunName %q; want 'special'", v)
+	}
+}
+
+type linkSelFunc func(ip netaddr.IP) string
+
+func (f linkSelFunc) PickLink(ip netaddr.IP) string { return f(ip) }

net/dns/wsl_windows.go

@@ -0,0 +1,254 @@
+// Copyright (c) 2021 Tailscale Inc & AUTHORS All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package dns
+
+import (
+	"bytes"
+	"fmt"
+	"os"
+	"os/exec"
+	"os/user"
+	"strings"
+	"syscall"
+	"unicode/utf16"
+
+	"golang.org/x/sys/windows"
+	"tailscale.com/types/logger"
+	"tailscale.com/util/winutil"
+)
+
+// wslDistros reports the names of the installed WSL2 linux distributions.
+func wslDistros() ([]string, error) {
+	b, err := wslCombinedOutput(exec.Command("wsl.exe", "-l"))
+	if err != nil {
+		return nil, fmt.Errorf("%v: %q", err, string(b))
+	}
+
+	// The first line of output is a WSL header. E.g.
+	//
+	//	C:\tsdev>wsl.exe -l
+	//	Windows Subsystem for Linux Distributions:
+	//	Ubuntu-20.04 (Default)
+	//
+	// We can skip it by passing '-q', but here we put it to work.
+	// It turns out wsl.exe -l is broken, and outputs UTF-16 names
+	// that nothing can read. (Try `wsl.exe -l | more`.)
+	// So we look at the header to see if it's UTF-16.
+	// If so, we run the rest through a UTF-16 parser.
+	//
+	// https://github.com/microsoft/WSL/issues/4607
+	var output string
+	if bytes.HasPrefix(b, []byte("W\x00i\x00n\x00d\x00o\x00w\x00s\x00")) {
+		output, err = decodeUTF16(b)
+		if err != nil {
+			return nil, fmt.Errorf("failed to decode wsl.exe -l output %q: %v", b, err)
+		}
+	} else {
+		output = string(b)
+	}
+	lines := strings.Split(output, "\n")
+	if len(lines) < 1 {
+		return nil, nil
+	}
+	lines = lines[1:] // drop "Windows Subsystem For Linux" header
+
+	var distros []string
+	for _, name := range lines {
+		name = strings.TrimSpace(name)
+		name = strings.TrimSuffix(name, " (Default)")
+		if name == "" {
+			continue
+		}
+		distros = append(distros, name)
+	}
+	return distros, nil
+}
+
+func decodeUTF16(b []byte) (string, error) {
+	if len(b) == 0 {
+		return "", nil
+	} else if len(b)%2 != 0 {
+		return "", fmt.Errorf("decodeUTF16: invalid length %d", len(b))
+	}
+	var u16 []uint16
+	for i := 0; i < len(b); i += 2 {
+		u16 = append(u16, uint16(b[i])+(uint16(b[i+1])<<8))
+	}
+	return string(utf16.Decode(u16)), nil
+}
+
+// wslManager is a DNS manager for WSL2 linux distributions.
+// It configures /etc/wsl.conf and /etc/resolv.conf.
+type wslManager struct {
+	logf logger.Logf
+}
+
+func newWSLManager(logf logger.Logf) *wslManager {
+	m := &wslManager{
+		logf: logf,
+	}
+	return m
+}
+
+func (wm *wslManager) SetDNS(cfg OSConfig) error {
+	distros, err := wslDistros()
+	if err != nil {
+		return err
+	} else if len(distros) == 0 {
+		return nil
+	}
+	managers := make(map[string]directManager)
+	for _, distro := range distros {
+		managers[distro] = newDirectManagerOnFS(wslFS{
+			user:   "root",
+			distro: distro,
+		})
+	}
+
+	if !cfg.IsZero() {
+		if wm.setWSLConf(managers) {
+			// What's this? So glad you asked.
+			//
+			// WSL2 writes the /etc/resolv.conf.
+			// It is aggressive about it. Every time you execute wsl.exe,
+			// it writes it. (Opening a terminal is done by running wsl.exe.)
+			// You can turn this off using /etc/wsl.conf! But: this wsl.conf
+			// file is only parsed when the VM boots up. To do that, we
+			// have to shut down WSL2.
+			//
+			// So we do it here, before we call wsl.exe to write resolv.conf.
+			if b, err := wslCombinedOutput(wslCommand("--shutdown")); err != nil {
+				wm.logf("WSL SetDNS shutdown: %v: %s", err, b)
+			}
+		}
+	}
+
+	for distro, m := range managers {
+		if err := m.SetDNS(cfg); err != nil {
+			wm.logf("WSL(%q) SetDNS: %v", distro, err)
+		}
+	}
+	return nil
+}
+
+const wslConf = "/etc/wsl.conf"
+const wslConfSection = `# added by tailscale
+[network]
+generateResolvConf = false
+`
+
+// setWSLConf attempts to disable generateResolvConf in each WSL2 linux.
+// If any are changed, it reports true.
+func (wm *wslManager) setWSLConf(managers map[string]directManager) (changed bool) {
+	for distro, m := range managers {
+		b, err := m.fs.ReadFile(wslConf)
+		if err != nil && !os.IsNotExist(err) {
+			wm.logf("WSL(%q) wsl.conf: read: %v", distro, err)
+			continue
+		}
+		ini := parseIni(string(b))
+		if v := ini["network"]["generateResolvConf"]; v == "" {
+			b = append(b, wslConfSection...)
+			if err := m.fs.WriteFile(wslConf, b, 0644); err != nil {
+				wm.logf("WSL(%q) wsl.conf: write: %v", distro, err)
+				continue
+			}
+			changed = true
+		}
+	}
+	return changed
+}
+
+func (m *wslManager) SupportsSplitDNS() bool { return false }
+func (m *wslManager) Close() error           { return m.SetDNS(OSConfig{}) }
+
+// wslFS is a pinholeFS implemented on top of wsl.exe.
+//
+// We access WSL2 file systems via wsl.exe instead of \\wsl$\ because
+// the netpath appears to operate as the standard user, not root.
+type wslFS struct {
+	user   string
+	distro string
+}
+
+func (fs wslFS) Stat(name string) (isRegular bool, err error) {
+	err = wslRun(fs.cmd("test", "-f", name))
+	if ee, _ := err.(*exec.ExitError); ee != nil {
+		if ee.ExitCode() == 1 {
+			return false, os.ErrNotExist
+		}
+		return false, err
+	}
+	return true, nil
+}
+
+func (fs wslFS) Rename(oldName, newName string) error {
+	return wslRun(fs.cmd("mv", "--", oldName, newName))
+}
+func (fs wslFS) Remove(name string) error { return wslRun(fs.cmd("rm", "--", name)) }
+
+func (fs wslFS) ReadFile(name string) ([]byte, error) {
+	b, err := wslCombinedOutput(fs.cmd("cat", "--", name))
+	if ee, _ := err.(*exec.ExitError); ee != nil && ee.ExitCode() == 1 {
+		return nil, os.ErrNotExist
+	}
+	return b, err
+}
+
+func (fs wslFS) WriteFile(name string, contents []byte, perm os.FileMode) error {
+	cmd := fs.cmd("tee", "--", name)
+	cmd.Stdin = bytes.NewReader(contents)
+	cmd.Stdout = nil
+	if err := wslRun(cmd); err != nil {
+		return err
+	}
+	return wslRun(fs.cmd("chmod", "--", fmt.Sprintf("%04o", perm), name))
+}
+
+func (fs wslFS) cmd(args ...string) *exec.Cmd {
+	cmd := wslCommand("-u", fs.user, "-d", fs.distro, "-e")
+	cmd.Args = append(cmd.Args, args...)
+	return cmd
+}
+
+func wslCommand(args ...string) *exec.Cmd {
+	cmd := exec.Command("wsl.exe", args...)
+	return cmd
+}
+
+func wslCombinedOutput(cmd *exec.Cmd) ([]byte, error) {
+	buf := new(bytes.Buffer)
+	cmd.Stdout = buf
+	cmd.Stderr = buf
+	err := wslRun(cmd)
+	return buf.Bytes(), err
+}
+
+func wslRun(cmd *exec.Cmd) (err error) {
+	defer func() {
+		if err != nil {
+			err = fmt.Errorf("wslRun(%v): %w", cmd.Args, err)
+		}
+	}()
+
+	var token windows.Token
+	if u, err := user.Current(); err == nil && u.Name == "SYSTEM" {
+		// We need to switch user to run wsl.exe.
+		// https://github.com/microsoft/WSL/issues/4803
+		sessionID := winutil.WTSGetActiveConsoleSessionId()
+		if sessionID != 0xFFFFFFFF {
+			if err := windows.WTSQueryUserToken(sessionID, &token); err != nil {
+				return err
+			}
+			defer token.Close()
+		}
+	}
+
+	cmd.SysProcAttr = &syscall.SysProcAttr{
+		Token:      syscall.Token(token),
+		HideWindow: true,
+	}
+	return cmd.Run()
+}

net/dnsfallback/dns-fallback-servers.json

@@ -0,0 +1,179 @@
+{
+	"Regions": {
+		"1": {
+			"RegionID": 1,
+			"RegionCode": "r1",
+			"RegionName": "r1",
+			"Nodes": [
+				{
+					"Name": "1a",
+					"RegionID": 1,
+					"HostName": "derp1.tailscale.com",
+					"IPv4": "159.89.225.99",
+					"IPv6": "2604:a880:400:d1::828:b001"
+				},
+				{
+					"Name": "1b",
+					"RegionID": 1,
+					"HostName": "derp1b.tailscale.com",
+					"IPv4": "45.55.35.93",
+					"IPv6": "2604:a880:800:a1::f:2001"
+				}
+			]
+		},
+		"10": {
+			"RegionID": 10,
+			"RegionCode": "r10",
+			"RegionName": "r10",
+			"Nodes": [
+				{
+					"Name": "10a",
+					"RegionID": 10,
+					"HostName": "derp10.tailscale.com",
+					"IPv4": "137.220.36.168",
+					"IPv6": "2001:19f0:8001:2d9:5400:2ff:feef:bbb1"
+				}
+			]
+		},
+		"11": {
+			"RegionID": 11,
+			"RegionCode": "r11",
+			"RegionName": "r11",
+			"Nodes": [
+				{
+					"Name": "11a",
+					"RegionID": 11,
+					"HostName": "derp11.tailscale.com",
+					"IPv4": "18.230.97.74",
+					"IPv6": "2600:1f1e:ee4:5611:ec5c:1736:d43b:a454"
+				}
+			]
+		},
+		"2": {
+			"RegionID": 2,
+			"RegionCode": "r2",
+			"RegionName": "r2",
+			"Nodes": [
+				{
+					"Name": "2a",
+					"RegionID": 2,
+					"HostName": "derp2.tailscale.com",
+					"IPv4": "167.172.206.31",
+					"IPv6": "2604:a880:2:d1::c5:7001"
+				},
+				{
+					"Name": "2b",
+					"RegionID": 2,
+					"HostName": "derp2b.tailscale.com",
+					"IPv4": "64.227.106.23",
+					"IPv6": "2604:a880:4:1d0::29:9000"
+				}
+			]
+		},
+		"3": {
+			"RegionID": 3,
+			"RegionCode": "r3",
+			"RegionName": "r3",
+			"Nodes": [
+				{
+					"Name": "3a",
+					"RegionID": 3,
+					"HostName": "derp3.tailscale.com",
+					"IPv4": "68.183.179.66",
+					"IPv6": "2400:6180:0:d1::67d:8001"
+				}
+			]
+		},
+		"4": {
+			"RegionID": 4,
+			"RegionCode": "r4",
+			"RegionName": "r4",
+			"Nodes": [
+				{
+					"Name": "4a",
+					"RegionID": 4,
+					"HostName": "derp4.tailscale.com",
+					"IPv4": "167.172.182.26",
+					"IPv6": "2a03:b0c0:3:e0::36e:9001"
+				},
+				{
+					"Name": "4b",
+					"RegionID": 4,
+					"HostName": "derp4b.tailscale.com",
+					"IPv4": "157.230.25.0",
+					"IPv6": "2a03:b0c0:3:e0::58f:3001"
+				}
+			]
+		},
+		"5": {
+			"RegionID": 5,
+			"RegionCode": "r5",
+			"RegionName": "r5",
+			"Nodes": [
+				{
+					"Name": "5a",
+					"RegionID": 5,
+					"HostName": "derp5.tailscale.com",
+					"IPv4": "103.43.75.49",
+					"IPv6": "2001:19f0:5801:10b7:5400:2ff:feaa:284c"
+				}
+			]
+		},
+		"6": {
+			"RegionID": 6,
+			"RegionCode": "r6",
+			"RegionName": "r6",
+			"Nodes": [
+				{
+					"Name": "6a",
+					"RegionID": 6,
+					"HostName": "derp6.tailscale.com",
+					"IPv4": "68.183.90.120",
+					"IPv6": "2400:6180:100:d0::982:d001"
+				}
+			]
+		},
+		"7": {
+			"RegionID": 7,
+			"RegionCode": "r7",
+			"RegionName": "r7",
+			"Nodes": [
+				{
+					"Name": "7a",
+					"RegionID": 7,
+					"HostName": "derp7.tailscale.com",
+					"IPv4": "167.179.89.145",
+					"IPv6": "2401:c080:1000:467f:5400:2ff:feee:22aa"
+				}
+			]
+		},
+		"8": {
+			"RegionID": 8,
+			"RegionCode": "r8",
+			"RegionName": "r8",
+			"Nodes": [
+				{
+					"Name": "8a",
+					"RegionID": 8,
+					"HostName": "derp8.tailscale.com",
+					"IPv4": "167.71.139.179",
+					"IPv6": "2a03:b0c0:1:e0::3cc:e001"
+				}
+			]
+		},
+		"9": {
+			"RegionID": 9,
+			"RegionCode": "r9",
+			"RegionName": "r9",
+			"Nodes": [
+				{
+					"Name": "9a",
+					"RegionID": 9,
+					"HostName": "derp9.tailscale.com",
+					"IPv4": "207.148.3.137",
+					"IPv6": "2001:19f0:6401:1d9c:5400:2ff:feef:bb82"
+				}
+			]
+		}
+	}
+}

net/dnsfallback/dnsfallback.go

@@ -2,12 +2,15 @@
 // Use of this source code is governed by a BSD-style
 // license that can be found in the LICENSE file.
 
+//go:generate go run update-dns-fallbacks.go
+
 // Package dnsfallback contains a DNS fallback mechanism
 // for starting up Tailscale when the system DNS is broken or otherwise unavailable.
 package dnsfallback
 
 import (
 	"context"
+	_ "embed"
 	"encoding/json"
 	"errors"
 	"fmt"
@@ -19,9 +22,9 @@ import (
 	"time"
 
 	"inet.af/netaddr"
-	"tailscale.com/derp/derpmap"
 	"tailscale.com/net/netns"
 	"tailscale.com/net/tshttpproxy"
+	"tailscale.com/tailcfg"
 )
 
 func Lookup(ctx context.Context, host string) ([]netaddr.IP, error) {
@@ -30,7 +33,7 @@ func Lookup(ctx context.Context, host string) ([]netaddr.IP, error) {
 		ip      netaddr.IP
 	}
 
-	dm := derpmap.Prod()
+	dm := getDERPMap()
 	var cands4, cands6 []nameIP
 	for _, dr := range dm.Regions {
 		for _, n := range dr.Nodes {
@@ -84,7 +87,7 @@ func Lookup(ctx context.Context, host string) ([]netaddr.IP, error) {
 }
 
 // serverName and serverIP of are, say, "derpN.tailscale.com".
-// queryName is the name being sought (e.g. "login.tailscale.com"), passed as hint.
+// queryName is the name being sought (e.g. "controlplane.tailscale.com"), passed as hint.
 func bootstrapDNSMap(ctx context.Context, serverName string, serverIP netaddr.IP, queryName string) (dnsMap, error) {
 	dialer := netns.NewDialer()
 	tr := http.DefaultTransport.(*http.Transport).Clone()
@@ -115,3 +118,22 @@ func bootstrapDNSMap(ctx context.Context, serverName string, serverIP netaddr.IP
 // dnsMap is the JSON type returned by the DERP /bootstrap-dns handler:
 // https://derp10.tailscale.com/bootstrap-dns
 type dnsMap map[string][]netaddr.IP
+
+// getDERPMap returns some DERP map. The DERP servers also run a fallback
+// DNS server.
+func getDERPMap() *tailcfg.DERPMap {
+	// TODO(bradfitz): try to read the last known DERP map from disk,
+	// at say /var/lib/tailscale/derpmap.txt and write it when it changes,
+	// and read it here.
+	// But ultimately the fallback will be to use a copy baked into the binary,
+	// which is this part:
+
+	dm := new(tailcfg.DERPMap)
+	if err := json.Unmarshal(staticDERPMapJSON, dm); err != nil {
+		panic(err)
+	}
+	return dm
+}
+
+//go:embed dns-fallback-servers.json
+var staticDERPMapJSON []byte

net/dnsfallback/dnsfallback_test.go

@@ -0,0 +1,17 @@
+// Copyright (c) 2021 Tailscale Inc & AUTHORS All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package dnsfallback
+
+import "testing"
+
+func TestGetDERPMap(t *testing.T) {
+	dm := getDERPMap()
+	if dm == nil {
+		t.Fatal("nil")
+	}
+	if len(dm.Regions) == 0 {
+		t.Fatal("no regions")
+	}
+}

net/dnsfallback/update-dns-fallbacks.go

@@ -0,0 +1,47 @@
+// Copyright (c) 2021 Tailscale Inc & AUTHORS All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+// +build ignore
+
+package main
+
+import (
+	"encoding/json"
+	"fmt"
+	"io/ioutil"
+	"log"
+	"net/http"
+	"os"
+
+	"tailscale.com/tailcfg"
+)
+
+func main() {
+	res, err := http.Get("https://login.tailscale.com/derpmap/default")
+	if err != nil {
+		log.Fatal(err)
+	}
+	if res.StatusCode != 200 {
+		res.Write(os.Stderr)
+		os.Exit(1)
+	}
+	dm := new(tailcfg.DERPMap)
+	if err := json.NewDecoder(res.Body).Decode(dm); err != nil {
+		log.Fatal(err)
+	}
+	for rid, r := range dm.Regions {
+		// Names misleading to check into git, as this is a
+		// static snapshot and doesn't reflect the live DERP
+		// map.
+		r.RegionCode = fmt.Sprintf("r%d", rid)
+		r.RegionName = r.RegionCode
+	}
+	out, err := json.MarshalIndent(dm, "", "\t")
+	if err != nil {
+		log.Fatal(err)
+	}
+	if err := ioutil.WriteFile("dns-fallback-servers.json", out, 0644); err != nil {
+		log.Fatal(err)
+	}
+}

net/interfaces/interfaces.go

@@ -15,13 +15,14 @@ import (
 	"strings"
 
 	"inet.af/netaddr"
+	"tailscale.com/hostinfo"
 	"tailscale.com/net/tsaddr"
 	"tailscale.com/net/tshttpproxy"
 )
 
 // LoginEndpointForProxyDetermination is the URL used for testing
 // which HTTP proxy the system should use.
-var LoginEndpointForProxyDetermination = "https://login.tailscale.com/"
+var LoginEndpointForProxyDetermination = "https://controlplane.tailscale.com/"
 
 // Tailscale returns the current machine's Tailscale interface, if any.
 // If none is found, all zero values are returned.
@@ -81,13 +82,16 @@ func isProblematicInterface(nif *net.Interface) bool {
 }
 
 // LocalAddresses returns the machine's IP addresses, separated by
-// whether they're loopback addresses.
+// whether they're loopback addresses. If there are no regular addresses
+// it will return any IPv4 linklocal or IPv6 unique local addresses because we
+// know of environments where these are used with NAT to provide connectivity.
 func LocalAddresses() (regular, loopback []netaddr.IP, err error) {
 	// TODO(crawshaw): don't serve interface addresses that we are routing
 	ifaces, err := net.Interfaces()
 	if err != nil {
 		return nil, nil, err
 	}
+	var regular4, regular6, linklocal4, ula6 []netaddr.IP
 	for i := range ifaces {
 		iface := &ifaces[i]
 		if !isUp(iface) || isProblematicInterface(iface) {
@@ -117,17 +121,44 @@ func LocalAddresses() (regular, loopback []netaddr.IP, err error) {
 				if tsaddr.IsTailscaleIP(ip) {
 					continue
 				}
-				if ip.IsLinkLocalUnicast() {
-					continue
-				}
 				if ip.IsLoopback() || ifcIsLoopback {
 					loopback = append(loopback, ip)
+				} else if ip.IsLinkLocalUnicast() {
+					if ip.Is4() {
+						linklocal4 = append(linklocal4, ip)
+					}
+
+					// We know of no cases where the IPv6 fe80:: addresses
+					// are used to provide WAN connectivity. It is also very
+					// common for users to have no IPv6 WAN connectivity,
+					// but their OS supports IPv6 so they have an fe80::
+					// address. We don't want to report all of those
+					// IPv6 LL to Control.
+				} else if ip.Is6() && tsaddr.IsULA(ip) {
+					// Google Cloud Run uses NAT with IPv6 Unique
+					// Local Addresses to provide IPv6 connectivity.
+					ula6 = append(ula6, ip)
 				} else {
-					regular = append(regular, ip)
+					if ip.Is4() {
+						regular4 = append(regular4, ip)
+					} else {
+						regular6 = append(regular6, ip)
+					}
 				}
 			}
 		}
 	}
+	if len(regular4) == 0 && len(regular6) == 0 {
+		// if we have no usable IP addresses then be willing to accept
+		// addresses we otherwise wouldn't, like:
+		//   + 169.254.x.x (AWS Lambda uses NAT with these)
+		//   + IPv6 ULA (Google Cloud Run uses these with address translation)
+		if hostinfo.GetEnvType() == hostinfo.AWSLambda {
+			regular4 = linklocal4
+		}
+		regular6 = ula6
+	}
+	regular = append(regular4, regular6...)
 	sortIPs(regular)
 	sortIPs(loopback)
 	return regular, loopback, nil
@@ -213,9 +244,9 @@ type State struct {
 	InterfaceIPs map[string][]netaddr.IPPrefix
 	Interface    map[string]Interface
 
-	// HaveV6Global is whether this machine has an IPv6 global address
-	// on some non-Tailscale interface that's up.
-	HaveV6Global bool
+	// HaveV6 is whether this machine has an IPv6 Global or Unique Local Address
+	// which might provide connectivity on a non-Tailscale interface that's up.
+	HaveV6 bool
 
 	// HaveV4 is whether the machine has some non-localhost,
 	// non-link-local IPv4 address on a non-Tailscale interface that's up.
@@ -289,7 +320,7 @@ func (s *State) String() string {
 	if s.PAC != "" {
 		fmt.Fprintf(&sb, " pac=%s", s.PAC)
 	}
-	fmt.Fprintf(&sb, " v4=%v v6global=%v}", s.HaveV4, s.HaveV6Global)
+	fmt.Fprintf(&sb, " v4=%v v6=%v}", s.HaveV4, s.HaveV6)
 	return sb.String()
 }
 
@@ -302,7 +333,7 @@ func (s *State) EqualFiltered(s2 *State, filter func(i Interface, ips []netaddr.
 	if s == nil || s2 == nil {
 		return false
 	}
-	if s.HaveV6Global != s2.HaveV6Global ||
+	if s.HaveV6 != s2.HaveV6 ||
 		s.HaveV4 != s2.HaveV4 ||
 		s.IsExpensive != s2.IsExpensive ||
 		s.DefaultRouteInterface != s2.DefaultRouteInterface ||
@@ -362,7 +393,7 @@ func (s *State) HasPAC() bool { return s != nil && s.PAC != "" }
 
 // AnyInterfaceUp reports whether any interface seems like it has Internet access.
 func (s *State) AnyInterfaceUp() bool {
-	return s != nil && (s.HaveV4 || s.HaveV6Global)
+	return s != nil && (s.HaveV4 || s.HaveV6)
 }
 
 func hasTailscaleIP(pfxs []netaddr.IPPrefix) bool {
@@ -407,11 +438,11 @@ func GetState() (*State, error) {
 			return
 		}
 		for _, pfx := range pfxs {
-			if pfx.IP().IsLoopback() || pfx.IP().IsLinkLocalUnicast() {
+			if pfx.IP().IsLoopback() {
 				continue
 			}
-			s.HaveV6Global = s.HaveV6Global || isGlobalV6(pfx.IP())
-			s.HaveV4 = s.HaveV4 || pfx.IP().Is4()
+			s.HaveV6 = s.HaveV6 || isUsableV6(pfx.IP())
+			s.HaveV4 = s.HaveV4 || isUsableV4(pfx.IP())
 		}
 	}); err != nil {
 		return nil, err
@@ -503,7 +534,25 @@ func isPrivateIP(ip netaddr.IP) bool {
 	return private1.Contains(ip) || private2.Contains(ip) || private3.Contains(ip)
 }
 
-func isGlobalV6(ip netaddr.IP) bool {
+// isUsableV4 reports whether ip is a usable IPv4 address which could
+// conceivably be used to get Internet connectivity. Globally routable and
+// private IPv4 addresses are always Usable, and link local 169.254.x.x
+// addresses are in some environments.
+func isUsableV4(ip netaddr.IP) bool {
+	if !ip.Is4() || ip.IsLoopback() {
+		return false
+	}
+	if ip.IsLinkLocalUnicast() {
+		return hostinfo.GetEnvType() == hostinfo.AWSLambda
+	}
+	return true
+}
+
+// isUsableV6 reports whether ip is a usable IPv6 address which could
+// conceivably be used to get Internet connectivity. Globally routable
+// IPv6 addresses are always Usable, and Unique Local Addresses
+// (fc00::/7) are in some environments used with address translation.
+func isUsableV6(ip netaddr.IP) bool {
 	return v6Global1.Contains(ip) ||
 		(tsaddr.IsULA(ip) && !tsaddr.TailscaleULARange().Contains(ip))
 }

net/interfaces/interfaces_test.go

@@ -46,7 +46,7 @@ func TestLikelyHomeRouterIP(t *testing.T) {
 	t.Logf("myIP = %v; gw = %v", my, gw)
 }
 
-func TestIsGlobalV6(t *testing.T) {
+func TestIsUsableV6(t *testing.T) {
 	tests := []struct {
 		name string
 		ip   string
@@ -61,8 +61,8 @@ func TestIsGlobalV6(t *testing.T) {
 	}
 
 	for _, test := range tests {
-		if got := isGlobalV6(netaddr.MustParseIP(test.ip)); got != test.want {
-			t.Errorf("isGlobalV6(%s) = %v, want %v", test.name, got, test.want)
+		if got := isUsableV6(netaddr.MustParseIP(test.ip)); got != test.want {
+			t.Errorf("isUsableV6(%s) = %v, want %v", test.name, got, test.want)
 		}
 	}
 }

net/netcheck/netcheck.go

@@ -336,7 +336,7 @@ func makeProbePlan(dm *tailcfg.DERPMap, ifState *interfaces.State, last *Report)
 	if last == nil || len(last.RegionLatency) == 0 {
 		return makeProbePlanInitial(dm, ifState)
 	}
-	have6if := ifState.HaveV6Global
+	have6if := ifState.HaveV6
 	have4if := ifState.HaveV4
 	plan = make(probePlan)
 	if !have4if && !have6if {
@@ -425,7 +425,7 @@ func makeProbePlanInitial(dm *tailcfg.DERPMap, ifState *interfaces.State) (plan
 			if ifState.HaveV4 && nodeMight4(n) {
 				p4 = append(p4, probe{delay: delay, node: n.Name, proto: probeIPv4})
 			}
-			if ifState.HaveV6Global && nodeMight6(n) {
+			if ifState.HaveV6 && nodeMight6(n) {
 				p6 = append(p6, probe{delay: delay, node: n.Name, proto: probeIPv6})
 			}
 		}
@@ -808,7 +808,7 @@ func (c *Client) GetReport(ctx context.Context, dm *tailcfg.DERPMap) (*Report, e
 		go c.readPackets(ctx, u4)
 	}
 
-	if ifState.HaveV6Global {
+	if ifState.HaveV6 {
 		if f := c.GetSTUNConn6; f != nil {
 			rs.pc6 = f()
 		} else {

net/netcheck/netcheck_test.go

@@ -443,8 +443,8 @@ func TestMakeProbePlan(t *testing.T) {
 	for _, tt := range tests {
 		t.Run(tt.name, func(t *testing.T) {
 			ifState := &interfaces.State{
-				HaveV6Global: tt.have6if,
-				HaveV4:       !tt.no4,
+				HaveV6: tt.have6if,
+				HaveV4: !tt.no4,
 			}
 			got := makeProbePlan(tt.dm, ifState, tt.last)
 			if !reflect.DeepEqual(got, tt.want) {

net/netns/netns_android.go

@@ -0,0 +1,64 @@
+// Copyright (c) 2021 Tailscale Inc & AUTHORS All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+// +build android
+
+package netns
+
+import (
+	"fmt"
+	"sync"
+	"syscall"
+)
+
+var (
+	androidProtectFuncMu sync.Mutex
+	androidProtectFunc   func(fd int) error
+)
+
+// SetAndroidProtectFunc register a func that Android provides that JNI calls into
+// https://developer.android.com/reference/android/net/VpnService#protect(int)
+// which is documented as:
+//
+// "Protect a socket from VPN connections. After protecting, data sent
+// through this socket will go directly to the underlying network, so
+// its traffic will not be forwarded through the VPN. This method is
+// useful if some connections need to be kept outside of VPN. For
+// example, a VPN tunnel should protect itself if its destination is
+// covered by VPN routes. Otherwise its outgoing packets will be sent
+// back to the VPN interface and cause an infinite loop. This method
+// will fail if the application is not prepared or is revoked."
+//
+// A nil func disables the use the hook.
+//
+// This indirection is necessary because this is the supported, stable
+// interface to use on Android, and doing the sockopts to set the
+// fwmark return errors on Android. The actual implementation of
+// VpnService.protect ends up doing an IPC to another process on
+// Android, asking for the fwmark to be set.
+func SetAndroidProtectFunc(f func(fd int) error) {
+	androidProtectFuncMu.Lock()
+	defer androidProtectFuncMu.Unlock()
+	androidProtectFunc = f
+}
+
+// control marks c as necessary to dial in a separate network namespace.
+//
+// It's intentionally the same signature as net.Dialer.Control
+// and net.ListenConfig.Control.
+func control(network, address string, c syscall.RawConn) error {
+	var sockErr error
+	err := c.Control(func(fd uintptr) {
+		androidProtectFuncMu.Lock()
+		f := androidProtectFunc
+		androidProtectFuncMu.Unlock()
+		if f != nil {
+			sockErr = f(int(fd))
+		}
+	})
+	if err != nil {
+		return fmt.Errorf("RawConn.Control on %T: %w", c, err)
+	}
+	return sockErr
+}

net/netns/netns_linux.go

@@ -2,6 +2,8 @@
 // Use of this source code is governed by a BSD-style
 // license that can be found in the LICENSE file.
 
+// +build linux,!android
+
 package netns
 
 import (

net/netns/netns_macios.go

@@ -0,0 +1,53 @@
+// Copyright (c) 2021 Tailscale Inc & AUTHORS All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+// +build darwin ios
+
+package netns
+
+import (
+	"errors"
+	"log"
+	"net"
+	"strings"
+	"syscall"
+
+	"golang.org/x/sys/unix"
+)
+
+// SetListenConfigInterfaceIndex sets lc.Control such that sockets are bound
+// to the provided interface index.
+func SetListenConfigInterfaceIndex(lc *net.ListenConfig, ifIndex int) error {
+	if lc == nil {
+		return errors.New("nil ListenConfig")
+	}
+	if lc.Control != nil {
+		return errors.New("ListenConfig.Control already set")
+	}
+	lc.Control = func(network, address string, c syscall.RawConn) error {
+		var sockErr error
+		err := c.Control(func(fd uintptr) {
+			sockErr = bindInterface(fd, network, address, ifIndex)
+			if sockErr != nil {
+				log.Printf("netns: bind(%q, %q) on index %v: %v", network, address, ifIndex, sockErr)
+			}
+		})
+		if err != nil {
+			return err
+		}
+		return sockErr
+	}
+	return nil
+}
+
+func bindInterface(fd uintptr, network, address string, ifIndex int) error {
+	v6 := strings.Contains(address, "]:") || strings.HasSuffix(network, "6") // hacky test for v6
+	proto := unix.IPPROTO_IP
+	opt := unix.IP_BOUND_IF
+	if v6 {
+		proto = unix.IPPROTO_IPV6
+		opt = unix.IPV6_BOUND_IF
+	}
+	return unix.SetsockoptInt(int(fd), proto, opt, ifIndex)
+}

net/portmapper/portmapper.go

@@ -577,8 +577,6 @@ func pcpAnnounceRequest(myIP netaddr.IP) []byte {
 	return pkt
 }
 
-//lint:ignore U1000 moved this code from netcheck's old PCP probing; will be needed when we add PCP mapping
-
 // pcpMapRequest generates a PCP packet with a MAP opcode.
 func pcpMapRequest(myIP netaddr.IP, mapToLocalPort int, delete bool) []byte {
 	const udpProtoNumber = 17

net/socks5/socks5.go

@@ -2,8 +2,15 @@
 // Use of this source code is governed by a BSD-style
 // license that can be found in the LICENSE file.
 
-// Package socks5 is a SOCKS5 server implementation
-// for userspace networking in Tailscale.
+// Package socks5 is a SOCKS5 server implementation.
+//
+// This is used for userspace networking in Tailscale. Specifically,
+// this is used for dialing out of the machine to other nodes, without
+// the host kernel's involvement, so it doesn't proper routing tables,
+// TUN, IPv6, etc. This package is meant to only handle the SOCKS5 protocol
+// details and not any integration with Tailscale internals itself.
+//
+// The glue between this package and Tailscale is in net/socks5/tssocks.
 package socks5
 
 import (
@@ -32,7 +39,7 @@ const (
 // that represent the kind of connection the client needs.
 type commandType byte
 
-// The set of valid SOCKS5 commans as described in RFC 1928.
+// The set of valid SOCKS5 commands as described in RFC 1928.
 const (
 	connect      commandType = 1
 	bind         commandType = 2

net/socks5/tssocks/tssocks.go

@@ -0,0 +1,79 @@
+// Copyright (c) 2021 Tailscale Inc & AUTHORS All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+// Package tssocks is the glue between Tailscale and the net/socks5 package.
+package tssocks
+
+import (
+	"context"
+	"net"
+	"sync"
+
+	"inet.af/netaddr"
+	"tailscale.com/net/socks5"
+	"tailscale.com/net/tsaddr"
+	"tailscale.com/types/logger"
+	"tailscale.com/types/netmap"
+	"tailscale.com/wgengine"
+	"tailscale.com/wgengine/netstack"
+)
+
+// NewServer returns a new SOCKS5 server configured to dial out to
+// Tailscale addresses.
+//
+// The returned server is not yet listening. The caller must call
+// Serve with a listener.
+//
+// If ns is non-nil, it is used for dialing when needed.
+func NewServer(logf logger.Logf, e wgengine.Engine, ns *netstack.Impl) *socks5.Server {
+	d := &dialer{ns: ns}
+	e.AddNetworkMapCallback(d.onNewNetmap)
+	return &socks5.Server{
+		Logf:   logf,
+		Dialer: d.DialContext,
+	}
+}
+
+// dialer is the Tailscale SOCKS5 dialer.
+type dialer struct {
+	ns *netstack.Impl
+
+	mu  sync.Mutex
+	dns netstack.DNSMap
+}
+
+func (d *dialer) onNewNetmap(nm *netmap.NetworkMap) {
+	d.mu.Lock()
+	defer d.mu.Unlock()
+	d.dns = netstack.DNSMapFromNetworkMap(nm)
+}
+
+func (d *dialer) resolve(ctx context.Context, addr string) (netaddr.IPPort, error) {
+	d.mu.Lock()
+	dns := d.dns
+	d.mu.Unlock()
+	return dns.Resolve(ctx, addr)
+}
+
+func (d *dialer) DialContext(ctx context.Context, network, addr string) (net.Conn, error) {
+	ipp, err := d.resolve(ctx, addr)
+	if err != nil {
+		return nil, err
+	}
+	if d.ns != nil && d.useNetstackForIP(ipp.IP()) {
+		return d.ns.DialContextTCP(ctx, ipp.String())
+	}
+	var stdDialer net.Dialer
+	return stdDialer.DialContext(ctx, network, ipp.String())
+}
+
+func (d *dialer) useNetstackForIP(ip netaddr.IP) bool {
+	if d.ns == nil {
+		return false
+	}
+	// TODO(bradfitz): this isn't exactly right.
+	// We should also support subnets when the
+	// prefs are configured as such.
+	return tsaddr.IsTailscaleIP(ip)
+}

net/tstun/wrap.go

@@ -72,12 +72,14 @@ type Wrapper struct {
 	// It is made a static buffer in order to avoid allocations.
 	buffer [maxBufferSize]byte
 	// bufferConsumed synchronizes access to buffer (shared by Read and poll).
+	//
+	// Close closes bufferConsumed. There may be outstanding sends to bufferConsumed
+	// when that happens; we catch any resulting panics.
+	// This lets us avoid expensive multi-case selects.
 	bufferConsumed chan struct{}
 
 	// closed signals poll (by closing) when the device is closed.
 	closed chan struct{}
-	// errors is the error queue populated by poll.
-	errors chan error
 	// outbound is the queue by which packets leave the TUN device.
 	//
 	// The directions are relative to the network, not the device:
@@ -88,7 +90,11 @@ type Wrapper struct {
 	//
 	// Empty reads are skipped by Wireguard, so it is always legal
 	// to discard an empty packet instead of sending it through t.outbound.
-	outbound chan []byte
+	//
+	// Close closes outbound. There may be outstanding sends to outbound
+	// when that happens; we catch any resulting panics.
+	// This lets us avoid expensive multi-case selects.
+	outbound chan tunReadResult
 
 	// eventsUpDown yields up and down tun.Events that arrive on a Wrapper's events channel.
 	eventsUpDown chan tun.Event
@@ -125,6 +131,14 @@ type Wrapper struct {
 	disableTSMPRejected bool
 }
 
+// tunReadResult is the result of a TUN read: Some data and an error.
+// The byte slice is not interpreted in the usual way for a Read method.
+// See the comment in the middle of Wrap.Read.
+type tunReadResult struct {
+	data []byte
+	err  error
+}
+
 func Wrap(logf logger.Logf, tdev tun.Device) *Wrapper {
 	tun := &Wrapper{
 		logf: logger.WithPrefix(logf, "tstun: "),
@@ -133,8 +147,7 @@ func Wrap(logf logger.Logf, tdev tun.Device) *Wrapper {
 		// a goroutine should not block when setting it, even with no listeners.
 		bufferConsumed: make(chan struct{}, 1),
 		closed:         make(chan struct{}),
-		errors:         make(chan error),
-		outbound:       make(chan []byte),
+		outbound:       make(chan tunReadResult),
 		eventsUpDown:   make(chan tun.Event),
 		eventsOther:    make(chan tun.Event),
 		// TODO(dmytro): (highly rate-limited) hexdumps should happen on unknown packets.
@@ -160,14 +173,24 @@ func (t *Wrapper) SetDestIPActivityFuncs(m map[netaddr.IP]func()) {
 func (t *Wrapper) Close() error {
 	var err error
 	t.closeOnce.Do(func() {
-		// Other channels need not be closed: poll will exit gracefully after this.
 		close(t.closed)
-
+		close(t.bufferConsumed)
+		close(t.outbound)
 		err = t.tdev.Close()
 	})
 	return err
 }
 
+// isClosed reports whether t is closed.
+func (t *Wrapper) isClosed() bool {
+	select {
+	case <-t.closed:
+		return true
+	default:
+		return false
+	}
+}
+
 // pumpEvents copies events from t.tdev to t.eventsUpDown and t.eventsOther.
 // pumpEvents exits when t.tdev.events or t.closed is closed.
 // pumpEvents closes t.eventsUpDown and t.eventsOther when it exits.
@@ -230,46 +253,47 @@ func (t *Wrapper) Name() (string, error) {
 	return t.tdev.Name()
 }
 
+// allowSendOnClosedChannel suppresses panics due to sending on a closed channel.
+// This allows us to avoid synchronization between poll and Close.
+// Such synchronization (particularly multi-case selects) is too expensive
+// for code like poll or Read that is on the hot path of every packet.
+// If this makes you sad or angry, you may want to join our
+// weekly Go Performance Delinquents Anonymous meetings on Monday nights.
+func allowSendOnClosedChannel() {
+	r := recover()
+	if r == nil {
+		return
+	}
+	e, _ := r.(error)
+	if e != nil && e.Error() == "send on closed channel" {
+		return
+	}
+	panic(r)
+}
+
 // poll polls t.tdev.Read, placing the oldest unconsumed packet into t.buffer.
 // This is needed because t.tdev.Read in general may block (it does on Windows),
 // so packets may be stuck in t.outbound if t.Read called t.tdev.Read directly.
 func (t *Wrapper) poll() {
-	for {
-		select {
-		case <-t.closed:
-			return
-		case <-t.bufferConsumed:
-			// continue
-		}
-
+	defer allowSendOnClosedChannel() // for send to t.outbound
+	for range t.bufferConsumed {
+		var n int
+		var err error
 		// Read may use memory in t.buffer before PacketStartOffset for mandatory headers.
 		// This is the rationale behind the tun.Wrapper.{Read,Write} interfaces
 		// and the reason t.buffer has size MaxMessageSize and not MaxContentSize.
-		n, err := t.tdev.Read(t.buffer[:], PacketStartOffset)
-		if err != nil {
-			select {
-			case <-t.closed:
+		// In principle, read errors are not fatal (but wireguard-go disagrees).
+		// We loop here until we get a non-empty (or failed) read.
+		// We don't need this loop for correctness,
+		// but wireguard-go will skip an empty read,
+		// so we might as well avoid the send through t.outbound.
+		for n == 0 && err == nil {
+			if t.isClosed() {
 				return
-			case t.errors <- err:
-				// In principle, read errors are not fatal (but wireguard-go disagrees).
-				t.bufferConsumed <- struct{}{}
 			}
-			continue
-		}
-
-		// Wireguard will skip an empty read,
-		// so we might as well do it here to avoid the send through t.outbound.
-		if n == 0 {
-			t.bufferConsumed <- struct{}{}
-			continue
-		}
-
-		select {
-		case <-t.closed:
-			return
-		case t.outbound <- t.buffer[PacketStartOffset : PacketStartOffset+n]:
-			// continue
+			n, err = t.tdev.Read(t.buffer[:], PacketStartOffset)
 		}
+		t.outbound <- tunReadResult{data: t.buffer[PacketStartOffset : PacketStartOffset+n], err: err}
 	}
 }
 
@@ -325,26 +349,24 @@ func (t *Wrapper) IdleDuration() time.Duration {
 }
 
 func (t *Wrapper) Read(buf []byte, offset int) (int, error) {
-	var n int
-
-	wasInjectedPacket := false
-
-	select {
-	case <-t.closed:
+	res, ok := <-t.outbound
+	if !ok {
+		// Wrapper is closed.
 		return 0, io.EOF
-	case err := <-t.errors:
-		return 0, err
-	case pkt := <-t.outbound:
-		n = copy(buf[offset:], pkt)
-		// t.buffer has a fixed location in memory,
-		// so this is the easiest way to tell when it has been consumed.
-		// &pkt[0] can be used because empty packets do not reach t.outbound.
-		if &pkt[0] == &t.buffer[PacketStartOffset] {
-			t.bufferConsumed <- struct{}{}
-		} else {
-			// If the packet is not from t.buffer, then it is an injected packet.
-			wasInjectedPacket = true
-		}
+	}
+	if res.err != nil {
+		return 0, res.err
+	}
+	defer allowSendOnClosedChannel() // for send to t.bufferConsumed
+	pkt := res.data
+	n := copy(buf[offset:], pkt)
+	// t.buffer has a fixed location in memory.
+	// If the packet is not from t.buffer, then it is an injected packet.
+	// &pkt[0] can be used because empty packets do not reach t.outbound.
+	isInjectedPacket := &pkt[0] != &t.buffer[PacketStartOffset]
+	if !isInjectedPacket {
+		// We are done with t.buffer. Let poll re-use it.
+		t.bufferConsumed <- struct{}{}
 	}
 
 	p := parsedPacketPool.Get().(*packet.Parsed)
@@ -357,13 +379,8 @@ func (t *Wrapper) Read(buf []byte, offset int) (int, error) {
 		}
 	}
 
-	// For injected packets, we return early to bypass filtering.
-	if wasInjectedPacket {
-		t.noteActivity()
-		return n, nil
-	}
-
-	if !t.disableFilter {
+	// Do not filter injected packets.
+	if !isInjectedPacket && !t.disableFilter {
 		response := t.filterOut(p)
 		if response != filter.Accept {
 			// Wireguard considers read errors fatal; pretend nothing was read
@@ -566,12 +583,9 @@ func (t *Wrapper) InjectOutbound(packet []byte) error {
 	if len(packet) == 0 {
 		return nil
 	}
-	select {
-	case <-t.closed:
-		return ErrClosed
-	case t.outbound <- packet:
-		return nil
-	}
+	defer allowSendOnClosedChannel() // for send to t.outbound
+	t.outbound <- tunReadResult{data: packet}
+	return nil
 }
 
 // Unwrap returns the underlying tun.Device.

portlist/portlist_windows.go

@@ -23,7 +23,7 @@ func listPorts() (List, error) {
 }
 
 func addProcesses(pl []Port) ([]Port, error) {
-	//lint:ignore SA1019 OpenCurrentProcessToken instead of GetCurrentProcessToken,
+	// OpenCurrentProcessToken instead of GetCurrentProcessToken,
 	// as GetCurrentProcessToken only works on Windows 8+.
 	tok, err := windows.OpenCurrentProcessToken()
 	if err != nil {

staticcheck.conf

@@ -0,0 +1,17 @@
+# Full list: https://staticcheck.io/docs/checks
+checks = [
+	"SA*", "-SA1019", "-SA2001", "-SA9003", # SA* are mostly legit code errors
+
+	# S1?? are "code simplifications" which we consider unnecessary
+
+	# ST1??? are stylistic issues, some of which are generally accepted
+	# In general, if it's listed in
+	# https://github.com/golang/go/wiki/CodeReviewComments, then it
+	# may be an acceptable check.
+
+	# TODO(crawshaw): enable when we have docs? "ST1000", # missing package docs
+	"ST1001", # discourage dot imports
+
+	"QF1004", # Use `strings.ReplaceAll` instead of `strings.Replace` with `n == 1`
+	"QF1006", # Lift if+break into loop condition
+]

syncs/locked_test.go

@@ -4,8 +4,6 @@
 
 // +build go1.13,!go1.16
 
-//lint:file-ignore SA2001 the empty critical sections are part of triggering different internal mutex states
-
 package syncs
 
 import (

syncs/syncs.go

@@ -83,6 +83,17 @@ func (b *AtomicBool) Get() bool {
 	return atomic.LoadInt32((*int32)(b)) != 0
 }
 
+// AtomicUint32 is an atomic uint32.
+type AtomicUint32 uint32
+
+func (b *AtomicUint32) Set(v uint32) {
+	atomic.StoreUint32((*uint32)(b), v)
+}
+
+func (b *AtomicUint32) Get() uint32 {
+	return atomic.LoadUint32((*uint32)(b))
+}
+
 // Semaphore is a counting semaphore.
 //
 // Use NewSemaphore to create one.

syncs/watchdog.go

@@ -52,7 +52,7 @@ func Watch(ctx context.Context, mu sync.Locker, tick, max time.Duration) chan ti
 		go func() {
 			start := time.Now()
 			mu.Lock()
-			mu.Unlock() //lint:ignore SA2001 ignore the empty critical section
+			mu.Unlock()
 			elapsed := time.Since(start)
 			if elapsed > max {
 				elapsed = max

tailcfg/derpmap.go

@@ -14,6 +14,10 @@ type DERPMap struct {
 	//
 	// The numbers are not necessarily contiguous.
 	Regions map[int]*DERPRegion
+
+	// OmitDefaultRegions specifies to not use Tailscale's DERP servers, and only use those
+	// specified in this DERPMap. If there are none set outside of the defaults, this is a noop.
+	OmitDefaultRegions bool `json:"omitDefaultRegions,omitempty"`
 }
 
 /// RegionIDs returns the sorted region IDs.

tailcfg/tailcfg.go

@@ -4,7 +4,7 @@
 
 package tailcfg
 
-//go:generate go run tailscale.com/cmd/cloner --type=User,Node,Hostinfo,NetInfo,Login,DNSConfig,DNSResolver,RegisterResponse --clonefunc=true --output=tailcfg_clone.go
+//go:generate go run tailscale.com/cmd/cloner --type=User,Node,Hostinfo,NetInfo,Login,DNSConfig,DNSResolver,RegisterResponse,DERPRegion,DERPMap,DERPNode --clonefunc=true --output=tailcfg_clone.go
 
 import (
 	"encoding/hex"
@@ -42,7 +42,10 @@ import (
 //    17: 2021-04-18: MapResponse.Domain empty means unchanged
 //    18: 2021-04-19: MapResponse.Node nil means unchanged (all fields now omitempty)
 //    19: 2021-04-21: MapResponse.Debug.SleepSeconds
-const CurrentMapRequestVersion = 19
+//    20: 2021-06-11: MapResponse.LastSeen used even less (https://github.com/tailscale/tailscale/issues/2107)
+//    21: 2021-06-15: added MapResponse.DNSConfig.CertDomains
+//    22: 2021-06-16: added MapResponse.DNSConfig.ExtraRecords
+const CurrentMapRequestVersion = 22
 
 type StableID string
 
@@ -872,6 +875,36 @@ type DNSConfig struct {
 
 	// PerDomain is not set by the control server, and does nothing.
 	PerDomain bool `json:",omitempty"`
+
+	// CertDomains are the set of DNS names for which the control
+	// plane server will assist with provisioning TLS
+	// certificates. See SetDNSRequest, which can be used to
+	// answer dns-01 ACME challenges for e.g. LetsEncrypt.
+	// These names are FQDNs without trailing periods, and without
+	// any "_acme-challenge." prefix.
+	CertDomains []string `json:",omitempty"`
+
+	// ExtraRecords contains extra DNS records to add to the
+	// MagicDNS config.
+	ExtraRecords []DNSRecord `json:",omitempty"`
+}
+
+// DNSRecord is an extra DNS record to add to MagicDNS.
+type DNSRecord struct {
+	// Name is the fully qualified domain name of
+	// the record to add. The trailing dot is optional.
+	Name string
+
+	// Type is the DNS record type.
+	// Empty means A or AAAA, depending on value.
+	// Other values are currently ignored.
+	Type string `json:",omitempty"`
+
+	// Value is the IP address in string form.
+	// TODO(bradfitz): if we ever add support for record types
+	// with non-UTF8 binary data, add ValueBytes []byte that
+	// would take precedence.
+	Value string
 }
 
 // PingRequest is a request to send an HTTP request to prove the
@@ -879,6 +912,9 @@ type DNSConfig struct {
 type PingRequest struct {
 	// URL is the URL to send a HEAD request to.
 	// It will be a unique URL each time. No auth headers are necessary.
+	//
+	// If the client sees multiple PingRequests with the same URL,
+	// subsequent ones should be ignored.
 	URL string
 
 	// Log is whether to log about this ping in the success case.
@@ -1019,6 +1055,11 @@ type Debug struct {
 	// The client can (and should) limit the value (such as 5
 	// minutes).
 	SleepSeconds float64 `json:",omitempty"`
+
+	// RandomizeClientPort is whether magicsock should UDP bind to
+	// :0 to get a random local port, ignoring any configured
+	// fixed port.
+	RandomizeClientPort bool `json:",omitempty"`
 }
 
 func (k MachineKey) String() string                   { return fmt.Sprintf("mkey:%x", k[:]) }
@@ -1193,6 +1234,9 @@ type SetDNSRequest struct {
 	NodeKey NodeKey
 
 	// Name is the domain name for which to create a record.
+	// For ACME DNS-01 challenges, it should be one of the domains
+	// in MapResponse.DNSConfig.CertDomains with the prefix
+	// "_acme-challenge.".
 	Name string
 
 	// Type is the DNS record type. For ACME DNS-01 challenges, it

tailcfg/tailcfg_clone.go

@@ -2,7 +2,7 @@
 // Use of this source code is governed by a BSD-style
 // license that can be found in the LICENSE file.
 
-// Code generated by tailscale.com/cmd/cloner -type User,Node,Hostinfo,NetInfo,Login,DNSConfig,DNSResolver,RegisterResponse; DO NOT EDIT.
+// Code generated by tailscale.com/cmd/cloner -type User,Node,Hostinfo,NetInfo,Login,DNSConfig,DNSResolver,RegisterResponse,DERPRegion,DERPMap,DERPNode; DO NOT EDIT.
 
 package tailcfg
 
@@ -26,7 +26,7 @@ func (src *User) Clone() *User {
 }
 
 // A compilation failure here means this code must be regenerated, with command:
-//   tailscale.com/cmd/cloner -type User,Node,Hostinfo,NetInfo,Login,DNSConfig,DNSResolver,RegisterResponse
+//   tailscale.com/cmd/cloner -type User,Node,Hostinfo,NetInfo,Login,DNSConfig,DNSResolver,RegisterResponse,DERPRegion,DERPMap,DERPNode
 var _UserNeedsRegeneration = User(struct {
 	ID            UserID
 	LoginName     string
@@ -62,7 +62,7 @@ func (src *Node) Clone() *Node {
 }
 
 // A compilation failure here means this code must be regenerated, with command:
-//   tailscale.com/cmd/cloner -type User,Node,Hostinfo,NetInfo,Login,DNSConfig,DNSResolver,RegisterResponse
+//   tailscale.com/cmd/cloner -type User,Node,Hostinfo,NetInfo,Login,DNSConfig,DNSResolver,RegisterResponse,DERPRegion,DERPMap,DERPNode
 var _NodeNeedsRegeneration = Node(struct {
 	ID                      NodeID
 	StableID                StableNodeID
@@ -105,7 +105,7 @@ func (src *Hostinfo) Clone() *Hostinfo {
 }
 
 // A compilation failure here means this code must be regenerated, with command:
-//   tailscale.com/cmd/cloner -type User,Node,Hostinfo,NetInfo,Login,DNSConfig,DNSResolver,RegisterResponse
+//   tailscale.com/cmd/cloner -type User,Node,Hostinfo,NetInfo,Login,DNSConfig,DNSResolver,RegisterResponse,DERPRegion,DERPMap,DERPNode
 var _HostinfoNeedsRegeneration = Hostinfo(struct {
 	IPNVersion    string
 	FrontendLogID string
@@ -142,7 +142,7 @@ func (src *NetInfo) Clone() *NetInfo {
 }
 
 // A compilation failure here means this code must be regenerated, with command:
-//   tailscale.com/cmd/cloner -type User,Node,Hostinfo,NetInfo,Login,DNSConfig,DNSResolver,RegisterResponse
+//   tailscale.com/cmd/cloner -type User,Node,Hostinfo,NetInfo,Login,DNSConfig,DNSResolver,RegisterResponse,DERPRegion,DERPMap,DERPNode
 var _NetInfoNeedsRegeneration = NetInfo(struct {
 	MappingVariesByDestIP opt.Bool
 	HairPinning           opt.Bool
@@ -169,7 +169,7 @@ func (src *Login) Clone() *Login {
 }
 
 // A compilation failure here means this code must be regenerated, with command:
-//   tailscale.com/cmd/cloner -type User,Node,Hostinfo,NetInfo,Login,DNSConfig,DNSResolver,RegisterResponse
+//   tailscale.com/cmd/cloner -type User,Node,Hostinfo,NetInfo,Login,DNSConfig,DNSResolver,RegisterResponse,DERPRegion,DERPMap,DERPNode
 var _LoginNeedsRegeneration = Login(struct {
 	_             structs.Incomparable
 	ID            LoginID
@@ -204,11 +204,13 @@ func (src *DNSConfig) Clone() *DNSConfig {
 	}
 	dst.Domains = append(src.Domains[:0:0], src.Domains...)
 	dst.Nameservers = append(src.Nameservers[:0:0], src.Nameservers...)
+	dst.CertDomains = append(src.CertDomains[:0:0], src.CertDomains...)
+	dst.ExtraRecords = append(src.ExtraRecords[:0:0], src.ExtraRecords...)
 	return dst
 }
 
 // A compilation failure here means this code must be regenerated, with command:
-//   tailscale.com/cmd/cloner -type User,Node,Hostinfo,NetInfo,Login,DNSConfig,DNSResolver,RegisterResponse
+//   tailscale.com/cmd/cloner -type User,Node,Hostinfo,NetInfo,Login,DNSConfig,DNSResolver,RegisterResponse,DERPRegion,DERPMap,DERPNode
 var _DNSConfigNeedsRegeneration = DNSConfig(struct {
 	Resolvers         []DNSResolver
 	Routes            map[string][]DNSResolver
@@ -217,6 +219,8 @@ var _DNSConfigNeedsRegeneration = DNSConfig(struct {
 	Proxied           bool
 	Nameservers       []netaddr.IP
 	PerDomain         bool
+	CertDomains       []string
+	ExtraRecords      []DNSRecord
 }{})
 
 // Clone makes a deep copy of DNSResolver.
@@ -232,7 +236,7 @@ func (src *DNSResolver) Clone() *DNSResolver {
 }
 
 // A compilation failure here means this code must be regenerated, with command:
-//   tailscale.com/cmd/cloner -type User,Node,Hostinfo,NetInfo,Login,DNSConfig,DNSResolver,RegisterResponse
+//   tailscale.com/cmd/cloner -type User,Node,Hostinfo,NetInfo,Login,DNSConfig,DNSResolver,RegisterResponse,DERPRegion,DERPMap,DERPNode
 var _DNSResolverNeedsRegeneration = DNSResolver(struct {
 	Addr                string
 	BootstrapResolution []netaddr.IP
@@ -251,7 +255,7 @@ func (src *RegisterResponse) Clone() *RegisterResponse {
 }
 
 // A compilation failure here means this code must be regenerated, with command:
-//   tailscale.com/cmd/cloner -type User,Node,Hostinfo,NetInfo,Login,DNSConfig,DNSResolver,RegisterResponse
+//   tailscale.com/cmd/cloner -type User,Node,Hostinfo,NetInfo,Login,DNSConfig,DNSResolver,RegisterResponse,DERPRegion,DERPMap,DERPNode
 var _RegisterResponseNeedsRegeneration = RegisterResponse(struct {
 	User              User
 	Login             Login
@@ -260,9 +264,84 @@ var _RegisterResponseNeedsRegeneration = RegisterResponse(struct {
 	AuthURL           string
 }{})
 
+// Clone makes a deep copy of DERPRegion.
+// The result aliases no memory with the original.
+func (src *DERPRegion) Clone() *DERPRegion {
+	if src == nil {
+		return nil
+	}
+	dst := new(DERPRegion)
+	*dst = *src
+	dst.Nodes = make([]*DERPNode, len(src.Nodes))
+	for i := range dst.Nodes {
+		dst.Nodes[i] = src.Nodes[i].Clone()
+	}
+	return dst
+}
+
+// A compilation failure here means this code must be regenerated, with command:
+//   tailscale.com/cmd/cloner -type User,Node,Hostinfo,NetInfo,Login,DNSConfig,DNSResolver,RegisterResponse,DERPRegion,DERPMap,DERPNode
+var _DERPRegionNeedsRegeneration = DERPRegion(struct {
+	RegionID   int
+	RegionCode string
+	RegionName string
+	Avoid      bool
+	Nodes      []*DERPNode
+}{})
+
+// Clone makes a deep copy of DERPMap.
+// The result aliases no memory with the original.
+func (src *DERPMap) Clone() *DERPMap {
+	if src == nil {
+		return nil
+	}
+	dst := new(DERPMap)
+	*dst = *src
+	if dst.Regions != nil {
+		dst.Regions = map[int]*DERPRegion{}
+		for k, v := range src.Regions {
+			dst.Regions[k] = v.Clone()
+		}
+	}
+	return dst
+}
+
+// A compilation failure here means this code must be regenerated, with command:
+//   tailscale.com/cmd/cloner -type User,Node,Hostinfo,NetInfo,Login,DNSConfig,DNSResolver,RegisterResponse,DERPRegion,DERPMap,DERPNode
+var _DERPMapNeedsRegeneration = DERPMap(struct {
+	Regions            map[int]*DERPRegion
+	OmitDefaultRegions bool
+}{})
+
+// Clone makes a deep copy of DERPNode.
+// The result aliases no memory with the original.
+func (src *DERPNode) Clone() *DERPNode {
+	if src == nil {
+		return nil
+	}
+	dst := new(DERPNode)
+	*dst = *src
+	return dst
+}
+
+// A compilation failure here means this code must be regenerated, with command:
+//   tailscale.com/cmd/cloner -type User,Node,Hostinfo,NetInfo,Login,DNSConfig,DNSResolver,RegisterResponse,DERPRegion,DERPMap,DERPNode
+var _DERPNodeNeedsRegeneration = DERPNode(struct {
+	Name         string
+	RegionID     int
+	HostName     string
+	CertName     string
+	IPv4         string
+	IPv6         string
+	STUNPort     int
+	STUNOnly     bool
+	DERPTestPort int
+	STUNTestIP   string
+}{})
+
 // Clone duplicates src into dst and reports whether it succeeded.
 // To succeed, <src, dst> must be of types <*T, *T> or <*T, **T>,
-// where T is one of User,Node,Hostinfo,NetInfo,Login,DNSConfig,DNSResolver,RegisterResponse.
+// where T is one of User,Node,Hostinfo,NetInfo,Login,DNSConfig,DNSResolver,RegisterResponse,DERPRegion,DERPMap,DERPNode.
 func Clone(dst, src interface{}) bool {
 	switch src := src.(type) {
 	case *User:
@@ -337,6 +416,33 @@ func Clone(dst, src interface{}) bool {
 			*dst = src.Clone()
 			return true
 		}
+	case *DERPRegion:
+		switch dst := dst.(type) {
+		case *DERPRegion:
+			*dst = *src.Clone()
+			return true
+		case **DERPRegion:
+			*dst = src.Clone()
+			return true
+		}
+	case *DERPMap:
+		switch dst := dst.(type) {
+		case *DERPMap:
+			*dst = *src.Clone()
+			return true
+		case **DERPMap:
+			*dst = src.Clone()
+			return true
+		}
+	case *DERPNode:
+		switch dst := dst.(type) {
+		case *DERPNode:
+			*dst = *src.Clone()
+			return true
+		case **DERPNode:
+			*dst = src.Clone()
+			return true
+		}
 	}
 	return false
 }

tstest/integration/integration.go

@@ -9,6 +9,11 @@
 package integration
 
 import (
+	"crypto/rand"
+	"crypto/tls"
+	"net"
+	"net/http"
+	"net/http/httptest"
 	"os"
 	"os/exec"
 	"path"
@@ -19,6 +24,13 @@ import (
 	"testing"
 	"time"
 
+	"tailscale.com/derp"
+	"tailscale.com/derp/derphttp"
+	"tailscale.com/net/stun/stuntest"
+	"tailscale.com/tailcfg"
+	"tailscale.com/types/key"
+	"tailscale.com/types/logger"
+	"tailscale.com/types/nettype"
 	"tailscale.com/version"
 )
 
@@ -98,3 +110,53 @@ func exe() string {
 	}
 	return ""
 }
+
+// RunDERPAndSTUN runs a local DERP and STUN server for tests, returning the derpMap
+// that clients should use. This creates resources that must be cleaned up with the
+// returned cleanup function.
+func RunDERPAndSTUN(t testing.TB, logf logger.Logf, ipAddress string) (derpMap *tailcfg.DERPMap) {
+	t.Helper()
+
+	var serverPrivateKey key.Private
+	if _, err := rand.Read(serverPrivateKey[:]); err != nil {
+		t.Fatal(err)
+	}
+	d := derp.NewServer(serverPrivateKey, logf)
+
+	httpsrv := httptest.NewUnstartedServer(derphttp.Handler(d))
+	httpsrv.Config.ErrorLog = logger.StdLogger(logf)
+	httpsrv.Config.TLSNextProto = make(map[string]func(*http.Server, *tls.Conn, http.Handler))
+	httpsrv.StartTLS()
+
+	stunAddr, stunCleanup := stuntest.ServeWithPacketListener(t, nettype.Std{})
+
+	m := &tailcfg.DERPMap{
+		Regions: map[int]*tailcfg.DERPRegion{
+			1: {
+				RegionID:   1,
+				RegionCode: "test",
+				Nodes: []*tailcfg.DERPNode{
+					{
+						Name:         "t1",
+						RegionID:     1,
+						HostName:     ipAddress,
+						IPv4:         ipAddress,
+						IPv6:         "none",
+						STUNPort:     stunAddr.Port,
+						DERPTestPort: httpsrv.Listener.Addr().(*net.TCPAddr).Port,
+						STUNTestIP:   stunAddr.IP.String(),
+					},
+				},
+			},
+		},
+	}
+
+	t.Cleanup(func() {
+		httpsrv.CloseClientConnections()
+		httpsrv.Close()
+		d.Close()
+		stunCleanup()
+	})
+
+	return m
+}

tstest/integration/integration_test.go

@@ -6,8 +6,7 @@ package integration
 
 import (
 	"bytes"
-	crand "crypto/rand"
-	"crypto/tls"
+	"context"
 	"encoding/json"
 	"errors"
 	"flag"
@@ -15,7 +14,6 @@ import (
 	"io"
 	"io/ioutil"
 	"log"
-	"net"
 	"net/http"
 	"net/http/httptest"
 	"os"
@@ -30,25 +28,24 @@ import (
 	"time"
 
 	"go4.org/mem"
-	"tailscale.com/derp"
-	"tailscale.com/derp/derphttp"
 	"tailscale.com/ipn/ipnstate"
-	"tailscale.com/net/stun/stuntest"
 	"tailscale.com/safesocket"
 	"tailscale.com/smallzstd"
 	"tailscale.com/tailcfg"
 	"tailscale.com/tstest"
 	"tailscale.com/tstest/integration/testcontrol"
-	"tailscale.com/types/key"
 	"tailscale.com/types/logger"
-	"tailscale.com/types/nettype"
 )
 
-var verbose = flag.Bool("verbose", false, "verbose debug logs")
+var (
+	verboseLogCatcher = flag.Bool("verbose-log-catcher", false, "verbose log catcher logging")
+	verboseTailscaled = flag.Bool("verbose-tailscaled", false, "verbose tailscaled logging")
+)
 
 var mainError atomic.Value // of error
 
 func TestMain(m *testing.M) {
+	flag.Parse()
 	v := m.Run()
 	if v != 0 {
 		os.Exit(v)
@@ -106,9 +103,10 @@ func TestOneNodeUp_Auth(t *testing.T) {
 	t.Parallel()
 	bins := BuildTestBinaries(t)
 
-	env := newTestEnv(t, bins)
+	env := newTestEnv(t, bins, configureControl(func(control *testcontrol.Server) {
+		control.RequireAuth = true
+	}))
 	defer env.Close()
-	env.Control.RequireAuth = true
 
 	n1 := newTestNode(t, env)
 	d1 := n1.StartDaemon(t)
@@ -158,13 +156,20 @@ func TestTwoNodes(t *testing.T) {
 
 	// Create two nodes:
 	n1 := newTestNode(t, env)
+	n1SocksAddrCh := n1.socks5AddrChan()
 	d1 := n1.StartDaemon(t)
 	defer d1.Kill()
 
 	n2 := newTestNode(t, env)
+	n2SocksAddrCh := n2.socks5AddrChan()
 	d2 := n2.StartDaemon(t)
 	defer d2.Kill()
 
+	n1Socks := n1.AwaitSocksAddr(t, n1SocksAddrCh)
+	n2Socks := n1.AwaitSocksAddr(t, n2SocksAddrCh)
+	t.Logf("node1 SOCKS5 addr: %v", n1Socks)
+	t.Logf("node2 SOCKS5 addr: %v", n2Socks)
+
 	n1.AwaitListening(t)
 	n2.AwaitListening(t)
 	n1.MustUp()
@@ -251,20 +256,34 @@ func TestAddPingRequest(t *testing.T) {
 	}
 
 	nodeKey := nodes[0].Key
-	pr := &tailcfg.PingRequest{URL: waitPing.URL, Log: true}
-	ok := env.Control.AddPingRequest(nodeKey, pr)
-	if !ok {
-		t.Fatalf("no node found with NodeKey %v in AddPingRequest", nodeKey)
-	}
 
-	// Wait for PingRequest to come back
-	pingTimeout := time.NewTimer(10 * time.Second)
-	select {
-	case <-gotPing:
-		pingTimeout.Stop()
-	case <-pingTimeout.C:
-		t.Error("didn't get PingRequest from tailscaled")
+	// Check that we get at least one ping reply after 10 tries.
+	for try := 1; try <= 10; try++ {
+		t.Logf("ping %v ...", try)
+		ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
+		if err := env.Control.AwaitNodeInMapRequest(ctx, nodeKey); err != nil {
+			t.Fatal(err)
+		}
+		cancel()
+
+		pr := &tailcfg.PingRequest{URL: fmt.Sprintf("%s/ping-%d", waitPing.URL, try), Log: true}
+		if !env.Control.AddPingRequest(nodeKey, pr) {
+			t.Logf("failed to AddPingRequest")
+			continue
+		}
+
+		// Wait for PingRequest to come back
+		pingTimeout := time.NewTimer(2 * time.Second)
+		defer pingTimeout.Stop()
+		select {
+		case <-gotPing:
+			t.Logf("got ping; success")
+			return
+		case <-pingTimeout.C:
+			// Try again.
+		}
 	}
+	t.Error("all ping attempts failed")
 }
 
 // testEnv contains the test environment (set of servers) used by one
@@ -281,23 +300,32 @@ type testEnv struct {
 
 	TrafficTrap       *trafficTrap
 	TrafficTrapServer *httptest.Server
+}
 
-	derpShutdown func()
+type testEnvOpt interface {
+	modifyTestEnv(*testEnv)
+}
+
+type configureControl func(*testcontrol.Server)
+
+func (f configureControl) modifyTestEnv(te *testEnv) {
+	f(te.Control)
 }
 
 // newTestEnv starts a bunch of services and returns a new test
 // environment.
 //
 // Call Close to shut everything down.
-func newTestEnv(t testing.TB, bins *Binaries) *testEnv {
+func newTestEnv(t testing.TB, bins *Binaries, opts ...testEnvOpt) *testEnv {
 	if runtime.GOOS == "windows" {
 		t.Skip("not tested/working on Windows yet")
 	}
-	derpMap, derpShutdown := runDERPAndStun(t, logger.Discard)
+	derpMap := RunDERPAndSTUN(t, logger.Discard, "127.0.0.1")
 	logc := new(logCatcher)
 	control := &testcontrol.Server{
 		DERPMap: derpMap,
 	}
+	control.HTTPTestServer = httptest.NewUnstartedServer(control)
 	trafficTrap := new(trafficTrap)
 	e := &testEnv{
 		t:                 t,
@@ -305,12 +333,14 @@ func newTestEnv(t testing.TB, bins *Binaries) *testEnv {
 		LogCatcher:        logc,
 		LogCatcherServer:  httptest.NewServer(logc),
 		Control:           control,
-		ControlServer:     httptest.NewServer(control),
+		ControlServer:     control.HTTPTestServer,
 		TrafficTrap:       trafficTrap,
 		TrafficTrapServer: httptest.NewServer(trafficTrap),
-		derpShutdown:      derpShutdown,
 	}
-	e.Control.BaseURL = e.ControlServer.URL
+	for _, o := range opts {
+		o.modifyTestEnv(e)
+	}
+	control.HTTPTestServer.Start()
 	return e
 }
 
@@ -323,7 +353,6 @@ func (e *testEnv) Close() error {
 	e.LogCatcherServer.Close()
 	e.TrafficTrapServer.Close()
 	e.ControlServer.Close()
-	e.derpShutdown()
 	return nil
 }
 
@@ -336,6 +365,9 @@ type testNode struct {
 	dir       string // temp dir for sock & state
 	sockFile  string
 	stateFile string
+
+	mu        sync.Mutex
+	onLogLine []func([]byte)
 }
 
 // newTestNode allocates a temp directory for a new test node.
@@ -350,6 +382,85 @@ func newTestNode(t *testing.T, env *testEnv) *testNode {
 	}
 }
 
+// addLogLineHook registers a hook f to be called on each tailscaled
+// log line output.
+func (n *testNode) addLogLineHook(f func([]byte)) {
+	n.mu.Lock()
+	defer n.mu.Unlock()
+	n.onLogLine = append(n.onLogLine, f)
+}
+
+// socks5AddrChan returns a channel that receives the address (e.g. "localhost:23874")
+// of the node's SOCKS5 listener, once started.
+func (n *testNode) socks5AddrChan() <-chan string {
+	ch := make(chan string, 1)
+	n.addLogLineHook(func(line []byte) {
+		const sub = "SOCKS5 listening on "
+		i := mem.Index(mem.B(line), mem.S(sub))
+		if i == -1 {
+			return
+		}
+		addr := string(line)[i+len(sub):]
+		select {
+		case ch <- addr:
+		default:
+		}
+	})
+	return ch
+}
+
+func (n *testNode) AwaitSocksAddr(t testing.TB, ch <-chan string) string {
+	t.Helper()
+	timer := time.NewTimer(10 * time.Second)
+	defer timer.Stop()
+	select {
+	case v := <-ch:
+		return v
+	case <-timer.C:
+		t.Fatal("timeout waiting for node to log its SOCK5 listening address")
+		panic("unreachable")
+	}
+}
+
+// nodeOutputParser parses stderr of tailscaled processes, calling the
+// per-line callbacks previously registered via
+// testNode.addLogLineHook.
+type nodeOutputParser struct {
+	buf bytes.Buffer
+	n   *testNode
+}
+
+func (op *nodeOutputParser) Write(p []byte) (n int, err error) {
+	n, err = op.buf.Write(p)
+	op.parseLines()
+	return
+}
+
+func (op *nodeOutputParser) parseLines() {
+	n := op.n
+	buf := op.buf.Bytes()
+	for len(buf) > 0 {
+		nl := bytes.IndexByte(buf, '\n')
+		if nl == -1 {
+			break
+		}
+		line := buf[:nl+1]
+		buf = buf[nl+1:]
+		lineTrim := bytes.TrimSpace(line)
+
+		n.mu.Lock()
+		for _, f := range n.onLogLine {
+			f(lineTrim)
+		}
+		n.mu.Unlock()
+	}
+	if len(buf) == 0 {
+		op.buf.Reset()
+	} else {
+		io.CopyN(ioutil.Discard, &op.buf, int64(op.buf.Len()-len(buf)))
+	}
+}
+
 type Daemon struct {
 	Process *os.Process
 }
@@ -376,12 +487,18 @@ func (n *testNode) StartDaemon(t testing.TB) *Daemon {
 		"--tun=userspace-networking",
 		"--state="+n.stateFile,
 		"--socket="+n.sockFile,
+		"--socks5-server=localhost:0",
 	)
 	cmd.Env = append(os.Environ(),
 		"TS_LOG_TARGET="+n.env.LogCatcherServer.URL,
 		"HTTP_PROXY="+n.env.TrafficTrapServer.URL,
 		"HTTPS_PROXY="+n.env.TrafficTrapServer.URL,
 	)
+	cmd.Stderr = &nodeOutputParser{n: n}
+	if *verboseTailscaled {
+		cmd.Stdout = os.Stdout
+		cmd.Stderr = io.MultiWriter(cmd.Stderr, os.Stderr)
+	}
 	if err := cmd.Start(); err != nil {
 		t.Fatalf("starting tailscaled: %v", err)
 	}
@@ -547,7 +664,7 @@ func (lc *logCatcher) ServeHTTP(w http.ResponseWriter, r *http.Request) {
 	} else {
 		for _, ent := range jreq {
 			fmt.Fprintf(&lc.buf, "%s\n", strings.TrimSpace(ent.Text))
-			if *verbose {
+			if *verboseLogCatcher {
 				fmt.Fprintf(os.Stderr, "%s\n", strings.TrimSpace(ent.Text))
 			}
 		}
@@ -582,51 +699,6 @@ func (tt *trafficTrap) ServeHTTP(w http.ResponseWriter, r *http.Request) {
 	w.WriteHeader(403)
 }
 
-func runDERPAndStun(t testing.TB, logf logger.Logf) (derpMap *tailcfg.DERPMap, cleanup func()) {
-	var serverPrivateKey key.Private
-	if _, err := crand.Read(serverPrivateKey[:]); err != nil {
-		t.Fatal(err)
-	}
-	d := derp.NewServer(serverPrivateKey, logf)
-
-	httpsrv := httptest.NewUnstartedServer(derphttp.Handler(d))
-	httpsrv.Config.ErrorLog = logger.StdLogger(logf)
-	httpsrv.Config.TLSNextProto = make(map[string]func(*http.Server, *tls.Conn, http.Handler))
-	httpsrv.StartTLS()
-
-	stunAddr, stunCleanup := stuntest.ServeWithPacketListener(t, nettype.Std{})
-
-	m := &tailcfg.DERPMap{
-		Regions: map[int]*tailcfg.DERPRegion{
-			1: {
-				RegionID:   1,
-				RegionCode: "test",
-				Nodes: []*tailcfg.DERPNode{
-					{
-						Name:         "t1",
-						RegionID:     1,
-						HostName:     "127.0.0.1", // to bypass HTTP proxy
-						IPv4:         "127.0.0.1",
-						IPv6:         "none",
-						STUNPort:     stunAddr.Port,
-						DERPTestPort: httpsrv.Listener.Addr().(*net.TCPAddr).Port,
-						STUNTestIP:   stunAddr.IP.String(),
-					},
-				},
-			},
-		},
-	}
-
-	cleanup = func() {
-		httpsrv.CloseClientConnections()
-		httpsrv.Close()
-		d.Close()
-		stunCleanup()
-	}
-
-	return m, cleanup
-}
-
 type authURLParserWriter struct {
 	buf bytes.Buffer
 	fn  func(urlStr string) error

tstest/integration/testcontrol/testcontrol.go

@@ -7,6 +7,7 @@ package testcontrol
 
 import (
 	"bytes"
+	"context"
 	crand "crypto/rand"
 	"encoding/binary"
 	"encoding/json"
@@ -17,6 +18,7 @@ import (
 	"log"
 	"math/rand"
 	"net/http"
+	"net/http/httptest"
 	"net/url"
 	"sort"
 	"strings"
@@ -26,7 +28,6 @@ import (
 	"github.com/klauspost/compress/zstd"
 	"golang.org/x/crypto/nacl/box"
 	"inet.af/netaddr"
-	"tailscale.com/derp/derpmap"
 	"tailscale.com/smallzstd"
 	"tailscale.com/tailcfg"
 	"tailscale.com/types/logger"
@@ -39,13 +40,17 @@ type Server struct {
 	Logf        logger.Logf      // nil means to use the log package
 	DERPMap     *tailcfg.DERPMap // nil means to use prod DERP map
 	RequireAuth bool
-	BaseURL     string // must be set to e.g. "http://127.0.0.1:1234" with no trailing URL
 	Verbose     bool
 
+	// ExplicitBaseURL or HTTPTestServer must be set.
+	ExplicitBaseURL string           // e.g. "http://127.0.0.1:1234" with no trailing URL
+	HTTPTestServer  *httptest.Server // if non-nil, used to get BaseURL
+
 	initMuxOnce sync.Once
 	mux         *http.ServeMux
 
 	mu            sync.Mutex
+	cond          *sync.Cond // lazily initialized by condLocked
 	pubKey        wgkey.Key
 	privKey       wgkey.Private
 	nodes         map[tailcfg.NodeKey]*tailcfg.Node
@@ -57,6 +62,20 @@ type Server struct {
 	pingReqsToAdd map[tailcfg.NodeKey]*tailcfg.PingRequest
 }
 
+// BaseURL returns the server's base URL, without trailing slash.
+func (s *Server) BaseURL() string {
+	if e := s.ExplicitBaseURL; e != "" {
+		return e
+	}
+	if hs := s.HTTPTestServer; hs != nil {
+		if hs.URL != "" {
+			return hs.URL
+		}
+		panic("Server.HTTPTestServer not started")
+	}
+	panic("Server ExplicitBaseURL and HTTPTestServer both unset")
+}
+
 // NumNodes returns the number of nodes in the testcontrol server.
 //
 // This is useful when connecting a bunch of virtual machines to a testcontrol
@@ -68,6 +87,47 @@ func (s *Server) NumNodes() int {
 	return len(s.nodes)
 }
 
+// condLocked lazily initializes and returns s.cond.
+// s.mu must be held.
+func (s *Server) condLocked() *sync.Cond {
+	if s.cond == nil {
+		s.cond = sync.NewCond(&s.mu)
+	}
+	return s.cond
+}
+
+// AwaitNodeInMapRequest waits for node k to be stuck in a map poll.
+// It returns an error if and only if the context is done first.
+func (s *Server) AwaitNodeInMapRequest(ctx context.Context, k tailcfg.NodeKey) error {
+	s.mu.Lock()
+	defer s.mu.Unlock()
+	cond := s.condLocked()
+
+	done := make(chan struct{})
+	defer close(done)
+	go func() {
+		select {
+		case <-done:
+		case <-ctx.Done():
+			cond.Broadcast()
+		}
+	}()
+
+	for {
+		node := s.nodeLocked(k)
+		if node == nil {
+			return errors.New("unknown node key")
+		}
+		if _, ok := s.updates[node.ID]; ok {
+			return nil
+		}
+		cond.Wait()
+		if err := ctx.Err(); err != nil {
+			return err
+		}
+	}
+}
+
 // AddPingRequest sends the ping pr to nodeKeyDst. It reports whether it did so. That is,
 // it reports whether nodeKeyDst was connected.
 func (s *Server) AddPingRequest(nodeKeyDst tailcfg.NodeKey, pr *tailcfg.PingRequest) bool {
@@ -85,8 +145,7 @@ func (s *Server) AddPingRequest(nodeKeyDst tailcfg.NodeKey, pr *tailcfg.PingRequ
 	s.pingReqsToAdd[nodeKeyDst] = pr
 	nodeID := node.ID
 	oldUpdatesCh := s.updates[nodeID]
-	sendUpdate(oldUpdatesCh, updateDebugInjection)
-	return true
+	return sendUpdate(oldUpdatesCh, updateDebugInjection)
 }
 
 type AuthPath struct {
@@ -373,7 +432,7 @@ func (s *Server) serveRegister(w http.ResponseWriter, r *http.Request, mkey tail
 		crand.Read(randHex)
 		authPath := fmt.Sprintf("/auth/%x", randHex)
 		s.addAuthPath(authPath, req.NodeKey)
-		authURL = s.BaseURL + authPath
+		authURL = s.BaseURL() + authPath
 	}
 
 	res, err := s.encode(mkey, false, tailcfg.RegisterResponse{
@@ -414,17 +473,19 @@ func (s *Server) updateLocked(source string, peers []tailcfg.NodeID) {
 }
 
 // sendUpdate sends updateType to dst if dst is non-nil and
-// has capacity.
-func sendUpdate(dst chan<- updateType, updateType updateType) {
+// has capacity. It reports whether a value was sent.
+func sendUpdate(dst chan<- updateType, updateType updateType) bool {
 	if dst == nil {
-		return
+		return false
 	}
 	// The dst channel has a buffer size of 1.
 	// If we fail to insert an update into the buffer that
 	// means there is already an update pending.
 	select {
 	case dst <- updateType:
+		return true
 	default:
+		return false
 	}
 }
 
@@ -489,6 +550,7 @@ func (s *Server) serveMap(w http.ResponseWriter, r *http.Request, mkey tailcfg.M
 		sendUpdate(oldUpdatesCh, updateSelfChanged)
 	}
 	s.updateLocked("serveMap", peersToUpdate)
+	s.condLocked().Broadcast()
 	s.mu.Unlock()
 
 	// ReadOnly implies no streaming, as it doesn't
@@ -553,8 +615,6 @@ var keepAliveMsg = &struct {
 	KeepAlive: true,
 }
 
-var prodDERPMap = derpmap.Prod()
-
 // MapResponse generates a MapResponse for a MapRequest.
 //
 // No updates to s are done here.
@@ -564,14 +624,10 @@ func (s *Server) MapResponse(req *tailcfg.MapRequest) (res *tailcfg.MapResponse,
 		// node key rotated away (once test server supports that)
 		return nil, nil
 	}
-	derpMap := s.DERPMap
-	if derpMap == nil {
-		derpMap = prodDERPMap
-	}
 	user, _ := s.getUser(req.NodeKey)
 	res = &tailcfg.MapResponse{
 		Node:            node,
-		DERPMap:         derpMap,
+		DERPMap:         s.DERPMap,
 		Domain:          string(user.Domain),
 		CollectServices: "true",
 		PacketFilter:    tailcfg.FilterAllowAll,

tstest/integration/vms/nixos_test.go

@@ -0,0 +1,216 @@
+// Copyright (c) 2021 Tailscale Inc & AUTHORS All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+// +build linux
+
+package vms
+
+import (
+	"flag"
+	"fmt"
+	"os"
+	"os/exec"
+	"path/filepath"
+	"strings"
+	"testing"
+	"text/template"
+
+	"tailscale.com/tstest/integration"
+	"tailscale.com/types/logger"
+)
+
+var (
+	verboseNixOutput = flag.Bool("verbose-nix-output", false, "if set, use verbose nix output (lots of noise)")
+)
+
+/*
+   NOTE(Xe): Okay, so, at a high level testing NixOS is a lot different than
+   other distros due to NixOS' determinism. Normally NixOS wants packages to
+   be defined in either an overlay, a custom packageOverrides or even
+   yolo-inline as a part of the system configuration. This is going to have
+   us take a different approach compared to other distributions. The overall
+   plan here is as following:
+
+   1. make the binaries as normal
+   2. template in their paths as raw strings to the nixos system module
+   3. run `nixos-generators -f qcow -o $CACHE_DIR/tailscale/nixos/version -c generated-config.nix`
+   4. pass that to the steps that make the virtual machine
+
+   It doesn't really make sense for us to use a premade virtual machine image
+   for this as that will make it harder to deterministically create the image.
+*/
+
+const nixosConfigTemplate = `
+# NOTE(Xe): This template is going to be heavily commented.
+
+# All NixOS modules are functions. Here is the function prelude for this NixOS
+# module that defines the system. It is a function that takes in an attribute
+# set (effectively a map[string]nix.Value) and destructures it to some variables:
+{
+  # other NixOS settings as defined in other modules
+  config,
+
+  # nixpkgs, which is basically the standard library of NixOS
+  pkgs,
+
+  # the path to some system-scoped NixOS modules that aren't imported by default
+  modulesPath,
+
+  # the rest of the arguments don't matter
+  ...
+}:
+
+# Nix's syntax was inspired by Haskell and other functional languages, so the
+# let .. in pattern is used to create scoped variables:
+let
+  # Define the package (derivation) for Tailscale based on the binaries we
+  # just built for this test:
+  testTailscale = pkgs.stdenv.mkDerivation {
+    # The name of the package. This usually includes a version however it
+    # doesn't matter here.
+    name = "tailscale-test";
+
+    # The path on disk to the "source code" of the package, in this case it is
+    # the path to the binaries that are built. This needs to be the raw
+    # unquoted slash-separated path, not a string contaning the path because Nix
+    # has a special path type.
+    src = {{.BinPath}};
+
+    # We only need to worry about the install phase because we've already
+    # built the binaries.
+    phases = "installPhase";
+
+    # We need to wrap tailscaled such that it has iptables in its $PATH.
+    nativeBuildInputs = [ pkgs.makeWrapper ];
+
+    # The install instructions for this package ('' ''defines a multi-line string).
+    # The with statement lets us bring in values into scope as if they were
+    # defined in the current scope.
+    installPhase = with pkgs; ''
+      # This is bash.
+
+      # Make the output folders for the package (systemd unit and binary folders).
+      mkdir -p $out/bin
+
+      # Install tailscale{,d}
+      cp $src/tailscale $out/bin/tailscale
+      cp $src/tailscaled $out/bin/tailscaled
+
+      # Wrap tailscaled with the ip and iptables commands.
+      wrapProgram $out/bin/tailscaled --prefix PATH : ${
+        lib.makeBinPath [ iproute iptables ]
+      }
+
+      # Install systemd unit.
+      cp $src/systemd/tailscaled.service .
+      sed -i -e "s#/usr/sbin#$out/bin#" -e "/^EnvironmentFile/d" ./tailscaled.service
+      install -D -m0444 -t $out/lib/systemd/system ./tailscaled.service
+    '';
+  };
+in {
+  # This is a QEMU VM. This module has a lot of common qemu VM settings so you
+  # don't have to set them manually.
+  imports = [ (modulesPath + "/profiles/qemu-guest.nix") ];
+
+  # We need virtio support to boot.
+  boot.initrd.availableKernelModules =
+    [ "ata_piix" "uhci_hcd" "virtio_pci" "sr_mod" "virtio_blk" ];
+  boot.initrd.kernelModules = [ ];
+  boot.kernelModules = [ ];
+  boot.extraModulePackages = [ ];
+
+  # Curl is needed for one of the steps in cloud-final
+  systemd.services.cloud-final.path = [ pkgs.curl ];
+
+  # yolo, this vm can sudo freely.
+  security.sudo.wheelNeedsPassword = false;
+
+  # Enable cloud-init so we can set VM hostnames and the like the same as other
+  # distros. This will also take care of SSH keys. It's pretty handy.
+  services.cloud-init = {
+    enable = true;
+    ext4.enable = true;
+  };
+
+  # We want sshd running.
+  services.openssh.enable = true;
+
+  # Tailscale settings:
+  services.tailscale = {
+    # We want Tailscale to start at boot.
+    enable = true;
+
+    # Use the Tailscale package we just assembled.
+    package = testTailscale;
+  };
+}`
+
+func copyUnit(t *testing.T, bins *integration.Binaries) {
+	t.Helper()
+
+	data, err := os.ReadFile("../../../cmd/tailscaled/tailscaled.service")
+	if err != nil {
+		t.Fatal(err)
+	}
+	os.MkdirAll(filepath.Join(bins.Dir, "systemd"), 0755)
+	err = os.WriteFile(filepath.Join(bins.Dir, "systemd", "tailscaled.service"), data, 0666)
+	if err != nil {
+		t.Fatal(err)
+	}
+}
+
+func makeNixOSImage(t *testing.T, d Distro, cdir string, bins *integration.Binaries) string {
+	copyUnit(t, bins)
+	dir := t.TempDir()
+	fname := filepath.Join(dir, d.name+".nix")
+	fout, err := os.Create(fname)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	tmpl := template.Must(template.New("base.nix").Parse(nixosConfigTemplate))
+	err = tmpl.Execute(fout, struct{ BinPath string }{BinPath: bins.Dir})
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	err = fout.Close()
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	outpath := filepath.Join(cdir, "nixos")
+	os.MkdirAll(outpath, 0755)
+
+	t.Cleanup(func() {
+		os.RemoveAll(filepath.Join(outpath, d.name)) // makes the disk image a candidate for GC
+	})
+
+	cmd := exec.Command("nixos-generate", "-f", "qcow", "-o", filepath.Join(outpath, d.name), "-c", fname)
+	if *verboseNixOutput {
+		cmd.Stdout = logger.FuncWriter(t.Logf)
+		cmd.Stderr = logger.FuncWriter(t.Logf)
+	} else {
+		fname := fmt.Sprintf("nix-build-%s-%s", os.Getenv("GITHUB_RUN_NUMBER"), strings.Replace(t.Name(), "/", "-", -1))
+		t.Logf("writing nix logs to %s", fname)
+		fout, err := os.Create(fname)
+		if err != nil {
+			t.Fatalf("can't make log file for nix build: %v", err)
+		}
+		cmd.Stdout = fout
+		cmd.Stderr = fout
+		defer fout.Close()
+	}
+	cmd.Env = append(os.Environ(), "NIX_PATH=nixpkgs="+d.url)
+	cmd.Dir = outpath
+	if err := cmd.Run(); err != nil {
+		t.Fatalf("error while making NixOS image for %s: %v", d.name, err)
+	}
+
+	if !*verboseNixOutput {
+		t.Log("done")
+	}
+
+	return filepath.Join(outpath, d.name, "nixos.qcow2")
+}

tstest/integration/vms/runner.nix

@@ -28,6 +28,18 @@
 
       # The C complier so cgo builds work.
       gcc
+
+      # The package manager Nix, just in case.
+      nix
+
+      # Used to generate a NixOS image for testing.
+      nixos-generators
+
+      # Used to extract things.
+      gnutar
+
+      # Used to decompress things.
+      lzma
     ];
 
     # Customize this to include your GitHub username so we can track

tstest/integration/vms/vms_test.go

@@ -43,6 +43,7 @@ import (
 	"tailscale.com/tstest"
 	"tailscale.com/tstest/integration"
 	"tailscale.com/tstest/integration/testcontrol"
+	"tailscale.com/types/logger"
 )
 
 const (
@@ -54,6 +55,7 @@ var (
 	runVMTests = flag.Bool("run-vm-tests", false, "if set, run expensive VM based integration tests")
 	noS3       = flag.Bool("no-s3", false, "if set, always download images from the public internet (risks breaking)")
 	vmRamLimit = flag.Int("ram-limit", 4096, "the maximum number of megabytes of ram that can be used for VMs, must be greater than or equal to 1024")
+	useVNC     = flag.Bool("use-vnc", false, "if set, display guest vms over VNC")
 	distroRex  = func() *regexValue {
 		result := &regexValue{r: regexp.MustCompile(`.*`)}
 		flag.Var(result, "distro-regex", "The regex that matches what distros should be run")
@@ -67,6 +69,7 @@ type Distro struct {
 	sha256sum      string // hex-encoded sha256 sum of contents of URL
 	mem            int    // VM memory in megabytes
 	packageManager string // yum/apt/dnf/zypper
+	initSystem     string // systemd/openrc
 }
 
 func (d *Distro) InstallPre() string {
@@ -85,7 +88,8 @@ func (d *Distro) InstallPre() string {
  - [ apt-get, "-y", install, curl, "apt-transport-https", gnupg2 ]`
 
 	case "apk":
-		return ` - [ apk, "-U", add, curl, "ca-certificates" ]`
+		return ` - [ apk, "-U", add, curl, "ca-certificates", iptables ]
+ - [ modprobe, tun ]`
 	}
 
 	return ""
@@ -96,6 +100,8 @@ func TestDownloadImages(t *testing.T) {
 		t.Skip("not running integration tests (need --run-vm-tests)")
 	}
 
+	bins := integration.BuildTestBinaries(t)
+
 	for _, d := range distros {
 		distro := d
 		t.Run(distro.name, func(t *testing.T) {
@@ -103,16 +109,20 @@ func TestDownloadImages(t *testing.T) {
 				t.Skipf("distro name %q doesn't match regex: %s", distro.name, distroRex)
 			}
 
+			if strings.HasPrefix(distro.name, "nixos") {
+				t.Skip("NixOS is built on the fly, no need to download it")
+			}
+
 			t.Parallel()
 
-			fetchDistro(t, distro)
+			fetchDistro(t, distro, bins)
 		})
 	}
 }
 
 var distros = []Distro{
-	// NOTE(Xe): If you run into issues getting the autoconfig to work, comment
-	// out all the other distros and uncomment this one. Connect with a VNC
+	// NOTE(Xe): If you run into issues getting the autoconfig to work, run
+	// this test with the flag `--distro-regex=alpine-edge`. Connect with a VNC
 	// client with a command like this:
 	//
 	//    $ vncviewer :0
@@ -124,24 +134,47 @@ var distros = []Distro{
 	// Login as root with the password root. Then look in
 	// /var/log/cloud-init-output.log for what you messed up.
 
-	// {"alpine-edge", "https://xena.greedo.xeserv.us/pkg/alpine/img/alpine-edge-2021-05-18-cloud-init-within.qcow2", "b3bb15311c0bd3beffa1b554f022b75d3b7309b5fdf76fb146fe7c72b83b16d0", 256, "apk"},
+	// NOTE(Xe): These images are not official images created by the Alpine Linux
+	// cloud team because the cloud team hasn't created any official images yet.
+	// These images were created under the guidance of the cloud team and contain
+	// few notable differences from what they would end up shipping. The Alpine
+	// Linux cloud team probably won't have official images up until a year or so
+	// after this comment is written (2021-06-11), but overall they will be
+	// compatible with these images. These images were created using the setup in
+	// this repo: https://github.com/Xe/alpine-image. I hereby promise to not break
+	// these links.
+	{"alpine-3-13-5", "https://xena.greedo.xeserv.us/pkg/alpine/img/alpine-3.13.5-cloud-init-within.qcow2", "a2665c16724e75899723e81d81126bd0254a876e5de286b0b21553734baec287", 256, "apk", "openrc"},
+	{"alpine-edge", "https://xena.greedo.xeserv.us/pkg/alpine/img/alpine-edge-2021-05-18-cloud-init-within.qcow2", "b3bb15311c0bd3beffa1b554f022b75d3b7309b5fdf76fb146fe7c72b83b16d0", 256, "apk", "openrc"},
 
-	{"amazon-linux", "https://cdn.amazonlinux.com/os-images/2.0.20210427.0/kvm/amzn2-kvm-2.0.20210427.0-x86_64.xfs.gpt.qcow2", "6ef9daef32cec69b2d0088626ec96410cd24afc504d57278bbf2f2ba2b7e529b", 512, "yum"},
-	{"arch", "https://mirror.pkgbuild.com/images/v20210515.22945/Arch-Linux-x86_64-cloudimg-20210515.22945.qcow2", "e4077f5ba3c5d545478f64834bc4852f9f7a2e05950fce8ecd0df84193162a27", 512, "pacman"},
-	{"centos-7", "https://cloud.centos.org/centos/7/images/CentOS-7-x86_64-GenericCloud-2003.qcow2c", "b7555ecf90b24111f2efbc03c1e80f7b38f1e1fc7e1b15d8fee277d1a4575e87", 512, "yum"},
-	{"centos-8", "https://cloud.centos.org/centos/8/x86_64/images/CentOS-8-GenericCloud-8.3.2011-20201204.2.x86_64.qcow2", "7ec97062618dc0a7ebf211864abf63629da1f325578868579ee70c495bed3ba0", 768, "dnf"},
-	{"debian-9", "http://cloud.debian.org/images/cloud/OpenStack/9.13.22-20210531/debian-9.13.22-20210531-openstack-amd64.qcow2", "c36e25f2ab0b5be722180db42ed9928476812f02d053620e1c287f983e9f6f1d", 512, "apt"},
-	{"debian-10", "https://cdimage.debian.org/images/cloud/buster/20210329-591/debian-10-generic-amd64-20210329-591.qcow2", "70c61956095870c4082103d1a7a1cb5925293f8405fc6cb348588ec97e8611b0", 768, "apt"},
-	{"fedora-34", "https://download.fedoraproject.org/pub/fedora/linux/releases/34/Cloud/x86_64/images/Fedora-Cloud-Base-34-1.2.x86_64.qcow2", "b9b621b26725ba95442d9a56cbaa054784e0779a9522ec6eafff07c6e6f717ea", 768, "dnf"},
-	{"opensuse-leap-15-1", "https://download.opensuse.org/repositories/Cloud:/Images:/Leap_15.1/images/openSUSE-Leap-15.1-OpenStack.x86_64.qcow2", "40bc72b8ee143364fc401f2c9c9a11ecb7341a29fa84c6f7bf42fc94acf19a02", 512, "zypper"},
-	{"opensuse-leap-15-2", "https://download.opensuse.org/repositories/Cloud:/Images:/Leap_15.2/images/openSUSE-Leap-15.2-OpenStack.x86_64.qcow2", "4df9cee9281d1f57d20f79dc65d76e255592b904760e73c0dd44ac753a54330f", 512, "zypper"},
-	{"opensuse-leap-15-3", "http://mirror.its.dal.ca/opensuse/distribution/leap/15.3/appliances/openSUSE-Leap-15.3-JeOS.x86_64-OpenStack-Cloud.qcow2", "22e0392e4d0becb523d1bc5f709366140b7ee20d6faf26de3d0f9046d1ee15d5", 512, "zypper"},
-	{"opensuse-tumbleweed", "https://download.opensuse.org/tumbleweed/appliances/openSUSE-Tumbleweed-JeOS.x86_64-OpenStack-Cloud.qcow2", "79e610bba3ed116556608f031c06e4b9260e3be2b193ce1727914ba213afac3f", 512, "zypper"},
-	{"ubuntu-16-04", "https://cloud-images.ubuntu.com/xenial/20210429/xenial-server-cloudimg-amd64-disk1.img", "50a21bc067c05e0c73bf5d8727ab61152340d93073b3dc32eff18b626f7d813b", 512, "apt"},
-	{"ubuntu-18-04", "https://cloud-images.ubuntu.com/bionic/20210526/bionic-server-cloudimg-amd64.img", "389ffd5d36bbc7a11bf384fd217cda9388ccae20e5b0cb7d4516733623c96022", 512, "apt"},
-	{"ubuntu-20-04", "https://cloud-images.ubuntu.com/focal/20210603/focal-server-cloudimg-amd64.img", "1c0969323b058ba8b91fec245527069c2f0502fc119b9138b213b6bfebd965cb", 512, "apt"},
-	{"ubuntu-20-10", "https://cloud-images.ubuntu.com/groovy/20210604/groovy-server-cloudimg-amd64.img", "2196df5f153faf96443e5502bfdbcaa0baaefbaec614348fec344a241855b0ef", 512, "apt"},
-	{"ubuntu-21-04", "https://cloud-images.ubuntu.com/hirsute/20210603/hirsute-server-cloudimg-amd64.img", "bf07f36fc99ff521d3426e7d257e28f0c81feebc9780b0c4f4e25ae594ff4d3b", 512, "apt"},
+	// NOTE(Xe): All of the following images are official images straight from each
+	// distribution's official documentation.
+	{"amazon-linux", "https://cdn.amazonlinux.com/os-images/2.0.20210427.0/kvm/amzn2-kvm-2.0.20210427.0-x86_64.xfs.gpt.qcow2", "6ef9daef32cec69b2d0088626ec96410cd24afc504d57278bbf2f2ba2b7e529b", 512, "yum", "systemd"},
+	{"arch", "https://mirror.pkgbuild.com/images/v20210515.22945/Arch-Linux-x86_64-cloudimg-20210515.22945.qcow2", "e4077f5ba3c5d545478f64834bc4852f9f7a2e05950fce8ecd0df84193162a27", 512, "pacman", "systemd"},
+	{"centos-7", "https://cloud.centos.org/centos/7/images/CentOS-7-x86_64-GenericCloud-2003.qcow2c", "b7555ecf90b24111f2efbc03c1e80f7b38f1e1fc7e1b15d8fee277d1a4575e87", 512, "yum", "systemd"},
+	{"centos-8", "https://cloud.centos.org/centos/8/x86_64/images/CentOS-8-GenericCloud-8.3.2011-20201204.2.x86_64.qcow2", "7ec97062618dc0a7ebf211864abf63629da1f325578868579ee70c495bed3ba0", 768, "dnf", "systemd"},
+	{"debian-9", "http://cloud.debian.org/images/cloud/OpenStack/9.13.22-20210531/debian-9.13.22-20210531-openstack-amd64.qcow2", "c36e25f2ab0b5be722180db42ed9928476812f02d053620e1c287f983e9f6f1d", 512, "apt", "systemd"},
+	{"debian-10", "https://cdimage.debian.org/images/cloud/buster/20210329-591/debian-10-generic-amd64-20210329-591.qcow2", "70c61956095870c4082103d1a7a1cb5925293f8405fc6cb348588ec97e8611b0", 768, "apt", "systemd"},
+	{"fedora-34", "https://download.fedoraproject.org/pub/fedora/linux/releases/34/Cloud/x86_64/images/Fedora-Cloud-Base-34-1.2.x86_64.qcow2", "b9b621b26725ba95442d9a56cbaa054784e0779a9522ec6eafff07c6e6f717ea", 768, "dnf", "systemd"},
+	{"opensuse-leap-15-1", "https://download.opensuse.org/repositories/Cloud:/Images:/Leap_15.1/images/openSUSE-Leap-15.1-OpenStack.x86_64.qcow2", "40bc72b8ee143364fc401f2c9c9a11ecb7341a29fa84c6f7bf42fc94acf19a02", 512, "zypper", "systemd"},
+	{"opensuse-leap-15-2", "https://download.opensuse.org/repositories/Cloud:/Images:/Leap_15.2/images/openSUSE-Leap-15.2-OpenStack.x86_64.qcow2", "4df9cee9281d1f57d20f79dc65d76e255592b904760e73c0dd44ac753a54330f", 512, "zypper", "systemd"},
+	{"opensuse-leap-15-3", "http://mirror.its.dal.ca/opensuse/distribution/leap/15.3/appliances/openSUSE-Leap-15.3-JeOS.x86_64-OpenStack-Cloud.qcow2", "22e0392e4d0becb523d1bc5f709366140b7ee20d6faf26de3d0f9046d1ee15d5", 512, "zypper", "systemd"},
+	{"opensuse-tumbleweed", "https://download.opensuse.org/tumbleweed/appliances/openSUSE-Tumbleweed-JeOS.x86_64-OpenStack-Cloud.qcow2", "79e610bba3ed116556608f031c06e4b9260e3be2b193ce1727914ba213afac3f", 512, "zypper", "systemd"},
+	{"ubuntu-16-04", "https://cloud-images.ubuntu.com/xenial/20210429/xenial-server-cloudimg-amd64-disk1.img", "50a21bc067c05e0c73bf5d8727ab61152340d93073b3dc32eff18b626f7d813b", 512, "apt", "systemd"},
+	{"ubuntu-18-04", "https://cloud-images.ubuntu.com/bionic/20210526/bionic-server-cloudimg-amd64.img", "389ffd5d36bbc7a11bf384fd217cda9388ccae20e5b0cb7d4516733623c96022", 512, "apt", "systemd"},
+	{"ubuntu-20-04", "https://cloud-images.ubuntu.com/focal/20210603/focal-server-cloudimg-amd64.img", "1c0969323b058ba8b91fec245527069c2f0502fc119b9138b213b6bfebd965cb", 512, "apt", "systemd"},
+	{"ubuntu-20-10", "https://cloud-images.ubuntu.com/groovy/20210604/groovy-server-cloudimg-amd64.img", "2196df5f153faf96443e5502bfdbcaa0baaefbaec614348fec344a241855b0ef", 512, "apt", "systemd"},
+	{"ubuntu-21-04", "https://cloud-images.ubuntu.com/hirsute/20210603/hirsute-server-cloudimg-amd64.img", "bf07f36fc99ff521d3426e7d257e28f0c81feebc9780b0c4f4e25ae594ff4d3b", 512, "apt", "systemd"},
+
+	// NOTE(Xe): We build fresh NixOS images for every test run, so the URL being
+	// used here is actually the URL of the NixOS channel being built from and the
+	// shasum is meaningless. This `channel:name` syntax is documented at [1].
+	//
+	// [1]: https://nixos.org/manual/nix/unstable/command-ref/env-common.html
+	{"nixos-21-05", "channel:nixos-21.05", "lolfakesha", 512, "nix", "systemd"},
+
+	// // NOTE(Xe): disabled until https://github.com/NixOS/nixpkgs/issues/128783
+	// // is fixed.
+	// {"nixos-unstable", "channel:nixos-unstable", "lolfakesha", 512, "nix", "systemd"},
 }
 
 // fetchFromS3 fetches a distribution image from Amazon S3 or reports whether
@@ -196,7 +229,7 @@ func fetchFromS3(t *testing.T, fout *os.File, d Distro) bool {
 
 // fetchDistro fetches a distribution from the internet if it doesn't already exist locally. It
 // also validates the sha256 sum from a known good hash.
-func fetchDistro(t *testing.T, resultDistro Distro) {
+func fetchDistro(t *testing.T, resultDistro Distro, bins *integration.Binaries) string {
 	t.Helper()
 
 	cdir, err := os.UserCacheDir()
@@ -205,6 +238,10 @@ func fetchDistro(t *testing.T, resultDistro Distro) {
 	}
 	cdir = filepath.Join(cdir, "tailscale", "vm-test")
 
+	if strings.HasPrefix(resultDistro.name, "nixos") {
+		return makeNixOSImage(t, resultDistro, cdir, bins)
+	}
+
 	qcowPath := filepath.Join(cdir, "qcow2", resultDistro.sha256sum)
 
 	_, err = os.Stat(qcowPath)
@@ -251,6 +288,8 @@ func fetchDistro(t *testing.T, resultDistro Distro) {
 			}
 		}
 	}
+
+	return qcowPath
 }
 
 func checkCachedImageHash(t *testing.T, d Distro, cacheDir string) (gotHash string) {
@@ -285,8 +324,8 @@ func run(t *testing.T, dir, prog string, args ...string) {
 	tstest.FixLogs(t)
 
 	cmd := exec.Command(prog, args...)
-	cmd.Stdout = log.Writer()
-	cmd.Stderr = log.Writer()
+	cmd.Stdout = logger.FuncWriter(t.Logf)
+	cmd.Stderr = logger.FuncWriter(t.Logf)
 	cmd.Dir = dir
 	if err := cmd.Run(); err != nil {
 		t.Fatal(err)
@@ -295,18 +334,12 @@ func run(t *testing.T, dir, prog string, args ...string) {
 
 // mkLayeredQcow makes a layered qcow image that allows us to keep the upstream
 // VM images pristine and only do our changes on an overlay.
-func mkLayeredQcow(t *testing.T, tdir string, d Distro) {
+func mkLayeredQcow(t *testing.T, tdir string, d Distro, qcowBase string) {
 	t.Helper()
 
-	cdir, err := os.UserCacheDir()
-	if err != nil {
-		t.Fatalf("can't find cache dir: %v", err)
-	}
-	cdir = filepath.Join(cdir, "tailscale", "vm-test")
-
 	run(t, tdir, "qemu-img", "create",
 		"-f", "qcow2",
-		"-o", "backing_file="+filepath.Join(cdir, "qcow2", d.sha256sum),
+		"-o", "backing_file="+qcowBase,
 		filepath.Join(tdir, d.name+".qcow2"),
 	)
 }
@@ -397,7 +430,7 @@ func mkSeed(t *testing.T, d Distro, sshKey, hostURL, tdir string, port int) {
 // mkVM makes a KVM-accelerated virtual machine and prepares it for introduction
 // to the testcontrol server. The function it returns is for killing the virtual
 // machine when it is time for it to die.
-func mkVM(t *testing.T, n int, d Distro, sshKey, hostURL, tdir string) func() {
+func mkVM(t *testing.T, n int, d Distro, sshKey, hostURL, tdir string, bins *integration.Binaries) {
 	t.Helper()
 
 	cdir, err := os.UserCacheDir()
@@ -407,10 +440,12 @@ func mkVM(t *testing.T, n int, d Distro, sshKey, hostURL, tdir string) func() {
 	cdir = filepath.Join(cdir, "tailscale", "vm-test")
 	os.MkdirAll(filepath.Join(cdir, "qcow2"), 0755)
 
-	port := 23100 + n
+	port, err := getProbablyFreePortNumber()
+	if err != nil {
+		t.Fatal(err)
+	}
 
-	fetchDistro(t, d)
-	mkLayeredQcow(t, tdir, d)
+	mkLayeredQcow(t, tdir, d, fetchDistro(t, d, bins))
 	mkSeed(t, d, sshKey, hostURL, tdir, port)
 
 	driveArg := fmt.Sprintf("file=%s,if=virtio", filepath.Join(tdir, d.name+".qcow2"))
@@ -423,15 +458,26 @@ func mkVM(t *testing.T, n int, d Distro, sshKey, hostURL, tdir string) func() {
 		"-boot", "c",
 		"-drive", driveArg,
 		"-cdrom", filepath.Join(tdir, d.name, "seed", "seed.iso"),
-		"-vnc", fmt.Sprintf(":%d", n),
 		"-smbios", "type=1,serial=ds=nocloud;h=" + d.name,
 	}
 
+	if *useVNC {
+		// test listening on VNC port
+		ln, err := net.Listen("tcp", net.JoinHostPort("0.0.0.0", strconv.Itoa(5900+n)))
+		if err != nil {
+			t.Fatalf("would not be able to listen on the VNC port for the VM: %v", err)
+		}
+		ln.Close()
+		args = append(args, "-vnc", fmt.Sprintf(":%d", n))
+	} else {
+		args = append(args, "-display", "none")
+	}
+
 	t.Logf("running: qemu-system-x86_64 %s", strings.Join(args, " "))
 
 	cmd := exec.Command("qemu-system-x86_64", args...)
-	cmd.Stdout = log.Writer()
-	cmd.Stderr = log.Writer()
+	cmd.Stdout = logger.FuncWriter(t.Logf)
+	cmd.Stderr = logger.FuncWriter(t.Logf)
 	err = cmd.Start()
 
 	if err != nil {
@@ -440,18 +486,21 @@ func mkVM(t *testing.T, n int, d Distro, sshKey, hostURL, tdir string) func() {
 
 	time.Sleep(time.Second)
 
+	// NOTE(Xe): In Unix if you do a kill with signal number 0, the kernel will do
+	// all of the access checking for the process (existence, permissions, etc) but
+	// nothing else. This is a way to ensure that qemu's process is active.
 	if err := cmd.Process.Signal(syscall.Signal(0)); err != nil {
-		t.Fatal("qemu is not running")
+		t.Fatalf("qemu is not running: %v", err)
 	}
 
-	return func() {
+	t.Cleanup(func() {
 		err := cmd.Process.Kill()
 		if err != nil {
 			t.Errorf("can't kill %s (%d): %v", d.name, cmd.Process.Pid, err)
 		}
 
 		cmd.Wait()
-	}
+	})
 }
 
 // ipMapping maps a hostname, SSH port and SSH IP together
@@ -461,6 +510,34 @@ type ipMapping struct {
 	ip   string
 }
 
+// getProbablyFreePortNumber does what it says on the tin, but as a side effect
+// it is a kind of racy function. Do not use this carelessly.
+//
+// This is racy because it does not "lock" the port number with the OS. The
+// "random" port number that is returned here is most likely free to use, however
+// it is difficult to be 100% sure. This function should be used with care. It
+// will probably do what you want, but it is very easy to hold this wrong.
+func getProbablyFreePortNumber() (int, error) {
+	l, err := net.Listen("tcp", ":0")
+	if err != nil {
+		return 0, err
+	}
+
+	defer l.Close()
+
+	_, port, err := net.SplitHostPort(l.Addr().String())
+	if err != nil {
+		return 0, err
+	}
+
+	portNum, err := strconv.Atoi(port)
+	if err != nil {
+		return 0, err
+	}
+
+	return portNum, nil
+}
+
 // TestVMIntegrationEndToEnd creates a virtual machine with qemu, installs
 // tailscale on it and then ensures that it connects to the network
 // successfully.
@@ -485,7 +562,8 @@ func TestVMIntegrationEndToEnd(t *testing.T) {
 
 	rex := distroRex.Unwrap()
 
-	ln, err := net.Listen("tcp", deriveBindhost(t)+":0")
+	bindHost := deriveBindhost(t)
+	ln, err := net.Listen("tcp", net.JoinHostPort(bindHost, "0"))
 	if err != nil {
 		t.Fatalf("can't make TCP listener: %v", err)
 	}
@@ -494,6 +572,9 @@ func TestVMIntegrationEndToEnd(t *testing.T) {
 
 	cs := &testcontrol.Server{}
 
+	derpMap := integration.RunDERPAndSTUN(t, t.Logf, bindHost)
+	cs.DERPMap = derpMap
+
 	var (
 		ipMu  sync.Mutex
 		ipMap = map[string]ipMapping{}
@@ -543,12 +624,11 @@ func TestVMIntegrationEndToEnd(t *testing.T) {
 	loginServer := fmt.Sprintf("http://%s", ln.Addr())
 	t.Logf("loginServer: %s", loginServer)
 
-	tstest.FixLogs(t)
-	defer tstest.UnfixLogs(t)
-
 	ramsem := semaphore.NewWeighted(int64(*vmRamLimit))
 	bins := integration.BuildTestBinaries(t)
 
+	makeTestNode(t, bins, loginServer)
+
 	t.Run("do", func(t *testing.T) {
 		for n, distro := range distros {
 			n, distro := n, distro
@@ -560,18 +640,19 @@ func TestVMIntegrationEndToEnd(t *testing.T) {
 
 			t.Run(distro.name, func(t *testing.T) {
 				ctx, done := context.WithCancel(context.Background())
-				defer done()
+				t.Cleanup(done)
 
 				t.Parallel()
 
+				dir := t.TempDir()
+
 				err := ramsem.Acquire(ctx, int64(distro.mem))
 				if err != nil {
 					t.Fatalf("can't acquire ram semaphore: %v", err)
 				}
 				defer ramsem.Release(int64(distro.mem))
 
-				cancel := mkVM(t, n, distro, string(pubkey), loginServer, dir)
-				defer cancel()
+				mkVM(t, n, distro, string(pubkey), loginServer, dir, bins)
 				var ipm ipMapping
 
 				t.Run("wait-for-start", func(t *testing.T) {
@@ -589,13 +670,13 @@ func TestVMIntegrationEndToEnd(t *testing.T) {
 					}
 				})
 
-				testDistro(t, loginServer, signer, ipm, bins)
+				testDistro(t, loginServer, distro, signer, ipm, bins)
 			})
 		}
 	})
 }
 
-func testDistro(t *testing.T, loginServer string, signer ssh.Signer, ipm ipMapping, bins *integration.Binaries) {
+func testDistro(t *testing.T, loginServer string, d Distro, signer ssh.Signer, ipm ipMapping, bins *integration.Binaries) {
 	t.Helper()
 	port := ipm.port
 	hostport := fmt.Sprintf("127.0.0.1:%d", port)
@@ -631,11 +712,18 @@ func testDistro(t *testing.T, loginServer string, signer ssh.Signer, ipm ipMappi
 	if err != nil {
 		t.Fatal(err)
 	}
-	copyBinaries(t, cli, bins)
+	copyBinaries(t, d, cli, bins)
 
-	timeout := 5 * time.Minute
+	timeout := 30 * time.Second
 
-	e, _, err := expect.SpawnSSH(cli, timeout, expect.Verbose(true), expect.VerboseWriter(log.Writer()))
+	e, _, err := expect.SpawnSSH(cli, timeout,
+		expect.Verbose(true),
+		expect.VerboseWriter(logger.FuncWriter(t.Logf)),
+
+		// // NOTE(Xe): if you get a timeout, uncomment this region to have the raw
+		// // output be sent to the test log quicker.
+		// expect.Tee(nopWriteCloser{logger.FuncWriter(t.Logf)}),
+	)
 	if err != nil {
 		t.Fatalf("%d: can't register a shell session: %v", port, err)
 	}
@@ -643,30 +731,57 @@ func testDistro(t *testing.T, loginServer string, signer ssh.Signer, ipm ipMappi
 
 	t.Log("opened session")
 
-	_, _, err = e.Expect(regexp.MustCompile(`(\#)`), timeout)
-	if err != nil {
-		t.Fatalf("%d: can't get a shell: %v", port, err)
+	var batch = []expect.Batcher{
+		&expect.BSnd{S: "PS1='# '\n"},
+		&expect.BExp{R: `(\#)`},
 	}
-	t.Logf("got shell for %d", port)
-	err = e.Send("systemctl start tailscaled.service\n")
-	if err != nil {
-		t.Fatalf("can't send command to start tailscaled: %v", err)
+
+	switch d.initSystem {
+	case "openrc":
+		// NOTE(Xe): this is a sin, however openrc doesn't really have the concept
+		// of service readiness. If this sleep is removed then tailscale will not be
+		// ready once the `tailscale up` command is sent. This is not ideal, but I
+		// am not really sure there is a good way around this without a delay of
+		// some kind.
+		batch = append(batch, &expect.BSnd{S: "rc-service tailscaled start && sleep 2\n"})
+	case "systemd":
+		batch = append(batch, &expect.BSnd{S: "systemctl start tailscaled.service\n"})
 	}
-	_, _, err = e.Expect(regexp.MustCompile(`(\#)`), timeout)
-	if err != nil {
-		t.Fatalf("%d: can't get a shell: %v", port, err)
-	}
-	err = e.Send(fmt.Sprintf("sudo tailscale up --login-server %s\n", loginServer))
-	if err != nil {
-		t.Fatalf("%d: can't send tailscale up command: %v", port, err)
-	}
-	_, _, err = e.Expect(regexp.MustCompile(`Success.`), timeout)
+
+	batch = append(batch,
+		&expect.BExp{R: `(\#)`},
+		&expect.BSnd{S: fmt.Sprintf("tailscale up --login-server=%s\n", loginServer)},
+		&expect.BExp{R: `Success.`},
+		&expect.BSnd{S: "sleep 5 && tailscale status\n"},
+		&expect.BExp{R: `100.64.0.1`},
+		&expect.BExp{R: `(\#)`},
+		&expect.BSnd{S: "tailscale ping -c 1 100.64.0.1\n"},
+		&expect.BExp{R: `pong from.*\(100.64.0.1\)`},
+		&expect.BSnd{S: "ping -c 1 100.64.0.1\n"},
+		&expect.BExp{R: `bytes`},
+	)
+
+	_, err = e.ExpectBatch(batch, timeout)
 	if err != nil {
+		sess, terr := cli.NewSession()
+		if terr != nil {
+			t.Fatalf("can't dump tailscaled logs on failed test: %v", err)
+		}
+		sess.Stdout = logger.FuncWriter(t.Logf)
+		sess.Stderr = logger.FuncWriter(t.Logf)
+		terr = sess.Run("journalctl -u tailscaled")
+		if terr != nil {
+			t.Fatalf("can't dump tailscaled logs on failed test: %v", err)
+		}
 		t.Fatalf("not successful: %v", err)
 	}
 }
 
-func copyBinaries(t *testing.T, conn *ssh.Client, bins *integration.Binaries) {
+func copyBinaries(t *testing.T, d Distro, conn *ssh.Client, bins *integration.Binaries) {
+	if strings.HasPrefix(d.name, "nixos") {
+		return
+	}
+
 	cli, err := sftp.NewClient(conn)
 	if err != nil {
 		t.Fatalf("can't connect over sftp to copy binaries: %v", err)
@@ -674,16 +789,23 @@ func copyBinaries(t *testing.T, conn *ssh.Client, bins *integration.Binaries) {
 
 	mkdir(t, cli, "/usr/bin")
 	mkdir(t, cli, "/usr/sbin")
-	mkdir(t, cli, "/etc/systemd/system")
 	mkdir(t, cli, "/etc/default")
+	mkdir(t, cli, "/var/lib/tailscale")
 
 	copyFile(t, cli, bins.Daemon, "/usr/sbin/tailscaled")
 	copyFile(t, cli, bins.CLI, "/usr/bin/tailscale")
 
-	// TODO(Xe): revisit this life decision, hopefully before this assumption
-	// breaks the test.
+	// TODO(Xe): revisit this assumption before it breaks the test.
 	copyFile(t, cli, "../../../cmd/tailscaled/tailscaled.defaults", "/etc/default/tailscaled")
-	copyFile(t, cli, "../../../cmd/tailscaled/tailscaled.service", "/etc/systemd/system/tailscaled.service")
+
+	switch d.initSystem {
+	case "openrc":
+		mkdir(t, cli, "/etc/init.d")
+		copyFile(t, cli, "../../../cmd/tailscaled/tailscaled.openrc", "/etc/init.d/tailscaled")
+	case "systemd":
+		mkdir(t, cli, "/etc/systemd/system")
+		copyFile(t, cli, "../../../cmd/tailscaled/tailscaled.service", "/etc/systemd/system/tailscaled.service")
+	}
 
 	t.Log("tailscale installed!")
 }
@@ -767,6 +889,62 @@ func TestDeriveBindhost(t *testing.T) {
 	t.Log(deriveBindhost(t))
 }
 
+func makeTestNode(t *testing.T, bins *integration.Binaries, controlURL string) {
+	dir := t.TempDir()
+	cmd := exec.Command(
+		bins.Daemon,
+		"--tun=userspace-networking",
+		"--state="+filepath.Join(dir, "state.json"),
+		"--socket="+filepath.Join(dir, "sock"),
+		"--socks5-server=localhost:0",
+	)
+
+	cmd.Env = append(os.Environ(), "NOTIFY_SOCKET="+filepath.Join(dir, "notify_socket"))
+
+	err := cmd.Start()
+	if err != nil {
+		t.Fatalf("can't start tailscaled: %v", err)
+	}
+
+	t.Cleanup(func() {
+		cmd.Process.Kill()
+	})
+
+	ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
+	defer cancel()
+	ticker := time.NewTicker(100 * time.Millisecond)
+
+outer:
+	for {
+		select {
+		case <-ctx.Done():
+			t.Fatal("timed out waiting for tailscaled to come up")
+			return
+		case <-ticker.C:
+			conn, err := net.Dial("unix", filepath.Join(dir, "sock"))
+			if err != nil {
+				continue
+			}
+
+			conn.Close()
+			break outer
+		}
+	}
+
+	run(t, dir, bins.CLI,
+		"--socket="+filepath.Join(dir, "sock"),
+		"up",
+		"--login-server="+controlURL,
+		"--hostname=tester",
+	)
+}
+
+type nopWriteCloser struct {
+	io.Writer
+}
+
+func (nwc nopWriteCloser) Close() error { return nil }
+
 const metaDataTemplate = `instance-id: {{.ID}}
 local-hostname: {{.Hostname}}`
 

tstest/natlab/natlab.go

@@ -2,9 +2,6 @@
 // Use of this source code is governed by a BSD-style
 // license that can be found in the LICENSE file.
 
-//lint:file-ignore U1000 in development
-//lint:file-ignore S1000 in development
-
 // Package natlab lets us simulate different types of networks all
 // in-memory without running VMs or requiring root, etc. Despite the
 // name, it does more than just NATs. But NATs are the most

tstest/resource.go

@@ -15,19 +15,19 @@ import (
 )
 
 func ResourceCheck(tb testing.TB) {
+	tb.Helper()
 	startN, startStacks := goroutines()
 	tb.Cleanup(func() {
 		if tb.Failed() {
 			// Something else went wrong.
 			return
 		}
-		tb.Helper()
 		// Goroutines might be still exiting.
 		for i := 0; i < 100; i++ {
 			if runtime.NumGoroutine() <= startN {
 				return
 			}
-			time.Sleep(1 * time.Millisecond)
+			time.Sleep(5 * time.Millisecond)
 		}
 		endN, endStacks := goroutines()
 		tb.Logf("goroutine diff:\n%v\n", cmp.Diff(startStacks, endStacks))

tsweb/debug.go

@@ -0,0 +1,136 @@
+// Copyright (c) 2021 Tailscale Inc & AUTHORS All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package tsweb
+
+import (
+	"expvar"
+	"fmt"
+	"html"
+	"io"
+	"net/http"
+	"net/http/pprof"
+	"net/url"
+	"os"
+	"runtime"
+
+	"tailscale.com/version"
+)
+
+// DebugHandler is an http.Handler that serves a debugging "homepage",
+// and provides helpers to register more debug endpoints and reports.
+//
+// The rendered page consists of three sections: informational
+// key/value pairs, links to other pages, and additional
+// program-specific HTML. Callers can add to these sections using the
+// KV, URL and Section helpers respectively.
+//
+// Additionally, the Handle method offers a shorthand for correctly
+// registering debug handlers and cross-linking them from /debug/.
+type DebugHandler struct {
+	mux      *http.ServeMux                   // where this handler is registered
+	kvs      []func(io.Writer)                // output one <li>...</li> each, see KV()
+	urls     []string                         // one <li>...</li> block with link each
+	sections []func(io.Writer, *http.Request) // invoked in registration order prior to outputting </body>
+}
+
+// Debugger returns the DebugHandler registered on mux at /debug/,
+// creating it if necessary.
+func Debugger(mux *http.ServeMux) *DebugHandler {
+	h, pat := mux.Handler(&http.Request{URL: &url.URL{Path: "/debug/"}})
+	if d, ok := h.(*DebugHandler); ok && pat == "/debug/" {
+		return d
+	}
+	ret := &DebugHandler{
+		mux: mux,
+	}
+	mux.Handle("/debug/", ret)
+
+	ret.KVFunc("Uptime", func() interface{} { return Uptime() })
+	ret.KV("Version", version.Long)
+	ret.Handle("vars", "Metrics (Go)", expvar.Handler())
+	ret.Handle("varz", "Metrics (Prometheus)", http.HandlerFunc(VarzHandler))
+	ret.Handle("pprof/", "pprof", http.HandlerFunc(pprof.Index))
+	ret.URL("/debug/pprof/goroutine?debug=1", "Goroutines (collapsed)")
+	ret.URL("/debug/pprof/goroutine?debug=2", "Goroutines (full)")
+	ret.Handle("gc", "force GC", http.HandlerFunc(gcHandler))
+	hostname, err := os.Hostname()
+	if err == nil {
+		ret.KV("Machine", hostname)
+	}
+	return ret
+}
+
+// ServeHTTP implements http.Handler.
+func (d *DebugHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
+	if !AllowDebugAccess(r) {
+		http.Error(w, "debug access denied", http.StatusForbidden)
+		return
+	}
+	if r.URL.Path != "/debug/" {
+		// Sub-handlers are handled by the parent mux directly.
+		http.NotFound(w, r)
+		return
+	}
+
+	f := func(format string, args ...interface{}) { fmt.Fprintf(w, format, args...) }
+	f("<html><body><h1>%s debug</h1><ul>", version.CmdName())
+	for _, kv := range d.kvs {
+		kv(w)
+	}
+	for _, url := range d.urls {
+		io.WriteString(w, url)
+	}
+	for _, section := range d.sections {
+		section(w, r)
+	}
+}
+
+// Handle registers handler at /debug/<slug> and creates a descriptive
+// entry in /debug/ for it.
+func (d *DebugHandler) Handle(slug, desc string, handler http.Handler) {
+	href := "/debug/" + slug
+	d.mux.Handle(href, Protected(handler))
+	d.URL(href, desc)
+}
+
+// KV adds a key/value list item to /debug/.
+func (d *DebugHandler) KV(k string, v interface{}) {
+	val := html.EscapeString(fmt.Sprintf("%v", v))
+	d.kvs = append(d.kvs, func(w io.Writer) {
+		fmt.Fprintf(w, "<li><b>%s:</b> %s</li>", k, val)
+	})
+}
+
+// KVFunc adds a key/value list item to /debug/. v is called on every
+// render of /debug/.
+func (d *DebugHandler) KVFunc(k string, v func() interface{}) {
+	d.kvs = append(d.kvs, func(w io.Writer) {
+		val := html.EscapeString(fmt.Sprintf("%v", v()))
+		fmt.Fprintf(w, "<li><b>%s:</b> %s</li>", k, val)
+	})
+}
+
+// URL adds a URL and description list item to /debug/.
+func (d *DebugHandler) URL(url, desc string) {
+	if desc != "" {
+		desc = " (" + desc + ")"
+	}
+	d.urls = append(d.urls, fmt.Sprintf(`<li><a href="%s">%s</a>%s</li>`, url, url, html.EscapeString(desc)))
+}
+
+// Section invokes f on every render of /debug/ to add supplemental
+// HTML to the page body.
+func (d *DebugHandler) Section(f func(w io.Writer, r *http.Request)) {
+	d.sections = append(d.sections, f)
+}
+
+func gcHandler(w http.ResponseWriter, r *http.Request) {
+	w.Write([]byte("running GC...\n"))
+	if f, ok := w.(http.Flusher); ok {
+		f.Flush()
+	}
+	runtime.GC()
+	w.Write([]byte("Done.\n"))
+}

tsweb/debug_test.go

@@ -0,0 +1,189 @@
+// Copyright (c) 2021 Tailscale Inc & AUTHORS All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package tsweb
+
+import (
+	"fmt"
+	"io"
+	"net/http"
+	"net/http/httptest"
+	"strings"
+	"testing"
+)
+
+func TestDebugger(t *testing.T) {
+	mux := http.NewServeMux()
+
+	dbg1 := Debugger(mux)
+	if dbg1 == nil {
+		t.Fatal("didn't get a debugger from mux")
+	}
+
+	dbg2 := Debugger(mux)
+	if dbg2 != dbg1 {
+		t.Fatal("Debugger returned different debuggers for the same mux")
+	}
+}
+
+func get(m http.Handler, path, srcIP string) (int, string) {
+	req := httptest.NewRequest("GET", path, nil)
+	req.RemoteAddr = srcIP + ":1234"
+	rec := httptest.NewRecorder()
+	m.ServeHTTP(rec, req)
+	return rec.Result().StatusCode, rec.Body.String()
+}
+
+const (
+	tsIP  = "100.100.100.100"
+	pubIP = "8.8.8.8"
+)
+
+func TestDebuggerKV(t *testing.T) {
+	mux := http.NewServeMux()
+	dbg := Debugger(mux)
+	dbg.KV("Donuts", 42)
+	dbg.KV("Secret code", "hunter2")
+	val := "red"
+	dbg.KVFunc("Condition", func() interface{} { return val })
+
+	code, _ := get(mux, "/debug/", pubIP)
+	if code != 403 {
+		t.Fatalf("debug access wasn't denied, got %v", code)
+	}
+
+	code, body := get(mux, "/debug/", tsIP)
+	if code != 200 {
+		t.Fatalf("debug access failed, got %v", code)
+	}
+	for _, want := range []string{"Donuts", "42", "Secret code", "hunter2", "Condition", "red"} {
+		if !strings.Contains(body, want) {
+			t.Errorf("want %q in output, not found", want)
+		}
+	}
+
+	val = "green"
+	code, body = get(mux, "/debug/", tsIP)
+	if code != 200 {
+		t.Fatalf("debug access failed, got %v", code)
+	}
+	for _, want := range []string{"Condition", "green"} {
+		if !strings.Contains(body, want) {
+			t.Errorf("want %q in output, not found", want)
+		}
+	}
+}
+
+func TestDebuggerURL(t *testing.T) {
+	mux := http.NewServeMux()
+	dbg := Debugger(mux)
+	dbg.URL("https://www.tailscale.com", "Homepage")
+
+	code, body := get(mux, "/debug/", tsIP)
+	if code != 200 {
+		t.Fatalf("debug access failed, got %v", code)
+	}
+	for _, want := range []string{"https://www.tailscale.com", "Homepage"} {
+		if !strings.Contains(body, want) {
+			t.Errorf("want %q in output, not found", want)
+		}
+	}
+}
+
+func TestDebuggerSection(t *testing.T) {
+	mux := http.NewServeMux()
+	dbg := Debugger(mux)
+	dbg.Section(func(w io.Writer, r *http.Request) {
+		fmt.Fprintf(w, "Test output %v", r.RemoteAddr)
+	})
+
+	code, body := get(mux, "/debug/", tsIP)
+	if code != 200 {
+		t.Fatalf("debug access failed, got %v", code)
+	}
+	want := `Test output 100.100.100.100:1234`
+	if !strings.Contains(body, want) {
+		t.Errorf("want %q in output, not found", want)
+	}
+}
+
+func TestDebuggerHandle(t *testing.T) {
+	mux := http.NewServeMux()
+	dbg := Debugger(mux)
+	dbg.Handle("check", "Consistency check", http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		fmt.Fprintf(w, "Test output %v", r.RemoteAddr)
+	}))
+
+	code, body := get(mux, "/debug/", tsIP)
+	if code != 200 {
+		t.Fatalf("debug access failed, got %v", code)
+	}
+	for _, want := range []string{"/debug/check", "Consistency check"} {
+		if !strings.Contains(body, want) {
+			t.Errorf("want %q in output, not found", want)
+		}
+	}
+
+	code, _ = get(mux, "/debug/check", pubIP)
+	if code != 403 {
+		t.Fatal("/debug/check should be protected, but isn't")
+	}
+
+	code, body = get(mux, "/debug/check", tsIP)
+	if code != 200 {
+		t.Fatal("/debug/check denied debug access")
+	}
+	want := "Test output " + tsIP
+	if !strings.Contains(body, want) {
+		t.Errorf("want %q in output, not found", want)
+	}
+}
+
+func ExampleDebugHandler_Handle() {
+	mux := http.NewServeMux()
+	dbg := Debugger(mux)
+	// Registers /debug/flushcache with the given handler, and adds a
+	// link to /debug/ with the description "Flush caches".
+	dbg.Handle("flushcache", "Flush caches", http.HandlerFunc(http.NotFound))
+}
+
+func ExampleDebugHandler_KV() {
+	mux := http.NewServeMux()
+	dbg := Debugger(mux)
+	// Adds two list items to /debug/, showing that the condition is
+	// red and there are 42 donuts.
+	dbg.KV("Conditon", "red")
+	dbg.KV("Donuts", 42)
+}
+
+func ExampleDebugHandler_KVFunc() {
+	mux := http.NewServeMux()
+	dbg := Debugger(mux)
+	// Adds an count of page renders to /debug/. Note this example
+	// isn't concurrency-safe.
+	views := 0
+	dbg.KVFunc("Debug pageviews", func() interface{} {
+		views = views + 1
+		return views
+	})
+	dbg.KV("Donuts", 42)
+}
+
+func ExampleDebugHandler_URL() {
+	mux := http.NewServeMux()
+	dbg := Debugger(mux)
+	// Links to the Tailscale website from /debug/.
+	dbg.URL("https://www.tailscale.com", "Homepage")
+}
+
+func ExampleDebugHandler_Section() {
+	mux := http.NewServeMux()
+	dbg := Debugger(mux)
+	// Adds a section to /debug/ that dumps the HTTP request of the
+	// visitor.
+	dbg.Section(func(w io.Writer, r *http.Request) {
+		io.WriteString(w, "<h3>Dump of your HTTP request</h3>")
+		fmt.Fprintf(w, "<code>%#v</code>", r)
+	})
+}

tsweb/tsweb.go

@@ -29,34 +29,13 @@ import (
 	"tailscale.com/types/logger"
 )
 
-// DevMode controls whether extra output in shown, for when the binary is being run in dev mode.
-var DevMode bool
-
-// NewMux returns a new ServeMux with debugHandler registered (and protected) at /debug/.
-func NewMux(debugHandler http.Handler) *http.ServeMux {
-	mux := http.NewServeMux()
-	registerCommonDebug(mux)
-	mux.Handle("/debug/", Protected(debugHandler))
-	return mux
-}
-
-func registerCommonDebug(mux *http.ServeMux) {
+func init() {
 	expvar.Publish("counter_uptime_sec", expvar.Func(func() interface{} { return int64(Uptime().Seconds()) }))
 	expvar.Publish("gauge_goroutines", expvar.Func(func() interface{} { return runtime.NumGoroutine() }))
-	mux.Handle("/debug/pprof/", Protected(http.DefaultServeMux)) // to net/http/pprof
-	mux.Handle("/debug/vars", Protected(http.DefaultServeMux))   // to expvar
-	mux.Handle("/debug/varz", Protected(http.HandlerFunc(VarzHandler)))
-	mux.Handle("/debug/gc", Protected(http.HandlerFunc(gcHandler)))
 }
 
-func gcHandler(w http.ResponseWriter, r *http.Request) {
-	w.Write([]byte("running GC...\n"))
-	if f, ok := w.(http.Flusher); ok {
-		f.Flush()
-	}
-	runtime.GC()
-	w.Write([]byte("Done.\n"))
-}
+// DevMode controls whether extra output in shown, for when the binary is being run in dev mode.
+var DevMode bool
 
 func DefaultCertDir(leafDir string) string {
 	cacheDir, err := os.UserCacheDir()
@@ -183,13 +162,6 @@ type HandlerOptions struct {
 	StatusCodeCounters *expvar.Map
 }
 
-// StdHandler converts a ReturnHandler into a standard http.Handler.
-// Handled requests are logged using logf, as are any errors. Errors
-// are handled as specified by the Handler interface.
-func StdHandler(h ReturnHandler, logf logger.Logf) http.Handler {
-	return StdHandlerOpts(h, HandlerOptions{Logf: logf, Now: time.Now})
-}
-
 // ReturnHandlerFunc is an adapter to allow the use of ordinary
 // functions as ReturnHandlers. If f is a function with the
 // appropriate signature, ReturnHandlerFunc(f) is a ReturnHandler that
@@ -201,22 +173,16 @@ func (f ReturnHandlerFunc) ServeHTTPReturn(w http.ResponseWriter, r *http.Reques
 	return f(w, r)
 }
 
-// StdHandlerNo200s is like StdHandler, but successfully handled HTTP
-// requests don't write an access log entry to logf.
-//
-// TODO(josharian): eliminate this and StdHandler in favor of StdHandlerOpts,
-// rename StdHandlerOpts to StdHandler. Will be a breaking API change.
-func StdHandlerNo200s(h ReturnHandler, logf logger.Logf) http.Handler {
-	return StdHandlerOpts(h, HandlerOptions{Logf: logf, Now: time.Now, Quiet200s: true})
-}
-
-// StdHandlerOpts converts a ReturnHandler into a standard http.Handler.
+// StdHandler converts a ReturnHandler into a standard http.Handler.
 // Handled requests are logged using opts.Logf, as are any errors.
 // Errors are handled as specified by the Handler interface.
-func StdHandlerOpts(h ReturnHandler, opts HandlerOptions) http.Handler {
+func StdHandler(h ReturnHandler, opts HandlerOptions) http.Handler {
 	if opts.Now == nil {
 		opts.Now = time.Now
 	}
+	if opts.Logf == nil {
+		opts.Logf = logger.Discard
+	}
 	return retHandler{h, opts}
 }
 

tsweb/tsweb_test.go

@@ -248,7 +248,7 @@ func TestStdHandler(t *testing.T) {
 			clock.Reset()
 
 			rec := noopHijacker{httptest.NewRecorder(), false}
-			h := StdHandlerOpts(test.rh, HandlerOptions{Logf: logf, Now: clock.Now})
+			h := StdHandler(test.rh, HandlerOptions{Logf: logf, Now: clock.Now})
 			h.ServeHTTP(&rec, test.r)
 			res := rec.Result()
 			if res.StatusCode != test.wantCode {
@@ -277,8 +277,7 @@ func BenchmarkLogNot200(b *testing.B) {
 		// Implicit 200 OK.
 		return nil
 	})
-	discardLogger := func(string, ...interface{}) {}
-	h := StdHandlerNo200s(rh, discardLogger)
+	h := StdHandler(rh, HandlerOptions{Quiet200s: true})
 	req := httptest.NewRequest("GET", "/", nil)
 	rw := new(httptest.ResponseRecorder)
 	for i := 0; i < b.N; i++ {
@@ -293,8 +292,7 @@ func BenchmarkLog(b *testing.B) {
 		// Implicit 200 OK.
 		return nil
 	})
-	discardLogger := func(string, ...interface{}) {}
-	h := StdHandler(rh, discardLogger)
+	h := StdHandler(rh, HandlerOptions{})
 	req := httptest.NewRequest("GET", "/", nil)
 	rw := new(httptest.ResponseRecorder)
 	for i := 0; i < b.N; i++ {

types/key/key.go

@@ -85,8 +85,6 @@ func (k Private) Public() Public {
 func (k Private) SharedSecret(pub Public) (ss [32]byte) {
 	apk := (*[32]byte)(&pub)
 	ask := (*[32]byte)(&k)
-	//lint:ignore SA1019 Code copied from wireguard-go, we aim for
-	//minimal changes from it.
 	curve25519.ScalarMult(&ss, ask, apk)
 	return ss
 }

types/logger/logger.go

@@ -106,7 +106,6 @@ func RateLimitedFnWithClock(logf Logf, f time.Duration, burst int, maxCache int,
 		}
 
 		mu.Lock()
-		defer mu.Unlock()
 		rl, ok := msgLim[format]
 		if ok {
 			msgCache.MoveToFront(rl.ele)
@@ -138,8 +137,8 @@ func RateLimitedFnWithClock(logf Logf, f time.Duration, burst int, maxCache int,
 			rl.nBlocked = 0
 		}
 		if rl.nBlocked == 0 && rl.bucket.Get() {
-			logf(format, args...)
-			if rl.bucket.remaining == 0 {
+			hitLimit := rl.bucket.remaining == 0
+			if hitLimit {
 				// Enter "blocked" mode immediately after
 				// reaching the burst limit. We want to
 				// always accompany the format() message
@@ -148,12 +147,16 @@ func RateLimitedFnWithClock(logf Logf, f time.Duration, burst int, maxCache int,
 				// message anyway. But this way they can
 				// be on two separate lines and we don't
 				// corrupt the original message.
-				logf("[RATELIMIT] format(%q)", format)
 				rl.nBlocked = 1
 			}
-			return
+			mu.Unlock() // release before calling logf
+			logf(format, args...)
+			if hitLimit {
+				logf("[RATELIMIT] format(%q)", format)
+			}
 		} else {
 			rl.nBlocked++
+			mu.Unlock()
 		}
 	}
 }

types/logger/logger_test.go

@@ -170,3 +170,16 @@ func TestSynchronization(t *testing.T) {
 		})
 	}
 }
+
+// test that RateLimitedFn is safe for reentrancy without deadlocking
+func TestRateLimitedFnReentrancy(t *testing.T) {
+	rlogf := RateLimitedFn(t.Logf, time.Nanosecond, 10, 10)
+	rlogf("Hello.")
+	rlogf("Hello, %v", ArgWriter(func(bw *bufio.Writer) {
+		bw.WriteString("world")
+	}))
+	rlogf("Hello, %v", ArgWriter(func(bw *bufio.Writer) {
+		bw.WriteString("bye")
+		rlogf("boom") // this used to deadlock
+	}))
+}

util/deephash/deephash.go

@@ -4,6 +4,9 @@
 
 // Package deephash hashes a Go value recursively, in a predictable
 // order, without looping.
+//
+// This package, like most of the tailscale.com Go module, should be
+// considered Tailscale-internal; we make no API promises.
 package deephash
 
 import (
@@ -12,34 +15,82 @@ import (
 	"encoding/hex"
 	"fmt"
 	"hash"
+	"math"
 	"reflect"
 	"strconv"
 	"sync"
 )
 
-func calcHash(v interface{}) string {
-	h := sha256.New()
-	b := bufio.NewWriterSize(h, h.BlockSize())
-	scratch := make([]byte, 0, 128)
-	printTo(b, v, scratch)
-	b.Flush()
-	scratch = h.Sum(scratch[:0])
-	hex.Encode(scratch[:cap(scratch)], scratch[:sha256.Size])
-	return string(scratch[:sha256.Size*2])
+// hasher is reusable state for hashing a value.
+// Get one via hasherPool.
+type hasher struct {
+	h       hash.Hash
+	bw      *bufio.Writer
+	scratch [128]byte
+	visited map[uintptr]bool
 }
 
-// UpdateHash sets last to the hash of v and reports whether its value changed.
-func UpdateHash(last *string, v ...interface{}) (changed bool) {
-	sig := calcHash(v)
-	if *last != sig {
-		*last = sig
-		return true
+// newHasher initializes a new hasher, for use by hasherPool.
+func newHasher() *hasher {
+	h := &hasher{
+		h:       sha256.New(),
+		visited: map[uintptr]bool{},
 	}
-	return false
+	h.bw = bufio.NewWriterSize(h.h, h.h.BlockSize())
+	return h
 }
 
-func printTo(w *bufio.Writer, v interface{}, scratch []byte) {
-	print(w, reflect.ValueOf(v), make(map[uintptr]bool), scratch)
+func (h *hasher) setBufioWriter(w *bufio.Writer) {
+	h.bw.Flush()
+	h.bw = w
+}
+
+// Hash returns the raw SHA-256 (not hex) of v.
+func (h *hasher) Hash(v interface{}) (hash [sha256.Size]byte) {
+	h.bw.Flush()
+	h.h.Reset()
+	h.print(reflect.ValueOf(v))
+	h.bw.Flush()
+	h.h.Sum(hash[:0])
+	return hash
+}
+
+var hasherPool = &sync.Pool{
+	New: func() interface{} { return newHasher() },
+}
+
+// Hash returns the raw SHA-256 hash of v.
+func Hash(v interface{}) [sha256.Size]byte {
+	hasher := hasherPool.Get().(*hasher)
+	defer hasherPool.Put(hasher)
+	return hasher.Hash(v)
+}
+
+// UpdateHash sets last to the hex-encoded hash of v and reports whether its value changed.
+func UpdateHash(last *string, v ...interface{}) (changed bool) {
+	sum := Hash(v)
+	if sha256EqualHex(sum, *last) {
+		// unchanged.
+		return false
+	}
+	*last = hex.EncodeToString(sum[:])
+	return true
+}
+
+// sha256EqualHex reports whether hx is the hex encoding of sum.
+func sha256EqualHex(sum [sha256.Size]byte, hx string) bool {
+	if len(hx) != len(sum)*2 {
+		return false
+	}
+	const hextable = "0123456789abcdef"
+	j := 0
+	for _, v := range sum {
+		if hx[j] != hextable[v>>4] || hx[j+1] != hextable[v&0x0f] {
+			return false
+		}
+		j += 2
+	}
+	return true
 }
 
 var appenderToType = reflect.TypeOf((*appenderTo)(nil)).Elem()
@@ -50,16 +101,19 @@ type appenderTo interface {
 
 // print hashes v into w.
 // It reports whether it was able to do so without hitting a cycle.
-func print(w *bufio.Writer, v reflect.Value, visited map[uintptr]bool, scratch []byte) (acyclic bool) {
+func (h *hasher) print(v reflect.Value) (acyclic bool) {
 	if !v.IsValid() {
 		return true
 	}
 
+	w := h.bw
+	visited := h.visited
+
 	if v.CanInterface() {
 		// Use AppendTo methods, if available and cheap.
 		if v.CanAddr() && v.Type().Implements(appenderToType) {
 			a := v.Addr().Interface().(appenderTo)
-			scratch = a.AppendTo(scratch[:0])
+			scratch := a.AppendTo(h.scratch[:0])
 			w.Write(scratch)
 			return true
 		}
@@ -75,13 +129,13 @@ func print(w *bufio.Writer, v reflect.Value, visited map[uintptr]bool, scratch [
 			return false
 		}
 		visited[ptr] = true
-		return print(w, v.Elem(), visited, scratch)
+		return h.print(v.Elem())
 	case reflect.Struct:
 		acyclic = true
 		w.WriteString("struct{\n")
 		for i, n := 0, v.NumField(); i < n; i++ {
 			fmt.Fprintf(w, " [%d]: ", i)
-			if !print(w, v.Field(i), visited, scratch) {
+			if !h.print(v.Field(i)) {
 				acyclic = false
 			}
 			w.WriteString("\n")
@@ -97,7 +151,7 @@ func print(w *bufio.Writer, v reflect.Value, visited map[uintptr]bool, scratch [
 		acyclic = true
 		for i, ln := 0, v.Len(); i < ln; i++ {
 			fmt.Fprintf(w, " [%d]: ", i)
-			if !print(w, v.Index(i), visited, scratch) {
+			if !h.print(v.Index(i)) {
 				acyclic = false
 			}
 			w.WriteString("\n")
@@ -105,23 +159,48 @@ func print(w *bufio.Writer, v reflect.Value, visited map[uintptr]bool, scratch [
 		w.WriteString("}\n")
 		return acyclic
 	case reflect.Interface:
-		return print(w, v.Elem(), visited, scratch)
+		return h.print(v.Elem())
 	case reflect.Map:
-		if hashMapAcyclic(w, v, visited, scratch) {
+		// TODO(bradfitz): ideally we'd avoid these map
+		// operations to detect cycles if we knew from the map
+		// element type that there no way to form a cycle,
+		// which is the common case. Notably, we don't care
+		// about hashing the same map+contents twice in
+		// different parts of the tree. In fact, we should
+		// ideally. (And this prevents it) We should only stop
+		// hashing when there's a cycle.  What we should
+		// probably do is make sure we enumerate the data
+		// structure tree is a fixed order and then give each
+		// pointer an increasing number, and when we hit a
+		// dup, rather than emitting nothing, we should emit a
+		// "value #12" reference. Which implies that all things
+		// emit to the bufio.Writer should be type-tagged so
+		// we can distinguish loop references without risk of
+		// collisions.
+		ptr := v.Pointer()
+		if visited[ptr] {
+			return false
+		}
+		visited[ptr] = true
+
+		vlen0 := len(visited)
+		if false && h.hashMapAcyclic(v) {
 			return true
 		}
-		return hashMapFallback(w, v, visited, scratch)
+		if len(visited) != vlen0 {
+			panic("it grew")
+		}
+		return h.hashMapFallback(v)
 	case reflect.String:
 		w.WriteString(v.String())
 	case reflect.Bool:
-		fmt.Fprintf(w, "%v", v.Bool())
+		w.Write(strconv.AppendBool(h.scratch[:0], v.Bool()))
 	case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64:
-		fmt.Fprintf(w, "%v", v.Int())
+		w.Write(strconv.AppendInt(h.scratch[:0], v.Int(), 10))
 	case reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64, reflect.Uintptr:
-		scratch = strconv.AppendUint(scratch[:0], v.Uint(), 10)
-		w.Write(scratch)
+		w.Write(strconv.AppendUint(h.scratch[:0], v.Uint(), 10))
 	case reflect.Float32, reflect.Float64:
-		fmt.Fprintf(w, "%v", v.Float())
+		w.Write(strconv.AppendUint(h.scratch[:0], math.Float64bits(v.Float()), 10))
 	case reflect.Complex64, reflect.Complex128:
 		fmt.Fprintf(w, "%v", v.Complex())
 	}
@@ -183,40 +262,48 @@ func (c valueCache) get(t reflect.Type) reflect.Value {
 // hashMapAcyclic is the faster sort-free version of map hashing. If
 // it detects a cycle it returns false and guarantees that nothing was
 // written to w.
-func hashMapAcyclic(w *bufio.Writer, v reflect.Value, visited map[uintptr]bool, scratch []byte) (acyclic bool) {
+func (h *hasher) hashMapAcyclic(v reflect.Value) (acyclic bool) {
 	mh := mapHasherPool.Get().(*mapHasher)
 	defer mapHasherPool.Put(mh)
 	mh.Reset()
 	iter := mapIter(mh.iter, v)
 	defer mapIter(mh.iter, reflect.Value{}) // avoid pinning v from mh.iter when we return
+
+	// Flush the current writer and temporarily restore the bufio
+	// writer we'll be using for future prints.
+	oldw := h.bw
+	defer h.setBufioWriter(oldw) // restore bufio writer
+	h.setBufioWriter(mh.bw)
+
 	k := mh.val.get(v.Type().Key())
 	e := mh.val.get(v.Type().Elem())
 	for iter.Next() {
 		key := iterKey(iter, k)
 		val := iterVal(iter, e)
 		mh.startEntry()
-		if !print(mh.bw, key, visited, scratch) {
+		if !h.print(key) {
 			return false
 		}
-		if !print(mh.bw, val, visited, scratch) {
+		if !h.print(val) {
 			return false
 		}
 		mh.endEntry()
 	}
-	w.Write(mh.xbuf[:])
+	oldw.Write(mh.xbuf[:])
 	return true
 }
 
-func hashMapFallback(w *bufio.Writer, v reflect.Value, visited map[uintptr]bool, scratch []byte) (acyclic bool) {
+func (h *hasher) hashMapFallback(v reflect.Value) (acyclic bool) {
 	acyclic = true
 	sm := newSortedMap(v)
+	w := h.bw
 	fmt.Fprintf(w, "map[%d]{\n", len(sm.Key))
 	for i, k := range sm.Key {
-		if !print(w, k, visited, scratch) {
+		if !h.print(k) {
 			acyclic = false
 		}
 		w.WriteString(": ")
-		if !print(w, sm.Value[i], visited, scratch) {
+		if !h.print(sm.Value[i]) {
 			acyclic = false
 		}
 		w.WriteString("\n")

util/deephash/deephash_test.go

@@ -7,6 +7,8 @@ package deephash
 import (
 	"bufio"
 	"bytes"
+	"crypto/sha256"
+	"encoding/hex"
 	"fmt"
 	"reflect"
 	"testing"
@@ -23,10 +25,10 @@ func TestDeepHash(t *testing.T) {
 	// Mostly we're just testing that we don't panic on handled types.
 	v := getVal()
 
-	hash1 := calcHash(v)
+	hash1 := Hash(v)
 	t.Logf("hash: %v", hash1)
 	for i := 0; i < 20; i++ {
-		hash2 := calcHash(getVal())
+		hash2 := Hash(getVal())
 		if hash1 != hash2 {
 			t.Error("second hash didn't match")
 		}
@@ -36,8 +38,8 @@ func TestDeepHash(t *testing.T) {
 func getVal() []interface{} {
 	return []interface{}{
 		&wgcfg.Config{
-			Name:      "foo",
-			Addresses: []netaddr.IPPrefix{netaddr.IPPrefixFrom(netaddr.IPFrom16([16]byte{3: 3}), 5)},
+			//Name:      "foo",
+			//Addresses: []netaddr.IPPrefix{netaddr.IPPrefixFrom(netaddr.IPFrom16([16]byte{3: 3}), 5)},
 			Peers: []wgcfg.Peer{
 				{
 					Endpoints: wgcfg.Endpoints{
@@ -46,6 +48,8 @@ func getVal() []interface{} {
 				},
 			},
 		},
+	}
+	return []interface{}{
 		&router.Config{
 			Routes: []netaddr.IPPrefix{
 				netaddr.MustParseIPPrefix("1.2.3.0/24"),
@@ -73,14 +77,66 @@ func getVal() []interface{} {
 			{2: 3}: true,
 			{3: 4}: false,
 		},
+		&tailcfg.MapResponse{
+			DERPMap: &tailcfg.DERPMap{
+				Regions: map[int]*tailcfg.DERPRegion{
+					1: &tailcfg.DERPRegion{
+						RegionID:   1,
+						RegionCode: "foo",
+						Nodes: []*tailcfg.DERPNode{
+							{
+								Name:     "n1",
+								RegionID: 1,
+								HostName: "foo.com",
+							},
+							{
+								Name:     "n2",
+								RegionID: 1,
+								HostName: "bar.com",
+							},
+						},
+					},
+				},
+			},
+			DNSConfig: &tailcfg.DNSConfig{
+				Resolvers: []tailcfg.DNSResolver{
+					{Addr: "10.0.0.1"},
+				},
+			},
+			PacketFilter: []tailcfg.FilterRule{
+				{
+					SrcIPs: []string{"1.2.3.4"},
+					DstPorts: []tailcfg.NetPortRange{
+						{
+							IP:    "1.2.3.4/32",
+							Ports: tailcfg.PortRange{First: 1, Last: 2},
+						},
+					},
+				},
+			},
+			Peers: []*tailcfg.Node{
+				{
+					ID: 1,
+				},
+				{
+					ID: 2,
+				},
+			},
+			UserProfiles: []tailcfg.UserProfile{
+				{ID: 1, LoginName: "foo@bar.com"},
+				{ID: 2, LoginName: "bar@foo.com"},
+			},
+		},
 	}
 }
 
+var sink = Hash("foo")
+
 func BenchmarkHash(b *testing.B) {
 	b.ReportAllocs()
 	v := getVal()
 	for i := 0; i < b.N; i++ {
-		calcHash(v)
+		sink = Hash(v)
 	}
 }
 
@@ -95,12 +151,14 @@ func TestHashMapAcyclic(t *testing.T) {
 	bw := bufio.NewWriter(&buf)
 
 	for i := 0; i < 20; i++ {
-		visited := map[uintptr]bool{}
-		scratch := make([]byte, 0, 64)
 		v := reflect.ValueOf(m)
 		buf.Reset()
 		bw.Reset(&buf)
-		if !hashMapAcyclic(bw, v, visited, scratch) {
+		h := &hasher{
+			bw:      bw,
+			visited: map[uintptr]bool{},
+		}
+		if !h.hashMapAcyclic(v) {
 			t.Fatal("returned false")
 		}
 		if got[string(buf.Bytes())] {
@@ -122,15 +180,76 @@ func BenchmarkHashMapAcyclic(b *testing.B) {
 
 	var buf bytes.Buffer
 	bw := bufio.NewWriter(&buf)
-	visited := map[uintptr]bool{}
-	scratch := make([]byte, 0, 64)
 	v := reflect.ValueOf(m)
 
+	h := &hasher{
+		bw:      bw,
+		visited: map[uintptr]bool{},
+	}
+
 	for i := 0; i < b.N; i++ {
 		buf.Reset()
 		bw.Reset(&buf)
-		if !hashMapAcyclic(bw, v, visited, scratch) {
+		if !h.hashMapAcyclic(v) {
 			b.Fatal("returned false")
 		}
 	}
 }
+
+func BenchmarkTailcfgNode(b *testing.B) {
+	b.ReportAllocs()
+
+	node := new(tailcfg.Node)
+	for i := 0; i < b.N; i++ {
+		sink = Hash(node)
+	}
+}
+
+func TestExhaustive(t *testing.T) {
+	seen := make(map[[sha256.Size]byte]bool)
+	for i := 0; i < 100000; i++ {
+		s := Hash(i)
+		if seen[s] {
+			t.Fatalf("hash collision %v", i)
+		}
+		seen[s] = true
+	}
+}
+
+func TestSHA256EqualHex(t *testing.T) {
+	for i := 0; i < 1000; i++ {
+		sum := Hash(i)
+		hx := hex.EncodeToString(sum[:])
+		if !sha256EqualHex(sum, hx) {
+			t.Fatal("didn't match, should've")
+		}
+		if sha256EqualHex(sum, hx[:len(hx)-1]) {
+			t.Fatal("matched on wrong length")
+		}
+	}
+}
+
+// verify this doesn't loop forever, as it used to (Issue 2340)
+func TestMapCyclicFallback(t *testing.T) {
+	type T struct {
+		M map[string]interface{}
+	}
+	v := &T{
+		M: map[string]interface{}{},
+	}
+	v.M["m"] = v.M
+	Hash(v)
+}
+
+func TestVisitedFallback(t *testing.T) {
+	t.Skip("known failure: https://github.com/tailscale/tailscale/issues/2342")
+	type V struct {
+		I int
+	}
+	m := map[int]*V{}
+	v1 := &V{1}
+	for i := 0; i < 1000; i++ {
+		m[i] = v1
+	}
+	Hash(m)
+}