VERSION.txt: this is v1.10.2.

The localapi was double-unescaping: once by net/http populating
the URL, and once by ourselves later. We need to start with the raw
escaped URL if we're doing it ourselves.

Started to write a test but it got invasive. Will have to add those
tests later in a commit that's not being cherry-picked to a release
branch.

Fixes #2288

Signed-off-by: Brad Fitzpatrick <bradfitz@tailscale.com>
(cherry picked from commit 98ad7f279c)

.github/workflows/xe-experimental-vm-test.yml

@@ -0,0 +1,36 @@
+name: "integration-vms"
+
+on:
+  # # NOTE(Xe): uncomment this region when testing the test
+  # pull_request:
+  #   branches:
+  #     - 'main'
+  release:
+    types: [ created ]
+  schedule:
+    # At minute 0 past hour 6 and 18
+    # https://crontab.guru/#00_6,18_*_*_*
+    - cron: '00 6,18 * * *'
+
+jobs:
+  experimental-linux-vm-test:
+    # To set up a new runner, see tstest/integration/vms/runner.nix
+    runs-on: [ self-hosted, linux, vm_integration_test ]
+
+    if: "!contains(github.event.head_commit.message, '[ci skip]')"
+
+    steps:
+      - name: Checkout Code
+        uses: actions/checkout@v1
+
+      - name: Download VM Images
+        run: go test ./tstest/integration/vms -run-vm-tests -run=Download -timeout=60m
+        env:
+          XDG_CACHE_HOME: "/var/lib/ghrunner/cache"
+
+      - name: Run VM tests
+        run: go test ./tstest/integration/vms -v -run-vm-tests
+        env:
+          TMPDIR: "/tmp"
+          XDG_CACHE_HOME: "/var/lib/ghrunner/cache"
+

VERSION.txt

@@ -1 +1 @@
-1.9.0
+1.10.2

build_dist.sh

@@ -11,6 +11,36 @@
 
 set -eu
 
-eval $(./version/version.sh)
+IFS=".$IFS" read -r major minor patch <VERSION.txt
+git_hash=$(git rev-parse HEAD)
+if ! git diff-index --quiet HEAD; then
+	git_hash="${git_hash}-dirty"
+fi
+base_hash=$(git rev-list --max-count=1 HEAD -- VERSION.txt)
+change_count=$(git rev-list --count HEAD "^$base_hash")
+short_hash=$(echo "$git_hash" | cut -c1-9)
 
-exec go build -tags xversion -ldflags "-X tailscale.com/version.Long=${VERSION_LONG} -X tailscale.com/version.Short=${VERSION_SHORT} -X tailscale.com/version.GitCommit=${VERSION_GIT_HASH}" "$@"
+if expr "$minor" : "[0-9]*[13579]$" >/dev/null; then
+	patch="$change_count"
+	change_suffix=""
+elif [ "$change_count" != "0" ]; then
+	change_suffix="-$change_count"
+else
+	change_suffix=""
+fi
+
+long_suffix="$change_suffix-t$short_hash"
+SHORT="$major.$minor.$patch"
+LONG="${SHORT}$long_suffix"
+GIT_HASH="$git_hash"
+
+if [ "$1" = "shellvars" ]; then
+	cat <<EOF
+VERSION_SHORT="$SHORT"
+VERSION_LONG="$LONG"
+VERSION_GIT_HASH="$GIT_HASH"
+EOF
+	exit 0
+fi
+
+exec go build -ldflags "-X tailscale.com/version.Long=${LONG} -X tailscale.com/version.Short=${SHORT} -X tailscale.com/version.GitCommit=${GIT_HASH}" "$@"

client/tailscale/tailscale.go

@@ -256,3 +256,25 @@ func Logout(ctx context.Context) error {
 	_, err := send(ctx, "POST", "/localapi/v0/logout", http.StatusNoContent, nil)
 	return err
 }
+
+// SetDNS adds a DNS TXT record for the given domain name, containing
+// the provided TXT value. The intended use case is answering
+// LetsEncrypt/ACME dns-01 challenges.
+//
+// The control plane will only permit SetDNS requests with very
+// specific names and values. The name should be
+// "_acme-challenge." + your node's MagicDNS name. It's expected that
+// clients cache the certs from LetsEncrypt (or whichever CA is
+// providing them) and only request new ones as needed; the control plane
+// rate limits SetDNS requests.
+//
+// This is a low-level interface; it's expected that most Tailscale
+// users use a higher level interface to getting/using TLS
+// certificates.
+func SetDNS(ctx context.Context, name, value string) error {
+	v := url.Values{}
+	v.Set("name", name)
+	v.Set("value", value)
+	_, err := send(ctx, "POST", "/localapi/v0/set-dns?"+v.Encode(), 200, nil)
+	return err
+}

cmd/cloner/cloner.go

@@ -246,7 +246,9 @@ func gen(buf *bytes.Buffer, imports map[string]struct{}, name string, typ *types
 					writef("\t\tdst.%s[k] = append([]%s{}, src.%s[k]...)", fname, n, fname)
 					writef("\t}")
 				} else if containsPointers(ft.Elem()) {
-					writef("\t\t" + `panic("TODO map value pointers")`)
+					writef("\tfor k, v := range src.%s {", fname)
+					writef("\t\tdst.%s[k] = v.Clone()", fname)
+					writef("\t}")
 				} else {
 					writef("\tfor k, v := range src.%s {", fname)
 					writef("\t\tdst.%s[k] = v", fname)

cmd/derper/derper.go

@@ -12,8 +12,6 @@ import (
 	"errors"
 	"expvar"
 	"flag"
-	"fmt"
-	"html"
 	"io"
 	"io/ioutil"
 	"log"
@@ -35,7 +33,6 @@ import (
 	"tailscale.com/tsweb"
 	"tailscale.com/types/key"
 	"tailscale.com/types/wgkey"
-	"tailscale.com/version"
 )
 
 var (
@@ -49,6 +46,7 @@ var (
 	meshPSKFile   = flag.String("mesh-psk-file", defaultMeshPSKFile(), "if non-empty, path to file containing the mesh pre-shared key file. It should contain some hex string; whitespace is trimmed.")
 	meshWith      = flag.String("mesh-with", "", "optional comma-separated list of hostnames to mesh with; the server's own hostname can be in the list")
 	bootstrapDNS  = flag.String("bootstrap-dns-names", "", "optional comma-separated list of hostnames to make available at /bootstrap-dns")
+	verifyClients = flag.Bool("verify-clients", false, "verify clients to this DERP server through a local tailscaled instance.")
 )
 
 type config struct {
@@ -125,6 +123,7 @@ func main() {
 	letsEncrypt := tsweb.IsProd443(*addr)
 
 	s := derp.NewServer(key.Private(cfg.PrivateKey), log.Printf)
+	s.SetVerifyClient(*verifyClients)
 
 	if *meshPSKFile != "" {
 		b, err := ioutil.ReadFile(*meshPSKFile)
@@ -143,8 +142,7 @@ func main() {
 	}
 	expvar.Publish("derp", s.ExpVar())
 
-	// Create our own mux so we don't expose /debug/ stuff to the world.
-	mux := tsweb.NewMux(debugHandler(s))
+	mux := http.NewServeMux()
 	mux.Handle("/derp", derphttp.Handler(s))
 	go refreshBootstrapDNSLoop()
 	mux.HandleFunc("/bootstrap-dns", handleBootstrapDNS)
@@ -164,6 +162,18 @@ func main() {
 			io.WriteString(w, "<p>Debug info at <a href='/debug/'>/debug/</a>.</p>\n")
 		}
 	}))
+	debug := tsweb.Debugger(mux)
+	debug.KV("TLS hostname", *hostname)
+	debug.KV("Mesh key", s.HasMeshKey())
+	debug.Handle("check", "Consistency check", http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		err := s.ConsistencyCheck()
+		if err != nil {
+			http.Error(w, err.Error(), 500)
+		} else {
+			io.WriteString(w, "derp.Server ConsistencyCheck okay")
+		}
+	}))
+	debug.Handle("traffic", "Traffic check", http.HandlerFunc(s.ServeDebugTraffic))
 
 	if *runSTUN {
 		go serveSTUN()
@@ -217,39 +227,6 @@ func main() {
 	}
 }
 
-func debugHandler(s *derp.Server) http.Handler {
-	return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
-		if r.RequestURI == "/debug/check" {
-			err := s.ConsistencyCheck()
-			if err != nil {
-				http.Error(w, err.Error(), 500)
-			} else {
-				io.WriteString(w, "derp.Server ConsistencyCheck okay")
-			}
-			return
-		}
-		f := func(format string, args ...interface{}) { fmt.Fprintf(w, format, args...) }
-		f(`<html><body>
-<h1>DERP debug</h1>
-<ul>
-`)
-		f("<li><b>Hostname:</b> %v</li>\n", html.EscapeString(*hostname))
-		f("<li><b>Uptime:</b> %v</li>\n", tsweb.Uptime())
-		f("<li><b>Mesh Key:</b> %v</li>\n", s.HasMeshKey())
-		f("<li><b>Version:</b> %v</li>\n", html.EscapeString(version.Long))
-
-		f(`<li><a href="/debug/vars">/debug/vars</a> (Go)</li>
-   <li><a href="/debug/varz">/debug/varz</a> (Prometheus)</li>
-   <li><a href="/debug/pprof/">/debug/pprof/</a></li>
-   <li><a href="/debug/pprof/goroutine?debug=1">/debug/pprof/goroutine</a> (collapsed)</li>
-   <li><a href="/debug/pprof/goroutine?debug=2">/debug/pprof/goroutine</a> (full)</li>
-   <li><a href="/debug/check">/debug/check</a> internal consistency check</li>
-<ul>
-</html>
-`)
-	})
-}
-
 func serveSTUN() {
 	pc, err := net.ListenPacket("udp", ":3478")
 	if err != nil {

cmd/microproxy/microproxy.go

@@ -55,9 +55,13 @@ func main() {
 		log.Fatalf("Couldn't parse URL %q: %v", *goVarsURL, err)
 	}
 
-	mux := tsweb.NewMux(http.HandlerFunc(debugHandler))
+	mux := http.NewServeMux()
+	tsweb.Debugger(mux) // registers /debug/*
 	mux.Handle("/metrics", tsweb.Protected(proxy))
-	mux.Handle("/varz", tsweb.Protected(tsweb.StdHandler(&goVarsHandler{*goVarsURL}, log.Printf)))
+	mux.Handle("/varz", tsweb.Protected(tsweb.StdHandler(&goVarsHandler{*goVarsURL}, tsweb.HandlerOptions{
+		Quiet200s: true,
+		Logf:      log.Printf,
+	})))
 
 	ch := &certHolder{
 		hostname: *hostname,
@@ -167,23 +171,3 @@ func (c *certHolder) loadLocked() error {
 	c.loaded = time.Now()
 	return nil
 }
-
-// debugHandler serves a page with links to tsweb-managed debug URLs
-// at /debug/.
-func debugHandler(w http.ResponseWriter, r *http.Request) {
-	f := func(format string, args ...interface{}) { fmt.Fprintf(w, format, args...) }
-	f(`<html><body>
-<h1>microproxy debug</h1>
-<ul>
-`)
-	f("<li><b>Hostname:</b> %v</li>\n", *hostname)
-	f("<li><b>Uptime:</b> %v</li>\n", tsweb.Uptime())
-	f(`<li><a href="/debug/vars">/debug/vars</a> (Go)</li>
-   <li><a href="/debug/varz">/debug/varz</a> (Prometheus)</li>
-   <li><a href="/debug/pprof/">/debug/pprof/</a></li>
-   <li><a href="/debug/pprof/goroutine?debug=1">/debug/pprof/goroutine</a> (collapsed)</li>
-   <li><a href="/debug/pprof/goroutine?debug=2">/debug/pprof/goroutine</a> (full)</li>
-<ul>
-</html>
-`)
-}

cmd/mkpkg/main.go

@@ -21,6 +21,9 @@ import (
 // into a map of filePathOnDisk -> filePathInPackage.
 func parseFiles(s string) (map[string]string, error) {
 	ret := map[string]string{}
+	if len(s) == 0 {
+		return ret, nil
+	}
 	for _, f := range strings.Split(s, ",") {
 		fs := strings.Split(f, ":")
 		if len(fs) != 2 {

cmd/tailscale/cli/auth-redirect.html

@@ -0,0 +1,57 @@
+<html>
+<head>
+	<title>Redirecting...</title>
+	<style>
+		html,
+		body {
+			height: 100%;
+		}
+
+		html {
+			background-color: rgb(249, 247, 246);
+			font-family: ui-sans-serif, system-ui, -apple-system, BlinkMacSystemFont, "Segoe UI", Roboto, "Helvetica Neue", Arial, "Noto Sans", sans-serif, "Apple Color Emoji", "Segoe UI Emoji", "Segoe UI Symbol", "Noto Color Emoji";
+			line-height: 1.5;
+			-webkit-text-size-adjust: 100%;
+			-webkit-font-smoothing: antialiased;
+			-moz-osx-font-smoothing: grayscale;
+		}
+
+		body {
+			display: flex;
+			flex-direction: column;
+			align-items: center;
+			justify-content: center;
+		}
+
+		.spinner {
+			margin-bottom: 2rem;
+			border: 4px rgba(112, 110, 109, 0.5) solid;
+			border-left-color: transparent;
+			border-radius: 9999px;
+			width: 4rem;
+			height: 4rem;
+			-webkit-animation: spin 700ms linear infinite;
+      animation: spin 800ms linear infinite;
+		}
+
+		.label {
+			color: rgb(112, 110, 109);
+			padding-left: 0.4rem;
+		}
+
+		@-webkit-keyframes spin {
+			to {
+				transform: rotate(360deg);
+			}
+		}
+
+		@keyframes spin {
+			to {
+				transform: rotate(360deg);
+			}
+		}
+	</style>
+</head> <body>
+	<div class="spinner"></div>
+	<div class="label">Redirecting...</div>
+</body>

cmd/tailscale/cli/file.go

@@ -74,7 +74,6 @@ func runCp(ctx context.Context, args []string) error {
 		return runCpTargets(ctx, args)
 	}
 	if len(args) < 2 {
-		//lint:ignore ST1005 no sorry need that colon at the end
 		return errors.New("usage: tailscale file cp <files...> <target>:")
 	}
 	files, target := args[:len(args)-1], args[len(args)-1]
@@ -97,14 +96,12 @@ func runCp(ctx context.Context, args []string) error {
 		return err
 	}
 
-	peerAPIBase, lastSeen, isOffline, err := discoverPeerAPIBase(ctx, ip)
+	peerAPIBase, isOffline, err := discoverPeerAPIBase(ctx, ip)
 	if err != nil {
 		return fmt.Errorf("can't send to %s: %v", target, err)
 	}
 	if isOffline {
 		fmt.Fprintf(os.Stderr, "# warning: %s is offline\n", target)
-	} else if !lastSeen.IsZero() && time.Since(lastSeen) > lastSeenOld {
-		fmt.Fprintf(os.Stderr, "# warning: %s last seen %v ago\n", target, time.Since(lastSeen).Round(time.Minute))
 	}
 
 	if len(files) > 1 {
@@ -182,14 +179,14 @@ func runCp(ctx context.Context, args []string) error {
 	return nil
 }
 
-func discoverPeerAPIBase(ctx context.Context, ipStr string) (base string, lastSeen time.Time, isOffline bool, err error) {
+func discoverPeerAPIBase(ctx context.Context, ipStr string) (base string, isOffline bool, err error) {
 	ip, err := netaddr.ParseIP(ipStr)
 	if err != nil {
-		return "", time.Time{}, false, err
+		return "", false, err
 	}
 	fts, err := tailscale.FileTargets(ctx)
 	if err != nil {
-		return "", time.Time{}, false, err
+		return "", false, err
 	}
 	for _, ft := range fts {
 		n := ft.Node
@@ -197,14 +194,11 @@ func discoverPeerAPIBase(ctx context.Context, ipStr string) (base string, lastSe
 			if a.IP() != ip {
 				continue
 			}
-			if n.LastSeen != nil {
-				lastSeen = *n.LastSeen
-			}
 			isOffline = n.Online != nil && !*n.Online
-			return ft.PeerAPIURL, lastSeen, isOffline, nil
+			return ft.PeerAPIURL, isOffline, nil
 		}
 	}
-	return "", time.Time{}, false, fileTargetErrorDetail(ctx, ip)
+	return "", false, fileTargetErrorDetail(ctx, ip)
 }
 
 // fileTargetErrorDetail returns a non-nil error saying why ip is an
@@ -274,8 +268,6 @@ func (r *slowReader) Read(p []byte) (n int, err error) {
 	return
 }
 
-const lastSeenOld = 20 * time.Minute
-
 func runCpTargets(ctx context.Context, args []string) error {
 	if len(args) > 0 {
 		return errors.New("invalid arguments with --targets")

cmd/tailscale/cli/web.go

@@ -9,12 +9,15 @@ import (
 	"context"
 	_ "embed"
 	"encoding/json"
+	"encoding/xml"
 	"flag"
 	"fmt"
 	"html/template"
+	"io/ioutil"
 	"log"
 	"net/http"
 	"net/http/cgi"
+	"net/url"
 	"os/exec"
 	"runtime"
 	"strings"
@@ -24,6 +27,7 @@ import (
 	"tailscale.com/ipn"
 	"tailscale.com/tailcfg"
 	"tailscale.com/types/preftype"
+	"tailscale.com/util/groupmember"
 	"tailscale.com/version/distro"
 )
 
@@ -33,6 +37,9 @@ var webHTML string
 //go:embed web.css
 var webCSS string
 
+//go:embed auth-redirect.html
+var authenticationRedirectHTML string
+
 var tmpl *template.Template
 
 func init() {
@@ -82,23 +89,114 @@ func runWeb(ctx context.Context, args []string) error {
 	return http.ListenAndServe(webArgs.listen, http.HandlerFunc(webHandler))
 }
 
-func auth() (string, error) {
-	if distro.Get() == distro.Synology {
-		cmd := exec.Command("/usr/syno/synoman/webman/modules/authenticate.cgi")
-		out, err := cmd.CombinedOutput()
+// authorize returns the name of the user accessing the web UI after verifying
+// whether the user has access to the web UI. The function will write the
+// error to the provided http.ResponseWriter.
+// Note: This is different from a tailscale user, and is typically the local
+// user on the node.
+func authorize(w http.ResponseWriter, r *http.Request) (string, error) {
+	switch distro.Get() {
+	case distro.Synology:
+		user, err := synoAuthn()
 		if err != nil {
-			return "", fmt.Errorf("auth: %v: %s", err, out)
+			http.Error(w, err.Error(), http.StatusUnauthorized)
+			return "", err
 		}
-		return string(out), nil
+		if err := authorizeSynology(user); err != nil {
+			http.Error(w, err.Error(), http.StatusForbidden)
+			return "", err
+		}
+		return user, nil
+	case distro.QNAP:
+		user, resp, err := qnapAuthn(r)
+		if err != nil {
+			http.Error(w, err.Error(), http.StatusUnauthorized)
+			return "", err
+		}
+		if resp.IsAdmin == 0 {
+			http.Error(w, err.Error(), http.StatusForbidden)
+			return "", err
+		}
+		return user, nil
 	}
-
 	return "", nil
 }
 
-func synoTokenRedirect(w http.ResponseWriter, r *http.Request) bool {
-	if distro.Get() != distro.Synology {
-		return false
+// authorizeSynology checks whether the provided user has access to the web UI
+// by consulting the membership of the "administrators" group.
+func authorizeSynology(name string) error {
+	yes, err := groupmember.IsMemberOfGroup("administrators", name)
+	if err != nil {
+		return err
 	}
+	if !yes {
+		return fmt.Errorf("not a member of administrators group")
+	}
+	return nil
+}
+
+type qnapAuthResponse struct {
+	AuthPassed int    `xml:"authPassed"`
+	IsAdmin    int    `xml:"isAdmin"`
+	AuthSID    string `xml:"authSid"`
+	ErrorValue int    `xml:"errorValue"`
+}
+
+func qnapAuthn(r *http.Request) (string, *qnapAuthResponse, error) {
+	user, err := r.Cookie("NAS_USER")
+	if err != nil {
+		return "", nil, err
+	}
+	token, err := r.Cookie("qtoken")
+	if err != nil {
+		return "", nil, err
+	}
+	query := url.Values{
+		"qtoken": []string{token.Value},
+		"user":   []string{user.Value},
+	}
+	u := url.URL{
+		Scheme:   r.URL.Scheme,
+		Host:     r.URL.Host,
+		Path:     "/cgi-bin/authLogin.cgi",
+		RawQuery: query.Encode(),
+	}
+	resp, err := http.Get(u.String())
+	if err != nil {
+		return "", nil, err
+	}
+	defer resp.Body.Close()
+	out, err := ioutil.ReadAll(resp.Body)
+	if err != nil {
+		return "", nil, err
+	}
+	authResp := &qnapAuthResponse{}
+	if err := xml.Unmarshal(out, authResp); err != nil {
+		return "", nil, err
+	}
+	if authResp.AuthPassed == 0 {
+		return "", nil, fmt.Errorf("not authenticated")
+	}
+	return user.Value, authResp, nil
+}
+
+func synoAuthn() (string, error) {
+	cmd := exec.Command("/usr/syno/synoman/webman/modules/authenticate.cgi")
+	out, err := cmd.CombinedOutput()
+	if err != nil {
+		return "", fmt.Errorf("auth: %v: %s", err, out)
+	}
+	return strings.TrimSpace(string(out)), nil
+}
+
+func authRedirect(w http.ResponseWriter, r *http.Request) bool {
+	if distro.Get() == distro.Synology {
+		return synoTokenRedirect(w, r)
+	}
+	return false
+}
+
+func synoTokenRedirect(w http.ResponseWriter, r *http.Request) bool {
 	if r.Header.Get("X-Syno-Token") != "" {
 		return false
 	}
@@ -132,75 +230,13 @@ req.send(null);
 </body></html>
 `
 
-const authenticationRedirectHTML = `
-<html>
-<head>
-	<title>Redirecting...</title>
-	<style>
-		html,
-		body {
-			height: 100%;
-		}
-
-		html {
-			background-color: rgb(249, 247, 246);
-			font-family: ui-sans-serif, system-ui, -apple-system, BlinkMacSystemFont, "Segoe UI", Roboto, "Helvetica Neue", Arial, "Noto Sans", sans-serif, "Apple Color Emoji", "Segoe UI Emoji", "Segoe UI Symbol", "Noto Color Emoji";
-			line-height: 1.5;
-			-webkit-text-size-adjust: 100%;
-			-webkit-font-smoothing: antialiased;
-			-moz-osx-font-smoothing: grayscale;
-		}
-
-		body {
-			display: flex;
-			flex-direction: column;
-			align-items: center;
-			justify-content: center;
-		}
-
-		.spinner {
-			margin-bottom: 2rem;
-			border: 4px rgba(112, 110, 109, 0.5) solid;
-			border-left-color: transparent;
-			border-radius: 9999px;
-			width: 4rem;
-			height: 4rem;
-			-webkit-animation: spin 700ms linear infinite;
-      animation: spin 800ms linear infinite;
-		}
-
-		.label {
-			color: rgb(112, 110, 109);
-			padding-left: 0.4rem;
-		}
-
-		@-webkit-keyframes spin {
-			to {
-				transform: rotate(360deg);
-			}
-		}
-
-		@keyframes spin {
-			to {
-				transform: rotate(360deg);
-			}
-		}
-	</style>
-</head>
-<body>
-	<div class="spinner"></div>
-	<div class="label">Redirecting...</div>
-</body>
-`
-
 func webHandler(w http.ResponseWriter, r *http.Request) {
-	if synoTokenRedirect(w, r) {
+	if authRedirect(w, r) {
 		return
 	}
 
-	user, err := auth()
+	user, err := authorize(w, r)
 	if err != nil {
-		http.Error(w, err.Error(), http.StatusForbidden)
 		return
 	}
 
@@ -214,7 +250,7 @@ func webHandler(w http.ResponseWriter, r *http.Request) {
 		w.Header().Set("Content-Type", "application/json")
 		url, err := tailscaleUpForceReauth(r.Context())
 		if err != nil {
-			w.WriteHeader(500)
+			w.WriteHeader(http.StatusInternalServerError)
 			json.NewEncoder(w).Encode(mi{"error": err.Error()})
 			return
 		}
@@ -224,7 +260,7 @@ func webHandler(w http.ResponseWriter, r *http.Request) {
 
 	st, err := tailscale.Status(r.Context())
 	if err != nil {
-		http.Error(w, err.Error(), 500)
+		http.Error(w, err.Error(), http.StatusInternalServerError)
 		return
 	}
 
@@ -242,7 +278,7 @@ func webHandler(w http.ResponseWriter, r *http.Request) {
 
 	buf := new(bytes.Buffer)
 	if err := tmpl.Execute(buf, data); err != nil {
-		http.Error(w, err.Error(), 500)
+		http.Error(w, err.Error(), http.StatusInternalServerError)
 		return
 	}
 	w.Write(buf.Bytes())

cmd/tailscale/depaware.txt

@@ -15,13 +15,14 @@ tailscale.com/cmd/tailscale dependencies: (generated by github.com/tailscale/dep
         inet.af/netaddr                                              from tailscale.com/cmd/tailscale/cli+
         rsc.io/goversion/version                                     from tailscale.com/version
         tailscale.com/atomicfile                                     from tailscale.com/ipn
-        tailscale.com/client/tailscale                               from tailscale.com/cmd/tailscale/cli
+        tailscale.com/client/tailscale                               from tailscale.com/cmd/tailscale/cli+
         tailscale.com/client/tailscale/apitype                       from tailscale.com/client/tailscale+
         tailscale.com/cmd/tailscale/cli                              from tailscale.com/cmd/tailscale
         tailscale.com/derp                                           from tailscale.com/derp/derphttp
         tailscale.com/derp/derphttp                                  from tailscale.com/net/netcheck
         tailscale.com/derp/derpmap                                   from tailscale.com/cmd/tailscale/cli
         tailscale.com/disco                                          from tailscale.com/derp
+        tailscale.com/hostinfo                                       from tailscale.com/net/interfaces
         tailscale.com/ipn                                            from tailscale.com/cmd/tailscale/cli+
         tailscale.com/ipn/ipnstate                                   from tailscale.com/cmd/tailscale/cli+
         tailscale.com/metrics                                        from tailscale.com/derp
@@ -53,9 +54,10 @@ tailscale.com/cmd/tailscale dependencies: (generated by github.com/tailscale/dep
         tailscale.com/types/wgkey                                    from tailscale.com/types/netmap+
         tailscale.com/util/dnsname                                   from tailscale.com/cmd/tailscale/cli+
    W    tailscale.com/util/endian                                    from tailscale.com/net/netns
-   L    tailscale.com/util/lineread                                  from tailscale.com/net/interfaces
+        tailscale.com/util/groupmember                               from tailscale.com/cmd/tailscale/cli
+        tailscale.com/util/lineread                                  from tailscale.com/net/interfaces+
         tailscale.com/version                                        from tailscale.com/cmd/tailscale/cli+
-        tailscale.com/version/distro                                 from tailscale.com/cmd/tailscale/cli
+        tailscale.com/version/distro                                 from tailscale.com/cmd/tailscale/cli+
         tailscale.com/wgengine/filter                                from tailscale.com/types/netmap
         golang.org/x/crypto/blake2b                                  from golang.org/x/crypto/nacl/box
         golang.org/x/crypto/chacha20                                 from golang.org/x/crypto/chacha20poly1305
@@ -118,13 +120,14 @@ tailscale.com/cmd/tailscale dependencies: (generated by github.com/tailscale/dep
         debug/macho                                                  from rsc.io/goversion/version
         debug/pe                                                     from rsc.io/goversion/version
         embed                                                        from tailscale.com/cmd/tailscale/cli
-        encoding                                                     from encoding/json
+        encoding                                                     from encoding/json+
         encoding/asn1                                                from crypto/x509+
         encoding/base64                                              from encoding/json+
         encoding/binary                                              from compress/gzip+
         encoding/hex                                                 from crypto/x509+
         encoding/json                                                from expvar+
         encoding/pem                                                 from crypto/tls+
+        encoding/xml                                                 from tailscale.com/cmd/tailscale/cli
         errors                                                       from bufio+
         expvar                                                       from tailscale.com/derp+
         flag                                                         from github.com/peterbourgon/ff/v2+
@@ -156,6 +159,7 @@ tailscale.com/cmd/tailscale dependencies: (generated by github.com/tailscale/dep
         os                                                           from crypto/rand+
         os/exec                                                      from github.com/toqueteos/webbrowser+
         os/signal                                                    from tailscale.com/cmd/tailscale/cli
+        os/user                                                      from tailscale.com/util/groupmember
         path                                                         from debug/dwarf+
         path/filepath                                                from crypto/x509+
         reflect                                                      from crypto/x509+

cmd/tailscaled/depaware.txt

@@ -17,14 +17,15 @@ tailscale.com/cmd/tailscaled dependencies: (generated by github.com/tailscale/de
         github.com/klauspost/compress/huff0                          from github.com/klauspost/compress/zstd
         github.com/klauspost/compress/zstd                           from tailscale.com/smallzstd
         github.com/klauspost/compress/zstd/internal/xxhash           from github.com/klauspost/compress/zstd
-   L 💣 github.com/mdlayher/netlink                                  from github.com/jsimonetti/rtnetlink+
-   L 💣 github.com/mdlayher/netlink/nlenc                            from github.com/jsimonetti/rtnetlink+
+   L 💣 github.com/mdlayher/netlink                                  from tailscale.com/wgengine/monitor+
+   L 💣 github.com/mdlayher/netlink/nlenc                            from github.com/mdlayher/netlink+
    L    github.com/mdlayher/sdnotify                                 from tailscale.com/util/systemd
+   L 💣 github.com/mdlayher/socket                                   from github.com/mdlayher/netlink
    W    github.com/pkg/errors                                        from github.com/tailscale/certstore
    W 💣 github.com/tailscale/certstore                               from tailscale.com/control/controlclient
         github.com/tcnksm/go-httpstat                                from tailscale.com/net/netcheck
      💣 go4.org/intern                                               from inet.af/netaddr
-     💣 go4.org/mem                                                  from tailscale.com/control/controlclient+
+     💣 go4.org/mem                                                  from tailscale.com/derp+
         go4.org/unsafe/assume-no-moving-gc                           from go4.org/intern
      💣 golang.zx2c4.com/wireguard/conn                              from golang.zx2c4.com/wireguard/device+
    W 💣 golang.zx2c4.com/wireguard/conn/winrio                       from golang.zx2c4.com/wireguard/conn
@@ -39,6 +40,8 @@ tailscale.com/cmd/tailscaled dependencies: (generated by github.com/tailscale/de
    W 💣 golang.zx2c4.com/wireguard/tun/wintun                        from golang.zx2c4.com/wireguard/tun+
    W 💣 golang.zx2c4.com/wireguard/windows/tunnel/winipcfg           from tailscale.com/net/interfaces+
         inet.af/netaddr                                              from tailscale.com/control/controlclient+
+        inet.af/netstack/atomicbitops                                from inet.af/netstack/tcpip+
+     💣 inet.af/netstack/buffer                                      from inet.af/netstack/tcpip/stack
      💣 inet.af/netstack/gohacks                                     from inet.af/netstack/state/wire+
         inet.af/netstack/linewriter                                  from inet.af/netstack/log
         inet.af/netstack/log                                         from inet.af/netstack/state+
@@ -47,7 +50,7 @@ tailscale.com/cmd/tailscaled dependencies: (generated by github.com/tailscale/de
      💣 inet.af/netstack/state                                       from inet.af/netstack/tcpip+
         inet.af/netstack/state/wire                                  from inet.af/netstack/state
      💣 inet.af/netstack/sync                                        from inet.af/netstack/linewriter+
-     💣 inet.af/netstack/tcpip                                       from inet.af/netstack/tcpip/adapters/gonet+
+        inet.af/netstack/tcpip                                       from inet.af/netstack/tcpip/adapters/gonet+
         inet.af/netstack/tcpip/adapters/gonet                        from tailscale.com/wgengine/netstack
      💣 inet.af/netstack/tcpip/buffer                                from inet.af/netstack/tcpip/adapters/gonet+
         inet.af/netstack/tcpip/hash/jenkins                          from inet.af/netstack/tcpip/stack+
@@ -73,6 +76,7 @@ tailscale.com/cmd/tailscaled dependencies: (generated by github.com/tailscale/de
    W 💣 inet.af/wf                                                   from tailscale.com/wf
         rsc.io/goversion/version                                     from tailscale.com/version
         tailscale.com/atomicfile                                     from tailscale.com/ipn+
+        tailscale.com/client/tailscale                               from tailscale.com/derp
         tailscale.com/client/tailscale/apitype                       from tailscale.com/ipn/ipnlocal+
         tailscale.com/control/controlclient                          from tailscale.com/ipn/ipnlocal+
         tailscale.com/derp                                           from tailscale.com/derp/derphttp+
@@ -80,6 +84,7 @@ tailscale.com/cmd/tailscaled dependencies: (generated by github.com/tailscale/de
         tailscale.com/derp/derpmap                                   from tailscale.com/cmd/tailscaled+
         tailscale.com/disco                                          from tailscale.com/derp+
         tailscale.com/health                                         from tailscale.com/control/controlclient+
+        tailscale.com/hostinfo                                       from tailscale.com/control/controlclient+
         tailscale.com/internal/deephash                              from tailscale.com/ipn/ipnlocal+
         tailscale.com/ipn                                            from tailscale.com/ipn/ipnserver+
         tailscale.com/ipn/ipnlocal                                   from tailscale.com/ipn/ipnserver+
@@ -113,7 +118,7 @@ tailscale.com/cmd/tailscaled dependencies: (generated by github.com/tailscale/de
         tailscale.com/net/tstun                                      from tailscale.com/cmd/tailscaled+
         tailscale.com/paths                                          from tailscale.com/cmd/tailscaled+
         tailscale.com/portlist                                       from tailscale.com/ipn/ipnlocal
-        tailscale.com/safesocket                                     from tailscale.com/ipn/ipnserver
+        tailscale.com/safesocket                                     from tailscale.com/ipn/ipnserver+
         tailscale.com/smallzstd                                      from tailscale.com/ipn/ipnserver+
         tailscale.com/syncs                                          from tailscale.com/net/interfaces+
         tailscale.com/tailcfg                                        from tailscale.com/control/controlclient+
@@ -134,7 +139,8 @@ tailscale.com/cmd/tailscaled dependencies: (generated by github.com/tailscale/de
    L    tailscale.com/util/cmpver                                    from tailscale.com/net/dns
         tailscale.com/util/dnsname                                   from tailscale.com/ipn/ipnstate+
   LW    tailscale.com/util/endian                                    from tailscale.com/net/netns+
-   L    tailscale.com/util/lineread                                  from tailscale.com/control/controlclient+
+        tailscale.com/util/groupmember                               from tailscale.com/ipn/ipnserver
+        tailscale.com/util/lineread                                  from tailscale.com/control/controlclient+
         tailscale.com/util/osshare                                   from tailscale.com/cmd/tailscaled+
         tailscale.com/util/pidowner                                  from tailscale.com/ipn/ipnserver
         tailscale.com/util/racebuild                                 from tailscale.com/logpolicy
@@ -179,7 +185,7 @@ tailscale.com/cmd/tailscaled dependencies: (generated by github.com/tailscale/de
         golang.org/x/sync/errgroup                                   from tailscale.com/derp
         golang.org/x/sync/singleflight                               from tailscale.com/net/dnscache
         golang.org/x/sys/cpu                                         from golang.org/x/crypto/blake2b+
-  LD    golang.org/x/sys/unix                                        from github.com/jsimonetti/rtnetlink/internal/unix+
+  LD    golang.org/x/sys/unix                                        from github.com/mdlayher/netlink+
    W    golang.org/x/sys/windows                                     from github.com/go-ole/go-ole+
    W    golang.org/x/sys/windows/registry                            from golang.zx2c4.com/wireguard/windows/tunnel/winipcfg+
    W    golang.org/x/sys/windows/svc                                 from tailscale.com/cmd/tailscaled+

cmd/tailscaled/tailscaled.openrc

@@ -0,0 +1,23 @@
+#!/sbin/openrc-run
+
+source /etc/default/tailscaled
+
+command="/usr/sbin/tailscaled"
+command_args="--state=/var/lib/tailscale/tailscaled.state --port=$PORT --socket=/var/run/tailscale/tailscaled.sock $FLAGS"
+command_background=true
+pidfile="/run/tailscaled.pid"
+start_stop_daemon_args="-1 /var/log/tailscaled.log -2 /var/log/tailscaled.log"
+
+depend() {
+    need net
+}
+
+start_pre() {
+    mkdir -p /var/run/tailscale
+    mkdir -p /var/lib/tailscale
+    $command --cleanup
+}
+
+stop_post() {
+    $command --cleanup
+}

cmd/tailscaled/tailscaled.service

@@ -2,7 +2,7 @@
 Description=Tailscale node agent
 Documentation=https://tailscale.com/kb/
 Wants=network-pre.target
-After=network-pre.target
+After=network-pre.target NetworkManager.service systemd-resolved.service
 
 [Service]
 EnvironmentFile=/etc/default/tailscaled

control/controlclient/auto.go

@@ -576,9 +576,12 @@ func (c *Auto) sendStatus(who string, err error, url string, nm *netmap.NetworkM
 	c.logf("[v1] sendStatus: %s: %v", who, state)
 
 	var p *persist.Persist
-	var fin *empty.Message
+	var loginFin, logoutFin *empty.Message
 	if state == StateAuthenticated {
-		fin = new(empty.Message)
+		loginFin = new(empty.Message)
+	}
+	if state == StateNotAuthenticated {
+		logoutFin = new(empty.Message)
 	}
 	if nm != nil && loggedIn && synced {
 		pp := c.direct.GetPersist()
@@ -589,12 +592,13 @@ func (c *Auto) sendStatus(who string, err error, url string, nm *netmap.NetworkM
 		nm = nil
 	}
 	new := Status{
-		LoginFinished: fin,
-		URL:           url,
-		Persist:       p,
-		NetMap:        nm,
-		Hostinfo:      hi,
-		State:         state,
+		LoginFinished:  loginFin,
+		LogoutFinished: logoutFin,
+		URL:            url,
+		Persist:        p,
+		NetMap:         nm,
+		Hostinfo:       hi,
+		State:          state,
 	}
 	if err != nil {
 		new.Err = err.Error()
@@ -712,3 +716,9 @@ func (c *Auto) TestOnlySetAuthKey(authkey string) {
 func (c *Auto) TestOnlyTimeNow() time.Time {
 	return c.timeNow()
 }
+
+// SetDNS sends the SetDNSRequest request to the control plane server,
+// requesting a DNS record be created or updated.
+func (c *Auto) SetDNS(ctx context.Context, req *tailcfg.SetDNSRequest) error {
+	return c.direct.SetDNS(ctx, req)
+}

control/controlclient/client.go

@@ -74,4 +74,7 @@ type Client interface {
 	// in a separate http request. It has nothing to do with the rest of
 	// the state machine.
 	UpdateEndpoints(localPort uint16, endpoints []tailcfg.Endpoint)
+	// SetDNS sends the SetDNSRequest request to the control plane server,
+	// requesting a DNS record be created or updated.
+	SetDNS(context.Context, *tailcfg.SetDNSRequest) error
 }

control/controlclient/controlclient_test.go

@@ -22,7 +22,7 @@ func fieldsOf(t reflect.Type) (fields []string) {
 
 func TestStatusEqual(t *testing.T) {
 	// Verify that the Equal method stays in sync with reality
-	equalHandles := []string{"LoginFinished", "Err", "URL", "NetMap", "State", "Persist", "Hostinfo"}
+	equalHandles := []string{"LoginFinished", "LogoutFinished", "Err", "URL", "NetMap", "State", "Persist", "Hostinfo"}
 	if have := fieldsOf(reflect.TypeOf(Status{})); !reflect.DeepEqual(have, equalHandles) {
 		t.Errorf("Status.Equal check might be out of sync\nfields: %q\nhandled: %q\n",
 			have, equalHandles)

control/controlclient/direct.go

@@ -32,6 +32,7 @@ import (
 	"golang.org/x/crypto/nacl/box"
 	"inet.af/netaddr"
 	"tailscale.com/health"
+	"tailscale.com/ipn/ipnstate"
 	"tailscale.com/log/logheap"
 	"tailscale.com/net/dnscache"
 	"tailscale.com/net/dnsfallback"
@@ -66,6 +67,7 @@ type Direct struct {
 	debugFlags             []string
 	keepSharerAndUserSplit bool
 	skipIPForwardingCheck  bool
+	pinger                 Pinger
 
 	mu           sync.Mutex // mutex guards the following fields
 	serverKey    wgkey.Key
@@ -78,6 +80,7 @@ type Direct struct {
 	endpoints     []tailcfg.Endpoint
 	everEndpoints bool   // whether we've ever had non-empty endpoints
 	localPort     uint16 // or zero to mean auto
+	lastPingURL   string // last PingRequest.URL received, for dup suppresion
 }
 
 type Options struct {
@@ -103,6 +106,18 @@ type Options struct {
 	// forwarding works and should not be double-checked by the
 	// controlclient package.
 	SkipIPForwardingCheck bool
+
+	// Pinger optionally specifies the Pinger to use to satisfy
+	// MapResponse.PingRequest queries from the control plane.
+	// If nil, PingRequest queries are not answered.
+	Pinger Pinger
+}
+
+// Pinger is a subset of the wgengine.Engine interface, containing just the Ping method.
+type Pinger interface {
+	// Ping is a request to start a discovery or TSMP ping with the peer handling
+	// the given IP and then call cb with its ping latency & method.
+	Ping(ip netaddr.IP, useTSMP bool, cb func(*ipnstate.PingResult))
 }
 
 type Decompressor interface {
@@ -165,6 +180,7 @@ func NewDirect(opts Options) (*Direct, error) {
 		keepSharerAndUserSplit: opts.KeepSharerAndUserSplit,
 		linkMon:                opts.LinkMonitor,
 		skipIPForwardingCheck:  opts.SkipIPForwardingCheck,
+		pinger:                 opts.Pinger,
 	}
 	if opts.Hostinfo == nil {
 		c.SetHostinfo(NewHostinfo())
@@ -760,7 +776,7 @@ func (c *Direct) sendMapRequest(ctx context.Context, maxPolls int, cb func(*netm
 			health.GotStreamedMapResponse()
 		}
 
-		if pr := resp.PingRequest; pr != nil {
+		if pr := resp.PingRequest; pr != nil && c.isUniquePingRequest(pr) {
 			go answerPing(c.logf, c.httpc, pr)
 		}
 
@@ -1155,6 +1171,23 @@ func ipForwardingBroken(routes []netaddr.IPPrefix, state *interfaces.State) bool
 	return false
 }
 
+// isUniquePingRequest reports whether pr contains a new PingRequest.URL
+// not already handled, noting its value when returning true.
+func (c *Direct) isUniquePingRequest(pr *tailcfg.PingRequest) bool {
+	if pr == nil || pr.URL == "" {
+		// Bogus.
+		return false
+	}
+
+	c.mu.Lock()
+	defer c.mu.Unlock()
+	if pr.URL == c.lastPingURL {
+		return false
+	}
+	c.lastPingURL = pr.URL
+	return true
+}
+
 func answerPing(logf logger.Logf, c *http.Client, pr *tailcfg.PingRequest) {
 	if pr.URL == "" {
 		logf("invalid PingRequest with no URL")
@@ -1211,3 +1244,50 @@ func sleepAsRequested(ctx context.Context, logf logger.Logf, timeoutReset chan<-
 		}
 	}
 }
+
+// SetDNS sends the SetDNSRequest request to the control plane server,
+// requesting a DNS record be created or updated.
+func (c *Direct) SetDNS(ctx context.Context, req *tailcfg.SetDNSRequest) error {
+	c.mu.Lock()
+	serverKey := c.serverKey
+	c.mu.Unlock()
+
+	if serverKey.IsZero() {
+		return errors.New("zero serverKey")
+	}
+	machinePrivKey, err := c.getMachinePrivKey()
+	if err != nil {
+		return fmt.Errorf("getMachinePrivKey: %w", err)
+	}
+	if machinePrivKey.IsZero() {
+		return errors.New("getMachinePrivKey returned zero key")
+	}
+
+	bodyData, err := encode(req, &serverKey, &machinePrivKey)
+	if err != nil {
+		return err
+	}
+	body := bytes.NewReader(bodyData)
+
+	u := fmt.Sprintf("%s/machine/%s/set-dns", c.serverURL, machinePrivKey.Public().HexString())
+	hreq, err := http.NewRequestWithContext(ctx, "POST", u, body)
+	if err != nil {
+		return err
+	}
+	res, err := c.httpc.Do(hreq)
+	if err != nil {
+		return err
+	}
+	defer res.Body.Close()
+	if res.StatusCode != 200 {
+		msg, _ := ioutil.ReadAll(res.Body)
+		return fmt.Errorf("set-dns response: %v, %.200s", res.Status, strings.TrimSpace(string(msg)))
+	}
+	var setDNSRes struct{} // no fields yet
+	if err := decode(res, &setDNSRes, &serverKey, &machinePrivKey); err != nil {
+		c.logf("error decoding SetDNSResponse with server key %s and machine key %s: %v", serverKey, machinePrivKey.Public(), err)
+		return fmt.Errorf("set-dns-response: %v", err)
+	}
+
+	return nil
+}

control/controlclient/hostinfo_linux.go

@@ -9,13 +9,11 @@ package controlclient
 import (
 	"bytes"
 	"fmt"
-	"io"
 	"io/ioutil"
-	"os"
 	"strings"
 	"syscall"
 
-	"go4.org/mem"
+	"tailscale.com/hostinfo"
 	"tailscale.com/util/lineread"
 	"tailscale.com/version/distro"
 )
@@ -56,11 +54,11 @@ func osVersionLinux() string {
 		}
 		attrBuf.WriteByte(byte(b))
 	}
-	if inContainer() {
+	if hostinfo.InContainer() {
 		attrBuf.WriteString("; container")
 	}
-	if inKnative() {
-		attrBuf.WriteString("; env=kn")
+	if env := hostinfo.GetEnvType(); env != "" {
+		fmt.Fprintf(&attrBuf, "; env=%s", env)
 	}
 	attr := attrBuf.String()
 
@@ -93,31 +91,3 @@ func osVersionLinux() string {
 	}
 	return fmt.Sprintf("Other%s", attr)
 }
-
-func inContainer() (ret bool) {
-	lineread.File("/proc/1/cgroup", func(line []byte) error {
-		if mem.Contains(mem.B(line), mem.S("/docker/")) ||
-			mem.Contains(mem.B(line), mem.S("/lxc/")) {
-			ret = true
-			return io.EOF // arbitrary non-nil error to stop loop
-		}
-		return nil
-	})
-	lineread.File("/proc/mounts", func(line []byte) error {
-		if mem.Contains(mem.B(line), mem.S("fuse.lxcfs")) {
-			ret = true
-			return io.EOF
-		}
-		return nil
-	})
-	return
-}
-
-func inKnative() bool {
-	// https://cloud.google.com/run/docs/reference/container-contract#env-vars
-	if os.Getenv("K_REVISION") != "" && os.Getenv("K_CONFIGURATION") != "" &&
-		os.Getenv("K_SERVICE") != "" && os.Getenv("PORT") != "" {
-		return true
-	}
-	return false
-}

control/controlclient/status.go

@@ -64,11 +64,12 @@ func (s State) String() string {
 }
 
 type Status struct {
-	_             structs.Incomparable
-	LoginFinished *empty.Message // nonempty when login finishes
-	Err           string
-	URL           string             // interactive URL to visit to finish logging in
-	NetMap        *netmap.NetworkMap // server-pushed configuration
+	_              structs.Incomparable
+	LoginFinished  *empty.Message // nonempty when login finishes
+	LogoutFinished *empty.Message // nonempty when logout finishes
+	Err            string
+	URL            string             // interactive URL to visit to finish logging in
+	NetMap         *netmap.NetworkMap // server-pushed configuration
 
 	// The internal state should not be exposed outside this
 	// package, but we have some automated tests elsewhere that need to
@@ -86,6 +87,7 @@ func (s *Status) Equal(s2 *Status) bool {
 	}
 	return s != nil && s2 != nil &&
 		(s.LoginFinished == nil) == (s2.LoginFinished == nil) &&
+		(s.LogoutFinished == nil) == (s2.LogoutFinished == nil) &&
 		s.Err == s2.Err &&
 		s.URL == s2.URL &&
 		reflect.DeepEqual(s.Persist, s2.Persist) &&

derp/derp_server.go

@@ -20,18 +20,24 @@ import (
 	"io"
 	"io/ioutil"
 	"log"
+	"math"
 	"math/big"
 	"math/rand"
+	"net/http"
 	"os"
+	"os/exec"
 	"runtime"
 	"strconv"
 	"strings"
 	"sync"
+	"sync/atomic"
 	"time"
 
 	"go4.org/mem"
 	"golang.org/x/crypto/nacl/box"
 	"golang.org/x/sync/errgroup"
+	"inet.af/netaddr"
+	"tailscale.com/client/tailscale"
 	"tailscale.com/disco"
 	"tailscale.com/metrics"
 	"tailscale.com/types/key"
@@ -120,6 +126,11 @@ type Server struct {
 	multiForwarderCreated    expvar.Int
 	multiForwarderDeleted    expvar.Int
 	removePktForwardOther    expvar.Int
+	avgQueueDuration         *uint64 // In milliseconds; accessed atomically
+
+	// verifyClients only accepts client connections to the DERP server if the clientKey is a
+	// known peer in the network, as specified by a running tailscaled's client's local api.
+	verifyClients bool
 
 	mu          sync.Mutex
 	closed      bool
@@ -138,6 +149,9 @@ type Server struct {
 	// because it includes intra-region forwarded packets as the
 	// src.
 	sentTo map[key.Public]map[key.Public]int64 // src => dst => dst's latest sclient.connNum
+
+	// maps from netaddr.IPPort to a client's public key
+	keyOfAddr map[netaddr.IPPort]key.Public
 }
 
 // PacketForwarder is something that can forward packets.
@@ -182,6 +196,8 @@ func NewServer(privateKey key.Private, logf logger.Logf) *Server {
 		memSys0:              ms.Sys,
 		watchers:             map[*sclient]bool{},
 		sentTo:               map[key.Public]map[key.Public]int64{},
+		avgQueueDuration:     new(uint64),
+		keyOfAddr:            map[netaddr.IPPort]key.Public{},
 	}
 	s.initMetacert()
 	s.packetsRecvDisco = s.packetsRecvByKind.Get("disco")
@@ -203,6 +219,13 @@ func (s *Server) SetMeshKey(v string) {
 	s.meshKey = v
 }
 
+// SetVerifyClients sets whether this DERP server verifies clients through tailscaled.
+//
+// It must be called before serving begins.
+func (s *Server) SetVerifyClient(v bool) {
+	s.verifyClients = v
+}
+
 // HasMeshKey reports whether the server is configured with a mesh key.
 func (s *Server) HasMeshKey() bool { return s.meshKey != "" }
 
@@ -339,6 +362,7 @@ func (s *Server) registerClient(c *sclient) {
 	if _, ok := s.clientsMesh[c.key]; !ok {
 		s.clientsMesh[c.key] = nil // just for varz of total users in cluster
 	}
+	s.keyOfAddr[c.remoteIPPort] = c.key
 	s.curClients.Add(1)
 	s.broadcastPeerStateChangeLocked(c.key, true)
 }
@@ -373,6 +397,8 @@ func (s *Server) unregisterClient(c *sclient) {
 		delete(s.watchers, c)
 	}
 
+	delete(s.keyOfAddr, c.remoteIPPort)
+
 	s.curClients.Add(-1)
 	if c.preferred {
 		s.curHomeClients.Add(-1)
@@ -446,20 +472,23 @@ func (s *Server) accept(nc Conn, brw *bufio.ReadWriter, remoteAddr string, connN
 	ctx, cancel := context.WithCancel(context.Background())
 	defer cancel()
 
+	remoteIPPort, _ := netaddr.ParseIPPort(remoteAddr)
+
 	c := &sclient{
-		connNum:     connNum,
-		s:           s,
-		key:         clientKey,
-		nc:          nc,
-		br:          br,
-		bw:          bw,
-		logf:        logger.WithPrefix(s.logf, fmt.Sprintf("derp client %v/%x: ", remoteAddr, clientKey)),
-		done:        ctx.Done(),
-		remoteAddr:  remoteAddr,
-		connectedAt: time.Now(),
-		sendQueue:   make(chan pkt, perClientSendQueueDepth),
-		peerGone:    make(chan key.Public),
-		canMesh:     clientInfo.MeshKey != "" && clientInfo.MeshKey == s.meshKey,
+		connNum:      connNum,
+		s:            s,
+		key:          clientKey,
+		nc:           nc,
+		br:           br,
+		bw:           bw,
+		logf:         logger.WithPrefix(s.logf, fmt.Sprintf("derp client %v/%x: ", remoteAddr, clientKey)),
+		done:         ctx.Done(),
+		remoteAddr:   remoteAddr,
+		remoteIPPort: remoteIPPort,
+		connectedAt:  time.Now(),
+		sendQueue:    make(chan pkt, perClientSendQueueDepth),
+		peerGone:     make(chan key.Public),
+		canMesh:      clientInfo.MeshKey != "" && clientInfo.MeshKey == s.meshKey,
 	}
 	if c.canMesh {
 		c.meshUpdate = make(chan struct{})
@@ -611,8 +640,9 @@ func (c *sclient) handleFrameForwardPacket(ft frameType, fl uint32) error {
 	}
 
 	return c.sendPkt(dst, pkt{
-		bs:  contents,
-		src: srcKey,
+		bs:         contents,
+		enqueuedAt: time.Now(),
+		src:        srcKey,
 	})
 }
 
@@ -665,8 +695,9 @@ func (c *sclient) handleFrameSendPacket(ft frameType, fl uint32) error {
 	}
 
 	p := pkt{
-		bs:  contents,
-		src: c.key,
+		bs:         contents,
+		enqueuedAt: time.Now(),
+		src:        c.key,
 	}
 	return c.sendPkt(dst, p)
 }
@@ -696,7 +727,7 @@ func (c *sclient) sendPkt(dst *sclient, p pkt) error {
 		}
 
 		select {
-		case <-dst.sendQueue:
+		case pkt := <-dst.sendQueue:
 			s.packetsDropped.Add(1)
 			s.packetsDroppedQueueHead.Add(1)
 			if verboseDropKeys[dstKey] {
@@ -705,6 +736,7 @@ func (c *sclient) sendPkt(dst *sclient, p pkt) error {
 				msg := fmt.Sprintf("tail drop %s -> %s", p.src.ShortString(), dstKey.ShortString())
 				c.s.limitedLogf(msg)
 			}
+			c.recordQueueTime(pkt.enqueuedAt)
 			if debug {
 				c.logf("dropping packet from client %x queue head", dstKey)
 			}
@@ -750,8 +782,17 @@ func (c *sclient) requestMeshUpdate() {
 }
 
 func (s *Server) verifyClient(clientKey key.Public, info *clientInfo) error {
-	// TODO(crawshaw): implement policy constraints on who can use the DERP server
-	// TODO(bradfitz): ... and at what rate.
+	if !s.verifyClients {
+		return nil
+	}
+	status, err := tailscale.Status(context.TODO())
+	if err != nil {
+		return fmt.Errorf("failed to query local tailscaled status: %w", err)
+	}
+	if _, exists := status.Peer[clientKey]; !exists {
+		return fmt.Errorf("client %v not in set of peers", clientKey)
+	}
+	// TODO(bradfitz): add policy for configurable bandwidth rate per client?
 	return nil
 }
 
@@ -885,18 +926,19 @@ func (s *Server) recvForwardPacket(br *bufio.Reader, frameLen uint32) (srcKey, d
 // (The "s" prefix is to more explicitly distinguish it from Client in derp_client.go)
 type sclient struct {
 	// Static after construction.
-	connNum    int64 // process-wide unique counter, incremented each Accept
-	s          *Server
-	nc         Conn
-	key        key.Public
-	info       clientInfo
-	logf       logger.Logf
-	done       <-chan struct{} // closed when connection closes
-	remoteAddr string          // usually ip:port from net.Conn.RemoteAddr().String()
-	sendQueue  chan pkt        // packets queued to this client; never closed
-	peerGone   chan key.Public // write request that a previous sender has disconnected (not used by mesh peers)
-	meshUpdate chan struct{}   // write request to write peerStateChange
-	canMesh    bool            // clientInfo had correct mesh token for inter-region routing
+	connNum      int64 // process-wide unique counter, incremented each Accept
+	s            *Server
+	nc           Conn
+	key          key.Public
+	info         clientInfo
+	logf         logger.Logf
+	done         <-chan struct{} // closed when connection closes
+	remoteAddr   string          // usually ip:port from net.Conn.RemoteAddr().String()
+	remoteIPPort netaddr.IPPort  // zero if remoteAddr is not ip:port.
+	sendQueue    chan pkt        // packets queued to this client; never closed
+	peerGone     chan key.Public // write request that a previous sender has disconnected (not used by mesh peers)
+	meshUpdate   chan struct{}   // write request to write peerStateChange
+	canMesh      bool            // clientInfo had correct mesh token for inter-region routing
 
 	// Owned by run, not thread-safe.
 	br          *bufio.Reader
@@ -927,11 +969,13 @@ type pkt struct {
 	// src is the who's the sender of the packet.
 	src key.Public
 
+	// enqueuedAt is when a packet was put onto a queue before it was sent,
+	// and is used for reporting metrics on the duration of packets in the queue.
+	enqueuedAt time.Time
+
 	// bs is the data packet bytes.
 	// The memory is owned by pkt.
 	bs []byte
-
-	// TODO(danderson): enqueue time, to measure queue latency?
 }
 
 func (c *sclient) setPreferred(v bool) {
@@ -959,6 +1003,25 @@ func (c *sclient) setPreferred(v bool) {
 	}
 }
 
+// expMovingAverage returns the new moving average given the previous average,
+// a new value, and an alpha decay factor.
+// https://en.wikipedia.org/wiki/Moving_average#Exponential_moving_average
+func expMovingAverage(prev, newValue, alpha float64) float64 {
+	return alpha*newValue + (1-alpha)*prev
+}
+
+// recordQueueTime updates the average queue duration metric after a packet has been sent.
+func (c *sclient) recordQueueTime(enqueuedAt time.Time) {
+	elapsed := float64(time.Since(enqueuedAt).Milliseconds())
+	for {
+		old := atomic.LoadUint64(c.s.avgQueueDuration)
+		newAvg := expMovingAverage(math.Float64frombits(old), elapsed, 0.1)
+		if atomic.CompareAndSwapUint64(c.s.avgQueueDuration, old, math.Float64bits(newAvg)) {
+			break
+		}
+	}
+}
+
 func (c *sclient) sendLoop(ctx context.Context) error {
 	defer func() {
 		// If the sender shuts down unilaterally due to an error, close so
@@ -1002,6 +1065,7 @@ func (c *sclient) sendLoop(ctx context.Context) error {
 			continue
 		case msg := <-c.sendQueue:
 			werr = c.sendPacket(msg.src, msg.bs)
+			c.recordQueueTime(msg.enqueuedAt)
 			continue
 		case <-keepAliveTick.C:
 			werr = c.sendKeepAlive()
@@ -1025,6 +1089,7 @@ func (c *sclient) sendLoop(ctx context.Context) error {
 			continue
 		case msg := <-c.sendQueue:
 			werr = c.sendPacket(msg.src, msg.bs)
+			c.recordQueueTime(msg.enqueuedAt)
 		case <-keepAliveTick.C:
 			werr = c.sendKeepAlive()
 		}
@@ -1290,6 +1355,9 @@ func (s *Server) ExpVar() expvar.Var {
 	m.Set("multiforwarder_created", &s.multiForwarderCreated)
 	m.Set("multiforwarder_deleted", &s.multiForwarderDeleted)
 	m.Set("packet_forwarder_delete_other_value", &s.removePktForwardOther)
+	m.Set("average_queue_duration_ms", expvar.Func(func() interface{} {
+		return math.Float64frombits(atomic.LoadUint64(s.avgQueueDuration))
+	}))
 	var expvarVersion expvar.String
 	expvarVersion.Set(version.Long)
 	m.Set("version", &expvarVersion)
@@ -1365,3 +1433,84 @@ func writePublicKey(bw *bufio.Writer, key *key.Public) error {
 	}
 	return nil
 }
+
+const minTimeBetweenLogs = 2 * time.Second
+
+// BytesSentRecv records the number of bytes that have been sent since the last traffic check
+// for a given process, as well as the public key of the process sending those bytes.
+type BytesSentRecv struct {
+	Sent uint64
+	Recv uint64
+	// Key is the public key of the client which sent/received these bytes.
+	Key key.Public
+}
+
+// parseSSOutput parses the output from the specific call to ss in ServeDebugTraffic.
+// Separated out for ease of testing.
+func parseSSOutput(raw string) map[netaddr.IPPort]BytesSentRecv {
+	newState := map[netaddr.IPPort]BytesSentRecv{}
+	// parse every 2 lines and get src and dst ips, and kv pairs
+	lines := strings.Split(raw, "\n")
+	for i := 0; i < len(lines); i += 2 {
+		ipInfo := strings.Fields(strings.TrimSpace(lines[i]))
+		if len(ipInfo) < 5 {
+			continue
+		}
+		src, err := netaddr.ParseIPPort(ipInfo[4])
+		if err != nil {
+			continue
+		}
+		stats := strings.Fields(strings.TrimSpace(lines[i+1]))
+		stat := BytesSentRecv{}
+		for _, s := range stats {
+			if strings.Contains(s, "bytes_sent") {
+				sent, err := strconv.Atoi(s[strings.Index(s, ":")+1:])
+				if err == nil {
+					stat.Sent = uint64(sent)
+				}
+			} else if strings.Contains(s, "bytes_received") {
+				recv, err := strconv.Atoi(s[strings.Index(s, ":")+1:])
+				if err == nil {
+					stat.Recv = uint64(recv)
+				}
+			}
+		}
+		newState[src] = stat
+	}
+	return newState
+}
+
+func (s *Server) ServeDebugTraffic(w http.ResponseWriter, r *http.Request) {
+	prevState := map[netaddr.IPPort]BytesSentRecv{}
+	enc := json.NewEncoder(w)
+	for r.Context().Err() == nil {
+		output, err := exec.Command("ss", "-i", "-H", "-t").Output()
+		if err != nil {
+			fmt.Fprintf(w, "ss failed: %v", err)
+			return
+		}
+		newState := parseSSOutput(string(output))
+		s.mu.Lock()
+		for k, next := range newState {
+			prev := prevState[k]
+			if prev.Sent < next.Sent || prev.Recv < next.Recv {
+				if pkey, ok := s.keyOfAddr[k]; ok {
+					next.Key = pkey
+					if err := enc.Encode(next); err != nil {
+						s.mu.Unlock()
+						return
+					}
+				}
+			}
+		}
+		s.mu.Unlock()
+		prevState = newState
+		if _, err := fmt.Fprintln(w); err != nil {
+			return
+		}
+		if f, ok := w.(http.Flusher); ok {
+			f.Flush()
+		}
+		time.Sleep(minTimeBetweenLogs)
+	}
+}

derp/derp_test.go

@@ -948,3 +948,14 @@ func waitConnect(t testing.TB, c *Client) {
 		t.Fatalf("client first Recv was unexpected type %T", v)
 	}
 }
+
+func TestParseSSOutput(t *testing.T) {
+	contents, err := ioutil.ReadFile("testdata/example_ss.txt")
+	if err != nil {
+		t.Errorf("ioutil.Readfile(example_ss.txt) failed: %v", err)
+	}
+	seen := parseSSOutput(string(contents))
+	if len(seen) == 0 {
+		t.Errorf("parseSSOutput expected non-empty map")
+	}
+}

derp/testdata/example_ss.txt

@@ -0,0 +1,8 @@
+ESTAB 0      0       10.255.1.11:35238  34.210.105.16:https
+         cubic wscale:7,7 rto:236 rtt:34.14/3.432 ato:40 mss:1448 pmtu:1500 rcvmss:1448 advmss:1448 cwnd:8 ssthresh:6 bytes_sent:38056577 bytes_retrans:2918 bytes_acked:38053660 bytes_received:6973211 segs_out:165090 segs_in:124227 data_segs_out:78018 data_segs_in:71645 send 2.71Mbps lastsnd:1156 lastrcv:1120 lastack:1120 pacing_rate 3.26Mbps delivery_rate 2.35Mbps delivered:78017 app_limited busy:2586132ms retrans:0/6 dsack_dups:4 reordering:5 reord_seen:15 rcv_rtt:126355 rcv_space:65780 rcv_ssthresh:541928 minrtt:26.632
+ESTAB 0      80     100.79.58.14:ssh    100.95.73.104:58145
+         cubic wscale:6,7 rto:224 rtt:23.051/2.03 ato:172 mss:1228 pmtu:1280 rcvmss:1228 advmss:1228 cwnd:10 ssthresh:94 bytes_sent:1591815 bytes_retrans:944 bytes_acked:1590791 bytes_received:158925 segs_out:8070 segs_in:8858 data_segs_out:7452 data_segs_in:3789 send 4.26Mbps lastsnd:4 lastrcv:4 lastack:4 pacing_rate 8.52Mbps delivery_rate 10.9Mbps delivered:7451 app_limited busy:61656ms unacked:2 retrans:0/10 dsack_dups:10 rcv_rtt:174712 rcv_space:65025 rcv_ssthresh:64296 minrtt:16.186                                   
+ESTAB 0      374     10.255.1.11:43254 167.172.206.31:https
+         cubic wscale:7,7 rto:224 rtt:22.55/1.941 ato:40 mss:1448 pmtu:1500 rcvmss:1448 advmss:1448 cwnd:6 ssthresh:4 bytes_sent:14594668 bytes_retrans:173314 bytes_acked:14420981 bytes_received:4207111 segs_out:80566 segs_in:70310 data_segs_out:24317 data_segs_in:20365 send 3.08Mbps lastsnd:4 lastrcv:4 lastack:4 pacing_rate 3.7Mbps delivery_rate 3.05Mbps delivered:24111 app_limited busy:184820ms unacked:2 retrans:0/185 dsack_dups:1 reord_seen:3 rcv_rtt:651.262 rcv_space:226657 rcv_ssthresh:1557136 minrtt:10.18           
+ESTAB 0      0       10.255.1.11:33036    3.121.18.47:https
+         cubic wscale:7,7 rto:372 rtt:168.408/2.044 ato:40 mss:1448 pmtu:1500 rcvmss:1448 advmss:1448 cwnd:10 bytes_sent:27500 bytes_acked:27501 bytes_received:1386524 segs_out:10990 segs_in:11037 data_segs_out:303 data_segs_in:3414 send 688kbps lastsnd:125776 lastrcv:9640 lastack:22760 pacing_rate 1.38Mbps delivery_rate 482kbps delivered:304 app_limited busy:43024ms rcv_rtt:3345.12 rcv_space:62431 rcv_ssthresh:760472 minrtt:168.867

go.mod

@@ -5,25 +5,28 @@ go 1.16
 require (
 	github.com/alexbrainman/sspi v0.0.0-20210105120005-909beea2cc74
 	github.com/anmitsu/go-shlex v0.0.0-20200514113438-38f4b401e2be // indirect
+	github.com/aws/aws-sdk-go v1.38.52
 	github.com/coreos/go-iptables v0.6.0
 	github.com/frankban/quicktest v1.13.0
-	github.com/tailscale/certstore v0.0.0-20210528134328-066c94b793d3
 	github.com/gliderlabs/ssh v0.3.2
 	github.com/go-multierror/multierror v1.0.2
 	github.com/go-ole/go-ole v1.2.5
 	github.com/godbus/dbus/v5 v5.0.4
-	github.com/google/go-cmp v0.5.5
+	github.com/google/go-cmp v0.5.6
 	github.com/google/goexpect v0.0.0-20210430020637-ab937bf7fd6f
+	github.com/google/uuid v1.1.2
 	github.com/goreleaser/nfpm v1.10.3
-	github.com/jsimonetti/rtnetlink v0.0.0-20210409061457-9561dc9288a7
+	github.com/jsimonetti/rtnetlink v0.0.0-20210525051524-4cc836578190
 	github.com/kballard/go-shellquote v0.0.0-20180428030007-95032a82bc51
 	github.com/klauspost/compress v1.12.2
 	github.com/kr/pty v1.1.8
-	github.com/mdlayher/netlink v1.4.0
+	github.com/mdlayher/netlink v1.4.1
 	github.com/mdlayher/sdnotify v0.0.0-20210228150836-ea3ec207d697
 	github.com/miekg/dns v1.1.42
 	github.com/pborman/getopt v1.1.0
 	github.com/peterbourgon/ff/v2 v2.0.0
+	github.com/pkg/sftp v1.13.0
+	github.com/tailscale/certstore v0.0.0-20210528134328-066c94b793d3
 	github.com/tailscale/depaware v0.0.0-20201214215404-77d1e9757027
 	github.com/tcnksm/go-httpstat v0.2.0
 	github.com/toqueteos/webbrowser v1.2.0
@@ -31,15 +34,15 @@ require (
 	golang.org/x/crypto v0.0.0-20210513164829-c07d793c2f9a
 	golang.org/x/net v0.0.0-20210525063256-abc453219eb5
 	golang.org/x/sync v0.0.0-20210220032951-036812b2e83c
-	golang.org/x/sys v0.0.0-20210525143221-35b2ab0089ea
+	golang.org/x/sys v0.0.0-20210616094352-59db8d763f22
 	golang.org/x/term v0.0.0-20210503060354-a79de5458b56
-	golang.org/x/time v0.0.0-20210220033141-f8bda1e9f3ba
+	golang.org/x/time v0.0.0-20210611083556-38a9dc6acbc6
 	golang.org/x/tools v0.1.2
 	golang.zx2c4.com/wireguard v0.0.0-20210525143454-64cb82f2b3f5
 	golang.zx2c4.com/wireguard/windows v0.3.15-0.20210525143335-94c0476d63e3
 	honnef.co/go/tools v0.1.4
-	inet.af/netaddr v0.0.0-20210523191804-d57edf19c517
-	inet.af/netstack v0.0.0-20210317161235-a1bf4e56ef22
+	inet.af/netaddr v0.0.0-20210602152128-50f8686885e3
+	inet.af/netstack v0.0.0-20210622165351-29b14ebc044e
 	inet.af/peercred v0.0.0-20210318190834-4259e17bb763
 	inet.af/wf v0.0.0-20210516214145-a5343001b756
 	rsc.io/goversion v1.2.0

go.sum

@@ -55,6 +55,8 @@ github.com/armon/go-metrics v0.0.0-20180917152333-f0300d1749da/go.mod h1:Q73ZrmV
 github.com/armon/go-radix v0.0.0-20180808171621-7fddfc383310/go.mod h1:ufUuZ+zHj4x4TnLV4JWEpy2hxWSpsRywHrMgIH9cCH8=
 github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5 h1:0CwZNZbxp69SHPdPJAN/hZIm0C4OItdklCFmMRWYpio=
 github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5/go.mod h1:wHh0iHkYZB8zMSxRWpUBQtwG5a7fFgvEO+odwuTv2gs=
+github.com/aws/aws-sdk-go v1.38.52 h1:7NKcUyTG/CyDX835kq04DDNe8vXaJhbGW8ThemHb18A=
+github.com/aws/aws-sdk-go v1.38.52/go.mod h1:hcU610XS61/+aQV88ixoOzUoG7v3b31pl2zKMmprdro=
 github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q=
 github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8=
 github.com/bgentry/speakeasy v0.1.0/go.mod h1:+zsyZBPWlz7T6j88CTgSN5bM796AkVf0kBD4zp0CCIs=
@@ -231,8 +233,9 @@ github.com/google/go-cmp v0.5.1/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/
 github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
 github.com/google/go-cmp v0.5.3/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
 github.com/google/go-cmp v0.5.4/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
-github.com/google/go-cmp v0.5.5 h1:Khx7svrCpmxxtHBq5j2mp/xVjsi8hQMfNLvJFAlrGgU=
 github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
+github.com/google/go-cmp v0.5.6 h1:BKbKCqvP6I+rmFHt06ZmyQtvB8xAkWdhFyr0ZUNZcxQ=
+github.com/google/go-cmp v0.5.6/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
 github.com/google/goexpect v0.0.0-20210430020637-ab937bf7fd6f h1:7MmqygqdeJtziBUpm4Z9ThROFZUaVGaePMfcDnluf1E=
 github.com/google/goexpect v0.0.0-20210430020637-ab937bf7fd6f/go.mod h1:n1ej5+FqyEytMt/mugVDZLIiqTMO+vsrgY+kM6ohzN0=
 github.com/google/goterm v0.0.0-20190703233501-fc88cf888a3f h1:5CjVwnuUcp5adK4gmY6i72gpVFVnZDP2h5TmPScB6u4=
@@ -308,6 +311,10 @@ github.com/jingyugao/rowserrcheck v0.0.0-20191204022205-72ab7603b68a/go.mod h1:x
 github.com/jirfag/go-printf-func-name v0.0.0-20191110105641-45db9963cdd3/go.mod h1:HEWGJkRDzjJY2sqdDwxccsGicWEf9BQOZsq2tV+xzM0=
 github.com/jirfag/go-printf-func-name v0.0.0-20200119135958-7558a9eaa5af h1:KA9BjwUk7KlCh6S9EAGWBt1oExIUv9WyNCiRz5amv48=
 github.com/jirfag/go-printf-func-name v0.0.0-20200119135958-7558a9eaa5af/go.mod h1:HEWGJkRDzjJY2sqdDwxccsGicWEf9BQOZsq2tV+xzM0=
+github.com/jmespath/go-jmespath v0.4.0 h1:BEgLn5cpjn8UN1mAw4NjwDrS35OdebyEtFe+9YPoQUg=
+github.com/jmespath/go-jmespath v0.4.0/go.mod h1:T8mJZnbsbmF+m6zOOFylbeCJqk5+pHWvzYPziyZiYoo=
+github.com/jmespath/go-jmespath/internal/testify v1.5.1 h1:shLQSRRSCCPj3f2gpwzGwWFoC7ycTf1rcQZHOlsJ6N8=
+github.com/jmespath/go-jmespath/internal/testify v1.5.1/go.mod h1:L3OGu8Wl2/fWfCI6z80xFu9LTZmf1ZRjMHUOPmWr69U=
 github.com/jmoiron/sqlx v1.2.0/go.mod h1:1FEQNm3xlJgrMD+FBdI9+xvCksHtbpVBBw5dYhBSsks=
 github.com/jmoiron/sqlx v1.2.1-0.20190826204134-d7d95172beb5/go.mod h1:1FEQNm3xlJgrMD+FBdI9+xvCksHtbpVBBw5dYhBSsks=
 github.com/jonboulle/clockwork v0.1.0/go.mod h1:Ii8DK3G1RaLaWxj9trq07+26W01tbo22gdxWY5EU2bo=
@@ -320,8 +327,8 @@ github.com/jsimonetti/rtnetlink v0.0.0-20201216134343-bde56ed16391/go.mod h1:cR7
 github.com/jsimonetti/rtnetlink v0.0.0-20201220180245-69540ac93943/go.mod h1:z4c53zj6Eex712ROyh8WI0ihysb5j2ROyV42iNogmAs=
 github.com/jsimonetti/rtnetlink v0.0.0-20210122163228-8d122574c736/go.mod h1:ZXpIyOK59ZnN7J0BV99cZUPmsqDRZ3eq5X+st7u/oSA=
 github.com/jsimonetti/rtnetlink v0.0.0-20210212075122-66c871082f2b/go.mod h1:8w9Rh8m+aHZIG69YPGGem1i5VzoyRC8nw2kA8B+ik5U=
-github.com/jsimonetti/rtnetlink v0.0.0-20210409061457-9561dc9288a7 h1:0pS4NUf9WPvydLWHx2VHafjEyfN8vQrAxl/n3Kt2K9c=
-github.com/jsimonetti/rtnetlink v0.0.0-20210409061457-9561dc9288a7/go.mod h1:+fPVEwpdpYDhPa086y6yIAwUno3cBJZw15Fds43LDRA=
+github.com/jsimonetti/rtnetlink v0.0.0-20210525051524-4cc836578190 h1:iycCSDo8EKVueI9sfVBBJmtNn9DnXV/K1YWwEJO+uOs=
+github.com/jsimonetti/rtnetlink v0.0.0-20210525051524-4cc836578190/go.mod h1:NmKSdU4VGSiv1bMsdqNALI4RSvvjtz65tTMCnD05qLo=
 github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU=
 github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU=
 github.com/jtolds/gls v4.20.0+incompatible h1:xdiiI2gbIgH/gLH7ADydsJ1uDOEzR8yvV7C0MuV77Wo=
@@ -341,6 +348,7 @@ github.com/klauspost/compress v1.12.2 h1:2KCfW3I9M7nSc5wOqXAlW2v2U6v+w6cbjvbfp+O
 github.com/klauspost/compress v1.12.2/go.mod h1:8dP1Hq4DHOhN9w426knH3Rhby4rFm6D8eO+e+Dq5Gzg=
 github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ=
 github.com/konsorten/go-windows-terminal-sequences v1.0.3/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ=
+github.com/kr/fs v0.1.0 h1:Jskdu9ieNAYnjxsi0LbQp1ulIKZV1LAFgK1tWhpZgl8=
 github.com/kr/fs v0.1.0/go.mod h1:FFnZGqtBN9Gxj7eW1uZ42v5BccTP0vu6NEaFoC2HwRg=
 github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc=
 github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo=
@@ -358,11 +366,7 @@ github.com/kyoh86/exportloopref v0.1.8 h1:5Ry/at+eFdkX9Vsdw3qU4YkvGtzuVfzT4X7S77
 github.com/kyoh86/exportloopref v0.1.8/go.mod h1:1tUcJeiioIs7VWe5gcOObrux3lb66+sBqGZrRkMwPgg=
 github.com/lib/pq v1.0.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo=
 github.com/logrusorgru/aurora v0.0.0-20181002194514-a7b3b318ed4e/go.mod h1:7rIyQOR62GCctdiQpZ/zOJlFyk6y+94wXzv6RNZgaR4=
-github.com/lxn/walk v0.0.0-20201110160827-18ea5e372cdb/go.mod h1:E23UucZGqpuUANJooIbHWCufXvOcT6E7Stq81gU+CSQ=
-github.com/lxn/walk v0.0.0-20210112085537-c389da54e794 h1:NVRJ0Uy0SOFcXSKLsS65OmI1sgCCfiDUPj+cwnH7GZw=
 github.com/lxn/walk v0.0.0-20210112085537-c389da54e794/go.mod h1:E23UucZGqpuUANJooIbHWCufXvOcT6E7Stq81gU+CSQ=
-github.com/lxn/win v0.0.0-20201111105847-2a20daff6a55/go.mod h1:KxxjdtRkfNoYDCUP5ryK7XJJNTnpC8atvtmTheChOtk=
-github.com/lxn/win v0.0.0-20210218163916-a377121e959e h1:H+t6A/QJMbhCSEH5rAuRxh+CtW96g0Or0Fxa9IKr4uc=
 github.com/lxn/win v0.0.0-20210218163916-a377121e959e/go.mod h1:KxxjdtRkfNoYDCUP5ryK7XJJNTnpC8atvtmTheChOtk=
 github.com/magiconair/properties v1.8.1/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ=
 github.com/magiconair/properties v1.8.4 h1:8KGKTcQQGm0Kv7vEbKFErAoAOFyyacLStRtQSeYtvkY=
@@ -398,10 +402,13 @@ github.com/mdlayher/netlink v1.2.0/go.mod h1:kwVW1io0AZy9A1E2YYgaD4Cj+C+GPkU6klX
 github.com/mdlayher/netlink v1.2.1/go.mod h1:bacnNlfhqHqqLo4WsYeXSqfyXkInQ9JneWI68v1KwSU=
 github.com/mdlayher/netlink v1.2.2-0.20210123213345-5cc92139ae3e/go.mod h1:bacnNlfhqHqqLo4WsYeXSqfyXkInQ9JneWI68v1KwSU=
 github.com/mdlayher/netlink v1.3.0/go.mod h1:xK/BssKuwcRXHrtN04UBkwQ6dY9VviGGuriDdoPSWys=
-github.com/mdlayher/netlink v1.4.0 h1:n3ARR+Fm0dDv37dj5wSWZXDKcy+U0zwcXS3zKMnSiT0=
 github.com/mdlayher/netlink v1.4.0/go.mod h1:dRJi5IABcZpBD2A3D0Mv/AiX8I9uDEu5oGkAVrekmf8=
+github.com/mdlayher/netlink v1.4.1 h1:I154BCU+mKlIf7BgcAJB2r7QjveNPty6uNY1g9ChVfI=
+github.com/mdlayher/netlink v1.4.1/go.mod h1:e4/KuJ+s8UhfUpO9z00/fDZZmhSrs+oxyqAS9cNgn6Q=
 github.com/mdlayher/sdnotify v0.0.0-20210228150836-ea3ec207d697 h1:PBb7ld5cQGfxHF2pKvb/ydtuPwdRaltGI4e0QSCuiNI=
 github.com/mdlayher/sdnotify v0.0.0-20210228150836-ea3ec207d697/go.mod h1:HtjVsQfsrBm1GDcDTUFn4ZXhftxTwO/hxrvEiRc61U4=
+github.com/mdlayher/socket v0.0.0-20210307095302-262dc9984e00 h1:qEtkL8n1DAHpi5/AOgAckwGQUlMe4+jhL/GMt+GKIks=
+github.com/mdlayher/socket v0.0.0-20210307095302-262dc9984e00/go.mod h1:GAFlyu4/XV68LkQKYzKhIo/WW7j3Zi0YRAz/BOoanUc=
 github.com/miekg/dns v1.0.14/go.mod h1:W1PPwlIAgtquWBMBEV9nkV9Cazfe8ScdGz/Lj7v3Nrg=
 github.com/miekg/dns v1.1.42 h1:gWGe42RGaIqXQZ+r3WUGEKBEtvPHY2SXo4dqixDNxuY=
 github.com/miekg/dns v1.1.42/go.mod h1:+evo5L0630/F6ca/Z9+GAqzhjGyn8/c+TBaOyfEl0V4=
@@ -470,6 +477,8 @@ github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINE
 github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4=
 github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
 github.com/pkg/sftp v1.10.1/go.mod h1:lYOWFsE0bwd1+KfKJaKeuokY15vzFx25BLbzYYoAxZI=
+github.com/pkg/sftp v1.13.0 h1:Riw6pgOKK41foc1I1Uu03CjvbLZDXeGpInycM4shXoI=
+github.com/pkg/sftp v1.13.0/go.mod h1:41g+FIPlQUTDCveupEmEA65IoiQFrtgCeDopC4ajGIM=
 github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
 github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
 github.com/polyfloyd/go-errorlint v0.0.0-20201006195004-351e25ade6e3/go.mod h1:wi9BfjxjF/bwiZ701TzmfKu6UKC357IOAtNr0Td0Lvw=
@@ -570,8 +579,6 @@ github.com/tailscale/certstore v0.0.0-20210528134328-066c94b793d3 h1:fEubocuQkrl
 github.com/tailscale/certstore v0.0.0-20210528134328-066c94b793d3/go.mod h1:2P+hpOwd53e7JMX/L4f3VXkv1G+33ES6IWZSrkIeWNs=
 github.com/tailscale/depaware v0.0.0-20201214215404-77d1e9757027 h1:lK99QQdH3yBWY6aGilF+IRlQIdmhzLrsEmF6JgN+Ryw=
 github.com/tailscale/depaware v0.0.0-20201214215404-77d1e9757027/go.mod h1:p9lPsd+cx33L3H9nNoecRRxPssFKUwwI50I3pZ0yT+8=
-github.com/tailscale/wireguard-go v0.0.0-20210522003738-46b531feb08a h1:ujoIjR8p8HEVy26RnOe6U5aJwaMYFrIa4cpGGeZF5oc=
-github.com/tailscale/wireguard-go v0.0.0-20210522003738-46b531feb08a/go.mod h1:ys4yUmhKncXy1jWP34qUHKipRjl322VVhxoh1Rkfo7c=
 github.com/tcnksm/go-httpstat v0.2.0 h1:rP7T5e5U2HfmOBmZzGgGZjBQ5/GluWUylujl0tJ04I0=
 github.com/tcnksm/go-httpstat v0.2.0/go.mod h1:s3JVJFtQxtBEBC9dwcdTTXS9xFnM3SXAZwPG41aurT8=
 github.com/tdakkota/asciicheck v0.0.0-20200416190851-d7f85be797a2/go.mod h1:yHp0ai0Z9gUljN3o0xMhYJnH/IcvkdTBOX2fmJ93JEM=
@@ -640,8 +647,8 @@ golang.org/x/crypto v0.0.0-20200302210943-78000ba7a073/go.mod h1:LzIPMQfyMNhhGPh
 golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
 golang.org/x/crypto v0.0.0-20200728195943-123391ffb6de/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
 golang.org/x/crypto v0.0.0-20201016220609-9e8e0b390897/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
-golang.org/x/crypto v0.0.0-20201112155050-0c6587e931a9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
 golang.org/x/crypto v0.0.0-20201208171446-5f87f3452ae9/go.mod h1:jdWPYTVW3xRLrWPugEBEK3UY2ZEsg3UU495nc5E+M+I=
+golang.org/x/crypto v0.0.0-20201221181555-eec23a3978ad/go.mod h1:jdWPYTVW3xRLrWPugEBEK3UY2ZEsg3UU495nc5E+M+I=
 golang.org/x/crypto v0.0.0-20210220033148-5ea612d1eb83/go.mod h1:jdWPYTVW3xRLrWPugEBEK3UY2ZEsg3UU495nc5E+M+I=
 golang.org/x/crypto v0.0.0-20210513164829-c07d793c2f9a h1:kr2P4QFmQr29mSLA43kwrOcgcReGTfbE9N577tCTuBc=
 golang.org/x/crypto v0.0.0-20210513164829-c07d793c2f9a/go.mod h1:P+XmwS30IXTQdn5tA2iutPOUgjI07+tq3H3K9MVA1s8=
@@ -709,8 +716,6 @@ golang.org/x/net v0.0.0-20201224014010-6772e930b67b/go.mod h1:m0MpNAwzfU5UDzcl9v
 golang.org/x/net v0.0.0-20210119194325-5f4716e94777/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg=
 golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg=
 golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM=
-golang.org/x/net v0.0.0-20210521195947-fe42d452be8f h1:Si4U+UcgJzya9kpiEUJKQvjr512OLli+gL4poHrz93U=
-golang.org/x/net v0.0.0-20210521195947-fe42d452be8f/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
 golang.org/x/net v0.0.0-20210525063256-abc453219eb5 h1:wjuX4b5yYQnEQHzd+CBcrcC6OVR2J1CN6mUy0oSxIPo=
 golang.org/x/net v0.0.0-20210525063256-abc453219eb5/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
 golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
@@ -765,9 +770,7 @@ golang.org/x/sys v0.0.0-20200602225109-6fdc65e7d980/go.mod h1:h1NjWce9XRLGQEsW7w
 golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20201009025420-dfb3f7c4e634/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20201018230417-eeed37f84f13/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20201107080550-4d91cf3a1aaf/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20201109165425-215b40eba54c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20201112073958-5cba982894dd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20201118182958-a01c418693c7/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20201207223542-d4d67f95c62d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
@@ -781,15 +784,13 @@ golang.org/x/sys v0.0.0-20210301091718-77cc2087c03b/go.mod h1:h1NjWce9XRLGQEsW7w
 golang.org/x/sys v0.0.0-20210303074136-134d130e1a04/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20210305230114-8fe3ee5dd75b/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20210309040221-94ec62e08169/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20210316164454-77fc1eacc6aa/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20210330210617-4fbd30eecc44/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20210403161142-5e06dd20ab57/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
-golang.org/x/sys v0.0.0-20210521203332-0cec03c779c1 h1:lCnv+lfrU9FRPGf8NeRuWAAPjNnema5WtBinMgs1fD8=
-golang.org/x/sys v0.0.0-20210521203332-0cec03c779c1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
-golang.org/x/sys v0.0.0-20210525143221-35b2ab0089ea h1:+WiDlPBBaO+h9vPNZi8uJ3k4BkKQB7Iow3aqwHVA5hI=
 golang.org/x/sys v0.0.0-20210525143221-35b2ab0089ea/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.0.0-20210616094352-59db8d763f22 h1:RqytpXGR1iVNX7psjB3ff8y7sNFinVFvkx1c8SjBkio=
+golang.org/x/sys v0.0.0-20210616094352-59db8d763f22/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw=
 golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
 golang.org/x/term v0.0.0-20210503060354-a79de5458b56 h1:b8jxX3zqjpqb2LklXPzKSGJhzyxCOZSz8ncv8Nv+y7w=
@@ -799,14 +800,13 @@ golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3
 golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk=
 golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
 golang.org/x/text v0.3.4/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
-golang.org/x/text v0.3.6 h1:aRYxNxv6iGQlyVaZmk6ZgYEDa+Jg18DxebPSrd6bg1M=
 golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
 golang.org/x/text v0.3.7-0.20210524175448-3115f89c4b99 h1:ZEXtoJu1S0ie/EmdYnjY3CqaCCZxnldL+K1ftMITD2Q=
 golang.org/x/text v0.3.7-0.20210524175448-3115f89c4b99/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ=
 golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
 golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
-golang.org/x/time v0.0.0-20210220033141-f8bda1e9f3ba h1:O8mE0/t419eoIwhTFpKVkHiTs/Igowgfkj25AcZrtiE=
-golang.org/x/time v0.0.0-20210220033141-f8bda1e9f3ba/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
+golang.org/x/time v0.0.0-20210611083556-38a9dc6acbc6 h1:Vv0JUPWTyeqUq42B2WJ1FeIDjjvGKoA2Ss+Ts0lAVbs=
+golang.org/x/time v0.0.0-20210611083556-38a9dc6acbc6/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
 golang.org/x/tools v0.0.0-20180221164845-07fd8470d635/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
 golang.org/x/tools v0.0.0-20180525024113-a5b4c53f6e8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
 golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
@@ -862,8 +862,6 @@ golang.org/x/tools v0.0.0-20201121010211-780cb80bd7fb/go.mod h1:emZCQorbCU4vsT4f
 golang.org/x/tools v0.0.0-20201208233053-a543418bbed2/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA=
 golang.org/x/tools v0.0.0-20201211185031-d93e913c1a58/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA=
 golang.org/x/tools v0.1.0/go.mod h1:xkSsbof2nBLbhDlRMhhhyNLN/zl3eTqcnHD5viDpcZ0=
-golang.org/x/tools v0.1.1 h1:wGiQel/hW0NnEkJUk8lbzkX2gFJU6PFxf1v5OlCfuOs=
-golang.org/x/tools v0.1.1/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk=
 golang.org/x/tools v0.1.2 h1:kRBLX7v7Af8W7Gdbbc908OJcdgtK8bOz9Uaj8/F1ACA=
 golang.org/x/tools v0.1.2/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk=
 golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
@@ -874,10 +872,6 @@ golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8T
 golang.zx2c4.com/wireguard v0.0.0-20210521230051-c27ff9b9f6f7/go.mod h1:a057zjmoc00UN7gVkaJt2sXVK523kMJcogDTEvPIasg=
 golang.zx2c4.com/wireguard v0.0.0-20210525143454-64cb82f2b3f5 h1:5D3v3AKu7ktIhDlqZhZ4+YeNKsW+dnc2+zfFAdhwa8M=
 golang.zx2c4.com/wireguard v0.0.0-20210525143454-64cb82f2b3f5/go.mod h1:laHzsbfMhGSobUmruXWAyMKKHSqvIcrqZJMyHD+/3O8=
-golang.zx2c4.com/wireguard v0.0.20200321-0.20201111175144-60b3766b89b9 h1:qowcZ56hhpeoESmWzI4Exhx4Y78TpCyXUJur4/c0CoE=
-golang.zx2c4.com/wireguard v0.0.20200321-0.20201111175144-60b3766b89b9/go.mod h1:LMeNfjlcPZTrBC1juwgbQyA4Zy2XVcsrdO/fIJxwyuA=
-golang.zx2c4.com/wireguard/windows v0.1.2-0.20201113162609-9b85be97fdf8 h1:nlXPqGA98n+qcq1pwZ28KjM5EsFQvamKS00A+VUeVjs=
-golang.zx2c4.com/wireguard/windows v0.1.2-0.20201113162609-9b85be97fdf8/go.mod h1:psva4yDnAHLuh7lUzOK7J7bLYxNFfo0iKWz+mi9gzkA=
 golang.zx2c4.com/wireguard/windows v0.3.15-0.20210525143335-94c0476d63e3 h1:Xw0ZuZcvq981iPGZoLrUXhrK2jOJAw/B6gZxc6g8FsU=
 golang.zx2c4.com/wireguard/windows v0.3.15-0.20210525143335-94c0476d63e3/go.mod h1:f/UVhQ6vXZKDodGB3Glgwu9B3djRxR14jIbcuxD8NBw=
 google.golang.org/api v0.4.0/go.mod h1:8k5glujaEP+g9n7WNsDg8QP6cUVNI86fCNMcbazEtwE=
@@ -918,8 +912,6 @@ google.golang.org/protobuf v1.20.1-0.20200309200217-e05f789c0967/go.mod h1:A+miE
 google.golang.org/protobuf v1.21.0/go.mod h1:47Nbq4nVaFHyn7ilMalzfO3qCViNmqZ2kzikPIcrTAo=
 google.golang.org/protobuf v1.23.0 h1:4MY060fB1DLGMB/7MBTLnwQUY6+F09GEiz6SsrNqyzM=
 google.golang.org/protobuf v1.23.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU=
-gopkg.in/Knetic/govaluate.v3 v3.0.0 h1:18mUyIt4ZlRlFZAAfVetz4/rzlJs9yhN+U02F4u1AOc=
-gopkg.in/Knetic/govaluate.v3 v3.0.0/go.mod h1:csKLBORsPbafmSCGTEh3U7Ozmsuq8ZSIlKk1bcqph0E=
 gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw=
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
 gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
@@ -957,10 +949,10 @@ honnef.co/go/tools v0.0.1-2020.1.6/go.mod h1:pyyisuGw24ruLjrr1ddx39WE0y9OooInRzE
 honnef.co/go/tools v0.1.4 h1:SadWOkti5uVN1FAMgxn165+Mw00fuQKyk4Gyn/inxNQ=
 honnef.co/go/tools v0.1.4/go.mod h1:NgwopIslSNH47DimFoV78dnkksY2EFtX0ajyb3K/las=
 inet.af/netaddr v0.0.0-20210515010201-ad03edc7c841/go.mod h1:z0nx+Dh+7N7CC8V5ayHtHGpZpxLQZZxkIaaz6HN65Ls=
-inet.af/netaddr v0.0.0-20210523191804-d57edf19c517 h1:gieHAlViNfjNt0m6gKr4aazCMXQobPMOqeyQ1ZN5ekw=
-inet.af/netaddr v0.0.0-20210523191804-d57edf19c517/go.mod h1:z0nx+Dh+7N7CC8V5ayHtHGpZpxLQZZxkIaaz6HN65Ls=
-inet.af/netstack v0.0.0-20210317161235-a1bf4e56ef22 h1:DNtszwGa6w76qlIr+PbPEnlBJdiRV8SaxeigOy0q1gg=
-inet.af/netstack v0.0.0-20210317161235-a1bf4e56ef22/go.mod h1:GVx+5OZtbG4TVOW5ilmyRZAZXr1cNwfqUEkTOtWK0PM=
+inet.af/netaddr v0.0.0-20210602152128-50f8686885e3 h1:RlarOdsmOUCCvy7Xm1JchJIGuQsuKwD/Lo1bjYmfuQI=
+inet.af/netaddr v0.0.0-20210602152128-50f8686885e3/go.mod h1:z0nx+Dh+7N7CC8V5ayHtHGpZpxLQZZxkIaaz6HN65Ls=
+inet.af/netstack v0.0.0-20210622165351-29b14ebc044e h1:z11NK94NQcI3DA+a3pUC/2dRYTph1kPX6B0FnCaMDzk=
+inet.af/netstack v0.0.0-20210622165351-29b14ebc044e/go.mod h1:fG3G1dekmK8oDX3iVzt8c0zICLMLSN8SjdxbXVt0WjU=
 inet.af/peercred v0.0.0-20210318190834-4259e17bb763 h1:gPSJmmVzmdy4kHhlCMx912GdiUz3k/RzJGg0ADqy1dg=
 inet.af/peercred v0.0.0-20210318190834-4259e17bb763/go.mod h1:FjawnflS/udxX+SvpsMgZfdqx2aykOlkISeAsADi5IU=
 inet.af/wf v0.0.0-20210516214145-a5343001b756 h1:muIT3C1rH3/xpvIH8blKkMvhctV7F+OtZqs7kcwHDBQ=

hostinfo/hostinfo.go

@@ -0,0 +1,117 @@
+// Copyright (c) 2020 Tailscale Inc & AUTHORS All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+// Package hostinfo answers questions about the host environment that Tailscale is
+// running on.
+//
+// TODO(bradfitz): move more of control/controlclient/hostinfo_* into this package.
+package hostinfo
+
+import (
+	"io"
+	"os"
+	"runtime"
+	"sync/atomic"
+
+	"go4.org/mem"
+	"tailscale.com/util/lineread"
+)
+
+// EnvType represents a known environment type.
+// The empty string, the default, means unknown.
+type EnvType string
+
+const (
+	KNative         = EnvType("kn")
+	AWSLambda       = EnvType("lm")
+	Heroku          = EnvType("hr")
+	AzureAppService = EnvType("az")
+)
+
+var envType atomic.Value // of EnvType
+
+func GetEnvType() EnvType {
+	if e, ok := envType.Load().(EnvType); ok {
+		return e
+	}
+	e := getEnvType()
+	envType.Store(e)
+	return e
+}
+
+func getEnvType() EnvType {
+	if inKnative() {
+		return KNative
+	}
+	if inAWSLambda() {
+		return AWSLambda
+	}
+	if inHerokuDyno() {
+		return Heroku
+	}
+	if inAzureAppService() {
+		return AzureAppService
+	}
+	return ""
+}
+
+// InContainer reports whether we're running in a container.
+func InContainer() bool {
+	if runtime.GOOS != "linux" {
+		return false
+	}
+	var ret bool
+	lineread.File("/proc/1/cgroup", func(line []byte) error {
+		if mem.Contains(mem.B(line), mem.S("/docker/")) ||
+			mem.Contains(mem.B(line), mem.S("/lxc/")) {
+			ret = true
+			return io.EOF // arbitrary non-nil error to stop loop
+		}
+		return nil
+	})
+	lineread.File("/proc/mounts", func(line []byte) error {
+		if mem.Contains(mem.B(line), mem.S("fuse.lxcfs")) {
+			ret = true
+			return io.EOF
+		}
+		return nil
+	})
+	return ret
+}
+
+func inKnative() bool {
+	// https://cloud.google.com/run/docs/reference/container-contract#env-vars
+	if os.Getenv("K_REVISION") != "" && os.Getenv("K_CONFIGURATION") != "" &&
+		os.Getenv("K_SERVICE") != "" && os.Getenv("PORT") != "" {
+		return true
+	}
+	return false
+}
+
+func inAWSLambda() bool {
+	// https://docs.aws.amazon.com/lambda/latest/dg/configuration-envvars.html
+	if os.Getenv("AWS_LAMBDA_FUNCTION_NAME") != "" &&
+		os.Getenv("AWS_LAMBDA_FUNCTION_VERSION") != "" &&
+		os.Getenv("AWS_LAMBDA_INITIALIZATION_TYPE") != "" &&
+		os.Getenv("AWS_LAMBDA_RUNTIME_API") != "" {
+		return true
+	}
+	return false
+}
+
+func inHerokuDyno() bool {
+	// https://devcenter.heroku.com/articles/dynos#local-environment-variables
+	if os.Getenv("PORT") != "" && os.Getenv("DYNO") != "" {
+		return true
+	}
+	return false
+}
+
+func inAzureAppService() bool {
+	if os.Getenv("APPSVC_RUN_ZIP") != "" && os.Getenv("WEBSITE_STACK") != "" &&
+		os.Getenv("WEBSITE_AUTH_AUTO_AAD") != "" {
+		return true
+	}
+	return false
+}

internal/deephash/deephash.go

@@ -24,8 +24,12 @@ func calcHash(v interface{}) string {
 	printTo(b, v, scratch)
 	b.Flush()
 	scratch = h.Sum(scratch[:0])
-	hex.Encode(scratch[:cap(scratch)], scratch[:sha256.Size])
-	return string(scratch[:sha256.Size*2])
+	// The first sha256.Size bytes contain the hash.
+	// Hex-encode that into the next sha256.Size*2 bytes.
+	src := scratch[:sha256.Size]
+	dst := scratch[sha256.Size:cap(scratch)]
+	n := hex.Encode(dst, src)
+	return string(dst[:n])
 }
 
 // UpdateHash sets last to the hash of v and reports whether its value changed.

internal/deephash/deephash_test.go

@@ -134,3 +134,14 @@ func BenchmarkHashMapAcyclic(b *testing.B) {
 		}
 	}
 }
+
+func TestExhaustive(t *testing.T) {
+	seen := make(map[string]bool)
+	for i := 0; i < 100000; i++ {
+		s := calcHash(i)
+		if seen[s] {
+			t.Fatalf("hash collision %v", i)
+		}
+		seen[s] = true
+	}
+}

ipn/ipnlocal/local.go

@@ -244,7 +244,7 @@ func (b *LocalBackend) linkChange(major bool, ifst *interfaces.State) {
 	// need updating to tweak default routes.
 	b.updateFilter(b.netMap, b.prefs)
 
-	if runtime.GOOS == "windows" && b.netMap != nil && b.state == ipn.Running {
+	if peerAPIListenAsync && b.netMap != nil && b.state == ipn.Running {
 		want := len(b.netMap.Addresses)
 		b.logf("linkChange: peerAPIListeners too low; trying again")
 		if len(b.peerAPIListeners) < want {
@@ -325,6 +325,7 @@ func (b *LocalBackend) updateStatus(sb *ipnstate.StatusBuilder, extraLocked func
 		s.AuthURL = b.authURLSticky
 		if b.netMap != nil {
 			s.MagicDNSSuffix = b.netMap.MagicDNSSuffix()
+			s.CertDomains = append([]string(nil), b.netMap.DNS.CertDomains...)
 		}
 	})
 	sb.MutateSelfStatus(func(ss *ipnstate.PeerStatus) {
@@ -453,6 +454,13 @@ func (b *LocalBackend) setClientStatus(st controlclient.Status) {
 	// Lock b once and do only the things that require locking.
 	b.mu.Lock()
 
+	if st.LogoutFinished != nil {
+		// Since we're logged out now, our netmap cache is invalid.
+		// Since st.NetMap==nil means "netmap is unchanged", there is
+		// no other way to represent this change.
+		b.setNetMapLocked(nil)
+	}
+
 	prefs := b.prefs
 	stateKey := b.stateKey
 	netMap := b.netMap
@@ -650,6 +658,12 @@ func (b *LocalBackend) getNewControlClientFunc() clientGen {
 // startIsNoopLocked reports whether a Start call on this LocalBackend
 // with the provided Start Options would be a useless no-op.
 //
+// TODO(apenwarr): we shouldn't need this.
+//  The state machine is now nearly clean enough where it can accept a new
+//  connection while in any state, not just Running, and on any platform.
+//  We'd want to add a few more tests to state_test.go to ensure this continues
+//  to work as expected.
+//
 // b.mu must be held.
 func (b *LocalBackend) startIsNoopLocked(opts ipn.Options) bool {
 	// Options has 5 fields; check all of them:
@@ -703,6 +717,7 @@ func (b *LocalBackend) Start(opts ipn.Options) error {
 		b.send(ipn.Notify{
 			State:         &state,
 			NetMap:        nm,
+			Prefs:         b.prefs,
 			LoginFinished: new(empty.Message),
 		})
 		return nil
@@ -742,6 +757,12 @@ func (b *LocalBackend) Start(opts ipn.Options) error {
 		newPrefs := opts.UpdatePrefs
 		newPrefs.Persist = b.prefs.Persist
 		b.prefs = newPrefs
+
+		if opts.StateKey != "" {
+			if err := b.store.WriteState(opts.StateKey, b.prefs.ToBytes()); err != nil {
+				b.logf("failed to save UpdatePrefs state: %v", err)
+			}
+		}
 	}
 
 	wantRunning := b.prefs.WantRunning
@@ -915,8 +936,8 @@ func (b *LocalBackend) updateFilter(netMap *netmap.NetworkMap, prefs *ipn.Prefs)
 			}
 		}
 	}
-	localNets := localNetsB.IPSet()
-	logNets := logNetsB.IPSet()
+	localNets, _ := localNetsB.IPSet()
+	logNets, _ := logNetsB.IPSet()
 
 	changed := deephash.UpdateHash(&b.filterHash, haveNetmap, addrs, packetFilter, localNets.Ranges(), logNets.Ranges(), shieldsUp)
 	if !changed {
@@ -973,7 +994,8 @@ func interfaceRoutes() (ips *netaddr.IPSet, hostIPs []netaddr.IP, err error) {
 		return nil, nil, err
 	}
 
-	return b.IPSet(), hostIPs, nil
+	ipSet, _ := b.IPSet()
+	return ipSet, hostIPs, nil
 }
 
 // shrinkDefaultRoute returns an IPSet representing the IPs in route,
@@ -1004,7 +1026,7 @@ func shrinkDefaultRoute(route netaddr.IPPrefix) (*netaddr.IPSet, error) {
 	for _, pfx := range removeFromDefaultRoute {
 		b.RemovePrefix(pfx)
 	}
-	return b.IPSet(), nil
+	return b.IPSet()
 }
 
 // dnsCIDRsEqual determines whether two CIDR lists are equal
@@ -1734,6 +1756,25 @@ func (b *LocalBackend) authReconfig() {
 	for _, peer := range nm.Peers {
 		set(peer.Name, peer.Addresses)
 	}
+	for _, rec := range nm.DNS.ExtraRecords {
+		switch rec.Type {
+		case "", "A", "AAAA":
+			// Treat these all the same for now: infer from the value
+		default:
+			// TODO: more
+			continue
+		}
+		ip, err := netaddr.ParseIP(rec.Value)
+		if err != nil {
+			// Ignore.
+			continue
+		}
+		fqdn, err := dnsname.ToFQDN(rec.Name)
+		if err != nil {
+			continue
+		}
+		dcfg.Hosts[fqdn] = append(dcfg.Hosts[fqdn], ip)
+	}
 
 	if uc.CorpDNS {
 		addDefault := func(resolvers []tailcfg.DNSResolver) {
@@ -1803,7 +1844,7 @@ func (b *LocalBackend) authReconfig() {
 		}
 	}
 
-	err = b.e.Reconfig(cfg, rcfg, &dcfg)
+	err = b.e.Reconfig(cfg, rcfg, &dcfg, nm.Debug)
 	if err == wgengine.ErrNoChanges {
 		return
 	}
@@ -1869,10 +1910,26 @@ func (b *LocalBackend) closePeerAPIListenersLocked() {
 	b.peerAPIListeners = nil
 }
 
+// peerAPIListenAsync is whether the operating system requires that we
+// retry listening on the peerAPI ip/port for whatever reason.
+//
+// On Windows, see Issue 1620.
+// On Android, see Issue 1960.
+const peerAPIListenAsync = runtime.GOOS == "windows" || runtime.GOOS == "android"
+
 func (b *LocalBackend) initPeerAPIListener() {
 	b.mu.Lock()
 	defer b.mu.Unlock()
 
+	if b.netMap == nil {
+		// We're called from authReconfig which checks that
+		// netMap is non-nil, but if a concurrent Logout,
+		// ResetForClientDisconnect, or Start happens when its
+		// mutex was released, the netMap could be
+		// nil'ed out (Issue 1996). Bail out early here if so.
+		return
+	}
+
 	if len(b.netMap.Addresses) == len(b.peerAPIListeners) {
 		allSame := true
 		for i, pln := range b.peerAPIListeners {
@@ -1923,9 +1980,8 @@ func (b *LocalBackend) initPeerAPIListener() {
 		if !skipListen {
 			ln, err = ps.listen(a.IP(), b.prevIfState)
 			if err != nil {
-				if runtime.GOOS == "windows" {
-					// Expected for now. See Issue 1620.
-					// But we fix it later in linkChange
+				if peerAPIListenAsync {
+					// Expected. But we fix it later in linkChange
 					// ("peerAPIListeners too low").
 					continue
 				}
@@ -2058,7 +2114,7 @@ func (b *LocalBackend) routerConfig(cfg *wgcfg.Config, prefs *ipn.Prefs) *router
 		if !default6 {
 			rs.Routes = append(rs.Routes, ipv6Default)
 		}
-		if runtime.GOOS == "linux" {
+		if runtime.GOOS == "linux" || runtime.GOOS == "darwin" {
 			// Only allow local lan access on linux machines for now.
 			ips, _, err := interfaceRoutes()
 			if err != nil {
@@ -2149,7 +2205,7 @@ func (b *LocalBackend) enterState(newState ipn.State) {
 		b.blockEngineUpdates(true)
 		fallthrough
 	case ipn.Stopped:
-		err := b.e.Reconfig(&wgcfg.Config{}, &router.Config{}, &dns.Config{})
+		err := b.e.Reconfig(&wgcfg.Config{}, &router.Config{}, &dns.Config{}, nil)
 		if err != nil {
 			b.logf("Reconfig(down): %v", err)
 		}
@@ -2163,7 +2219,7 @@ func (b *LocalBackend) enterState(newState ipn.State) {
 		b.e.RequestStatus()
 	case ipn.Running:
 		var addrs []string
-		for _, addr := range b.netMap.Addresses {
+		for _, addr := range netMap.Addresses {
 			addrs = append(addrs, addr.IP().String())
 		}
 		systemd.Status("Connected; %s; %s", activeLogin, strings.Join(addrs, " "))
@@ -2269,7 +2325,7 @@ func (b *LocalBackend) stateMachine() {
 // a status update that predates the "I've shut down" update.
 func (b *LocalBackend) stopEngineAndWait() {
 	b.logf("stopEngineAndWait...")
-	b.e.Reconfig(&wgcfg.Config{}, &router.Config{}, &dns.Config{})
+	b.e.Reconfig(&wgcfg.Config{}, &router.Config{}, &dns.Config{}, nil)
 	b.requestEngineStatusAndWait()
 	b.logf("stopEngineAndWait: done.")
 }
@@ -2326,7 +2382,6 @@ func (b *LocalBackend) LogoutSync(ctx context.Context) error {
 func (b *LocalBackend) logout(ctx context.Context, sync bool) error {
 	b.mu.Lock()
 	cc := b.cc
-	b.setNetMapLocked(nil)
 	b.mu.Unlock()
 
 	b.EditPrefs(&ipn.MaskedPrefs{
@@ -2353,10 +2408,6 @@ func (b *LocalBackend) logout(ctx context.Context, sync bool) error {
 		cc.StartLogout()
 	}
 
-	b.mu.Lock()
-	b.setNetMapLocked(nil)
-	b.mu.Unlock()
-
 	b.stateMachine()
 	return err
 }
@@ -2558,6 +2609,42 @@ func (b *LocalBackend) FileTargets() ([]*apitype.FileTarget, error) {
 	return ret, nil
 }
 
+// SetDNS adds a DNS record for the given domain name & TXT record
+// value.
+//
+// It's meant for use with dns-01 ACME (LetsEncrypt) challenges.
+//
+// This is the low-level interface. Other layers will provide more
+// friendly options to get HTTPS certs.
+func (b *LocalBackend) SetDNS(ctx context.Context, name, value string) error {
+	req := &tailcfg.SetDNSRequest{
+		Version: 1,
+		Type:    "TXT",
+		Name:    name,
+		Value:   value,
+	}
+
+	b.mu.Lock()
+	cc := b.cc
+	if prefs := b.prefs; prefs != nil {
+		req.NodeKey = tailcfg.NodeKey(prefs.Persist.PrivateNodeKey.Public())
+	}
+	b.mu.Unlock()
+	if cc == nil {
+		return errors.New("not connected")
+	}
+	if req.NodeKey.IsZero() {
+		return errors.New("no nodekey")
+	}
+	if name == "" {
+		return errors.New("missing 'name'")
+	}
+	if value == "" {
+		return errors.New("missing 'value'")
+	}
+	return cc.SetDNS(ctx, req)
+}
+
 func (b *LocalBackend) registerIncomingFile(inf *incomingFile, active bool) {
 	b.mu.Lock()
 	defer b.mu.Unlock()
@@ -2630,7 +2717,6 @@ func (b *LocalBackend) CheckIPForwarding() error {
 		return nil
 	}
 	if isBSD(runtime.GOOS) {
-		//lint:ignore ST1005 output to users as is
 		return fmt.Errorf("Subnet routing and exit nodes only work with additional manual configuration on %v, and is not currently officially supported.", runtime.GOOS)
 	}
 
@@ -2647,16 +2733,13 @@ func (b *LocalBackend) CheckIPForwarding() error {
 	for _, key := range keys {
 		bs, err := exec.Command("sysctl", "-n", key).Output()
 		if err != nil {
-			//lint:ignore ST1005 output to users as is
 			return fmt.Errorf("couldn't check %s (%v).\nSubnet routes won't work without IP forwarding.", key, err)
 		}
 		on, err := strconv.ParseBool(string(bytes.TrimSpace(bs)))
 		if err != nil {
-			//lint:ignore ST1005 output to users as is
 			return fmt.Errorf("couldn't parse %s (%v).\nSubnet routes won't work without IP forwarding.", key, err)
 		}
 		if !on {
-			//lint:ignore ST1005 output to users as is
 			return fmt.Errorf("%s is disabled. Subnet routes won't work.", key)
 		}
 	}

ipn/ipnlocal/peerapi_macios_ext.go

@@ -2,21 +2,19 @@
 // Use of this source code is governed by a BSD-style
 // license that can be found in the LICENSE file.
 
-// +build darwin,redo ios,redo
+// +build darwin,ts_macext ios,ts_macext
 
 package ipnlocal
 
 import (
 	"errors"
 	"fmt"
-	"log"
 	"net"
-	"strings"
 	"syscall"
 
-	"golang.org/x/sys/unix"
 	"inet.af/netaddr"
 	"tailscale.com/net/interfaces"
+	"tailscale.com/net/netns"
 )
 
 func init() {
@@ -32,29 +30,7 @@ func initListenConfigNetworkExtension(nc *net.ListenConfig, ip netaddr.IP, st *i
 	if !ok {
 		return fmt.Errorf("no interface with name %q", tunIfName)
 	}
-	nc.Control = func(network, address string, c syscall.RawConn) error {
-		var sockErr error
-		err := c.Control(func(fd uintptr) {
-			sockErr = bindIf(fd, network, address, tunIf.Index)
-			log.Printf("peerapi: bind(%q, %q) on index %v = %v", network, address, tunIf.Index, sockErr)
-		})
-		if err != nil {
-			return err
-		}
-		return sockErr
-	}
-	return nil
-}
-
-func bindIf(fd uintptr, network, address string, ifIndex int) error {
-	v6 := strings.Contains(address, "]:") || strings.HasSuffix(network, "6") // hacky test for v6
-	proto := unix.IPPROTO_IP
-	opt := unix.IP_BOUND_IF
-	if v6 {
-		proto = unix.IPPROTO_IPV6
-		opt = unix.IPV6_BOUND_IF
-	}
-	return unix.SetsockoptInt(int(fd), proto, opt, ifIndex)
+	return netns.SetListenConfigInterfaceIndex(nc, tunIf.Index)
 }
 
 func peerDialControlFuncNetworkExtension(b *LocalBackend) func(network, address string, c syscall.RawConn) error {
@@ -68,17 +44,12 @@ func peerDialControlFuncNetworkExtension(b *LocalBackend) func(network, address
 			index = tunIf.Index
 		}
 	}
+	var lc net.ListenConfig
+	netns.SetListenConfigInterfaceIndex(&lc, index)
 	return func(network, address string, c syscall.RawConn) error {
 		if index == -1 {
 			return errors.New("failed to find TUN interface to bind to")
 		}
-		var sockErr error
-		err := c.Control(func(fd uintptr) {
-			sockErr = bindIf(fd, network, address, index)
-		})
-		if err != nil {
-			return err
-		}
-		return sockErr
+		return lc.Control(network, address, c)
 	}
 }

ipn/ipnlocal/state_test.go

@@ -140,6 +140,8 @@ func (cc *mockControl) send(err error, url string, loginFinished bool, nm *netma
 		}
 		if loginFinished {
 			s.LoginFinished = &empty.Message{}
+		} else if url == "" && err == nil && nm == nil {
+			s.LogoutFinished = &empty.Message{}
 		}
 		cc.statusFunc(s)
 	}
@@ -246,6 +248,10 @@ func (cc *mockControl) UpdateEndpoints(localPort uint16, endpoints []tailcfg.End
 	cc.called("UpdateEndpoints")
 }
 
+func (*mockControl) SetDNS(context.Context, *tailcfg.SetDNSRequest) error {
+	panic("unexpected SetDNS call")
+}
+
 // A very precise test of the sequence of function calls generated by
 // ipnlocal.Local into its controlclient instance, and the events it
 // produces upstream into the UI.
@@ -548,10 +554,7 @@ func TestStateMachine(t *testing.T) {
 		c.Assert(nn[0].State, qt.Not(qt.IsNil))
 		c.Assert(nn[0].LoginFinished, qt.Not(qt.IsNil))
 		c.Assert(nn[0].NetMap, qt.Not(qt.IsNil))
-		// BUG: Prefs should be sent too, or the UI could end up in
-		// a bad state. (iOS, the only current user of this feature,
-		// probably wouldn't notice because it happens to not display
-		// any prefs. Maybe exit nodes will look weird?)
+		c.Assert(nn[0].Prefs, qt.Not(qt.IsNil))
 	}
 
 	// undo the state hack above.
@@ -563,24 +566,25 @@ func TestStateMachine(t *testing.T) {
 	b.Logout()
 	{
 		nn := notifies.drain(2)
-		// BUG: now is not the time to unpause.
-		c.Assert([]string{"unpause", "StartLogout"}, qt.DeepEquals, cc.getCalls())
+		c.Assert([]string{"pause", "StartLogout"}, qt.DeepEquals, cc.getCalls())
 		c.Assert(nn[0].State, qt.Not(qt.IsNil))
 		c.Assert(nn[1].Prefs, qt.Not(qt.IsNil))
-		c.Assert(ipn.NeedsLogin, qt.Equals, *nn[0].State)
+		c.Assert(ipn.Stopped, qt.Equals, *nn[0].State)
 		c.Assert(nn[1].Prefs.LoggedOut, qt.IsTrue)
 		c.Assert(nn[1].Prefs.WantRunning, qt.IsFalse)
-		c.Assert(ipn.NeedsLogin, qt.Equals, b.State())
+		c.Assert(ipn.Stopped, qt.Equals, b.State())
 	}
 
 	// Let's make the logout succeed.
 	t.Logf("\n\nLogout (async) - succeed")
-	notifies.expect(0)
+	notifies.expect(1)
 	cc.setAuthBlocked(true)
 	cc.send(nil, "", false, nil)
 	{
-		notifies.drain(0)
-		c.Assert(cc.getCalls(), qt.HasLen, 0)
+		nn := notifies.drain(1)
+		c.Assert([]string{"unpause"}, qt.DeepEquals, cc.getCalls())
+		c.Assert(nn[0].State, qt.Not(qt.IsNil))
+		c.Assert(ipn.NeedsLogin, qt.Equals, *nn[0].State)
 		c.Assert(b.Prefs().LoggedOut, qt.IsTrue)
 		c.Assert(b.Prefs().WantRunning, qt.IsFalse)
 		c.Assert(ipn.NeedsLogin, qt.Equals, b.State())

ipn/ipnserver/server.go

@@ -24,7 +24,6 @@ import (
 	"strconv"
 	"strings"
 	"sync"
-	"sync/atomic"
 	"syscall"
 	"time"
 
@@ -41,9 +40,11 @@ import (
 	"tailscale.com/safesocket"
 	"tailscale.com/smallzstd"
 	"tailscale.com/types/logger"
+	"tailscale.com/util/groupmember"
 	"tailscale.com/util/pidowner"
 	"tailscale.com/util/systemd"
 	"tailscale.com/version"
+	"tailscale.com/version/distro"
 	"tailscale.com/wgengine"
 )
 
@@ -347,51 +348,32 @@ func isReadonlyConn(ci connIdentity, operatorUID string, logf logger.Logf) bool
 		logf("connection from userid %v; is configured operator", uid)
 		return rw
 	}
-	var adminGroupID string
-	switch runtime.GOOS {
-	case "darwin":
-		adminGroupID = darwinAdminGroupID()
-	default:
-		logf("connection from userid %v; read-only", uid)
+	if yes, err := isLocalAdmin(uid); err != nil {
+		logf("connection from userid %v; read-only; %v", uid, err)
 		return ro
-	}
-	if adminGroupID == "" {
-		logf("connection from userid %v; no system admin group found, read-only", uid)
-		return ro
-	}
-	u, err := user.LookupId(uid)
-	if err != nil {
-		logf("connection from userid %v; failed to look up user; read-only", uid)
-		return ro
-	}
-	gids, err := u.GroupIds()
-	if err != nil {
-		logf("connection from userid %v; failed to look up groups; read-only", uid)
-		return ro
-	}
-	for _, gid := range gids {
-		if gid == adminGroupID {
-			logf("connection from userid %v; is local admin, has access", uid)
-			return rw
-		}
+	} else if yes {
+		logf("connection from userid %v; is local admin, has access", uid)
+		return rw
 	}
 	logf("connection from userid %v; read-only", uid)
 	return ro
 }
 
-var darwinAdminGroupIDCache atomic.Value // of string
-
-func darwinAdminGroupID() string {
-	s, _ := darwinAdminGroupIDCache.Load().(string)
-	if s != "" {
-		return s
-	}
-	g, err := user.LookupGroup("admin")
+func isLocalAdmin(uid string) (bool, error) {
+	u, err := user.LookupId(uid)
 	if err != nil {
-		return ""
+		return false, err
 	}
-	darwinAdminGroupIDCache.Store(g.Gid)
-	return g.Gid
+	var adminGroup string
+	switch {
+	case runtime.GOOS == "darwin":
+		adminGroup = "admin"
+	case distro.Get() == distro.QNAP:
+		adminGroup = "administrators"
+	default:
+		return false, fmt.Errorf("no system admin group found")
+	}
+	return groupmember.IsMemberOfGroup(adminGroup, u.Username)
 }
 
 // inUseOtherUserError is the error type for when the server is in use
@@ -415,12 +397,10 @@ func (s *server) checkConnIdentityLocked(ci connIdentity) error {
 			break
 		}
 		if ci.UserID != active.UserID {
-			//lint:ignore ST1005 we want to capitalize Tailscale here
 			return inUseOtherUserError{fmt.Errorf("Tailscale already in use by %s, pid %d", active.User.Username, active.Pid)}
 		}
 	}
 	if su := s.serverModeUser; su != nil && ci.UserID != su.Uid {
-		//lint:ignore ST1005 we want to capitalize Tailscale here
 		return inUseOtherUserError{fmt.Errorf("Tailscale already in use by %s", su.Username)}
 	}
 	return nil

ipn/ipnstate/ipnstate.go

@@ -45,6 +45,13 @@ type Status struct {
 	// has MagicDNS enabled.
 	MagicDNSSuffix string
 
+	// CertDomains are the set of DNS names for which the control
+	// plane server will assist with provisioning TLS
+	// certificates. See SetDNSRequest for dns-01 ACME challenges
+	// for e.g. LetsEncrypt. These names are FQDNs without
+	// trailing periods, and without any "_acme-challenge." prefix.
+	CertDomains []string
+
 	Peer map[key.Public]*PeerStatus
 	User map[tailcfg.UserID]tailcfg.UserProfile
 }

ipn/localapi/localapi.go

@@ -100,6 +100,8 @@ func (h *Handler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
 		h.serveBugReport(w, r)
 	case "/localapi/v0/file-targets":
 		h.serveFileTargets(w, r)
+	case "/localapi/v0/set-dns":
+		h.serveSetDNS(w, r)
 	case "/":
 		io.WriteString(w, "tailscaled\n")
 	default:
@@ -262,7 +264,7 @@ func (h *Handler) serveFiles(w http.ResponseWriter, r *http.Request) {
 		http.Error(w, "file access denied", http.StatusForbidden)
 		return
 	}
-	suffix := strings.TrimPrefix(r.URL.Path, "/localapi/v0/files/")
+	suffix := strings.TrimPrefix(r.URL.EscapedPath(), "/localapi/v0/files/")
 	if suffix == "" {
 		if r.Method != "GET" {
 			http.Error(w, "want GET to list files", 400)
@@ -382,6 +384,25 @@ func (h *Handler) serveFilePut(w http.ResponseWriter, r *http.Request) {
 	rp.ServeHTTP(w, outReq)
 }
 
+func (h *Handler) serveSetDNS(w http.ResponseWriter, r *http.Request) {
+	if !h.PermitWrite {
+		http.Error(w, "access denied", http.StatusForbidden)
+		return
+	}
+	if r.Method != "POST" {
+		http.Error(w, "want POST", 400)
+		return
+	}
+	ctx := r.Context()
+	err := h.b.SetDNS(ctx, r.FormValue("name"), r.FormValue("value"))
+	if err != nil {
+		writeErrorJSON(w, err)
+		return
+	}
+	w.Header().Set("Content-Type", "application/json")
+	json.NewEncoder(w).Encode(struct{}{})
+}
+
 var dialPeerTransportOnce struct {
 	sync.Once
 	v *http.Transport
@@ -390,7 +411,7 @@ var dialPeerTransportOnce struct {
 func getDialPeerTransport(b *ipnlocal.LocalBackend) *http.Transport {
 	dialPeerTransportOnce.Do(func() {
 		t := http.DefaultTransport.(*http.Transport).Clone()
-		t.Dial = nil //lint:ignore SA1019 yes I know I'm setting it to nil defensively
+		t.Dial = nil
 		dialer := net.Dialer{
 			Timeout:   30 * time.Second,
 			KeepAlive: 30 * time.Second,

logtail/filch/filch.go

@@ -15,7 +15,6 @@ import (
 	"sync"
 )
 
-//lint:ignore U1000 work around false positive: https://github.com/dominikh/go-tools/issues/983
 var stderrFD = 2 // a variable for testing
 
 type Options struct {

net/dns/manager.go

@@ -20,8 +20,6 @@ import (
 // the lint exception is necessary and on others it is not,
 // and plain ignore complains if the exception is unnecessary.
 
-//lint:file-ignore U1000 reconfigTimeout is used on some platforms but not others
-
 // reconfigTimeout is the time interval within which Manager.{Up,Down} should complete.
 //
 // This is particularly useful because certain conditions can cause indefinite hangs
@@ -40,11 +38,11 @@ type Manager struct {
 }
 
 // NewManagers created a new manager from the given config.
-func NewManager(logf logger.Logf, oscfg OSConfigurator, linkMon *monitor.Mon) *Manager {
+func NewManager(logf logger.Logf, oscfg OSConfigurator, linkMon *monitor.Mon, linkSel resolver.ForwardLinkSelector) *Manager {
 	logf = logger.WithPrefix(logf, "dns: ")
 	m := &Manager{
 		logf:     logf,
-		resolver: resolver.New(logf, linkMon),
+		resolver: resolver.New(logf, linkMon, linkSel),
 		os:       oscfg,
 	}
 	m.logf("using %T", m.os)
@@ -209,7 +207,7 @@ func Cleanup(logf logger.Logf, interfaceName string) {
 		logf("creating dns cleanup: %v", err)
 		return
 	}
-	dns := NewManager(logf, oscfg, nil)
+	dns := NewManager(logf, oscfg, nil, nil)
 	if err := dns.Down(); err != nil {
 		logf("dns down: %v", err)
 	}

net/dns/manager_linux.go

@@ -5,7 +5,6 @@
 package dns
 
 import (
-	"bytes"
 	"context"
 	"errors"
 	"fmt"
@@ -15,6 +14,7 @@ import (
 	"time"
 
 	"github.com/godbus/dbus/v5"
+	"inet.af/netaddr"
 	"tailscale.com/types/logger"
 	"tailscale.com/util/cmpver"
 )
@@ -51,6 +51,15 @@ func NewOSConfigurator(logf logger.Logf, interfaceName string) (ret OSConfigurat
 	switch resolvOwner(bs) {
 	case "systemd-resolved":
 		dbg("rc", "resolved")
+		// Some systems, for reasons known only to them, have a
+		// resolv.conf that has the word "systemd-resolved" in its
+		// header, but doesn't actually point to resolved. We mustn't
+		// try to program resolved in that case.
+		// https://github.com/tailscale/tailscale/issues/2136
+		if err := resolvedIsActuallyResolver(); err != nil {
+			dbg("resolved", "not-in-use")
+			return newDirectManager()
+		}
 		if err := dbusPing("org.freedesktop.resolve1", "/org/freedesktop/resolve1"); err != nil {
 			dbg("resolved", "no")
 			return newDirectManager()
@@ -79,40 +88,41 @@ func NewOSConfigurator(logf logger.Logf, interfaceName string) (ret OSConfigurat
 		// "unmanaged" interfaces - meaning NM 1.26.6 and later
 		// actively ignore DNS configuration we give it. So, for those
 		// NM versions, we can and must use resolved directly.
-		old, err := nmVersionOlderThan("1.26.6")
+		//
+		// Even more fun, even-older versions of NM won't let us set
+		// DNS settings if the interface isn't managed by NM, with a
+		// hard failure on DBus requests. Empirically, NM 1.22 does
+		// this. Based on the versions popular distros shipped, we
+		// conservatively decree that only 1.26.0 through 1.26.5 are
+		// "safe" to use for our purposes. This roughly matches
+		// distros released in the latter half of 2020.
+		//
+		// In a perfect world, we'd avoid this by replacing
+		// configuration out from under NM entirely (e.g. using
+		// directManager to overwrite resolv.conf), but in a world
+		// where resolved runs, we need to get correct configuration
+		// into resolved regardless of what's in resolv.conf (because
+		// resolved can also be queried over dbus, or via an NSS
+		// module that bypasses /etc/resolv.conf). Given that we must
+		// get correct configuration into resolved, we have no choice
+		// but to use NM, and accept the loss of IPv6 configuration
+		// that comes with it (see
+		// https://github.com/tailscale/tailscale/issues/1699,
+		// https://github.com/tailscale/tailscale/pull/1945)
+		safe, err := nmVersionBetween("1.26.0", "1.26.5")
 		if err != nil {
 			// Failed to figure out NM's version, can't make a correct
 			// decision.
 			return nil, fmt.Errorf("checking NetworkManager version: %v", err)
 		}
-		if old {
-			dbg("nm-old", "yes")
+		if safe {
+			dbg("nm-safe", "yes")
 			return newNMManager(interfaceName)
 		}
-		dbg("nm-old", "no")
+		dbg("nm-safe", "no")
 		return newResolvedManager(logf, interfaceName)
 	case "resolvconf":
 		dbg("rc", "resolvconf")
-		if err := resolvconfSourceIsNM(bs); err == nil {
-			dbg("src-is-nm", "yes")
-			if err := dbusPing("org.freedesktop.NetworkManager", "/org/freedesktop/NetworkManager/DnsManager"); err == nil {
-				dbg("nm", "yes")
-				old, err := nmVersionOlderThan("1.26.6")
-				if err != nil {
-					return nil, fmt.Errorf("checking NetworkManager version: %v", err)
-				}
-				if old {
-					dbg("nm-old", "yes")
-					return newNMManager(interfaceName)
-				} else {
-					dbg("nm-old", "no")
-				}
-			} else {
-				dbg("nm", "no")
-			}
-		} else {
-			dbg("src-is-nm", "no")
-		}
 		if _, err := exec.LookPath("resolvconf"); err != nil {
 			dbg("resolvconf", "no")
 			return newDirectManager()
@@ -120,21 +130,19 @@ func NewOSConfigurator(logf logger.Logf, interfaceName string) (ret OSConfigurat
 		dbg("resolvconf", "yes")
 		return newResolvconfManager(logf)
 	case "NetworkManager":
+		// You'd think we would use newNMManager somewhere in
+		// here. However, as explained in
+		// https://github.com/tailscale/tailscale/issues/1699 , using
+		// NetworkManager for DNS configuration carries with it the
+		// cost of losing IPv6 configuration on the Tailscale network
+		// interface. So, when we can avoid it, we bypass
+		// NetworkManager by replacing resolv.conf directly.
+		//
+		// If you ever try to put NMManager back here, keep in mind
+		// that versions >=1.26.6 will ignore DNS configuration
+		// anyway, so you still need a fallback path that uses
+		// directManager.
 		dbg("rc", "nm")
-		if err := dbusPing("org.freedesktop.NetworkManager", "/org/freedesktop/NetworkManager/DnsManager"); err != nil {
-			dbg("nm", "no")
-			return newDirectManager()
-		}
-		dbg("nm", "yes")
-		old, err := nmVersionOlderThan("1.26.6")
-		if err != nil {
-			return nil, fmt.Errorf("checking NetworkManager version: %v", err)
-		}
-		if old {
-			dbg("nm-old", "yes")
-			return newNMManager(interfaceName)
-		}
-		dbg("nm-old", "no")
 		return newDirectManager()
 	default:
 		dbg("rc", "unknown")
@@ -142,46 +150,7 @@ func NewOSConfigurator(logf logger.Logf, interfaceName string) (ret OSConfigurat
 	}
 }
 
-func resolvconfSourceIsNM(resolvDotConf []byte) error {
-	b := bytes.NewBuffer(resolvDotConf)
-	cfg, err := readResolv(b)
-	if err != nil {
-		return fmt.Errorf("parsing /etc/resolv.conf: %w", err)
-	}
-
-	var (
-		paths = []string{
-			"/etc/resolvconf/run/interface/NetworkManager",
-			"/run/resolvconf/interface/NetworkManager",
-			"/var/run/resolvconf/interface/NetworkManager",
-			"/run/resolvconf/interfaces/NetworkManager",
-			"/var/run/resolvconf/interfaces/NetworkManager",
-		}
-		nmCfg OSConfig
-		found bool
-	)
-	for _, path := range paths {
-		nmCfg, err = readResolvFile(path)
-		if os.IsNotExist(err) {
-			continue
-		} else if err != nil {
-			return err
-		}
-		found = true
-		break
-	}
-	if !found {
-		return errors.New("NetworkManager resolvconf snippet not found")
-	}
-
-	if !nmCfg.Equal(cfg) {
-		return errors.New("NetworkManager config not applied by resolvconf")
-	}
-
-	return nil
-}
-
-func nmVersionOlderThan(want string) (bool, error) {
+func nmVersionBetween(first, last string) (bool, error) {
 	conn, err := dbus.SystemBus()
 	if err != nil {
 		// DBus probably not running.
@@ -199,7 +168,8 @@ func nmVersionOlderThan(want string) (bool, error) {
 		return false, fmt.Errorf("unexpected type %T for NM version", v.Value())
 	}
 
-	return cmpver.Compare(version, want) < 0, nil
+	outside := cmpver.Compare(version, first) < 0 || cmpver.Compare(version, last) > 0
+	return !outside, nil
 }
 
 func nmIsUsingResolved() error {
@@ -224,6 +194,17 @@ func nmIsUsingResolved() error {
 	return nil
 }
 
+func resolvedIsActuallyResolver() error {
+	cfg, err := readResolvConf()
+	if err != nil {
+		return err
+	}
+	if len(cfg.Nameservers) != 1 || cfg.Nameservers[0] != netaddr.IPv4(127, 0, 0, 53) {
+		return errors.New("resolv.conf doesn't point to systemd-resolved")
+	}
+	return nil
+}
+
 func dbusPing(name, objectPath string) error {
 	ctx, cancel := context.WithTimeout(context.Background(), time.Second)
 	defer cancel()

net/dns/manager_test.go

@@ -376,7 +376,7 @@ func TestManager(t *testing.T) {
 				SplitDNS:   test.split,
 				BaseConfig: test.bs,
 			}
-			m := NewManager(t.Logf, &f, nil)
+			m := NewManager(t.Logf, &f, nil, nil)
 			m.resolver.TestOnlySetHook(f.SetResolver)
 
 			if err := m.Set(test.in); err != nil {

net/dns/manager_windows.go

@@ -44,7 +44,7 @@ func NewOSConfigurator(logf logger.Logf, interfaceName string) (OSConfigurator,
 	ret := windowsManager{
 		logf:      logf,
 		guid:      interfaceName,
-		nrptWorks: !isWindows7(),
+		nrptWorks: isWindows10OrBetter(),
 	}
 
 	// Best-effort: if our NRPT rule exists, try to delete it. Unlike
@@ -407,22 +407,16 @@ var siteLocalResolvers = []netaddr.IP{
 	netaddr.MustParseIP("fec0:0:0:ffff::3"),
 }
 
-func isWindows7() bool {
+func isWindows10OrBetter() bool {
 	key, err := registry.OpenKey(registry.LOCAL_MACHINE, versionKey, registry.READ)
 	if err != nil {
-		// Fail safe, assume Windows 7.
-		return true
+		// Fail safe, assume old Windows.
+		return false
 	}
-	ver, _, err := key.GetStringValue("CurrentVersion")
-	if err != nil {
-		return true
+	// This key above only exists in Windows 10 and above. Its mere
+	// presence is good enough.
+	if _, _, err := key.GetIntegerValue("CurrentMajorVersionNumber"); err != nil {
+		return false
 	}
-	// Careful to not assume anything about version numbers beyond
-	// 6.3, Microsoft deprecated this registry key and locked its
-	// value to what it was in Windows 8.1. We can only use this to
-	// probe for versions before that. Good thing we only need Windows
-	// 7 (so far).
-	//
-	// And yes, Windows 7 is version 6.1. Don't ask.
-	return ver == "6.1"
+	return true
 }

net/dns/nm.go

@@ -4,8 +4,6 @@
 
 // +build linux
 
-//lint:file-ignore U1000 refactoring, temporarily unused code.
-
 package dns
 
 import (

net/dns/resolved.go

@@ -4,8 +4,6 @@
 
 // +build linux
 
-//lint:file-ignore U1000 refactoring, temporarily unused code.
-
 package dns
 
 import (

net/dns/resolver/forwarder.go

@@ -9,8 +9,8 @@ import (
 	"context"
 	"encoding/binary"
 	"errors"
-	"fmt"
 	"hash/crc32"
+	"io"
 	"math/rand"
 	"net"
 	"sync"
@@ -18,31 +18,21 @@ import (
 
 	dns "golang.org/x/net/dns/dnsmessage"
 	"inet.af/netaddr"
-	"tailscale.com/logtail/backoff"
 	"tailscale.com/types/logger"
 	"tailscale.com/util/dnsname"
+	"tailscale.com/wgengine/monitor"
 )
 
 // headerBytes is the number of bytes in a DNS message header.
 const headerBytes = 12
 
-// connCount is the number of UDP connections to use for forwarding.
-const connCount = 32
-
 const (
-	// cleanupInterval is the interval between purged of timed-out entries from txMap.
-	cleanupInterval = 30 * time.Second
 	// responseTimeout is the maximal amount of time to wait for a DNS response.
 	responseTimeout = 5 * time.Second
 )
 
 var errNoUpstreams = errors.New("upstream nameservers not set")
 
-type forwardingRecord struct {
-	src       netaddr.IPPort
-	createdAt time.Time
-}
-
 // txid identifies a DNS transaction.
 //
 // As the standard DNS Request ID is only 16 bits, we extend it:
@@ -99,159 +89,164 @@ func getTxID(packet []byte) txid {
 }
 
 type route struct {
-	suffix    dnsname.FQDN
-	resolvers []netaddr.IPPort
+	Suffix    dnsname.FQDN
+	Resolvers []netaddr.IPPort
 }
 
 // forwarder forwards DNS packets to a number of upstream nameservers.
 type forwarder struct {
-	logf logger.Logf
+	logf    logger.Logf
+	linkMon *monitor.Mon
+	linkSel ForwardLinkSelector
+
+	ctx       context.Context    // good until Close
+	ctxCancel context.CancelFunc // closes ctx
 
 	// responses is a channel by which responses are returned.
 	responses chan packet
-	// closed signals all goroutines to stop.
-	closed chan struct{}
-	// wg signals when all goroutines have stopped.
-	wg sync.WaitGroup
 
-	// conns are the UDP connections used for forwarding.
-	// A random one is selected for each request, regardless of the target upstream.
-	conns []*fwdConn
+	mu sync.Mutex // guards following
 
-	mu sync.Mutex
-	// routes are per-suffix resolvers to use.
-	routes []route                   // most specific routes first
-	txMap  map[txid]forwardingRecord // txids to in-flight requests
+	// routes are per-suffix resolvers to use, with
+	// the most specific routes first.
+	routes []route
 }
 
 func init() {
 	rand.Seed(time.Now().UnixNano())
 }
 
-func newForwarder(logf logger.Logf, responses chan packet) *forwarder {
-	ret := &forwarder{
+func newForwarder(logf logger.Logf, responses chan packet, linkMon *monitor.Mon, linkSel ForwardLinkSelector) *forwarder {
+	f := &forwarder{
 		logf:      logger.WithPrefix(logf, "forward: "),
+		linkMon:   linkMon,
+		linkSel:   linkSel,
 		responses: responses,
-		closed:    make(chan struct{}),
-		conns:     make([]*fwdConn, connCount),
-		txMap:     make(map[txid]forwardingRecord),
 	}
-
-	ret.wg.Add(connCount + 1)
-	for idx := range ret.conns {
-		ret.conns[idx] = newFwdConn(ret.logf, idx)
-		go ret.recv(ret.conns[idx])
-	}
-	go ret.cleanMap()
-
-	return ret
+	f.ctx, f.ctxCancel = context.WithCancel(context.Background())
+	return f
 }
 
-func (f *forwarder) Close() {
-	select {
-	case <-f.closed:
-		return
-	default:
-		// continue
-	}
-	close(f.closed)
-
-	for _, conn := range f.conns {
-		conn.close()
-	}
-
-	f.wg.Wait()
-}
-
-func (f *forwarder) rebindFromNetworkChange() {
-	for _, c := range f.conns {
-		c.mu.Lock()
-		c.reconnectLocked()
-		c.mu.Unlock()
-	}
+func (f *forwarder) Close() error {
+	f.ctxCancel()
+	return nil
 }
 
 func (f *forwarder) setRoutes(routes []route) {
 	f.mu.Lock()
+	defer f.mu.Unlock()
 	f.routes = routes
-	f.mu.Unlock()
+}
+
+var stdNetPacketListener packetListener = new(net.ListenConfig)
+
+type packetListener interface {
+	ListenPacket(ctx context.Context, network, address string) (net.PacketConn, error)
+}
+
+func (f *forwarder) packetListener(ip netaddr.IP) (packetListener, error) {
+	if f.linkSel == nil || initListenConfig == nil {
+		return stdNetPacketListener, nil
+	}
+	linkName := f.linkSel.PickLink(ip)
+	if linkName == "" {
+		return stdNetPacketListener, nil
+	}
+	lc := new(net.ListenConfig)
+	if err := initListenConfig(lc, f.linkMon, linkName); err != nil {
+		return nil, err
+	}
+	return lc, nil
 }
 
 // send sends packet to dst. It is best effort.
-func (f *forwarder) send(packet []byte, dst netaddr.IPPort) {
-	connIdx := rand.Intn(connCount)
-	conn := f.conns[connIdx]
-	conn.send(packet, dst)
-}
+//
+// send expects the reply to have the same txid as txidOut.
+//
+// The provided closeOnCtxDone lets send register values to Close if
+// the caller's ctx expires. This avoids send from allocating its own
+// waiting goroutine to interrupt the ReadFrom, as memory is tight on
+// iOS and we want the number of pending DNS lookups to be bursty
+// without too much associated goroutine/memory cost.
+func (f *forwarder) send(ctx context.Context, txidOut txid, closeOnCtxDone *closePool, packet []byte, dst netaddr.IPPort) ([]byte, error) {
+	// TODO(bradfitz): if dst.IP is 8.8.8.8 or 8.8.4.4 or 1.1.1.1, etc, or
+	// something dynamically probed earlier to support DoH or DoT,
+	// do that here instead.
 
-func (f *forwarder) recv(conn *fwdConn) {
-	defer f.wg.Done()
+	ln, err := f.packetListener(dst.IP())
+	if err != nil {
+		return nil, err
+	}
+	conn, err := ln.ListenPacket(ctx, "udp", ":0")
+	if err != nil {
+		f.logf("ListenPacket failed: %v", err)
+		return nil, err
+	}
+	defer conn.Close()
 
-	for {
-		select {
-		case <-f.closed:
-			return
-		default:
+	closeOnCtxDone.Add(conn)
+	defer closeOnCtxDone.Remove(conn)
+
+	if _, err := conn.WriteTo(packet, dst.UDPAddr()); err != nil {
+		if err := ctx.Err(); err != nil {
+			return nil, err
 		}
-		out := make([]byte, maxResponseBytes)
-		n := conn.read(out)
-		if n == 0 {
-			continue
+		return nil, err
+	}
+
+	// The 1 extra byte is to detect packet truncation.
+	out := make([]byte, maxResponseBytes+1)
+	n, _, err := conn.ReadFrom(out)
+	if err != nil {
+		if err := ctx.Err(); err != nil {
+			return nil, err
 		}
-		if n < headerBytes {
-			f.logf("recv: packet too small (%d bytes)", n)
-		}
-
-		out = out[:n]
-		txid := getTxID(out)
-
-		f.mu.Lock()
-
-		record, found := f.txMap[txid]
-		// At most one nameserver will return a response:
-		// the first one to do so will delete txid from the map.
-		if !found {
-			f.mu.Unlock()
-			continue
-		}
-		delete(f.txMap, txid)
-
-		f.mu.Unlock()
-
-		pkt := packet{out, record.src}
-		select {
-		case <-f.closed:
-			return
-		case f.responses <- pkt:
-			// continue
+		if packetWasTruncated(err) {
+			err = nil
+		} else {
+			return nil, err
 		}
 	}
+	truncated := n > maxResponseBytes
+	if truncated {
+		n = maxResponseBytes
+	}
+	if n < headerBytes {
+		f.logf("recv: packet too small (%d bytes)", n)
+	}
+	out = out[:n]
+	txid := getTxID(out)
+	if txid != txidOut {
+		return nil, errors.New("txid doesn't match")
+	}
+
+	if truncated {
+		const dnsFlagTruncated = 0x200
+		flags := binary.BigEndian.Uint16(out[2:4])
+		flags |= dnsFlagTruncated
+		binary.BigEndian.PutUint16(out[2:4], flags)
+
+		// TODO(#2067): Remove any incomplete records? RFC 1035 section 6.2
+		// states that truncation should head drop so that the authority
+		// section can be preserved if possible. However, the UDP read with
+		// a too-small buffer has already dropped the end, so that's the
+		// best we can do.
+	}
+
+	return out, nil
 }
 
-// cleanMap periodically deletes timed-out forwarding records from f.txMap to bound growth.
-func (f *forwarder) cleanMap() {
-	defer f.wg.Done()
-
-	t := time.NewTicker(cleanupInterval)
-	defer t.Stop()
-
-	var now time.Time
-	for {
-		select {
-		case <-f.closed:
-			return
-		case now = <-t.C:
-			// continue
+// resolvers returns the resolvers to use for domain.
+func (f *forwarder) resolvers(domain dnsname.FQDN) []netaddr.IPPort {
+	f.mu.Lock()
+	routes := f.routes
+	f.mu.Unlock()
+	for _, route := range routes {
+		if route.Suffix == "." || route.Suffix.Contains(domain) {
+			return route.Resolvers
 		}
-
-		f.mu.Lock()
-		for k, v := range f.txMap {
-			if now.Sub(v.createdAt) > responseTimeout {
-				delete(f.txMap, k)
-			}
-		}
-		f.mu.Unlock()
 	}
+	return nil
 }
 
 // forward forwards the query to all upstream nameservers and returns the first response.
@@ -263,217 +258,60 @@ func (f *forwarder) forward(query packet) error {
 
 	txid := getTxID(query.bs)
 
-	f.mu.Lock()
-	routes := f.routes
-	f.mu.Unlock()
-
-	var resolvers []netaddr.IPPort
-	for _, route := range routes {
-		if route.suffix != "." && !route.suffix.Contains(domain) {
-			continue
-		}
-		resolvers = route.resolvers
-		break
-	}
+	resolvers := f.resolvers(domain)
 	if len(resolvers) == 0 {
 		return errNoUpstreams
 	}
 
-	f.mu.Lock()
-	f.txMap[txid] = forwardingRecord{
-		src:       query.addr,
-		createdAt: time.Now(),
-	}
-	f.mu.Unlock()
+	closeOnCtxDone := new(closePool)
+	defer closeOnCtxDone.Close()
 
-	for _, resolver := range resolvers {
-		f.send(query.bs, resolver)
-	}
+	ctx, cancel := context.WithTimeout(f.ctx, responseTimeout)
+	defer cancel()
 
-	return nil
-}
+	resc := make(chan []byte, 1)
+	var (
+		mu       sync.Mutex
+		firstErr error
+	)
 
-// A fwdConn manages a single connection used to forward DNS requests.
-// Net link changes can cause a *net.UDPConn to become permanently unusable, particularly on macOS.
-// fwdConn detects such situations and transparently creates new connections.
-type fwdConn struct {
-	// logf allows a fwdConn to log.
-	logf logger.Logf
-
-	// change allows calls to read to block until a the network connection has been replaced.
-	change *sync.Cond
-
-	// mu protects fields that follow it; it is also change's Locker.
-	mu sync.Mutex
-	// closed tracks whether fwdConn has been permanently closed.
-	closed bool
-	// conn is the current active connection.
-	conn net.PacketConn
-}
-
-func newFwdConn(logf logger.Logf, idx int) *fwdConn {
-	c := new(fwdConn)
-	c.logf = logger.WithPrefix(logf, fmt.Sprintf("fwdConn %d: ", idx))
-	c.change = sync.NewCond(&c.mu)
-	// c.conn is created lazily in send
-	return c
-}
-
-// send sends packet to dst using c's connection.
-// It is best effort. It is UDP, after all. Failures are logged.
-func (c *fwdConn) send(packet []byte, dst netaddr.IPPort) {
-	var b *backoff.Backoff // lazily initialized, since it is not needed in the common case
-	backOff := func(err error) {
-		if b == nil {
-			b = backoff.NewBackoff("dns-fwdConn-send", c.logf, 30*time.Second)
-		}
-		b.BackOff(context.Background(), err)
-	}
-
-	for {
-		// Gather the current connection.
-		// We can't hold the lock while we call WriteTo.
-		c.mu.Lock()
-		conn := c.conn
-		closed := c.closed
-		if closed {
-			c.mu.Unlock()
-			return
-		}
-		if conn == nil {
-			c.reconnectLocked()
-			c.mu.Unlock()
-			continue
-		}
-		c.mu.Unlock()
-
-		_, err := conn.WriteTo(packet, dst.UDPAddr())
-		if err == nil {
-			// Success
-			return
-		}
-		if errors.Is(err, net.ErrClosed) {
-			// We intentionally closed this connection.
-			// It has been replaced by a new connection. Try again.
-			continue
-		}
-		// Something else went wrong.
-		// We have three choices here: try again, give up, or create a new connection.
-		var opErr *net.OpError
-		if !errors.As(err, &opErr) {
-			// Weird. All errors from the net package should be *net.OpError. Bail.
-			c.logf("send: non-*net.OpErr %v (%T)", err, err)
-			return
-		}
-		if opErr.Temporary() || opErr.Timeout() {
-			// I doubt that either of these can happen (this is UDP),
-			// but go ahead and try again.
-			backOff(err)
-			continue
-		}
-		if networkIsDown(err) {
-			// Fail.
-			c.logf("send: network is down")
-			return
-		}
-		if networkIsUnreachable(err) {
-			// This can be caused by a link change.
-			// Replace the existing connection with a new one.
-			c.mu.Lock()
-			// It's possible that multiple senders discovered simultaneously
-			// that the network is unreachable. Avoid reconnecting multiple times:
-			// Only reconnect if the current connection is the one that we
-			// discovered to be problematic.
-			if c.conn == conn {
-				backOff(err)
-				c.reconnectLocked()
+	for _, ipp := range resolvers {
+		go func(ipp netaddr.IPPort) {
+			resb, err := f.send(ctx, txid, closeOnCtxDone, query.bs, ipp)
+			if err != nil {
+				mu.Lock()
+				defer mu.Unlock()
+				if firstErr == nil {
+					firstErr = err
+				}
+				return
 			}
-			c.mu.Unlock()
-			// Try again with our new network connection.
-			continue
+			select {
+			case resc <- resb:
+			default:
+			}
+		}(ipp)
+	}
+
+	select {
+	case v := <-resc:
+		select {
+		case <-ctx.Done():
+			return ctx.Err()
+		case f.responses <- packet{v, query.addr}:
+			return nil
 		}
-		// Unrecognized error. Fail.
-		c.logf("send: unrecognized error: %v", err)
-		return
+	case <-ctx.Done():
+		mu.Lock()
+		defer mu.Unlock()
+		if firstErr != nil {
+			return firstErr
+		}
+		return ctx.Err()
 	}
 }
 
-// read waits for a response from c's connection.
-// It returns the number of bytes read, which may be 0
-// in case of an error or a closed connection.
-func (c *fwdConn) read(out []byte) int {
-	for {
-		// Gather the current connection.
-		// We can't hold the lock while we call ReadFrom.
-		c.mu.Lock()
-		conn := c.conn
-		closed := c.closed
-		if closed {
-			c.mu.Unlock()
-			return 0
-		}
-		if conn == nil {
-			// There is no current connection.
-			// Wait for the connection to change, then try again.
-			c.change.Wait()
-			c.mu.Unlock()
-			continue
-		}
-		c.mu.Unlock()
-
-		n, _, err := conn.ReadFrom(out)
-		if err == nil {
-			// Success.
-			return n
-		}
-		if errors.Is(err, net.ErrClosed) {
-			// We intentionally closed this connection.
-			// It has been replaced by a new connection. Try again.
-			continue
-		}
-
-		c.logf("read: unrecognized error: %v", err)
-		return 0
-	}
-}
-
-// reconnectLocked replaces the current connection with a new one.
-// c.mu must be locked.
-func (c *fwdConn) reconnectLocked() {
-	c.closeConnLocked()
-	// Make a new connection.
-	conn, err := net.ListenPacket("udp", "")
-	if err != nil {
-		c.logf("ListenPacket failed: %v", err)
-	} else {
-		c.conn = conn
-	}
-	// Broadcast that a new connection is available.
-	c.change.Broadcast()
-}
-
-// closeCurrentConn closes the current connection.
-// c.mu must be locked.
-func (c *fwdConn) closeConnLocked() {
-	if c.conn == nil {
-		return
-	}
-	c.conn.Close() // unblocks all readers/writers, they'll pick up the next connection.
-	c.conn = nil
-}
-
-// close permanently closes c.
-func (c *fwdConn) close() {
-	c.mu.Lock()
-	defer c.mu.Unlock()
-	if c.closed {
-		return
-	}
-	c.closed = true
-	c.closeConnLocked()
-	// Unblock any remaining readers.
-	c.change.Broadcast()
-}
+var initListenConfig func(_ *net.ListenConfig, _ *monitor.Mon, tunName string) error
 
 // nameFromQuery extracts the normalized query name from bs.
 func nameFromQuery(bs []byte) (dnsname.FQDN, error) {
@@ -495,3 +333,48 @@ func nameFromQuery(bs []byte) (dnsname.FQDN, error) {
 	n := q.Name.Data[:q.Name.Length]
 	return dnsname.ToFQDN(rawNameToLower(n))
 }
+
+// closePool is a dynamic set of io.Closers to close as a group.
+// It's intended to be Closed at most once.
+//
+// The zero value is ready for use.
+type closePool struct {
+	mu     sync.Mutex
+	m      map[io.Closer]bool
+	closed bool
+}
+
+func (p *closePool) Add(c io.Closer) {
+	p.mu.Lock()
+	defer p.mu.Unlock()
+	if p.closed {
+		c.Close()
+		return
+	}
+	if p.m == nil {
+		p.m = map[io.Closer]bool{}
+	}
+	p.m[c] = true
+}
+
+func (p *closePool) Remove(c io.Closer) {
+	p.mu.Lock()
+	defer p.mu.Unlock()
+	if p.closed {
+		return
+	}
+	delete(p.m, c)
+}
+
+func (p *closePool) Close() error {
+	p.mu.Lock()
+	defer p.mu.Unlock()
+	if p.closed {
+		return nil
+	}
+	p.closed = true
+	for c := range p.m {
+		c.Close()
+	}
+	return nil
+}

net/dns/resolver/macios_ext.go

@@ -0,0 +1,27 @@
+// Copyright (c) 2021 Tailscale Inc & AUTHORS All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+// +build darwin,ts_macext ios,ts_macext
+
+package resolver
+
+import (
+	"errors"
+	"net"
+
+	"tailscale.com/net/netns"
+	"tailscale.com/wgengine/monitor"
+)
+
+func init() {
+	initListenConfig = initListenConfigNetworkExtension
+}
+
+func initListenConfigNetworkExtension(nc *net.ListenConfig, mon *monitor.Mon, tunName string) error {
+	nif, ok := mon.InterfaceState().Interface[tunName]
+	if !ok {
+		return errors.New("utun not found")
+	}
+	return netns.SetListenConfigInterfaceIndex(nc, nif.Interface.Index)
+}

net/dns/resolver/neterr_darwin.go

@@ -23,3 +23,8 @@ func networkIsDown(err error) bool {
 func networkIsUnreachable(err error) bool {
 	return errors.Is(err, networkUnreachable)
 }
+
+// packetWasTruncated returns true if err indicates truncation but the RecvFrom
+// that generated err was otherwise successful. It always returns false on this
+// platform.
+func packetWasTruncated(err error) bool { return false }

net/dns/resolver/neterr_other.go

@@ -8,3 +8,8 @@ package resolver
 
 func networkIsDown(err error) bool        { return false }
 func networkIsUnreachable(err error) bool { return false }
+
+// packetWasTruncated returns true if err indicates truncation but the RecvFrom
+// that generated err was otherwise successful. It always returns false on this
+// platform.
+func packetWasTruncated(err error) bool   { return false }

net/dns/resolver/neterr_windows.go

@@ -5,6 +5,7 @@
 package resolver
 
 import (
+	"errors"
 	"net"
 	"os"
 
@@ -27,3 +28,16 @@ func networkIsUnreachable(err error) bool {
 	// difference between down and unreachable? Add comments.
 	return false
 }
+
+// packetWasTruncated returns true if err indicates truncation but the RecvFrom
+// that generated err was otherwise successful. On Windows, Go's UDP RecvFrom
+// calls WSARecvFrom which returns the WSAEMSGSIZE error code when the received
+// datagram is larger than the provided buffer. When that happens, both a valid
+// size and an error are returned (as per the partial fix for golang/go#14074).
+// If the WSAEMSGSIZE error is returned, then we ignore the error to get
+// semantics similar to the POSIX operating systems. One caveat is that it
+// appears that the source address is not returned when WSAEMSGSIZE occurs, but
+// we do not currently look at the source address.
+func packetWasTruncated(err error) bool {
+	return errors.Is(err, windows.WSAEMSGSIZE)
+}

net/dns/resolver/tsdns.go

@@ -9,26 +9,39 @@ package resolver
 import (
 	"encoding/hex"
 	"errors"
+	"runtime"
 	"sort"
 	"strings"
 	"sync"
+	"sync/atomic"
 	"time"
 
 	dns "golang.org/x/net/dns/dnsmessage"
 	"inet.af/netaddr"
-	"tailscale.com/net/interfaces"
 	"tailscale.com/types/logger"
 	"tailscale.com/util/dnsname"
 	"tailscale.com/wgengine/monitor"
 )
 
-// maxResponseBytes is the maximum size of a response from a Resolver.
-const maxResponseBytes = 512
+// maxResponseBytes is the maximum size of a response from a Resolver. The
+// actual buffer size will be one larger than this so that we can detect
+// truncation in a platform-agnostic way.
+const maxResponseBytes = 4095
 
-// queueSize is the maximal number of DNS requests that can await polling.
+// maxActiveQueries returns the maximal number of DNS requests that be
+// can running.
 // If EnqueueRequest is called when this many requests are already pending,
 // the request will be dropped to avoid blocking the caller.
-const queueSize = 64
+func maxActiveQueries() int32 {
+	if runtime.GOOS == "ios" {
+		// For memory paranoia reasons on iOS, match the
+		// historical Tailscale 1.x..1.8 behavior for now
+		// (just before the 1.10 release).
+		return 64
+	}
+	// But for other platforms, allow more burstiness:
+	return 256
+}
 
 // defaultTTL is the TTL of all responses from Resolver.
 const defaultTTL = 600 * time.Second
@@ -73,13 +86,12 @@ type Config struct {
 type Resolver struct {
 	logf               logger.Logf
 	linkMon            *monitor.Mon     // or nil
-	unregLinkMon       func()           // or nil
 	saveConfigForTests func(cfg Config) // used in tests to capture resolver config
 	// forwarder forwards requests to upstream nameservers.
 	forwarder *forwarder
 
-	// queue is a buffered channel holding DNS requests queued for resolution.
-	queue chan packet
+	activeQueriesAtomic int32 // number of DNS queries in flight
+
 	// responses is an unbuffered channel to which responses are returned.
 	responses chan packet
 	// errors is an unbuffered channel to which errors are returned.
@@ -96,27 +108,26 @@ type Resolver struct {
 	ipToHost     map[netaddr.IP]dnsname.FQDN
 }
 
+type ForwardLinkSelector interface {
+	// PickLink returns which network device should be used to query
+	// the DNS server at the given IP.
+	// The empty string means to use an unspecified default.
+	PickLink(netaddr.IP) (linkName string)
+}
+
 // New returns a new resolver.
 // linkMon optionally specifies a link monitor to use for socket rebinding.
-func New(logf logger.Logf, linkMon *monitor.Mon) *Resolver {
+func New(logf logger.Logf, linkMon *monitor.Mon, linkSel ForwardLinkSelector) *Resolver {
 	r := &Resolver{
 		logf:      logger.WithPrefix(logf, "dns: "),
 		linkMon:   linkMon,
-		queue:     make(chan packet, queueSize),
 		responses: make(chan packet),
 		errors:    make(chan error),
 		closed:    make(chan struct{}),
 		hostToIP:  map[dnsname.FQDN][]netaddr.IP{},
 		ipToHost:  map[netaddr.IP]dnsname.FQDN{},
 	}
-	r.forwarder = newForwarder(r.logf, r.responses)
-	if r.linkMon != nil {
-		r.unregLinkMon = r.linkMon.RegisterChangeCallback(r.onLinkMonitorChange)
-	}
-
-	r.wg.Add(1)
-	go r.poll()
-
+	r.forwarder = newForwarder(r.logf, r.responses, linkMon, linkSel)
 	return r
 }
 
@@ -138,13 +149,13 @@ func (r *Resolver) SetConfig(cfg Config) error {
 
 	for suffix, ips := range cfg.Routes {
 		routes = append(routes, route{
-			suffix:    suffix,
-			resolvers: ips,
+			Suffix:    suffix,
+			Resolvers: ips,
 		})
 	}
 	// Sort from longest prefix to shortest.
 	sort.Slice(routes, func(i, j int) bool {
-		return routes[i].suffix.NumLabels() > routes[j].suffix.NumLabels()
+		return routes[i].Suffix.NumLabels() > routes[j].Suffix.NumLabels()
 	})
 
 	r.forwarder.setRoutes(routes)
@@ -168,19 +179,7 @@ func (r *Resolver) Close() {
 	}
 	close(r.closed)
 
-	if r.unregLinkMon != nil {
-		r.unregLinkMon()
-	}
-
 	r.forwarder.Close()
-	r.wg.Wait()
-}
-
-func (r *Resolver) onLinkMonitorChange(changed bool, state *interfaces.State) {
-	if !changed {
-		return
-	}
-	r.forwarder.rebindFromNetworkChange()
 }
 
 // EnqueueRequest places the given DNS request in the resolver's queue.
@@ -190,11 +189,14 @@ func (r *Resolver) EnqueueRequest(bs []byte, from netaddr.IPPort) error {
 	select {
 	case <-r.closed:
 		return ErrClosed
-	case r.queue <- packet{bs, from}:
-		return nil
 	default:
+	}
+	if n := atomic.AddInt32(&r.activeQueriesAtomic, 1); n > maxActiveQueries() {
+		atomic.AddInt32(&r.activeQueriesAtomic, -1)
 		return errFullQueue
 	}
+	go r.handleQuery(packet{bs, from})
+	return nil
 }
 
 // NextResponse returns a DNS response to a previously enqueued request.
@@ -289,53 +291,34 @@ func (r *Resolver) resolveLocal(domain dnsname.FQDN, typ dns.Type) (netaddr.IP,
 // resolveReverse returns the unique domain name that maps to the given address.
 func (r *Resolver) resolveLocalReverse(ip netaddr.IP) (dnsname.FQDN, dns.RCode) {
 	r.mu.Lock()
-	ips := r.ipToHost
-	r.mu.Unlock()
-
-	name, found := ips[ip]
-	if !found {
+	defer r.mu.Unlock()
+	name, ok := r.ipToHost[ip]
+	if !ok {
 		return "", dns.RCodeNameError
 	}
 	return name, dns.RCodeSuccess
 }
 
-func (r *Resolver) poll() {
-	defer r.wg.Done()
+func (r *Resolver) handleQuery(pkt packet) {
+	defer atomic.AddInt32(&r.activeQueriesAtomic, -1)
 
-	var pkt packet
-	for {
+	out, err := r.respond(pkt.bs)
+	if err == errNotOurName {
+		err = r.forwarder.forward(pkt)
+		if err == nil {
+			// forward will send response into r.responses, nothing to do.
+			return
+		}
+	}
+	if err != nil {
 		select {
 		case <-r.closed:
-			return
-		case pkt = <-r.queue:
-			// continue
+		case r.errors <- err:
 		}
-
-		out, err := r.respond(pkt.bs)
-
-		if err == errNotOurName {
-			err = r.forwarder.forward(pkt)
-			if err == nil {
-				// forward will send response into r.responses, nothing to do.
-				continue
-			}
-		}
-
-		if err != nil {
-			select {
-			case <-r.closed:
-				return
-			case r.errors <- err:
-				// continue
-			}
-		} else {
-			pkt.bs = out
-			select {
-			case <-r.closed:
-				return
-			case r.responses <- pkt:
-				// continue
-			}
+	} else {
+		select {
+		case <-r.closed:
+		case r.responses <- packet{out, pkt.addr}:
 		}
 	}
 }
@@ -349,28 +332,44 @@ type response struct {
 	IP netaddr.IP
 }
 
-// parseQuery parses the query in given packet into a response struct.
-// if the parse is successful, resp.Name contains the normalized name being queried.
-// TODO: stuffing the query name in resp.Name temporarily is a hack. Clean it up.
-func parseQuery(query []byte, resp *response) error {
-	var parser dns.Parser
-	var err error
+var dnsParserPool = &sync.Pool{
+	New: func() interface{} {
+		return new(dnsParser)
+	},
+}
 
-	resp.Header, err = parser.Start(query)
+// dnsParser parses DNS queries using x/net/dns/dnsmessage.
+// These structs are pooled with dnsParserPool.
+type dnsParser struct {
+	Header   dns.Header
+	Question dns.Question
+
+	parser dns.Parser
+}
+
+func (p *dnsParser) response() *response {
+	return &response{Header: p.Header, Question: p.Question}
+}
+
+// zeroParser clears parser so it doesn't retain its most recently
+// parsed DNS query's []byte while it's sitting in a sync.Pool.
+// It's not useful to keep anyway: the next Start will do the same.
+func (p *dnsParser) zeroParser() { p.parser = dns.Parser{} }
+
+// parseQuery parses the query in given packet into p.Header and
+// p.Question.
+func (p *dnsParser) parseQuery(query []byte) error {
+	defer p.zeroParser()
+	var err error
+	p.Header, err = p.parser.Start(query)
 	if err != nil {
 		return err
 	}
-
-	if resp.Header.Response {
+	if p.Header.Response {
 		return errNotQuery
 	}
-
-	resp.Question, err = parser.Question()
-	if err != nil {
-		return err
-	}
-
-	return nil
+	p.Question, err = p.parser.Question()
+	return err
 }
 
 // marshalARecord serializes an A record into an active builder.
@@ -622,12 +621,13 @@ func (r *Resolver) respondReverse(query []byte, name dnsname.FQDN, resp *respons
 // respond returns a DNS response to query if it can be resolved locally.
 // Otherwise, it returns errNotOurName.
 func (r *Resolver) respond(query []byte) ([]byte, error) {
-	resp := new(response)
+	parser := dnsParserPool.Get().(*dnsParser)
+	defer dnsParserPool.Put(parser)
 
 	// ParseQuery is sufficiently fast to run on every DNS packet.
 	// This is considerably simpler than extracting the name by hand
 	// to shave off microseconds in case of delegation.
-	err := parseQuery(query, resp)
+	err := parser.parseQuery(query)
 	// We will not return this error: it is the sender's fault.
 	if err != nil {
 		if errors.Is(err, dns.ErrSectionDone) {
@@ -635,13 +635,15 @@ func (r *Resolver) respond(query []byte) ([]byte, error) {
 		} else {
 			r.logf("parseQuery(%02x): %v", query, err)
 		}
+		resp := parser.response()
 		resp.Header.RCode = dns.RCodeFormatError
 		return marshalResponse(resp)
 	}
-	rawName := resp.Question.Name.Data[:resp.Question.Name.Length]
+	rawName := parser.Question.Name.Data[:parser.Question.Name.Length]
 	name, err := dnsname.ToFQDN(rawNameToLower(rawName))
 	if err != nil {
 		// DNS packet unexpectedly contains an invalid FQDN.
+		resp := parser.response()
 		resp.Header.RCode = dns.RCodeFormatError
 		return marshalResponse(resp)
 	}
@@ -649,15 +651,17 @@ func (r *Resolver) respond(query []byte) ([]byte, error) {
 	// Always try to handle reverse lookups; delegate inside when not found.
 	// This way, queries for existent nodes do not leak,
 	// but we behave gracefully if non-Tailscale nodes exist in CGNATRange.
-	if resp.Question.Type == dns.TypePTR {
-		return r.respondReverse(query, name, resp)
+	if parser.Question.Type == dns.TypePTR {
+		return r.respondReverse(query, name, parser.response())
 	}
 
-	resp.IP, resp.Header.RCode = r.resolveLocal(name, resp.Question.Type)
-	// This return code is special: it requests forwarding.
-	if resp.Header.RCode == dns.RCodeRefused {
-		return nil, errNotOurName
+	ip, rcode := r.resolveLocal(name, parser.Question.Type)
+	if rcode == dns.RCodeRefused {
+		return nil, errNotOurName // sentinel error return value: it requests forwarding
 	}
 
+	resp := parser.response()
+	resp.Header.RCode = rcode
+	resp.IP = ip
 	return marshalResponse(resp)
 }

net/dns/resolver/tsdns_server_test.go

@@ -66,6 +66,39 @@ func resolveToIP(ipv4, ipv6 netaddr.IP, ns string) dns.HandlerFunc {
 	}
 }
 
+// resolveToTXT returns a handler function which responds to queries of type TXT
+// it receives with the strings in txts.
+func resolveToTXT(txts []string) dns.HandlerFunc {
+	return func(w dns.ResponseWriter, req *dns.Msg) {
+		m := new(dns.Msg)
+		m.SetReply(req)
+
+		if len(req.Question) != 1 {
+			panic("not a single-question request")
+		}
+		question := req.Question[0]
+
+		if question.Qtype != dns.TypeTXT {
+			w.WriteMsg(m)
+			return
+		}
+
+		ans := &dns.TXT{
+			Hdr: dns.RR_Header{
+				Name:   question.Name,
+				Rrtype: dns.TypeTXT,
+				Class:  dns.ClassINET,
+			},
+			Txt: txts,
+		}
+
+		m.Answer = append(m.Answer, ans)
+		if err := w.WriteMsg(m); err != nil {
+			panic(err)
+		}
+	}
+}
+
 var resolveToNXDOMAIN = dns.HandlerFunc(func(w dns.ResponseWriter, req *dns.Msg) {
 	m := new(dns.Msg)
 	m.SetRcode(req, dns.RcodeNameError)

net/dns/resolver/tsdns_test.go

@@ -6,14 +6,19 @@ package resolver
 
 import (
 	"bytes"
+	"encoding/hex"
 	"errors"
+	"fmt"
+	"math/rand"
 	"net"
+	"runtime"
 	"testing"
 
 	dns "golang.org/x/net/dns/dnsmessage"
 	"inet.af/netaddr"
 	"tailscale.com/tstest"
 	"tailscale.com/util/dnsname"
+	"tailscale.com/wgengine/monitor"
 )
 
 var testipv4 = netaddr.MustParseIP("1.2.3.4")
@@ -44,9 +49,11 @@ func dnspacket(domain dnsname.FQDN, tp dns.Type) []byte {
 }
 
 type dnsResponse struct {
-	ip    netaddr.IP
-	name  dnsname.FQDN
-	rcode dns.RCode
+	ip        netaddr.IP
+	txt       []string
+	name      dnsname.FQDN
+	rcode     dns.RCode
+	truncated bool
 }
 
 func unpackResponse(payload []byte) (dnsResponse, error) {
@@ -67,6 +74,16 @@ func unpackResponse(payload []byte) (dnsResponse, error) {
 		return response, nil
 	}
 
+	response.truncated = h.Truncated
+	if response.truncated {
+		// TODO(#2067): Ideally, answer processing should still succeed when
+		// dealing with a truncated message, but currently when we truncate
+		// a packet, it's caused by the buffer being too small and usually that
+		// means the data runs out mid-record. dns.Parser does not like it when
+		// that happens. We can improve this by trimming off incomplete records.
+		return response, nil
+	}
+
 	err = parser.SkipAllQuestions()
 	if err != nil {
 		return response, err
@@ -90,6 +107,12 @@ func unpackResponse(payload []byte) (dnsResponse, error) {
 			return response, err
 		}
 		response.ip = netaddr.IPv6Raw(res.AAAA)
+	case dns.TypeTXT:
+		res, err := parser.TXTResource()
+		if err != nil {
+			return response, err
+		}
+		response.txt = res.TXT
 	case dns.TypeNS:
 		res, err := parser.NSResource()
 		if err != nil {
@@ -107,7 +130,9 @@ func unpackResponse(payload []byte) (dnsResponse, error) {
 }
 
 func syncRespond(r *Resolver, query []byte) ([]byte, error) {
-	r.EnqueueRequest(query, netaddr.IPPort{})
+	if err := r.EnqueueRequest(query, netaddr.IPPort{}); err != nil {
+		return nil, fmt.Errorf("EnqueueRequest: %w", err)
+	}
 	payload, _, err := r.NextResponse()
 	return payload, err
 }
@@ -190,8 +215,12 @@ func TestRDNSNameToIPv6(t *testing.T) {
 	}
 }
 
+func newResolver(t testing.TB) *Resolver {
+	return New(t.Logf, nil /* no link monitor */, nil /* no link selector */)
+}
+
 func TestResolveLocal(t *testing.T) {
-	r := New(t.Logf, nil)
+	r := newResolver(t)
 	defer r.Close()
 
 	r.SetConfig(dnsCfg)
@@ -231,7 +260,7 @@ func TestResolveLocal(t *testing.T) {
 }
 
 func TestResolveLocalReverse(t *testing.T) {
-	r := New(t.Logf, nil)
+	r := newResolver(t)
 	defer r.Close()
 
 	r.SetConfig(dnsCfg)
@@ -269,6 +298,32 @@ func ipv6Works() bool {
 	return true
 }
 
+func generateTXT(size int, source rand.Source) []string {
+	const sizePerTXT = 120
+
+	if size%2 != 0 {
+		panic("even lengths only")
+	}
+
+	rng := rand.New(source)
+
+	txts := make([]string, 0, size/sizePerTXT+1)
+
+	raw := make([]byte, sizePerTXT/2)
+
+	rem := size
+	for ; rem > sizePerTXT; rem -= sizePerTXT {
+		rng.Read(raw)
+		txts = append(txts, hex.EncodeToString(raw))
+	}
+	if rem > 0 {
+		rng.Read(raw[:rem/2])
+		txts = append(txts, hex.EncodeToString(raw[:rem/2]))
+	}
+
+	return txts
+}
+
 func TestDelegate(t *testing.T) {
 	tstest.ResourceCheck(t)
 
@@ -276,16 +331,46 @@ func TestDelegate(t *testing.T) {
 		t.Skip("skipping test that requires localhost IPv6")
 	}
 
+	randSource := rand.NewSource(4)
+
+	// smallTXT does not require EDNS
+	smallTXT := generateTXT(300, randSource)
+
+	// medTXT and largeTXT are responses that require EDNS but we would like to
+	// support these sizes of response without truncation because they are
+	// moderately common.
+	medTXT := generateTXT(1200, randSource)
+	largeTXT := generateTXT(4000, randSource)
+
+	// xlargeTXT is slightly above the maximum response size that we support,
+	// so there should be truncation.
+	xlargeTXT := generateTXT(5000, randSource)
+
+	// hugeTXT is significantly larger than any typical MTU and will require
+	// significant fragmentation. For buffer management reasons, we do not
+	// intend to handle responses this large, so there should be truncation.
+	hugeTXT := generateTXT(64000, randSource)
+
 	v4server := serveDNS(t, "127.0.0.1:0",
 		"test.site.", resolveToIP(testipv4, testipv6, "dns.test.site."),
-		"nxdomain.site.", resolveToNXDOMAIN)
+		"nxdomain.site.", resolveToNXDOMAIN,
+		"small.txt.", resolveToTXT(smallTXT),
+		"med.txt.", resolveToTXT(medTXT),
+		"large.txt.", resolveToTXT(largeTXT),
+		"xlarge.txt.", resolveToTXT(xlargeTXT),
+		"huge.txt.", resolveToTXT(hugeTXT))
 	defer v4server.Shutdown()
 	v6server := serveDNS(t, "[::1]:0",
 		"test.site.", resolveToIP(testipv4, testipv6, "dns.test.site."),
-		"nxdomain.site.", resolveToNXDOMAIN)
+		"nxdomain.site.", resolveToNXDOMAIN,
+		"small.txt.", resolveToTXT(smallTXT),
+		"med.txt.", resolveToTXT(medTXT),
+		"large.txt.", resolveToTXT(largeTXT),
+		"xlarge.txt.", resolveToTXT(xlargeTXT),
+		"huge.txt.", resolveToTXT(hugeTXT))
 	defer v6server.Shutdown()
 
-	r := New(t.Logf, nil)
+	r := newResolver(t)
 	defer r.Close()
 
 	cfg := dnsCfg
@@ -322,10 +407,38 @@ func TestDelegate(t *testing.T) {
 			dnspacket("nxdomain.site.", dns.TypeA),
 			dnsResponse{rcode: dns.RCodeNameError},
 		},
+		{
+			"smalltxt",
+			dnspacket("small.txt.", dns.TypeTXT),
+			dnsResponse{txt: smallTXT, rcode: dns.RCodeSuccess},
+		},
+		{
+			"medtxt",
+			dnspacket("med.txt.", dns.TypeTXT),
+			dnsResponse{txt: medTXT, rcode: dns.RCodeSuccess},
+		},
+		{
+			"largetxt",
+			dnspacket("large.txt.", dns.TypeTXT),
+			dnsResponse{txt: largeTXT, rcode: dns.RCodeSuccess},
+		},
+		{
+			"xlargetxt",
+			dnspacket("xlarge.txt.", dns.TypeTXT),
+			dnsResponse{rcode: dns.RCodeSuccess, truncated: true},
+		},
+		{
+			"hugetxt",
+			dnspacket("huge.txt.", dns.TypeTXT),
+			dnsResponse{rcode: dns.RCodeSuccess, truncated: true},
+		},
 	}
 
 	for _, tt := range tests {
 		t.Run(tt.title, func(t *testing.T) {
+			if tt.title == "hugetxt" && runtime.GOOS == "darwin" {
+				t.Skip("known to not work on macOS: https://github.com/tailscale/tailscale/issues/2229")
+			}
 			payload, err := syncRespond(r, tt.query)
 			if err != nil {
 				t.Errorf("err = %v; want nil", err)
@@ -345,6 +458,15 @@ func TestDelegate(t *testing.T) {
 			if response.name != tt.response.name {
 				t.Errorf("name = %v; want %v", response.name, tt.response.name)
 			}
+			if len(response.txt) != len(tt.response.txt) {
+				t.Errorf("%v txt records, want %v txt records", len(response.txt), len(tt.response.txt))
+			} else {
+				for i := range response.txt {
+					if response.txt[i] != tt.response.txt[i] {
+						t.Errorf("txt record %v is %s, want %s", i, response.txt[i], tt.response.txt[i])
+					}
+				}
+			}
 		})
 	}
 }
@@ -360,7 +482,7 @@ func TestDelegateSplitRoute(t *testing.T) {
 		"test.other.", resolveToIP(test4, test6, "dns.other."))
 	defer server2.Shutdown()
 
-	r := New(t.Logf, nil)
+	r := newResolver(t)
 	defer r.Close()
 
 	cfg := dnsCfg
@@ -417,7 +539,7 @@ func TestDelegateCollision(t *testing.T) {
 		"test.site.", resolveToIP(testipv4, testipv6, "dns.test.site."))
 	defer server.Shutdown()
 
-	r := New(t.Logf, nil)
+	r := newResolver(t)
 	defer r.Close()
 
 	cfg := dnsCfg
@@ -631,7 +753,7 @@ var emptyResponse = []byte{
 }
 
 func TestFull(t *testing.T) {
-	r := New(t.Logf, nil)
+	r := newResolver(t)
 	defer r.Close()
 
 	r.SetConfig(dnsCfg)
@@ -667,7 +789,7 @@ func TestFull(t *testing.T) {
 }
 
 func TestAllocs(t *testing.T) {
-	r := New(t.Logf, nil)
+	r := newResolver(t)
 	defer r.Close()
 	r.SetConfig(dnsCfg)
 
@@ -721,7 +843,7 @@ func BenchmarkFull(b *testing.B) {
 		"test.site.", resolveToIP(testipv4, testipv6, "dns.test.site."))
 	defer server.Shutdown()
 
-	r := New(b.Logf, nil)
+	r := newResolver(b)
 	defer r.Close()
 
 	cfg := dnsCfg
@@ -758,3 +880,58 @@ func TestMarshalResponseFormatError(t *testing.T) {
 	}
 	t.Logf("response: %q", v)
 }
+
+func TestForwardLinkSelection(t *testing.T) {
+	old := initListenConfig
+	defer func() { initListenConfig = old }()
+
+	configCall := make(chan string, 1)
+	initListenConfig = func(nc *net.ListenConfig, mon *monitor.Mon, tunName string) error {
+		select {
+		case configCall <- tunName:
+			return nil
+		default:
+			t.Error("buffer full")
+			return errors.New("buffer full")
+		}
+	}
+
+	// specialIP is some IP we pretend that our link selector
+	// routes differently.
+	specialIP := netaddr.IPv4(1, 2, 3, 4)
+
+	fwd := newForwarder(t.Logf, nil, nil, linkSelFunc(func(ip netaddr.IP) string {
+		if ip == netaddr.IPv4(1, 2, 3, 4) {
+			return "special"
+		}
+		return ""
+	}))
+
+	// Test non-special IP.
+	if got, err := fwd.packetListener(netaddr.IP{}); err != nil {
+		t.Fatal(err)
+	} else if got != stdNetPacketListener {
+		t.Errorf("for IP zero value, didn't get expected packet listener")
+	}
+	select {
+	case v := <-configCall:
+		t.Errorf("unexpected ListenConfig call, with tunName %q", v)
+	default:
+	}
+
+	// Test that our special IP generates a call to initListenConfig.
+	if got, err := fwd.packetListener(specialIP); err != nil {
+		t.Fatal(err)
+	} else if got == stdNetPacketListener {
+		t.Errorf("special IP returned std packet listener; expected unique one")
+	}
+	if v, ok := <-configCall; !ok {
+		t.Errorf("didn't get ListenConfig call")
+	} else if v != "special" {
+		t.Errorf("got tunName %q; want 'special'", v)
+	}
+}
+
+type linkSelFunc func(ip netaddr.IP) string
+
+func (f linkSelFunc) PickLink(ip netaddr.IP) string { return f(ip) }

net/interfaces/interfaces.go

@@ -15,6 +15,7 @@ import (
 	"strings"
 
 	"inet.af/netaddr"
+	"tailscale.com/hostinfo"
 	"tailscale.com/net/tsaddr"
 	"tailscale.com/net/tshttpproxy"
 )
@@ -81,13 +82,16 @@ func isProblematicInterface(nif *net.Interface) bool {
 }
 
 // LocalAddresses returns the machine's IP addresses, separated by
-// whether they're loopback addresses.
+// whether they're loopback addresses. If there are no regular addresses
+// it will return any IPv4 linklocal or IPv6 unique local addresses because we
+// know of environments where these are used with NAT to provide connectivity.
 func LocalAddresses() (regular, loopback []netaddr.IP, err error) {
 	// TODO(crawshaw): don't serve interface addresses that we are routing
 	ifaces, err := net.Interfaces()
 	if err != nil {
 		return nil, nil, err
 	}
+	var regular4, regular6, linklocal4, ula6 []netaddr.IP
 	for i := range ifaces {
 		iface := &ifaces[i]
 		if !isUp(iface) || isProblematicInterface(iface) {
@@ -117,17 +121,44 @@ func LocalAddresses() (regular, loopback []netaddr.IP, err error) {
 				if tsaddr.IsTailscaleIP(ip) {
 					continue
 				}
-				if ip.IsLinkLocalUnicast() {
-					continue
-				}
 				if ip.IsLoopback() || ifcIsLoopback {
 					loopback = append(loopback, ip)
+				} else if ip.IsLinkLocalUnicast() {
+					if ip.Is4() {
+						linklocal4 = append(linklocal4, ip)
+					}
+
+					// We know of no cases where the IPv6 fe80:: addresses
+					// are used to provide WAN connectivity. It is also very
+					// common for users to have no IPv6 WAN connectivity,
+					// but their OS supports IPv6 so they have an fe80::
+					// address. We don't want to report all of those
+					// IPv6 LL to Control.
+				} else if ip.Is6() && tsaddr.IsULA(ip) {
+					// Google Cloud Run uses NAT with IPv6 Unique
+					// Local Addresses to provide IPv6 connectivity.
+					ula6 = append(ula6, ip)
 				} else {
-					regular = append(regular, ip)
+					if ip.Is4() {
+						regular4 = append(regular4, ip)
+					} else {
+						regular6 = append(regular6, ip)
+					}
 				}
 			}
 		}
 	}
+	if len(regular4) == 0 && len(regular6) == 0 {
+		// if we have no usable IP addresses then be willing to accept
+		// addresses we otherwise wouldn't, like:
+		//   + 169.254.x.x (AWS Lambda uses NAT with these)
+		//   + IPv6 ULA (Google Cloud Run uses these with address translation)
+		if hostinfo.GetEnvType() == hostinfo.AWSLambda {
+			regular4 = linklocal4
+		}
+		regular6 = ula6
+	}
+	regular = append(regular4, regular6...)
 	sortIPs(regular)
 	sortIPs(loopback)
 	return regular, loopback, nil
@@ -213,9 +244,9 @@ type State struct {
 	InterfaceIPs map[string][]netaddr.IPPrefix
 	Interface    map[string]Interface
 
-	// HaveV6Global is whether this machine has an IPv6 global address
-	// on some non-Tailscale interface that's up.
-	HaveV6Global bool
+	// HaveV6 is whether this machine has an IPv6 Global or Unique Local Address
+	// which might provide connectivity on a non-Tailscale interface that's up.
+	HaveV6 bool
 
 	// HaveV4 is whether the machine has some non-localhost,
 	// non-link-local IPv4 address on a non-Tailscale interface that's up.
@@ -289,7 +320,7 @@ func (s *State) String() string {
 	if s.PAC != "" {
 		fmt.Fprintf(&sb, " pac=%s", s.PAC)
 	}
-	fmt.Fprintf(&sb, " v4=%v v6global=%v}", s.HaveV4, s.HaveV6Global)
+	fmt.Fprintf(&sb, " v4=%v v6=%v}", s.HaveV4, s.HaveV6)
 	return sb.String()
 }
 
@@ -302,7 +333,7 @@ func (s *State) EqualFiltered(s2 *State, filter func(i Interface, ips []netaddr.
 	if s == nil || s2 == nil {
 		return false
 	}
-	if s.HaveV6Global != s2.HaveV6Global ||
+	if s.HaveV6 != s2.HaveV6 ||
 		s.HaveV4 != s2.HaveV4 ||
 		s.IsExpensive != s2.IsExpensive ||
 		s.DefaultRouteInterface != s2.DefaultRouteInterface ||
@@ -362,7 +393,7 @@ func (s *State) HasPAC() bool { return s != nil && s.PAC != "" }
 
 // AnyInterfaceUp reports whether any interface seems like it has Internet access.
 func (s *State) AnyInterfaceUp() bool {
-	return s != nil && (s.HaveV4 || s.HaveV6Global)
+	return s != nil && (s.HaveV4 || s.HaveV6)
 }
 
 func hasTailscaleIP(pfxs []netaddr.IPPrefix) bool {
@@ -407,11 +438,11 @@ func GetState() (*State, error) {
 			return
 		}
 		for _, pfx := range pfxs {
-			if pfx.IP().IsLoopback() || pfx.IP().IsLinkLocalUnicast() {
+			if pfx.IP().IsLoopback() {
 				continue
 			}
-			s.HaveV6Global = s.HaveV6Global || isGlobalV6(pfx.IP())
-			s.HaveV4 = s.HaveV4 || pfx.IP().Is4()
+			s.HaveV6 = s.HaveV6 || isUsableV6(pfx.IP())
+			s.HaveV4 = s.HaveV4 || isUsableV4(pfx.IP())
 		}
 	}); err != nil {
 		return nil, err
@@ -503,7 +534,25 @@ func isPrivateIP(ip netaddr.IP) bool {
 	return private1.Contains(ip) || private2.Contains(ip) || private3.Contains(ip)
 }
 
-func isGlobalV6(ip netaddr.IP) bool {
+// isUsableV4 reports whether ip is a usable IPv4 address which could
+// conceivably be used to get Internet connectivity. Globally routable and
+// private IPv4 addresses are always Usable, and link local 169.254.x.x
+// addresses are in some environments.
+func isUsableV4(ip netaddr.IP) bool {
+	if !ip.Is4() || ip.IsLoopback() {
+		return false
+	}
+	if ip.IsLinkLocalUnicast() {
+		return hostinfo.GetEnvType() == hostinfo.AWSLambda
+	}
+	return true
+}
+
+// isUsableV6 reports whether ip is a usable IPv6 address which could
+// conceivably be used to get Internet connectivity. Globally routable
+// IPv6 addresses are always Usable, and Unique Local Addresses
+// (fc00::/7) are in some environments used with address translation.
+func isUsableV6(ip netaddr.IP) bool {
 	return v6Global1.Contains(ip) ||
 		(tsaddr.IsULA(ip) && !tsaddr.TailscaleULARange().Contains(ip))
 }

net/interfaces/interfaces_default_route_test.go

@@ -2,15 +2,11 @@
 // Use of this source code is governed by a BSD-style
 // license that can be found in the LICENSE file.
 
-// +build linux,!redo
+// +build linux darwin,!ts_macext
 
 package interfaces
 
 import (
-	"fmt"
-	"io/ioutil"
-	"os"
-	"path/filepath"
 	"testing"
 )
 
@@ -23,64 +19,3 @@ func TestDefaultRouteInterface(t *testing.T) {
 	}
 	t.Logf("got %q", v)
 }
-
-// test the specific /proc/net/route path as found on Google Cloud Run instances
-func TestGoogleCloudRunDefaultRouteInterface(t *testing.T) {
-	dir := t.TempDir()
-	savedProcNetRoutePath := procNetRoutePath
-	defer func() { procNetRoutePath = savedProcNetRoutePath }()
-	procNetRoutePath = filepath.Join(dir, "CloudRun")
-	buf := []byte("Iface\tDestination\tGateway\tFlags\tRefCnt\tUse\tMetric\tMask\tMTU\tWindow\tIRTT\n" +
-		"eth0\t8008FEA9\t00000000\t0001\t0\t0\t0\t01FFFFFF\t0\t0\t0\n" +
-		"eth1\t00000000\t00000000\t0001\t0\t0\t0\t00000000\t0\t0\t0\n")
-	err := ioutil.WriteFile(procNetRoutePath, buf, 0644)
-	if err != nil {
-		t.Fatal(err)
-	}
-	got, err := DefaultRouteInterface()
-	if err != nil {
-		t.Fatal(err)
-	}
-
-	if got != "eth1" {
-		t.Fatalf("got %s, want eth1", got)
-	}
-}
-
-// we read chunks of /proc/net/route at a time, test that files longer than the chunk
-// size can be handled.
-func TestExtremelyLongProcNetRoute(t *testing.T) {
-	dir := t.TempDir()
-	savedProcNetRoutePath := procNetRoutePath
-	defer func() { procNetRoutePath = savedProcNetRoutePath }()
-	procNetRoutePath = filepath.Join(dir, "VeryLong")
-	f, err := os.Create(procNetRoutePath)
-	if err != nil {
-		t.Fatal(err)
-	}
-	_, err = f.Write([]byte("Iface\tDestination\tGateway\tFlags\tRefCnt\tUse\tMetric\tMask\tMTU\tWindow\tIRTT\n"))
-	if err != nil {
-		t.Fatal(err)
-	}
-
-	for n := 0; n <= 1000; n++ {
-		line := fmt.Sprintf("eth%d\t8008FEA9\t00000000\t0001\t0\t0\t0\t01FFFFFF\t0\t0\t0\n", n)
-		_, err := f.Write([]byte(line))
-		if err != nil {
-			t.Fatal(err)
-		}
-	}
-	_, err = f.Write([]byte("tokenring1\t00000000\t00000000\t0001\t0\t0\t0\t00000000\t0\t0\t0\n"))
-	if err != nil {
-		t.Fatal(err)
-	}
-
-	got, err := DefaultRouteInterface()
-	if err != nil {
-		t.Fatal(err)
-	}
-
-	if got != "tokenring1" {
-		t.Fatalf("got %q, want tokenring1", got)
-	}
-}

net/interfaces/interfaces_linux_test.go

@@ -4,7 +4,74 @@
 
 package interfaces
 
-import "testing"
+import (
+	"fmt"
+	"io/ioutil"
+	"os"
+	"path/filepath"
+	"testing"
+)
+
+// test the specific /proc/net/route path as found on Google Cloud Run instances
+func TestGoogleCloudRunDefaultRouteInterface(t *testing.T) {
+	dir := t.TempDir()
+	savedProcNetRoutePath := procNetRoutePath
+	defer func() { procNetRoutePath = savedProcNetRoutePath }()
+	procNetRoutePath = filepath.Join(dir, "CloudRun")
+	buf := []byte("Iface\tDestination\tGateway\tFlags\tRefCnt\tUse\tMetric\tMask\tMTU\tWindow\tIRTT\n" +
+		"eth0\t8008FEA9\t00000000\t0001\t0\t0\t0\t01FFFFFF\t0\t0\t0\n" +
+		"eth1\t00000000\t00000000\t0001\t0\t0\t0\t00000000\t0\t0\t0\n")
+	err := ioutil.WriteFile(procNetRoutePath, buf, 0644)
+	if err != nil {
+		t.Fatal(err)
+	}
+	got, err := DefaultRouteInterface()
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	if got != "eth1" {
+		t.Fatalf("got %s, want eth1", got)
+	}
+}
+
+// we read chunks of /proc/net/route at a time, test that files longer than the chunk
+// size can be handled.
+func TestExtremelyLongProcNetRoute(t *testing.T) {
+	dir := t.TempDir()
+	savedProcNetRoutePath := procNetRoutePath
+	defer func() { procNetRoutePath = savedProcNetRoutePath }()
+	procNetRoutePath = filepath.Join(dir, "VeryLong")
+	f, err := os.Create(procNetRoutePath)
+	if err != nil {
+		t.Fatal(err)
+	}
+	_, err = f.Write([]byte("Iface\tDestination\tGateway\tFlags\tRefCnt\tUse\tMetric\tMask\tMTU\tWindow\tIRTT\n"))
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	for n := 0; n <= 1000; n++ {
+		line := fmt.Sprintf("eth%d\t8008FEA9\t00000000\t0001\t0\t0\t0\t01FFFFFF\t0\t0\t0\n", n)
+		_, err := f.Write([]byte(line))
+		if err != nil {
+			t.Fatal(err)
+		}
+	}
+	_, err = f.Write([]byte("tokenring1\t00000000\t00000000\t0001\t0\t0\t0\t00000000\t0\t0\t0\n"))
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	got, err := DefaultRouteInterface()
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	if got != "tokenring1" {
+		t.Fatalf("got %q, want tokenring1", got)
+	}
+}
 
 func BenchmarkDefaultRouteInterface(b *testing.B) {
 	b.ReportAllocs()

net/interfaces/interfaces_test.go

@@ -46,7 +46,7 @@ func TestLikelyHomeRouterIP(t *testing.T) {
 	t.Logf("myIP = %v; gw = %v", my, gw)
 }
 
-func TestIsGlobalV6(t *testing.T) {
+func TestIsUsableV6(t *testing.T) {
 	tests := []struct {
 		name string
 		ip   string
@@ -61,8 +61,8 @@ func TestIsGlobalV6(t *testing.T) {
 	}
 
 	for _, test := range tests {
-		if got := isGlobalV6(netaddr.MustParseIP(test.ip)); got != test.want {
-			t.Errorf("isGlobalV6(%s) = %v, want %v", test.name, got, test.want)
+		if got := isUsableV6(netaddr.MustParseIP(test.ip)); got != test.want {
+			t.Errorf("isUsableV6(%s) = %v, want %v", test.name, got, test.want)
 		}
 	}
 }

net/netcheck/netcheck.go

@@ -336,7 +336,7 @@ func makeProbePlan(dm *tailcfg.DERPMap, ifState *interfaces.State, last *Report)
 	if last == nil || len(last.RegionLatency) == 0 {
 		return makeProbePlanInitial(dm, ifState)
 	}
-	have6if := ifState.HaveV6Global
+	have6if := ifState.HaveV6
 	have4if := ifState.HaveV4
 	plan = make(probePlan)
 	if !have4if && !have6if {
@@ -425,7 +425,7 @@ func makeProbePlanInitial(dm *tailcfg.DERPMap, ifState *interfaces.State) (plan
 			if ifState.HaveV4 && nodeMight4(n) {
 				p4 = append(p4, probe{delay: delay, node: n.Name, proto: probeIPv4})
 			}
-			if ifState.HaveV6Global && nodeMight6(n) {
+			if ifState.HaveV6 && nodeMight6(n) {
 				p6 = append(p6, probe{delay: delay, node: n.Name, proto: probeIPv6})
 			}
 		}
@@ -808,7 +808,7 @@ func (c *Client) GetReport(ctx context.Context, dm *tailcfg.DERPMap) (*Report, e
 		go c.readPackets(ctx, u4)
 	}
 
-	if ifState.HaveV6Global {
+	if ifState.HaveV6 {
 		if f := c.GetSTUNConn6; f != nil {
 			rs.pc6 = f()
 		} else {

net/netcheck/netcheck_test.go

@@ -443,8 +443,8 @@ func TestMakeProbePlan(t *testing.T) {
 	for _, tt := range tests {
 		t.Run(tt.name, func(t *testing.T) {
 			ifState := &interfaces.State{
-				HaveV6Global: tt.have6if,
-				HaveV4:       !tt.no4,
+				HaveV6: tt.have6if,
+				HaveV4: !tt.no4,
 			}
 			got := makeProbePlan(tt.dm, ifState, tt.last)
 			if !reflect.DeepEqual(got, tt.want) {

net/netns/netns_android.go

@@ -0,0 +1,64 @@
+// Copyright (c) 2021 Tailscale Inc & AUTHORS All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+// +build android
+
+package netns
+
+import (
+	"fmt"
+	"sync"
+	"syscall"
+)
+
+var (
+	androidProtectFuncMu sync.Mutex
+	androidProtectFunc   func(fd int) error
+)
+
+// SetAndroidProtectFunc register a func that Android provides that JNI calls into
+// https://developer.android.com/reference/android/net/VpnService#protect(int)
+// which is documented as:
+//
+// "Protect a socket from VPN connections. After protecting, data sent
+// through this socket will go directly to the underlying network, so
+// its traffic will not be forwarded through the VPN. This method is
+// useful if some connections need to be kept outside of VPN. For
+// example, a VPN tunnel should protect itself if its destination is
+// covered by VPN routes. Otherwise its outgoing packets will be sent
+// back to the VPN interface and cause an infinite loop. This method
+// will fail if the application is not prepared or is revoked."
+//
+// A nil func disables the use the hook.
+//
+// This indirection is necessary because this is the supported, stable
+// interface to use on Android, and doing the sockopts to set the
+// fwmark return errors on Android. The actual implementation of
+// VpnService.protect ends up doing an IPC to another process on
+// Android, asking for the fwmark to be set.
+func SetAndroidProtectFunc(f func(fd int) error) {
+	androidProtectFuncMu.Lock()
+	defer androidProtectFuncMu.Unlock()
+	androidProtectFunc = f
+}
+
+// control marks c as necessary to dial in a separate network namespace.
+//
+// It's intentionally the same signature as net.Dialer.Control
+// and net.ListenConfig.Control.
+func control(network, address string, c syscall.RawConn) error {
+	var sockErr error
+	err := c.Control(func(fd uintptr) {
+		androidProtectFuncMu.Lock()
+		f := androidProtectFunc
+		androidProtectFuncMu.Unlock()
+		if f != nil {
+			sockErr = f(int(fd))
+		}
+	})
+	if err != nil {
+		return fmt.Errorf("RawConn.Control on %T: %w", c, err)
+	}
+	return sockErr
+}

net/netns/netns_darwin_tailscaled.go

@@ -2,7 +2,7 @@
 // Use of this source code is governed by a BSD-style
 // license that can be found in the LICENSE file.
 
-// +build darwin,!redo
+// +build darwin,!ts_macext
 
 package netns
 

net/netns/netns_default.go

@@ -2,7 +2,7 @@
 // Use of this source code is governed by a BSD-style
 // license that can be found in the LICENSE file.
 
-// +build !linux,!windows,!darwin darwin,redo
+// +build !linux,!windows,!darwin darwin,ts_macext
 
 package netns
 

net/netns/netns_linux.go

@@ -2,6 +2,8 @@
 // Use of this source code is governed by a BSD-style
 // license that can be found in the LICENSE file.
 
+// +build linux,!android
+
 package netns
 
 import (

net/netns/netns_macios.go

@@ -0,0 +1,53 @@
+// Copyright (c) 2021 Tailscale Inc & AUTHORS All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+// +build darwin ios
+
+package netns
+
+import (
+	"errors"
+	"log"
+	"net"
+	"strings"
+	"syscall"
+
+	"golang.org/x/sys/unix"
+)
+
+// SetListenConfigInterfaceIndex sets lc.Control such that sockets are bound
+// to the provided interface index.
+func SetListenConfigInterfaceIndex(lc *net.ListenConfig, ifIndex int) error {
+	if lc == nil {
+		return errors.New("nil ListenConfig")
+	}
+	if lc.Control != nil {
+		return errors.New("ListenConfig.Control already set")
+	}
+	lc.Control = func(network, address string, c syscall.RawConn) error {
+		var sockErr error
+		err := c.Control(func(fd uintptr) {
+			sockErr = bindInterface(fd, network, address, ifIndex)
+			if sockErr != nil {
+				log.Printf("netns: bind(%q, %q) on index %v: %v", network, address, ifIndex, sockErr)
+			}
+		})
+		if err != nil {
+			return err
+		}
+		return sockErr
+	}
+	return nil
+}
+
+func bindInterface(fd uintptr, network, address string, ifIndex int) error {
+	v6 := strings.Contains(address, "]:") || strings.HasSuffix(network, "6") // hacky test for v6
+	proto := unix.IPPROTO_IP
+	opt := unix.IP_BOUND_IF
+	if v6 {
+		proto = unix.IPPROTO_IPV6
+		opt = unix.IPV6_BOUND_IF
+	}
+	return unix.SetsockoptInt(int(fd), proto, opt, ifIndex)
+}

net/portmapper/portmapper.go

@@ -577,8 +577,6 @@ func pcpAnnounceRequest(myIP netaddr.IP) []byte {
 	return pkt
 }
 
-//lint:ignore U1000 moved this code from netcheck's old PCP probing; will be needed when we add PCP mapping
-
 // pcpMapRequest generates a PCP packet with a MAP opcode.
 func pcpMapRequest(myIP netaddr.IP, mapToLocalPort int, delete bool) []byte {
 	const udpProtoNumber = 17

net/socks5/socks5.go

@@ -32,7 +32,7 @@ const (
 // that represent the kind of connection the client needs.
 type commandType byte
 
-// The set of valid SOCKS5 commans as described in RFC 1928.
+// The set of valid SOCKS5 commands as described in RFC 1928.
 const (
 	connect      commandType = 1
 	bind         commandType = 2

net/tsaddr/tsaddr.go

@@ -41,12 +41,9 @@ var (
 // TailscaleServiceIP returns the listen address of services
 // provided by Tailscale itself such as the MagicDNS proxy.
 func TailscaleServiceIP() netaddr.IP {
-	serviceIP.Do(func() { mustIP(&serviceIP.v, "100.100.100.100") })
-	return serviceIP.v
+	return netaddr.IPv4(100, 100, 100, 100) // "100.100.100.100" for those grepping
 }
 
-var serviceIP onceIP
-
 // IsTailscaleIP reports whether ip is an IP address in a range that
 // Tailscale assigns from.
 func IsTailscaleIP(ip netaddr.IP) bool {
@@ -126,19 +123,6 @@ type oncePrefix struct {
 	v netaddr.IPPrefix
 }
 
-func mustIP(v *netaddr.IP, ip string) {
-	var err error
-	*v, err = netaddr.ParseIP(ip)
-	if err != nil {
-		panic(err)
-	}
-}
-
-type onceIP struct {
-	sync.Once
-	v netaddr.IP
-}
-
 // NewContainsIPFunc returns a func that reports whether ip is in addrs.
 //
 // It's optimized for the cases of addrs being empty and addrs

net/tsaddr/tsaddr_test.go

@@ -93,3 +93,11 @@ func TestNewContainsIPFunc(t *testing.T) {
 		t.Fatal("bad")
 	}
 }
+
+var sinkIP netaddr.IP
+
+func BenchmarkTailscaleServiceAddr(b *testing.B) {
+	for i := 0; i < b.N; i++ {
+		sinkIP = TailscaleServiceIP()
+	}
+}

net/tstun/tun.go

@@ -11,6 +11,7 @@ import (
 	"os"
 	"os/exec"
 	"runtime"
+	"strconv"
 	"time"
 
 	"golang.zx2c4.com/wireguard/tun"
@@ -18,20 +19,26 @@ import (
 	"tailscale.com/version/distro"
 )
 
-// minimalMTU is the MTU we set on tailscale's TUN
-// interface. wireguard-go defaults to 1420 bytes, which only works if
-// the "outer" MTU is 1500 bytes. This breaks on DSL connections
-// (typically 1492 MTU) and on GCE (1460 MTU?!).
+// tunMTU is the MTU we set on tailscale's TUN interface. wireguard-go
+// defaults to 1420 bytes, which only works if the "outer" MTU is 1500
+// bytes. This breaks on DSL connections (typically 1492 MTU) and on
+// GCE (1460 MTU?!).
 //
 // 1280 is the smallest MTU allowed for IPv6, which is a sensible
 // "probably works everywhere" setting until we develop proper PMTU
 // discovery.
-const minimalMTU = 1280
+var tunMTU = 1280
+
+func init() {
+	if mtu, _ := strconv.Atoi(os.Getenv("TS_DEBUG_MTU")); mtu != 0 {
+		tunMTU = mtu
+	}
+}
 
 // New returns a tun.Device for the requested device name, along with
 // the OS-dependent name that was allocated to the device.
 func New(logf logger.Logf, tunName string) (tun.Device, string, error) {
-	dev, err := tun.CreateTUN(tunName, minimalMTU)
+	dev, err := tun.CreateTUN(tunName, tunMTU)
 	if err != nil {
 		return nil, "", err
 	}

net/tstun/wrap_test.go

@@ -146,7 +146,8 @@ func setfilter(logf logger.Logf, tun *Wrapper) {
 	}
 	var sb netaddr.IPSetBuilder
 	sb.AddPrefix(netaddr.MustParseIPPrefix("1.2.0.0/16"))
-	tun.SetFilter(filter.New(matches, sb.IPSet(), sb.IPSet(), nil, logf))
+	ipSet, _ := sb.IPSet()
+	tun.SetFilter(filter.New(matches, ipSet, ipSet, nil, logf))
 }
 
 func newChannelTUN(logf logger.Logf, secure bool) (*tuntest.ChannelTUN, *Wrapper) {

packages/deb/deb.go

@@ -0,0 +1,184 @@
+// Copyright (c) 2021 Tailscale Inc & AUTHORS All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+// Package deb extracts metadata from Debian packages.
+package deb
+
+import (
+	"archive/tar"
+	"bufio"
+	"bytes"
+	"compress/gzip"
+	"crypto/md5"
+	"crypto/sha1"
+	"crypto/sha256"
+	"errors"
+	"fmt"
+	"io"
+	"io/ioutil"
+	"os"
+	"path/filepath"
+	"strconv"
+	"strings"
+)
+
+// Info is the Debian package metadata needed to integrate the package
+// into a repository.
+type Info struct {
+	// Version is the version of the package, as reported by dpkg.
+	Version string
+	// Arch is the Debian CPU architecture the package is for.
+	Arch string
+	// Control is the entire contents of the package's control file,
+	// with leading and trailing whitespace removed.
+	Control []byte
+	// MD5 is the MD5 hash of the package file.
+	MD5 []byte
+	// SHA1 is the SHA1 hash of the package file.
+	SHA1 []byte
+	// SHA256 is the SHA256 hash of the package file.
+	SHA256 []byte
+}
+
+// ReadFile returns Debian package metadata from the .deb file at path.
+func ReadFile(path string) (*Info, error) {
+	f, err := os.Open(path)
+	if err != nil {
+		return nil, err
+	}
+	return Read(f)
+}
+
+// Read returns Debian package metadata from the .deb file in r.
+func Read(r io.Reader) (*Info, error) {
+	b := bufio.NewReader(r)
+
+	m5, s1, s256 := md5.New(), sha1.New(), sha256.New()
+	summers := io.MultiWriter(m5, s1, s256)
+	r = io.TeeReader(b, summers)
+
+	t, err := findControlTar(r)
+	if err != nil {
+		return nil, fmt.Errorf("searching for control.tar.gz: %w", err)
+	}
+
+	control, err := findControlFile(t)
+	if err != nil {
+		return nil, fmt.Errorf("searching for control file in control.tar.gz: %w", err)
+	}
+
+	arch, version, err := findArchAndVersion(control)
+	if err != nil {
+		return nil, fmt.Errorf("extracting version and architecture from control file: %w", err)
+	}
+
+	// Exhaust the remainder of r, so that the summers see the entire file.
+	if _, err := io.Copy(ioutil.Discard, r); err != nil {
+		return nil, fmt.Errorf("hashing file: %w", err)
+	}
+
+	return &Info{
+		Version: version,
+		Arch:    arch,
+		Control: control,
+		MD5:     m5.Sum(nil),
+		SHA1:    s1.Sum(nil),
+		SHA256:  s256.Sum(nil),
+	}, nil
+}
+
+// findControlTar reads r as an `ar` archive, finds a tarball named
+// `control.tar.gz` within, and returns a reader for that file.
+func findControlTar(r io.Reader) (tarReader io.Reader, err error) {
+	var magic [8]byte
+	if _, err := io.ReadFull(r, magic[:]); err != nil {
+		return nil, fmt.Errorf("reading ar magic: %w", err)
+	}
+	if string(magic[:]) != "!<arch>\n" {
+		return nil, fmt.Errorf("not an ar file (bad magic %q)", magic)
+	}
+
+	for {
+		var hdr [60]byte
+		if _, err := io.ReadFull(r, hdr[:]); err != nil {
+			return nil, fmt.Errorf("reading file header: %w", err)
+		}
+		filename := strings.TrimSpace(string(hdr[:16]))
+		size, err := strconv.ParseInt(strings.TrimSpace(string(hdr[48:58])), 10, 64)
+		if err != nil {
+			return nil, fmt.Errorf("reading size of file %q: %w", filename, err)
+		}
+		if filename == "control.tar.gz" {
+			return io.LimitReader(r, size), nil
+		}
+
+		// files in ar are padded out to 2 bytes.
+		if size%2 == 1 {
+			size++
+		}
+		if _, err := io.CopyN(ioutil.Discard, r, size); err != nil {
+			return nil, fmt.Errorf("seeking past file %q: %w", filename, err)
+		}
+	}
+}
+
+// findControlFile reads r as a tar.gz archive, finds a file named
+// `control` within, and returns its contents.
+func findControlFile(r io.Reader) (control []byte, err error) {
+	gz, err := gzip.NewReader(r)
+	if err != nil {
+		return nil, fmt.Errorf("decompressing control.tar.gz: %w", err)
+	}
+	defer gz.Close()
+
+	tr := tar.NewReader(gz)
+	for {
+		hdr, err := tr.Next()
+		if err != nil {
+			if errors.Is(err, io.EOF) {
+				return nil, errors.New("EOF while looking for control file in control.tar.gz")
+			}
+			return nil, fmt.Errorf("reading tar header: %w", err)
+		}
+
+		if filepath.Clean(hdr.Name) != "control" {
+			continue
+		}
+
+		// Found control file
+		break
+	}
+
+	bs, err := ioutil.ReadAll(tr)
+	if err != nil {
+		return nil, fmt.Errorf("reading control file: %w", err)
+	}
+
+	return bytes.TrimSpace(bs), nil
+}
+
+var (
+	archKey    = []byte("Architecture:")
+	versionKey = []byte("Version:")
+)
+
+// findArchAndVersion extracts the architecture and version strings
+// from the given control file.
+func findArchAndVersion(control []byte) (arch string, version string, err error) {
+	b := bytes.NewBuffer(control)
+	for {
+		l, err := b.ReadBytes('\n')
+		if err != nil {
+			return "", "", err
+		}
+		if bytes.HasPrefix(l, archKey) {
+			arch = string(bytes.TrimSpace(l[len(archKey):]))
+		} else if bytes.HasPrefix(l, versionKey) {
+			version = string(bytes.TrimSpace(l[len(versionKey):]))
+		}
+		if arch != "" && version != "" {
+			return arch, version, nil
+		}
+	}
+}

packages/deb/deb_test.go

@@ -0,0 +1,202 @@
+// Copyright (c) 2021 Tailscale Inc & AUTHORS All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package deb
+
+import (
+	"bytes"
+	"crypto/md5"
+	"crypto/sha1"
+	"crypto/sha256"
+	"encoding/hex"
+	"fmt"
+	"hash"
+	"strings"
+	"testing"
+
+	"github.com/google/go-cmp/cmp"
+	"github.com/goreleaser/nfpm"
+	_ "github.com/goreleaser/nfpm/deb"
+)
+
+func TestDebInfo(t *testing.T) {
+	tests := []struct {
+		name    string
+		in      []byte
+		want    *Info
+		wantErr bool
+	}{
+		{
+			name: "simple",
+			in:   mkTestDeb("1.2.3", "amd64"),
+			want: &Info{
+				Version: "1.2.3",
+				Arch:    "amd64",
+				Control: mkControl(
+					"Package", "tailscale",
+					"Version", "1.2.3",
+					"Section", "net",
+					"Priority", "extra",
+					"Architecture", "amd64",
+					"Installed-Size", "0",
+					"Description", "test package"),
+			},
+		},
+		{
+			name: "arm64",
+			in:   mkTestDeb("1.2.3", "arm64"),
+			want: &Info{
+				Version: "1.2.3",
+				Arch:    "arm64",
+				Control: mkControl(
+					"Package", "tailscale",
+					"Version", "1.2.3",
+					"Section", "net",
+					"Priority", "extra",
+					"Architecture", "arm64",
+					"Installed-Size", "0",
+					"Description", "test package"),
+			},
+		},
+		{
+			name: "unstable",
+			in:   mkTestDeb("1.7.25", "amd64"),
+			want: &Info{
+				Version: "1.7.25",
+				Arch:    "amd64",
+				Control: mkControl(
+					"Package", "tailscale",
+					"Version", "1.7.25",
+					"Section", "net",
+					"Priority", "extra",
+					"Architecture", "amd64",
+					"Installed-Size", "0",
+					"Description", "test package"),
+			},
+		},
+
+		// These truncation tests assume the structure of a .deb
+		// package, which is as follows:
+		//  magic: 8 bytes
+		//  file header: 60 bytes, before each file blob
+		//
+		// The first file in a .deb ar is "debian-binary", which is 4
+		// bytes long and consists of "2.0\n".
+		// The second file is control.tar.gz, which is what we care
+		// about introspecting for metadata.
+		// The final file is data.tar.gz, which we don't care about.
+		//
+		// The first file in control.tar.gz is the "control" file we
+		// want to read for metadata.
+		{
+			name:    "truncated_ar_magic",
+			in:      mkTestDeb("1.7.25", "amd64")[:4],
+			wantErr: true,
+		},
+		{
+			name:    "truncated_ar_header",
+			in:      mkTestDeb("1.7.25", "amd64")[:30],
+			wantErr: true,
+		},
+		{
+			name: "missing_control_tgz",
+			// Truncate right after the "debian-binary" file, which
+			// makes the file a valid 1-file archive that's missing
+			// control.tar.gz.
+			in:      mkTestDeb("1.7.25", "amd64")[:72],
+			wantErr: true,
+		},
+		{
+			name:    "truncated_tgz",
+			in:      mkTestDeb("1.7.25", "amd64")[:172],
+			wantErr: true,
+		},
+	}
+
+	for _, test := range tests {
+		// mkTestDeb returns non-deterministic output due to
+		// timestamps embedded in the package file, so compute the
+		// wanted hashes on the fly here.
+		if test.want != nil {
+			test.want.MD5 = mkHash(test.in, md5.New)
+			test.want.SHA1 = mkHash(test.in, sha1.New)
+			test.want.SHA256 = mkHash(test.in, sha256.New)
+		}
+
+		t.Run(test.name, func(t *testing.T) {
+			b := bytes.NewBuffer(test.in)
+			got, err := Read(b)
+			if err != nil {
+				if test.wantErr {
+					t.Logf("got expected error: %v", err)
+					return
+				}
+				t.Fatalf("reading deb info: %v", err)
+			}
+			if diff := diff(got, test.want); diff != "" {
+				t.Fatalf("parsed info diff (-got+want):\n%s", diff)
+			}
+		})
+	}
+}
+
+func diff(got, want interface{}) string {
+	matchField := func(name string) func(p cmp.Path) bool {
+		return func(p cmp.Path) bool {
+			if len(p) != 3 {
+				return false
+			}
+			return p[2].String() == "."+name
+		}
+	}
+	toLines := cmp.Transformer("lines", func(b []byte) []string { return strings.Split(string(b), "\n") })
+	toHex := cmp.Transformer("hex", func(b []byte) string { return hex.EncodeToString(b) })
+	return cmp.Diff(got, want,
+		cmp.FilterPath(matchField("Control"), toLines),
+		cmp.FilterPath(matchField("MD5"), toHex),
+		cmp.FilterPath(matchField("SHA1"), toHex),
+		cmp.FilterPath(matchField("SHA256"), toHex))
+}
+
+func mkTestDeb(version, arch string) []byte {
+	info := nfpm.WithDefaults(&nfpm.Info{
+		Name:        "tailscale",
+		Description: "test package",
+		Arch:        arch,
+		Platform:    "linux",
+		Version:     version,
+		Section:     "net",
+		Priority:    "extra",
+	})
+
+	pkg, err := nfpm.Get("deb")
+	if err != nil {
+		panic(fmt.Sprintf("getting deb packager: %v", err))
+	}
+
+	var b bytes.Buffer
+	if err := pkg.Package(info, &b); err != nil {
+		panic(fmt.Sprintf("creating deb package: %v", err))
+	}
+
+	return b.Bytes()
+}
+
+func mkControl(fs ...string) []byte {
+	if len(fs)%2 != 0 {
+		panic("odd number of control file fields")
+	}
+	var b bytes.Buffer
+	for i := 0; i < len(fs); i = i + 2 {
+		k, v := fs[i], fs[i+1]
+		fmt.Fprintf(&b, "%s: %s\n", k, v)
+	}
+	return bytes.TrimSpace(b.Bytes())
+}
+
+func mkHash(b []byte, hasher func() hash.Hash) []byte {
+	h := hasher()
+	h.Write(b)
+	return h.Sum(nil)
+}

paths/paths.go

@@ -11,6 +11,8 @@ import (
 	"path/filepath"
 	"runtime"
 	"sync/atomic"
+
+	"tailscale.com/version/distro"
 )
 
 // AppSharedDir is a string set by the iOS or Android app on start
@@ -26,11 +28,15 @@ func DefaultTailscaledSocket() string {
 	if runtime.GOOS == "darwin" {
 		return "/var/run/tailscaled.socket"
 	}
-	if runtime.GOOS == "linux" {
-		// TODO(crawshaw): does this path change with DSM7?
-		const synologySock = "/volume1/@appstore/Tailscale/var/tailscaled.sock" // SYNOPKG_PKGDEST in scripts/installer
-		if fi, err := os.Stat(filepath.Dir(synologySock)); err == nil && fi.IsDir() {
-			return synologySock
+	if distro.Get() == distro.Synology {
+		// TODO(maisem): be smarter about this. We can parse /etc/VERSION.
+		const dsm6Sock = "/var/packages/Tailscale/etc/tailscaled.sock"
+		const dsm7Sock = "/var/packages/Tailscale/var/tailscaled.sock"
+		if fi, err := os.Stat(dsm6Sock); err == nil && !fi.IsDir() {
+			return dsm6Sock
+		}
+		if fi, err := os.Stat(dsm7Sock); err == nil && !fi.IsDir() {
+			return dsm7Sock
 		}
 	}
 	if fi, err := os.Stat("/var/run"); err == nil && fi.IsDir() {

portlist/portlist_windows.go

@@ -23,7 +23,7 @@ func listPorts() (List, error) {
 }
 
 func addProcesses(pl []Port) ([]Port, error) {
-	//lint:ignore SA1019 OpenCurrentProcessToken instead of GetCurrentProcessToken,
+	// OpenCurrentProcessToken instead of GetCurrentProcessToken,
 	// as GetCurrentProcessToken only works on Windows 8+.
 	tok, err := windows.OpenCurrentProcessToken()
 	if err != nil {

staticcheck.conf

@@ -0,0 +1,17 @@
+# Full list: https://staticcheck.io/docs/checks
+checks = [
+	"SA*", "-SA1019", "-SA2001", "-SA9003", # SA* are mostly legit code errors
+
+	# S1?? are "code simplifications" which we consider unnecessary
+
+	# ST1??? are stylistic issues, some of which are generally accepted
+	# In general, if it's listed in
+	# https://github.com/golang/go/wiki/CodeReviewComments, then it
+	# may be an acceptable check.
+
+	# TODO(crawshaw): enable when we have docs? "ST1000", # missing package docs
+	"ST1001", # discourage dot imports
+
+	"QF1004", # Use `strings.ReplaceAll` instead of `strings.Replace` with `n == 1`
+	"QF1006", # Lift if+break into loop condition
+]

syncs/locked_test.go

@@ -4,8 +4,6 @@
 
 // +build go1.13,!go1.16
 
-//lint:file-ignore SA2001 the empty critical sections are part of triggering different internal mutex states
-
 package syncs
 
 import (

syncs/syncs.go

@@ -83,6 +83,17 @@ func (b *AtomicBool) Get() bool {
 	return atomic.LoadInt32((*int32)(b)) != 0
 }
 
+// AtomicUint32 is an atomic uint32.
+type AtomicUint32 uint32
+
+func (b *AtomicUint32) Set(v uint32) {
+	atomic.StoreUint32((*uint32)(b), v)
+}
+
+func (b *AtomicUint32) Get() uint32 {
+	return atomic.LoadUint32((*uint32)(b))
+}
+
 // Semaphore is a counting semaphore.
 //
 // Use NewSemaphore to create one.

syncs/watchdog.go

@@ -52,7 +52,7 @@ func Watch(ctx context.Context, mu sync.Locker, tick, max time.Duration) chan ti
 		go func() {
 			start := time.Now()
 			mu.Lock()
-			mu.Unlock() //lint:ignore SA2001 ignore the empty critical section
+			mu.Unlock()
 			elapsed := time.Since(start)
 			if elapsed > max {
 				elapsed = max

tailcfg/derpmap.go

@@ -14,6 +14,10 @@ type DERPMap struct {
 	//
 	// The numbers are not necessarily contiguous.
 	Regions map[int]*DERPRegion
+
+	// OmitDefaultRegions specifies to not use Tailscale's DERP servers, and only use those
+	// specified in this DERPMap. If there are none set outside of the defaults, this is a noop.
+	OmitDefaultRegions bool `json:"omitDefaultRegions,omitempty"`
 }
 
 /// RegionIDs returns the sorted region IDs.

tailcfg/tailcfg.go

@@ -4,7 +4,7 @@
 
 package tailcfg
 
-//go:generate go run tailscale.com/cmd/cloner --type=User,Node,Hostinfo,NetInfo,Login,DNSConfig,DNSResolver,RegisterResponse --clonefunc=true --output=tailcfg_clone.go
+//go:generate go run tailscale.com/cmd/cloner --type=User,Node,Hostinfo,NetInfo,Login,DNSConfig,DNSResolver,RegisterResponse,DERPRegion,DERPMap,DERPNode --clonefunc=true --output=tailcfg_clone.go
 
 import (
 	"encoding/hex"
@@ -42,7 +42,10 @@ import (
 //    17: 2021-04-18: MapResponse.Domain empty means unchanged
 //    18: 2021-04-19: MapResponse.Node nil means unchanged (all fields now omitempty)
 //    19: 2021-04-21: MapResponse.Debug.SleepSeconds
-const CurrentMapRequestVersion = 19
+//    20: 2021-06-11: MapResponse.LastSeen used even less (https://github.com/tailscale/tailscale/issues/2107)
+//    21: 2021-06-15: added MapResponse.DNSConfig.CertDomains
+//    22: 2021-06-16: added MapResponse.DNSConfig.ExtraRecords
+const CurrentMapRequestVersion = 22
 
 type StableID string
 
@@ -872,6 +875,36 @@ type DNSConfig struct {
 
 	// PerDomain is not set by the control server, and does nothing.
 	PerDomain bool `json:",omitempty"`
+
+	// CertDomains are the set of DNS names for which the control
+	// plane server will assist with provisioning TLS
+	// certificates. See SetDNSRequest, which can be used to
+	// answer dns-01 ACME challenges for e.g. LetsEncrypt.
+	// These names are FQDNs without trailing periods, and without
+	// any "_acme-challenge." prefix.
+	CertDomains []string `json:",omitempty"`
+
+	// ExtraRecords contains extra DNS records to add to the
+	// MagicDNS config.
+	ExtraRecords []DNSRecord `json:",omitempty"`
+}
+
+// DNSRecord is an extra DNS record to add to MagicDNS.
+type DNSRecord struct {
+	// Name is the fully qualified domain name of
+	// the record to add. The trailing dot is optional.
+	Name string
+
+	// Type is the DNS record type.
+	// Empty means A or AAAA, depending on value.
+	// Other values are currently ignored.
+	Type string `json:",omitempty"`
+
+	// Value is the IP address in string form.
+	// TODO(bradfitz): if we ever add support for record types
+	// with non-UTF8 binary data, add ValueBytes []byte that
+	// would take precedence.
+	Value string
 }
 
 // PingRequest is a request to send an HTTP request to prove the
@@ -879,6 +912,9 @@ type DNSConfig struct {
 type PingRequest struct {
 	// URL is the URL to send a HEAD request to.
 	// It will be a unique URL each time. No auth headers are necessary.
+	//
+	// If the client sees multiple PingRequests with the same URL,
+	// subsequent ones should be ignored.
 	URL string
 
 	// Log is whether to log about this ping in the success case.
@@ -1019,6 +1055,11 @@ type Debug struct {
 	// The client can (and should) limit the value (such as 5
 	// minutes).
 	SleepSeconds float64 `json:",omitempty"`
+
+	// RandomizeClientPort is whether magicsock should UDP bind to
+	// :0 to get a random local port, ignoring any configured
+	// fixed port.
+	RandomizeClientPort bool `json:",omitempty"`
 }
 
 func (k MachineKey) String() string                   { return fmt.Sprintf("mkey:%x", k[:]) }
@@ -1174,3 +1215,34 @@ const (
 	CapabilityFileSharing = "https://tailscale.com/cap/file-sharing"
 	CapabilityAdmin       = "https://tailscale.com/cap/is-admin"
 )
+
+// SetDNSRequest is a request to add a DNS record.
+//
+// This is used for ACME DNS-01 challenges (so people can use
+// LetsEncrypt, etc).
+//
+// The request is encoded to JSON, encrypted with golang.org/x/crypto/nacl/box,
+// using the local machine key, and sent to:
+//	https://login.tailscale.com/machine/<mkey hex>/set-dns
+type SetDNSRequest struct {
+	// Version indicates what level of SetDNSRequest functionality
+	// the client understands. Currently this type only has
+	// one version; this field should always be 1 for now.
+	Version int
+
+	// NodeKey is the client's current node key.
+	NodeKey NodeKey
+
+	// Name is the domain name for which to create a record.
+	// For ACME DNS-01 challenges, it should be one of the domains
+	// in MapResponse.DNSConfig.CertDomains with the prefix
+	// "_acme-challenge.".
+	Name string
+
+	// Type is the DNS record type. For ACME DNS-01 challenges, it
+	// should be "TXT".
+	Type string
+
+	// Value is the value to add.
+	Value string
+}

tailcfg/tailcfg_clone.go

@@ -2,7 +2,7 @@
 // Use of this source code is governed by a BSD-style
 // license that can be found in the LICENSE file.
 
-// Code generated by tailscale.com/cmd/cloner -type User,Node,Hostinfo,NetInfo,Login,DNSConfig,DNSResolver,RegisterResponse; DO NOT EDIT.
+// Code generated by tailscale.com/cmd/cloner -type User,Node,Hostinfo,NetInfo,Login,DNSConfig,DNSResolver,RegisterResponse,DERPRegion,DERPMap,DERPNode; DO NOT EDIT.
 
 package tailcfg
 
@@ -26,7 +26,7 @@ func (src *User) Clone() *User {
 }
 
 // A compilation failure here means this code must be regenerated, with command:
-//   tailscale.com/cmd/cloner -type User,Node,Hostinfo,NetInfo,Login,DNSConfig,DNSResolver,RegisterResponse
+//   tailscale.com/cmd/cloner -type User,Node,Hostinfo,NetInfo,Login,DNSConfig,DNSResolver,RegisterResponse,DERPRegion,DERPMap,DERPNode
 var _UserNeedsRegeneration = User(struct {
 	ID            UserID
 	LoginName     string
@@ -62,7 +62,7 @@ func (src *Node) Clone() *Node {
 }
 
 // A compilation failure here means this code must be regenerated, with command:
-//   tailscale.com/cmd/cloner -type User,Node,Hostinfo,NetInfo,Login,DNSConfig,DNSResolver,RegisterResponse
+//   tailscale.com/cmd/cloner -type User,Node,Hostinfo,NetInfo,Login,DNSConfig,DNSResolver,RegisterResponse,DERPRegion,DERPMap,DERPNode
 var _NodeNeedsRegeneration = Node(struct {
 	ID                      NodeID
 	StableID                StableNodeID
@@ -105,7 +105,7 @@ func (src *Hostinfo) Clone() *Hostinfo {
 }
 
 // A compilation failure here means this code must be regenerated, with command:
-//   tailscale.com/cmd/cloner -type User,Node,Hostinfo,NetInfo,Login,DNSConfig,DNSResolver,RegisterResponse
+//   tailscale.com/cmd/cloner -type User,Node,Hostinfo,NetInfo,Login,DNSConfig,DNSResolver,RegisterResponse,DERPRegion,DERPMap,DERPNode
 var _HostinfoNeedsRegeneration = Hostinfo(struct {
 	IPNVersion    string
 	FrontendLogID string
@@ -142,7 +142,7 @@ func (src *NetInfo) Clone() *NetInfo {
 }
 
 // A compilation failure here means this code must be regenerated, with command:
-//   tailscale.com/cmd/cloner -type User,Node,Hostinfo,NetInfo,Login,DNSConfig,DNSResolver,RegisterResponse
+//   tailscale.com/cmd/cloner -type User,Node,Hostinfo,NetInfo,Login,DNSConfig,DNSResolver,RegisterResponse,DERPRegion,DERPMap,DERPNode
 var _NetInfoNeedsRegeneration = NetInfo(struct {
 	MappingVariesByDestIP opt.Bool
 	HairPinning           opt.Bool
@@ -169,7 +169,7 @@ func (src *Login) Clone() *Login {
 }
 
 // A compilation failure here means this code must be regenerated, with command:
-//   tailscale.com/cmd/cloner -type User,Node,Hostinfo,NetInfo,Login,DNSConfig,DNSResolver,RegisterResponse
+//   tailscale.com/cmd/cloner -type User,Node,Hostinfo,NetInfo,Login,DNSConfig,DNSResolver,RegisterResponse,DERPRegion,DERPMap,DERPNode
 var _LoginNeedsRegeneration = Login(struct {
 	_             structs.Incomparable
 	ID            LoginID
@@ -204,11 +204,13 @@ func (src *DNSConfig) Clone() *DNSConfig {
 	}
 	dst.Domains = append(src.Domains[:0:0], src.Domains...)
 	dst.Nameservers = append(src.Nameservers[:0:0], src.Nameservers...)
+	dst.CertDomains = append(src.CertDomains[:0:0], src.CertDomains...)
+	dst.ExtraRecords = append(src.ExtraRecords[:0:0], src.ExtraRecords...)
 	return dst
 }
 
 // A compilation failure here means this code must be regenerated, with command:
-//   tailscale.com/cmd/cloner -type User,Node,Hostinfo,NetInfo,Login,DNSConfig,DNSResolver,RegisterResponse
+//   tailscale.com/cmd/cloner -type User,Node,Hostinfo,NetInfo,Login,DNSConfig,DNSResolver,RegisterResponse,DERPRegion,DERPMap,DERPNode
 var _DNSConfigNeedsRegeneration = DNSConfig(struct {
 	Resolvers         []DNSResolver
 	Routes            map[string][]DNSResolver
@@ -217,6 +219,8 @@ var _DNSConfigNeedsRegeneration = DNSConfig(struct {
 	Proxied           bool
 	Nameservers       []netaddr.IP
 	PerDomain         bool
+	CertDomains       []string
+	ExtraRecords      []DNSRecord
 }{})
 
 // Clone makes a deep copy of DNSResolver.
@@ -232,7 +236,7 @@ func (src *DNSResolver) Clone() *DNSResolver {
 }
 
 // A compilation failure here means this code must be regenerated, with command:
-//   tailscale.com/cmd/cloner -type User,Node,Hostinfo,NetInfo,Login,DNSConfig,DNSResolver,RegisterResponse
+//   tailscale.com/cmd/cloner -type User,Node,Hostinfo,NetInfo,Login,DNSConfig,DNSResolver,RegisterResponse,DERPRegion,DERPMap,DERPNode
 var _DNSResolverNeedsRegeneration = DNSResolver(struct {
 	Addr                string
 	BootstrapResolution []netaddr.IP
@@ -251,7 +255,7 @@ func (src *RegisterResponse) Clone() *RegisterResponse {
 }
 
 // A compilation failure here means this code must be regenerated, with command:
-//   tailscale.com/cmd/cloner -type User,Node,Hostinfo,NetInfo,Login,DNSConfig,DNSResolver,RegisterResponse
+//   tailscale.com/cmd/cloner -type User,Node,Hostinfo,NetInfo,Login,DNSConfig,DNSResolver,RegisterResponse,DERPRegion,DERPMap,DERPNode
 var _RegisterResponseNeedsRegeneration = RegisterResponse(struct {
 	User              User
 	Login             Login
@@ -260,9 +264,84 @@ var _RegisterResponseNeedsRegeneration = RegisterResponse(struct {
 	AuthURL           string
 }{})
 
+// Clone makes a deep copy of DERPRegion.
+// The result aliases no memory with the original.
+func (src *DERPRegion) Clone() *DERPRegion {
+	if src == nil {
+		return nil
+	}
+	dst := new(DERPRegion)
+	*dst = *src
+	dst.Nodes = make([]*DERPNode, len(src.Nodes))
+	for i := range dst.Nodes {
+		dst.Nodes[i] = src.Nodes[i].Clone()
+	}
+	return dst
+}
+
+// A compilation failure here means this code must be regenerated, with command:
+//   tailscale.com/cmd/cloner -type User,Node,Hostinfo,NetInfo,Login,DNSConfig,DNSResolver,RegisterResponse,DERPRegion,DERPMap,DERPNode
+var _DERPRegionNeedsRegeneration = DERPRegion(struct {
+	RegionID   int
+	RegionCode string
+	RegionName string
+	Avoid      bool
+	Nodes      []*DERPNode
+}{})
+
+// Clone makes a deep copy of DERPMap.
+// The result aliases no memory with the original.
+func (src *DERPMap) Clone() *DERPMap {
+	if src == nil {
+		return nil
+	}
+	dst := new(DERPMap)
+	*dst = *src
+	if dst.Regions != nil {
+		dst.Regions = map[int]*DERPRegion{}
+		for k, v := range src.Regions {
+			dst.Regions[k] = v.Clone()
+		}
+	}
+	return dst
+}
+
+// A compilation failure here means this code must be regenerated, with command:
+//   tailscale.com/cmd/cloner -type User,Node,Hostinfo,NetInfo,Login,DNSConfig,DNSResolver,RegisterResponse,DERPRegion,DERPMap,DERPNode
+var _DERPMapNeedsRegeneration = DERPMap(struct {
+	Regions            map[int]*DERPRegion
+	OmitDefaultRegions bool
+}{})
+
+// Clone makes a deep copy of DERPNode.
+// The result aliases no memory with the original.
+func (src *DERPNode) Clone() *DERPNode {
+	if src == nil {
+		return nil
+	}
+	dst := new(DERPNode)
+	*dst = *src
+	return dst
+}
+
+// A compilation failure here means this code must be regenerated, with command:
+//   tailscale.com/cmd/cloner -type User,Node,Hostinfo,NetInfo,Login,DNSConfig,DNSResolver,RegisterResponse,DERPRegion,DERPMap,DERPNode
+var _DERPNodeNeedsRegeneration = DERPNode(struct {
+	Name         string
+	RegionID     int
+	HostName     string
+	CertName     string
+	IPv4         string
+	IPv6         string
+	STUNPort     int
+	STUNOnly     bool
+	DERPTestPort int
+	STUNTestIP   string
+}{})
+
 // Clone duplicates src into dst and reports whether it succeeded.
 // To succeed, <src, dst> must be of types <*T, *T> or <*T, **T>,
-// where T is one of User,Node,Hostinfo,NetInfo,Login,DNSConfig,DNSResolver,RegisterResponse.
+// where T is one of User,Node,Hostinfo,NetInfo,Login,DNSConfig,DNSResolver,RegisterResponse,DERPRegion,DERPMap,DERPNode.
 func Clone(dst, src interface{}) bool {
 	switch src := src.(type) {
 	case *User:
@@ -337,6 +416,33 @@ func Clone(dst, src interface{}) bool {
 			*dst = src.Clone()
 			return true
 		}
+	case *DERPRegion:
+		switch dst := dst.(type) {
+		case *DERPRegion:
+			*dst = *src.Clone()
+			return true
+		case **DERPRegion:
+			*dst = src.Clone()
+			return true
+		}
+	case *DERPMap:
+		switch dst := dst.(type) {
+		case *DERPMap:
+			*dst = *src.Clone()
+			return true
+		case **DERPMap:
+			*dst = src.Clone()
+			return true
+		}
+	case *DERPNode:
+		switch dst := dst.(type) {
+		case *DERPNode:
+			*dst = *src.Clone()
+			return true
+		case **DERPNode:
+			*dst = src.Clone()
+			return true
+		}
 	}
 	return false
 }

tstest/integration/integration.go

@@ -0,0 +1,100 @@
+// Copyright (c) 2021 Tailscale Inc & AUTHORS All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+// Package integration contains Tailscale integration tests.
+//
+// This package is considered internal and the public API is subject
+// to change without notice.
+package integration
+
+import (
+	"os"
+	"os/exec"
+	"path"
+	"path/filepath"
+	"runtime"
+	"strings"
+	"sync"
+	"testing"
+	"time"
+
+	"tailscale.com/version"
+)
+
+// Binaries are the paths to a tailscaled and tailscale binary.
+// These can be shared by multiple nodes.
+type Binaries struct {
+	Dir    string // temp dir for tailscale & tailscaled
+	Daemon string // tailscaled
+	CLI    string // tailscale
+}
+
+// BuildTestBinaries builds tailscale and tailscaled, failing the test
+// if they fail to compile.
+func BuildTestBinaries(t testing.TB) *Binaries {
+	td := t.TempDir()
+	build(t, td, "tailscale.com/cmd/tailscaled", "tailscale.com/cmd/tailscale")
+	return &Binaries{
+		Dir:    td,
+		Daemon: filepath.Join(td, "tailscaled"+exe()),
+		CLI:    filepath.Join(td, "tailscale"+exe()),
+	}
+}
+
+// buildMu limits our use of "go build" to one at a time, so we don't
+// fight Go's built-in caching trying to do the same build concurrently.
+var buildMu sync.Mutex
+
+func build(t testing.TB, outDir string, targets ...string) {
+	buildMu.Lock()
+	defer buildMu.Unlock()
+
+	t0 := time.Now()
+	defer func() { t.Logf("built %s in %v", targets, time.Since(t0).Round(time.Millisecond)) }()
+
+	goBin := findGo(t)
+	cmd := exec.Command(goBin, "install")
+	if version.IsRace() {
+		cmd.Args = append(cmd.Args, "-race")
+	}
+	cmd.Args = append(cmd.Args, targets...)
+	cmd.Env = append(os.Environ(), "GOARCH="+runtime.GOARCH, "GOBIN="+outDir)
+	errOut, err := cmd.CombinedOutput()
+	if err == nil {
+		return
+	}
+	if strings.Contains(string(errOut), "when GOBIN is set") {
+		// Fallback slow path for cross-compiled binaries.
+		for _, target := range targets {
+			outFile := filepath.Join(outDir, path.Base(target)+exe())
+			cmd := exec.Command(goBin, "build", "-o", outFile, target)
+			cmd.Env = append(os.Environ(), "GOARCH="+runtime.GOARCH)
+			if errOut, err := cmd.CombinedOutput(); err != nil {
+				t.Fatalf("failed to build %v with %v: %v, %s", target, goBin, err, errOut)
+			}
+		}
+		return
+	}
+	t.Fatalf("failed to build %v with %v: %v, %s", targets, goBin, err, errOut)
+}
+
+func findGo(t testing.TB) string {
+	goBin := filepath.Join(runtime.GOROOT(), "bin", "go"+exe())
+	if fi, err := os.Stat(goBin); err != nil {
+		if os.IsNotExist(err) {
+			t.Fatalf("failed to find go at %v", goBin)
+		}
+		t.Fatalf("looking for go binary: %v", err)
+	} else if !fi.Mode().IsRegular() {
+		t.Fatalf("%v is unexpected %v", goBin, fi.Mode())
+	}
+	return goBin
+}
+
+func exe() string {
+	if runtime.GOOS == "windows" {
+		return ".exe"
+	}
+	return ""
+}

tstest/integration/integration_test.go

@@ -2,11 +2,11 @@
 // Use of this source code is governed by a BSD-style
 // license that can be found in the LICENSE file.
 
-// Package integration contains Tailscale integration tests.
 package integration
 
 import (
 	"bytes"
+	"context"
 	crand "crypto/rand"
 	"crypto/tls"
 	"encoding/json"
@@ -21,7 +21,6 @@ import (
 	"net/http/httptest"
 	"os"
 	"os/exec"
-	"path"
 	"path/filepath"
 	"regexp"
 	"runtime"
@@ -44,14 +43,17 @@ import (
 	"tailscale.com/types/key"
 	"tailscale.com/types/logger"
 	"tailscale.com/types/nettype"
-	"tailscale.com/version"
 )
 
-var verbose = flag.Bool("verbose", false, "verbose debug logs")
+var (
+	verboseLogCatcher = flag.Bool("verbose-log-catcher", false, "verbose log catcher logging")
+	verboseTailscaled = flag.Bool("verbose-tailscaled", false, "verbose tailscaled logging")
+)
 
 var mainError atomic.Value // of error
 
 func TestMain(m *testing.M) {
+	flag.Parse()
 	v := m.Run()
 	if v != 0 {
 		os.Exit(v)
@@ -65,7 +67,7 @@ func TestMain(m *testing.M) {
 
 func TestOneNodeUp_NoAuth(t *testing.T) {
 	t.Parallel()
-	bins := buildTestBinaries(t)
+	bins := BuildTestBinaries(t)
 
 	env := newTestEnv(t, bins)
 	defer env.Close()
@@ -107,11 +109,12 @@ func TestOneNodeUp_NoAuth(t *testing.T) {
 
 func TestOneNodeUp_Auth(t *testing.T) {
 	t.Parallel()
-	bins := buildTestBinaries(t)
+	bins := BuildTestBinaries(t)
 
-	env := newTestEnv(t, bins)
+	env := newTestEnv(t, bins, configureControl(func(control *testcontrol.Server) {
+		control.RequireAuth = true
+	}))
 	defer env.Close()
-	env.Control.RequireAuth = true
 
 	n1 := newTestNode(t, env)
 	d1 := n1.StartDaemon(t)
@@ -154,7 +157,7 @@ func TestOneNodeUp_Auth(t *testing.T) {
 
 func TestTwoNodes(t *testing.T) {
 	t.Parallel()
-	bins := buildTestBinaries(t)
+	bins := BuildTestBinaries(t)
 
 	env := newTestEnv(t, bins)
 	defer env.Close()
@@ -198,7 +201,7 @@ func TestTwoNodes(t *testing.T) {
 
 func TestNodeAddressIPFields(t *testing.T) {
 	t.Parallel()
-	bins := buildTestBinaries(t)
+	bins := BuildTestBinaries(t)
 
 	env := newTestEnv(t, bins)
 	defer env.Close()
@@ -227,31 +230,68 @@ func TestNodeAddressIPFields(t *testing.T) {
 	d1.MustCleanShutdown(t)
 }
 
-// testBinaries are the paths to a tailscaled and tailscale binary.
-// These can be shared by multiple nodes.
-type testBinaries struct {
-	dir    string // temp dir for tailscale & tailscaled
-	daemon string // tailscaled
-	cli    string // tailscale
-}
+func TestAddPingRequest(t *testing.T) {
+	t.Parallel()
+	bins := BuildTestBinaries(t)
 
-// buildTestBinaries builds tailscale and tailscaled, failing the test
-// if they fail to compile.
-func buildTestBinaries(t testing.TB) *testBinaries {
-	td := t.TempDir()
-	build(t, td, "tailscale.com/cmd/tailscaled", "tailscale.com/cmd/tailscale")
-	return &testBinaries{
-		dir:    td,
-		daemon: filepath.Join(td, "tailscaled"+exe()),
-		cli:    filepath.Join(td, "tailscale"+exe()),
+	env := newTestEnv(t, bins)
+	defer env.Close()
+
+	n1 := newTestNode(t, env)
+	d1 := n1.StartDaemon(t)
+	defer d1.Kill()
+
+	n1.AwaitListening(t)
+	n1.MustUp()
+	n1.AwaitRunning(t)
+
+	gotPing := make(chan bool, 1)
+	waitPing := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		gotPing <- true
+	}))
+	defer waitPing.Close()
+
+	nodes := env.Control.AllNodes()
+	if len(nodes) != 1 {
+		t.Fatalf("expected 1 node, got %d nodes", len(nodes))
 	}
+
+	nodeKey := nodes[0].Key
+
+	// Check that we get at least one ping reply after 10 tries.
+	for try := 1; try <= 10; try++ {
+		t.Logf("ping %v ...", try)
+		ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
+		if err := env.Control.AwaitNodeInMapRequest(ctx, nodeKey); err != nil {
+			t.Fatal(err)
+		}
+		cancel()
+
+		pr := &tailcfg.PingRequest{URL: fmt.Sprintf("%s/ping-%d", waitPing.URL, try), Log: true}
+		if !env.Control.AddPingRequest(nodeKey, pr) {
+			t.Logf("failed to AddPingRequest")
+			continue
+		}
+
+		// Wait for PingRequest to come back
+		pingTimeout := time.NewTimer(2 * time.Second)
+		defer pingTimeout.Stop()
+		select {
+		case <-gotPing:
+			t.Logf("got ping; success")
+			return
+		case <-pingTimeout.C:
+			// Try again.
+		}
+	}
+	t.Error("all ping attempts failed")
 }
 
 // testEnv contains the test environment (set of servers) used by one
 // or more nodes.
 type testEnv struct {
 	t        testing.TB
-	Binaries *testBinaries
+	Binaries *Binaries
 
 	LogCatcher       *logCatcher
 	LogCatcherServer *httptest.Server
@@ -265,11 +305,21 @@ type testEnv struct {
 	derpShutdown func()
 }
 
+type testEnvOpt interface {
+	modifyTestEnv(*testEnv)
+}
+
+type configureControl func(*testcontrol.Server)
+
+func (f configureControl) modifyTestEnv(te *testEnv) {
+	f(te.Control)
+}
+
 // newTestEnv starts a bunch of services and returns a new test
 // environment.
 //
 // Call Close to shut everything down.
-func newTestEnv(t testing.TB, bins *testBinaries) *testEnv {
+func newTestEnv(t testing.TB, bins *Binaries, opts ...testEnvOpt) *testEnv {
 	if runtime.GOOS == "windows" {
 		t.Skip("not tested/working on Windows yet")
 	}
@@ -278,6 +328,7 @@ func newTestEnv(t testing.TB, bins *testBinaries) *testEnv {
 	control := &testcontrol.Server{
 		DERPMap: derpMap,
 	}
+	control.HTTPTestServer = httptest.NewUnstartedServer(control)
 	trafficTrap := new(trafficTrap)
 	e := &testEnv{
 		t:                 t,
@@ -285,12 +336,15 @@ func newTestEnv(t testing.TB, bins *testBinaries) *testEnv {
 		LogCatcher:        logc,
 		LogCatcherServer:  httptest.NewServer(logc),
 		Control:           control,
-		ControlServer:     httptest.NewServer(control),
+		ControlServer:     control.HTTPTestServer,
 		TrafficTrap:       trafficTrap,
 		TrafficTrapServer: httptest.NewServer(trafficTrap),
 		derpShutdown:      derpShutdown,
 	}
-	e.Control.BaseURL = e.ControlServer.URL
+	for _, o := range opts {
+		o.modifyTestEnv(e)
+	}
+	control.HTTPTestServer.Start()
 	return e
 }
 
@@ -352,7 +406,7 @@ func (d *Daemon) MustCleanShutdown(t testing.TB) {
 // StartDaemon starts the node's tailscaled, failing if it fails to
 // start.
 func (n *testNode) StartDaemon(t testing.TB) *Daemon {
-	cmd := exec.Command(n.env.Binaries.daemon,
+	cmd := exec.Command(n.env.Binaries.Daemon,
 		"--tun=userspace-networking",
 		"--state="+n.stateFile,
 		"--socket="+n.sockFile,
@@ -362,6 +416,10 @@ func (n *testNode) StartDaemon(t testing.TB) *Daemon {
 		"HTTP_PROXY="+n.env.TrafficTrapServer.URL,
 		"HTTPS_PROXY="+n.env.TrafficTrapServer.URL,
 	)
+	if *verboseTailscaled {
+		cmd.Stdout = os.Stdout
+		cmd.Stderr = os.Stdout
+	}
 	if err := cmd.Start(); err != nil {
 		t.Fatalf("starting tailscaled: %v", err)
 	}
@@ -430,7 +488,7 @@ func (n *testNode) AwaitRunning(t testing.TB) {
 // Tailscale returns a command that runs the tailscale CLI with the provided arguments.
 // It does not start the process.
 func (n *testNode) Tailscale(arg ...string) *exec.Cmd {
-	cmd := exec.Command(n.env.Binaries.cli, "--socket="+n.sockFile)
+	cmd := exec.Command(n.env.Binaries.CLI, "--socket="+n.sockFile)
 	cmd.Args = append(cmd.Args, arg...)
 	cmd.Dir = n.dir
 	return cmd
@@ -457,63 +515,6 @@ func (n *testNode) MustStatus(tb testing.TB) *ipnstate.Status {
 	return st
 }
 
-func exe() string {
-	if runtime.GOOS == "windows" {
-		return ".exe"
-	}
-	return ""
-}
-
-func findGo(t testing.TB) string {
-	goBin := filepath.Join(runtime.GOROOT(), "bin", "go"+exe())
-	if fi, err := os.Stat(goBin); err != nil {
-		if os.IsNotExist(err) {
-			t.Fatalf("failed to find go at %v", goBin)
-		}
-		t.Fatalf("looking for go binary: %v", err)
-	} else if !fi.Mode().IsRegular() {
-		t.Fatalf("%v is unexpected %v", goBin, fi.Mode())
-	}
-	return goBin
-}
-
-// buildMu limits our use of "go build" to one at a time, so we don't
-// fight Go's built-in caching trying to do the same build concurrently.
-var buildMu sync.Mutex
-
-func build(t testing.TB, outDir string, targets ...string) {
-	buildMu.Lock()
-	defer buildMu.Unlock()
-
-	t0 := time.Now()
-	defer func() { t.Logf("built %s in %v", targets, time.Since(t0).Round(time.Millisecond)) }()
-
-	goBin := findGo(t)
-	cmd := exec.Command(goBin, "install")
-	if version.IsRace() {
-		cmd.Args = append(cmd.Args, "-race")
-	}
-	cmd.Args = append(cmd.Args, targets...)
-	cmd.Env = append(os.Environ(), "GOARCH="+runtime.GOARCH, "GOBIN="+outDir)
-	errOut, err := cmd.CombinedOutput()
-	if err == nil {
-		return
-	}
-	if strings.Contains(string(errOut), "when GOBIN is set") {
-		// Fallback slow path for cross-compiled binaries.
-		for _, target := range targets {
-			outFile := filepath.Join(outDir, path.Base(target)+exe())
-			cmd := exec.Command(goBin, "build", "-o", outFile, target)
-			cmd.Env = append(os.Environ(), "GOARCH="+runtime.GOARCH)
-			if errOut, err := cmd.CombinedOutput(); err != nil {
-				t.Fatalf("failed to build %v with %v: %v, %s", target, goBin, err, errOut)
-			}
-		}
-		return
-	}
-	t.Fatalf("failed to build %v with %v: %v, %s", targets, goBin, err, errOut)
-}
-
 // logCatcher is a minimal logcatcher for the logtail upload client.
 type logCatcher struct {
 	mu     sync.Mutex
@@ -584,7 +585,7 @@ func (lc *logCatcher) ServeHTTP(w http.ResponseWriter, r *http.Request) {
 	} else {
 		for _, ent := range jreq {
 			fmt.Fprintf(&lc.buf, "%s\n", strings.TrimSpace(ent.Text))
-			if *verbose {
+			if *verboseLogCatcher {
 				fmt.Fprintf(os.Stderr, "%s\n", strings.TrimSpace(ent.Text))
 			}
 		}

tstest/integration/testcontrol/testcontrol.go

@@ -7,6 +7,7 @@ package testcontrol
 
 import (
 	"bytes"
+	"context"
 	crand "crypto/rand"
 	"encoding/binary"
 	"encoding/json"
@@ -17,6 +18,7 @@ import (
 	"log"
 	"math/rand"
 	"net/http"
+	"net/http/httptest"
 	"net/url"
 	"sort"
 	"strings"
@@ -39,13 +41,17 @@ type Server struct {
 	Logf        logger.Logf      // nil means to use the log package
 	DERPMap     *tailcfg.DERPMap // nil means to use prod DERP map
 	RequireAuth bool
-	BaseURL     string // must be set to e.g. "http://127.0.0.1:1234" with no trailing URL
 	Verbose     bool
 
+	// ExplicitBaseURL or HTTPTestServer must be set.
+	ExplicitBaseURL string           // e.g. "http://127.0.0.1:1234" with no trailing URL
+	HTTPTestServer  *httptest.Server // if non-nil, used to get BaseURL
+
 	initMuxOnce sync.Once
 	mux         *http.ServeMux
 
 	mu            sync.Mutex
+	cond          *sync.Cond // lazily initialized by condLocked
 	pubKey        wgkey.Key
 	privKey       wgkey.Private
 	nodes         map[tailcfg.NodeKey]*tailcfg.Node
@@ -54,6 +60,21 @@ type Server struct {
 	updates       map[tailcfg.NodeID]chan updateType
 	authPath      map[string]*AuthPath
 	nodeKeyAuthed map[tailcfg.NodeKey]bool // key => true once authenticated
+	pingReqsToAdd map[tailcfg.NodeKey]*tailcfg.PingRequest
+}
+
+// BaseURL returns the server's base URL, without trailing slash.
+func (s *Server) BaseURL() string {
+	if e := s.ExplicitBaseURL; e != "" {
+		return e
+	}
+	if hs := s.HTTPTestServer; hs != nil {
+		if hs.URL != "" {
+			return hs.URL
+		}
+		panic("Server.HTTPTestServer not started")
+	}
+	panic("Server ExplicitBaseURL and HTTPTestServer both unset")
 }
 
 // NumNodes returns the number of nodes in the testcontrol server.
@@ -67,6 +88,67 @@ func (s *Server) NumNodes() int {
 	return len(s.nodes)
 }
 
+// condLocked lazily initializes and returns s.cond.
+// s.mu must be held.
+func (s *Server) condLocked() *sync.Cond {
+	if s.cond == nil {
+		s.cond = sync.NewCond(&s.mu)
+	}
+	return s.cond
+}
+
+// AwaitNodeInMapRequest waits for node k to be stuck in a map poll.
+// It returns an error if and only if the context is done first.
+func (s *Server) AwaitNodeInMapRequest(ctx context.Context, k tailcfg.NodeKey) error {
+	s.mu.Lock()
+	defer s.mu.Unlock()
+	cond := s.condLocked()
+
+	done := make(chan struct{})
+	defer close(done)
+	go func() {
+		select {
+		case <-done:
+		case <-ctx.Done():
+			cond.Broadcast()
+		}
+	}()
+
+	for {
+		node := s.nodeLocked(k)
+		if node == nil {
+			return errors.New("unknown node key")
+		}
+		if _, ok := s.updates[node.ID]; ok {
+			return nil
+		}
+		cond.Wait()
+		if err := ctx.Err(); err != nil {
+			return err
+		}
+	}
+}
+
+// AddPingRequest sends the ping pr to nodeKeyDst. It reports whether it did so. That is,
+// it reports whether nodeKeyDst was connected.
+func (s *Server) AddPingRequest(nodeKeyDst tailcfg.NodeKey, pr *tailcfg.PingRequest) bool {
+	s.mu.Lock()
+	defer s.mu.Unlock()
+	if s.pingReqsToAdd == nil {
+		s.pingReqsToAdd = map[tailcfg.NodeKey]*tailcfg.PingRequest{}
+	}
+	// Now send the update to the channel
+	node := s.nodeLocked(nodeKeyDst)
+	if node == nil {
+		return false
+	}
+
+	s.pingReqsToAdd[nodeKeyDst] = pr
+	nodeID := node.ID
+	oldUpdatesCh := s.updates[nodeID]
+	return sendUpdate(oldUpdatesCh, updateDebugInjection)
+}
+
 type AuthPath struct {
 	nodeKey tailcfg.NodeKey
 
@@ -176,6 +258,13 @@ func (s *Server) serveMachine(w http.ResponseWriter, r *http.Request) {
 func (s *Server) Node(nodeKey tailcfg.NodeKey) *tailcfg.Node {
 	s.mu.Lock()
 	defer s.mu.Unlock()
+	return s.nodeLocked(nodeKey)
+}
+
+// nodeLocked returns the node for nodeKey. It's always nil or cloned memory.
+//
+// s.mu must be held.
+func (s *Server) nodeLocked(nodeKey tailcfg.NodeKey) *tailcfg.Node {
 	return s.nodes[nodeKey].Clone()
 }
 
@@ -344,7 +433,7 @@ func (s *Server) serveRegister(w http.ResponseWriter, r *http.Request, mkey tail
 		crand.Read(randHex)
 		authPath := fmt.Sprintf("/auth/%x", randHex)
 		s.addAuthPath(authPath, req.NodeKey)
-		authURL = s.BaseURL + authPath
+		authURL = s.BaseURL() + authPath
 	}
 
 	res, err := s.encode(mkey, false, tailcfg.RegisterResponse{
@@ -373,6 +462,9 @@ const (
 	// via a lite endpoint update. These ones are never dup-suppressed,
 	// as the client is expecting an answer regardless.
 	updateSelfChanged
+
+	// updateDebugInjection is an update used for PingRequests
+	updateDebugInjection
 )
 
 func (s *Server) updateLocked(source string, peers []tailcfg.NodeID) {
@@ -382,17 +474,19 @@ func (s *Server) updateLocked(source string, peers []tailcfg.NodeID) {
 }
 
 // sendUpdate sends updateType to dst if dst is non-nil and
-// has capacity.
-func sendUpdate(dst chan<- updateType, updateType updateType) {
+// has capacity. It reports whether a value was sent.
+func sendUpdate(dst chan<- updateType, updateType updateType) bool {
 	if dst == nil {
-		return
+		return false
 	}
 	// The dst channel has a buffer size of 1.
 	// If we fail to insert an update into the buffer that
 	// means there is already an update pending.
 	select {
 	case dst <- updateType:
+		return true
 	default:
+		return false
 	}
 }
 
@@ -457,6 +551,7 @@ func (s *Server) serveMap(w http.ResponseWriter, r *http.Request, mkey tailcfg.M
 		sendUpdate(oldUpdatesCh, updateSelfChanged)
 	}
 	s.updateLocked("serveMap", peersToUpdate)
+	s.condLocked().Broadcast()
 	s.mu.Unlock()
 
 	// ReadOnly implies no streaming, as it doesn't
@@ -554,6 +649,14 @@ func (s *Server) MapResponse(req *tailcfg.MapRequest) (res *tailcfg.MapResponse,
 		netaddr.MustParseIPPrefix(fmt.Sprintf("100.64.%d.%d/32", uint8(node.ID>>8), uint8(node.ID))),
 	}
 	res.Node.AllowedIPs = res.Node.Addresses
+
+	// Consume the PingRequest while protected by mutex if it exists
+	s.mu.Lock()
+	if pr, ok := s.pingReqsToAdd[node.Key]; ok {
+		res.PingRequest = pr
+		delete(s.pingReqsToAdd, node.Key)
+	}
+	s.mu.Unlock()
 	return res, nil
 }
 

tstest/integration/vms/README.md

@@ -0,0 +1,98 @@
+# End-to-End VM-based Integration Testing
+
+This test spins up a bunch of common linux distributions and then tries to get
+them to connect to a
+[`testcontrol`](https://pkg.go.dev/tailscale.com/tstest/integration/testcontrol)
+server.
+
+## Running
+
+This test currently only runs on Linux.
+
+This test depends on the following command line tools:
+
+- [qemu](https://www.qemu.org/)
+- [cdrkit](https://en.wikipedia.org/wiki/Cdrkit)
+- [openssh](https://www.openssh.com/)
+
+This test also requires the following:
+
+- about 10 GB of temporary storage
+- about 10 GB of cached VM images
+- at least 4 GB of ram for virtual machines
+- hardware virtualization support
+  ([KVM](https://www.linux-kvm.org/page/Main_Page)) enabled in the BIOS
+- the `kvm` module to be loaded (`modprobe kvm`)
+- the user running these tests must have access to `/dev/kvm` (being in the
+  `kvm` group should suffice)
+
+This optionally requires an AWS profile to be configured at the [default
+path](https://docs.aws.amazon.com/cli/latest/userguide/cli-configure-files.html).
+The S3 bucket is set so that the requester pays. Please keep this in mind when
+running these tests on your machine. If you are uncomfortable with the cost from
+downloading from S3, you should pass the `-no-s3` flag to disable downloads from
+S3. However keep in mind that some distributions do not use stable URLs for each
+individual image artifact, so there may be spurious test failures as a result.
+
+If you are using [Nix](https://nixos.org), you can run all of the tests with the
+correct command line tools using this command:
+
+```console
+$ nix-shell -p openssh -p go -p qemu -p cdrkit --run "go test . --run-vm-tests --v --timeout 30m"
+```
+
+Keep the timeout high for the first run, especially if you are not downloading
+VM images from S3. The mirrors we pull images from have download rate limits and
+will take a while to download.
+
+Because of the hardware requirements of this test, this test will not run
+without the `--run-vm-tests` flag set.
+
+## Other Fun Flags
+
+This test's behavior is customized with command line flags.
+
+### Don't Download Images From S3
+
+If you pass the `-no-s3` flag to `go test`, the S3 step will be skipped in favor
+of downloading the images directly from upstream sources, which may cause the
+test to fail in odd places.
+
+### Distribution Picking
+
+This test runs on a large number of distributions. By default it tries to run
+everything, which may or may not be ideal for you. If you only want to test a
+subset of distributions, you can use the `--distro-regex` flag to match a subset
+of distributions using a [regular expression](https://golang.org/pkg/regexp/)
+such as like this:
+
+```console
+$ go test -run-vm-tests -distro-regex centos
+```
+
+This would run all tests on all versions of CentOS.
+
+```console
+$ go test -run-vm-tests -distro-regex '(debian|ubuntu)'
+```
+
+This would run all tests on all versions of Debian and Ubuntu.
+
+### Ram Limiting
+
+This test uses a lot of memory. In order to avoid making machines run out of
+memory running this test, a semaphore is used to limit how many megabytes of ram
+are being used at once. By default this semaphore is set to 4096 MB of ram
+(about 4 gigabytes). You can customize this with the `--ram-limit` flag:
+
+```console
+$ go test --run-vm-tests --ram-limit 2048
+$ go test --run-vm-tests --ram-limit 65536
+```
+
+The first example will set the limit to 2048 MB of ram (about 2 gigabytes). The
+second example will set the limit to 65536 MB of ram (about 65 gigabytes).
+Please be careful with this flag, improper usage of it is known to cause the
+Linux out-of-memory killer to engage. Try to keep it within 50-75% of your
+machine's available ram (there is some overhead involved with the
+virtualization) to be on the safe side.

tstest/integration/vms/opensuse_leap_15_1_test.go

@@ -0,0 +1,86 @@
+// Copyright (c) 2021 Tailscale Inc & AUTHORS All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+// +build linux
+
+package vms
+
+import (
+	"encoding/json"
+	"os"
+	"path/filepath"
+	"testing"
+
+	"github.com/google/uuid"
+)
+
+/*
+   The images that we use for OpenSUSE Leap 15.1 have an issue that makes the
+   nocloud backend[1] for cloud-init just not work. As a distro-specific
+   workaround, we're gonna pretend to be OpenStack.
+
+   TODO(Xe): delete once we no longer need to support OpenSUSE Leap 15.1.
+
+   [1]: https://cloudinit.readthedocs.io/en/latest/topics/datasources/nocloud.html
+*/
+
+type openSUSELeap151MetaData struct {
+	Zone        string                      `json:"availability_zone"` // nova
+	Hostname    string                      `json:"hostname"`          // opensuse-leap-15-1
+	LaunchIndex string                      `json:"launch_index"`      // 0
+	Meta        openSUSELeap151MetaDataMeta `json:"meta"`              // some openstack metadata we don't need to care about
+	Name        string                      `json:"name"`              // opensuse-leap-15-1
+	UUID        string                      `json:"uuid"`              // e9c664cd-b116-433b-aa61-7ff420163dcd
+}
+
+type openSUSELeap151MetaDataMeta struct {
+	Role      string `json:"role"`      // server
+	DSMode    string `json:"dsmode"`    // local
+	Essential string `json:"essential"` // essential
+}
+
+func hackOpenSUSE151UserData(t *testing.T, d Distro, dir string) bool {
+	if d.name != "opensuse-leap-15-1" {
+		return false
+	}
+
+	t.Log("doing OpenSUSE Leap 15.1 hack")
+	osDir := filepath.Join(dir, "openstack", "latest")
+	err := os.MkdirAll(osDir, 0755)
+	if err != nil {
+		t.Fatalf("can't make metadata home: %v", err)
+	}
+
+	metadata, err := json.Marshal(openSUSELeap151MetaData{
+		Zone:        "nova",
+		Hostname:    d.name,
+		LaunchIndex: "0",
+		Meta: openSUSELeap151MetaDataMeta{
+			Role:      "server",
+			DSMode:    "local",
+			Essential: "false",
+		},
+		Name: d.name,
+		UUID: uuid.New().String(),
+	})
+	if err != nil {
+		t.Fatalf("can't encode metadata: %v", err)
+	}
+	err = os.WriteFile(filepath.Join(osDir, "meta_data.json"), metadata, 0666)
+	if err != nil {
+		t.Fatalf("can't write to meta_data.json: %v", err)
+	}
+
+	data, err := os.ReadFile(filepath.Join(dir, "user-data"))
+	if err != nil {
+		t.Fatalf("can't read user_data: %v", err)
+	}
+
+	err = os.WriteFile(filepath.Join(osDir, "user_data"), data, 0666)
+	if err != nil {
+		t.Fatalf("can't create output user_data: %v", err)
+	}
+
+	return true
+}

tstest/integration/vms/regex_flag.go

@@ -0,0 +1,30 @@
+// Copyright (c) 2021 Tailscale Inc & AUTHORS All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package vms
+
+import "regexp"
+
+type regexValue struct {
+	r *regexp.Regexp
+}
+
+func (r *regexValue) String() string {
+	if r.r == nil {
+		return ""
+	}
+
+	return r.r.String()
+}
+
+func (r *regexValue) Set(val string) error {
+	if rex, err := regexp.Compile(val); err != nil {
+		return err
+	} else {
+		r.r = rex
+		return nil
+	}
+}
+
+func (r regexValue) Unwrap() *regexp.Regexp { return r.r }

tstest/integration/vms/regex_flag_test.go

@@ -0,0 +1,22 @@
+// Copyright (c) 2021 Tailscale Inc & AUTHORS All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package vms
+
+import (
+	"flag"
+	"testing"
+)
+
+func TestRegexFlag(t *testing.T) {
+	var v regexValue
+	fs := flag.NewFlagSet(t.Name(), flag.PanicOnError)
+	fs.Var(&v, "regex", "regex to parse")
+
+	const want = `.*`
+	fs.Parse([]string{"-regex", want})
+	if v.Unwrap().String() != want {
+		t.Fatalf("got wrong regex: %q, wanted: %q", v.Unwrap().String(), want)
+	}
+}

tstest/integration/vms/runner.nix

@@ -0,0 +1,77 @@
+# This is a NixOS module to allow a machine to act as an integration test
+# runner. This is used for the end-to-end VM test suite.
+
+{ lib, config, pkgs, ... }:
+
+{
+  # The GitHub Actions self-hosted runner service.
+  services.github-runner = {
+    enable = true;
+    url = "https://github.com/tailscale/tailscale";
+    replace = true;
+    extraLabels = [ "vm_integration_test" ];
+
+    # Justifications for the packages:
+    extraPackages = with pkgs; [
+      # The test suite is written in Go.
+      go
+
+      # This contains genisoimage, which is needed to create cloud-init
+      # seeds.
+      cdrkit
+
+      # This package is the virtual machine hypervisor we use in tests.
+      qemu
+
+      # This package contains tools like `ssh-keygen`.
+      openssh
+
+      # The C complier so cgo builds work.
+      gcc
+    ];
+
+    # Customize this to include your GitHub username so we can track
+    # who is running which node.
+    name = "YOUR-GITHUB-USERNAME-tstest-integration-vms";
+
+    # Replace this with the path to the GitHub Actions runner token on
+    # your disk.
+    tokenFile = "/run/decrypted/ts-oss-ghaction-token";
+  };
+
+  # A user account so there is a home directory and so they have kvm
+  # access. Please don't change this account name.
+  users.users.ghrunner = {
+    createHome = true;
+    isSystemUser = true;
+    extraGroups = [ "kvm" ];
+  };
+
+  # The default github-runner service sets a lot of isolation features
+  # that attempt to limit the damage that malicious code can use.
+  # Unfortunately we rely on some "dangerous" features to do these tests,
+  # so this shim will peel some of them away.
+  systemd.services.github-runner = {
+    serviceConfig = {
+      # We need access to /dev to poke /dev/kvm.
+      PrivateDevices = lib.mkForce false;
+
+      # /dev/kvm is how qemu creates a virtual machine with KVM.
+      DeviceAllow = lib.mkForce [ "/dev/kvm" ];
+
+      # Ensure the service has KVM permissions with the `kvm` group.
+      ExtraGroups = [ "kvm" ];
+
+      # The service runs as a dynamic user by default. This makes it hard
+      # to persistently store things in /var/lib/ghrunner. This line
+      # disables the dynamic user feature.
+      DynamicUser = lib.mkForce false;
+
+      # Run this service as our ghrunner user.
+      User = "ghrunner";
+
+      # We need access to /var/lib/ghrunner to store VM images.
+      ProtectSystem = lib.mkForce null;
+    };
+  };
+}

tstest/integration/vms/vms_test.go

@@ -7,11 +7,14 @@
 package vms
 
 import (
+	"context"
 	"crypto/sha256"
 	"encoding/hex"
+	"errors"
 	"flag"
 	"fmt"
 	"io"
+	"log"
 	"net"
 	"net/http"
 	"os"
@@ -19,6 +22,7 @@ import (
 	"path"
 	"path/filepath"
 	"regexp"
+	"strconv"
 	"strings"
 	"sync"
 	"syscall"
@@ -26,13 +30,37 @@ import (
 	"text/template"
 	"time"
 
+	"github.com/aws/aws-sdk-go/aws"
+	"github.com/aws/aws-sdk-go/aws/session"
+	"github.com/aws/aws-sdk-go/service/s3"
+	"github.com/aws/aws-sdk-go/service/s3/s3manager"
 	expect "github.com/google/goexpect"
+	"github.com/pkg/sftp"
 	"golang.org/x/crypto/ssh"
+	"golang.org/x/sync/semaphore"
 	"inet.af/netaddr"
+	"tailscale.com/net/interfaces"
+	"tailscale.com/tstest"
+	"tailscale.com/tstest/integration"
 	"tailscale.com/tstest/integration/testcontrol"
+	"tailscale.com/types/logger"
 )
 
-var runVMTests = flag.Bool("run-vm-tests", false, "if set, run expensive (10G+ ram) VM based integration tests")
+const (
+	securePassword = "hunter2"
+	bucketName     = "tailscale-integration-vm-images"
+)
+
+var (
+	runVMTests = flag.Bool("run-vm-tests", false, "if set, run expensive VM based integration tests")
+	noS3       = flag.Bool("no-s3", false, "if set, always download images from the public internet (risks breaking)")
+	vmRamLimit = flag.Int("ram-limit", 4096, "the maximum number of megabytes of ram that can be used for VMs, must be greater than or equal to 1024")
+	distroRex  = func() *regexValue {
+		result := &regexValue{r: regexp.MustCompile(`.*`)}
+		flag.Var(result, "distro-regex", "The regex that matches what distros should be run")
+		return result
+	}()
+)
 
 type Distro struct {
 	name           string // amazon-linux
@@ -40,22 +68,147 @@ type Distro struct {
 	sha256sum      string // hex-encoded sha256 sum of contents of URL
 	mem            int    // VM memory in megabytes
 	packageManager string // yum/apt/dnf/zypper
+	initSystem     string // systemd/openrc
 }
 
 func (d *Distro) InstallPre() string {
 	switch d.packageManager {
 	case "yum":
 		return ` - [ yum, update, gnupg2 ]
-`
+ - [ yum, "-y", install, iptables ]`
+	case "zypper":
+		return ` - [ zypper, in, "-y", iptables ]`
+
+	case "dnf":
+		return ` - [ dnf, install, "-y", iptables ]`
+
 	case "apt":
 		return ` - [ apt-get, update ]
- - [ apt-get, "-y", install, curl, "apt-transport-https", gnupg2 ]
-`
+ - [ apt-get, "-y", install, curl, "apt-transport-https", gnupg2 ]`
+
+	case "apk":
+		return ` - [ apk, "-U", add, curl, "ca-certificates", iptables ]
+ - [ modprobe, tun ]`
 	}
 
 	return ""
 }
 
+func TestDownloadImages(t *testing.T) {
+	if !*runVMTests {
+		t.Skip("not running integration tests (need --run-vm-tests)")
+	}
+
+	for _, d := range distros {
+		distro := d
+		t.Run(distro.name, func(t *testing.T) {
+			if !distroRex.Unwrap().MatchString(distro.name) {
+				t.Skipf("distro name %q doesn't match regex: %s", distro.name, distroRex)
+			}
+
+			t.Parallel()
+
+			fetchDistro(t, distro)
+		})
+	}
+}
+
+var distros = []Distro{
+	// NOTE(Xe): If you run into issues getting the autoconfig to work, run
+	// this test with the flag `--distro-regex=alpine-edge`. Connect with a VNC
+	// client with a command like this:
+	//
+	//    $ vncviewer :0
+	//
+	// On NixOS you can get away with something like this:
+	//
+	//    $ env NIXPKGS_ALLOW_UNFREE=1 nix-shell -p tigervnc --run 'vncviewer :0'
+	//
+	// Login as root with the password root. Then look in
+	// /var/log/cloud-init-output.log for what you messed up.
+
+	// NOTE(Xe): These images are not official images created by the Alpine Linux
+	// cloud team because the cloud team hasn't created any official images yet.
+	// These images were created under the guidance of the cloud team and contain
+	// few notable differences from what they would end up shipping. The Alpine
+	// Linux cloud team probably won't have official images up until a year or so
+	// after this comment is written (2021-06-11), but overall they will be
+	// compatible with these images. These images were created using the setup in
+	// this repo: https://github.com/Xe/alpine-image. I hereby promise to not break
+	// these links.
+	{"alpine-3-13-5", "https://xena.greedo.xeserv.us/pkg/alpine/img/alpine-3.13.5-cloud-init-within.qcow2", "a2665c16724e75899723e81d81126bd0254a876e5de286b0b21553734baec287", 256, "apk", "openrc"},
+	{"alpine-edge", "https://xena.greedo.xeserv.us/pkg/alpine/img/alpine-edge-2021-05-18-cloud-init-within.qcow2", "b3bb15311c0bd3beffa1b554f022b75d3b7309b5fdf76fb146fe7c72b83b16d0", 256, "apk", "openrc"},
+
+	// NOTE(Xe): All of the following images are official images straight from each
+	// distribution's official documentation.
+	{"amazon-linux", "https://cdn.amazonlinux.com/os-images/2.0.20210427.0/kvm/amzn2-kvm-2.0.20210427.0-x86_64.xfs.gpt.qcow2", "6ef9daef32cec69b2d0088626ec96410cd24afc504d57278bbf2f2ba2b7e529b", 512, "yum", "systemd"},
+	{"arch", "https://mirror.pkgbuild.com/images/v20210515.22945/Arch-Linux-x86_64-cloudimg-20210515.22945.qcow2", "e4077f5ba3c5d545478f64834bc4852f9f7a2e05950fce8ecd0df84193162a27", 512, "pacman", "systemd"},
+	{"centos-7", "https://cloud.centos.org/centos/7/images/CentOS-7-x86_64-GenericCloud-2003.qcow2c", "b7555ecf90b24111f2efbc03c1e80f7b38f1e1fc7e1b15d8fee277d1a4575e87", 512, "yum", "systemd"},
+	{"centos-8", "https://cloud.centos.org/centos/8/x86_64/images/CentOS-8-GenericCloud-8.3.2011-20201204.2.x86_64.qcow2", "7ec97062618dc0a7ebf211864abf63629da1f325578868579ee70c495bed3ba0", 768, "dnf", "systemd"},
+	{"debian-9", "http://cloud.debian.org/images/cloud/OpenStack/9.13.22-20210531/debian-9.13.22-20210531-openstack-amd64.qcow2", "c36e25f2ab0b5be722180db42ed9928476812f02d053620e1c287f983e9f6f1d", 512, "apt", "systemd"},
+	{"debian-10", "https://cdimage.debian.org/images/cloud/buster/20210329-591/debian-10-generic-amd64-20210329-591.qcow2", "70c61956095870c4082103d1a7a1cb5925293f8405fc6cb348588ec97e8611b0", 768, "apt", "systemd"},
+	{"fedora-34", "https://download.fedoraproject.org/pub/fedora/linux/releases/34/Cloud/x86_64/images/Fedora-Cloud-Base-34-1.2.x86_64.qcow2", "b9b621b26725ba95442d9a56cbaa054784e0779a9522ec6eafff07c6e6f717ea", 768, "dnf", "systemd"},
+	{"opensuse-leap-15-1", "https://download.opensuse.org/repositories/Cloud:/Images:/Leap_15.1/images/openSUSE-Leap-15.1-OpenStack.x86_64.qcow2", "40bc72b8ee143364fc401f2c9c9a11ecb7341a29fa84c6f7bf42fc94acf19a02", 512, "zypper", "systemd"},
+	{"opensuse-leap-15-2", "https://download.opensuse.org/repositories/Cloud:/Images:/Leap_15.2/images/openSUSE-Leap-15.2-OpenStack.x86_64.qcow2", "4df9cee9281d1f57d20f79dc65d76e255592b904760e73c0dd44ac753a54330f", 512, "zypper", "systemd"},
+	{"opensuse-leap-15-3", "http://mirror.its.dal.ca/opensuse/distribution/leap/15.3/appliances/openSUSE-Leap-15.3-JeOS.x86_64-OpenStack-Cloud.qcow2", "22e0392e4d0becb523d1bc5f709366140b7ee20d6faf26de3d0f9046d1ee15d5", 512, "zypper", "systemd"},
+	{"opensuse-tumbleweed", "https://download.opensuse.org/tumbleweed/appliances/openSUSE-Tumbleweed-JeOS.x86_64-OpenStack-Cloud.qcow2", "79e610bba3ed116556608f031c06e4b9260e3be2b193ce1727914ba213afac3f", 512, "zypper", "systemd"},
+	{"ubuntu-16-04", "https://cloud-images.ubuntu.com/xenial/20210429/xenial-server-cloudimg-amd64-disk1.img", "50a21bc067c05e0c73bf5d8727ab61152340d93073b3dc32eff18b626f7d813b", 512, "apt", "systemd"},
+	{"ubuntu-18-04", "https://cloud-images.ubuntu.com/bionic/20210526/bionic-server-cloudimg-amd64.img", "389ffd5d36bbc7a11bf384fd217cda9388ccae20e5b0cb7d4516733623c96022", 512, "apt", "systemd"},
+	{"ubuntu-20-04", "https://cloud-images.ubuntu.com/focal/20210603/focal-server-cloudimg-amd64.img", "1c0969323b058ba8b91fec245527069c2f0502fc119b9138b213b6bfebd965cb", 512, "apt", "systemd"},
+	{"ubuntu-20-10", "https://cloud-images.ubuntu.com/groovy/20210604/groovy-server-cloudimg-amd64.img", "2196df5f153faf96443e5502bfdbcaa0baaefbaec614348fec344a241855b0ef", 512, "apt", "systemd"},
+	{"ubuntu-21-04", "https://cloud-images.ubuntu.com/hirsute/20210603/hirsute-server-cloudimg-amd64.img", "bf07f36fc99ff521d3426e7d257e28f0c81feebc9780b0c4f4e25ae594ff4d3b", 512, "apt", "systemd"},
+}
+
+// fetchFromS3 fetches a distribution image from Amazon S3 or reports whether
+// it is unable to. It can fail to fetch from S3 if there is either no AWS
+// configuration (in ~/.aws/credentials) or if the `-no-s3` flag is passed. In
+// that case the test will fall back to downloading distribution images from the
+// public internet.
+//
+// Like fetching from HTTP, the test will fail if an error is encountered during
+// the downloading process.
+//
+// This function writes the distribution image to fout. It is always closed. Do
+// not expect fout to remain writable.
+func fetchFromS3(t *testing.T, fout *os.File, d Distro) bool {
+	t.Helper()
+
+	if *noS3 {
+		t.Log("you asked to not use S3, not using S3")
+		return false
+	}
+
+	sess, err := session.NewSession(&aws.Config{
+		Region: aws.String("us-east-1"),
+	})
+	if err != nil {
+		t.Logf("can't make AWS session: %v", err)
+		return false
+	}
+
+	dler := s3manager.NewDownloader(sess, func(d *s3manager.Downloader) {
+		d.PartSize = 64 * 1024 * 1024 // 64MB per part
+	})
+
+	t.Logf("fetching s3://%s/%s", bucketName, d.sha256sum)
+
+	_, err = dler.Download(fout, &s3.GetObjectInput{
+		Bucket: aws.String(bucketName),
+		Key:    aws.String(d.sha256sum),
+	})
+	if err != nil {
+		fout.Close()
+		t.Fatalf("can't get s3://%s/%s: %v", bucketName, d.sha256sum, err)
+	}
+
+	err = fout.Close()
+	if err != nil {
+		t.Fatalf("can't close fout: %v", err)
+	}
+
+	return true
+}
+
 // fetchDistro fetches a distribution from the internet if it doesn't already exist locally. It
 // also validates the sha256 sum from a known good hash.
 func fetchDistro(t *testing.T, resultDistro Distro) {
@@ -70,70 +223,93 @@ func fetchDistro(t *testing.T, resultDistro Distro) {
 	qcowPath := filepath.Join(cdir, "qcow2", resultDistro.sha256sum)
 
 	_, err = os.Stat(qcowPath)
+	if err == nil {
+		hash := checkCachedImageHash(t, resultDistro, cdir)
+		if hash != resultDistro.sha256sum {
+			t.Logf("hash for %s (%s) doesn't match expected %s, re-downloading", resultDistro.name, qcowPath, resultDistro.sha256sum)
+			err = errors.New("some fake non-nil error to force a redownload")
+
+			if err := os.Remove(qcowPath); err != nil {
+				t.Fatalf("can't delete wrong cached image: %v", err)
+			}
+		}
+	}
+
 	if err != nil {
 		t.Logf("downloading distro image %s to %s", resultDistro.url, qcowPath)
 		fout, err := os.Create(qcowPath)
 		if err != nil {
 			t.Fatal(err)
 		}
-		resp, err := http.Get(resultDistro.url)
-		if err != nil {
-			t.Fatalf("can't fetch qcow2 for %s (%s): %v", resultDistro.name, resultDistro.url, err)
-		}
 
-		if resp.StatusCode != http.StatusOK {
+		if !fetchFromS3(t, fout, resultDistro) {
+			resp, err := http.Get(resultDistro.url)
+			if err != nil {
+				t.Fatalf("can't fetch qcow2 for %s (%s): %v", resultDistro.name, resultDistro.url, err)
+			}
+
+			if resp.StatusCode != http.StatusOK {
+				resp.Body.Close()
+				t.Fatalf("%s replied %s", resultDistro.url, resp.Status)
+			}
+
+			_, err = io.Copy(fout, resp.Body)
 			resp.Body.Close()
-			t.Fatalf("%s replied %s", resultDistro.url, resp.Status)
-		}
+			if err != nil {
+				t.Fatalf("download of %s failed: %v", resultDistro.url, err)
+			}
 
-		_, err = io.Copy(fout, resp.Body)
-		resp.Body.Close()
-		if err != nil {
-			t.Fatalf("download of %s failed: %v", resultDistro.url, err)
-		}
+			hash := checkCachedImageHash(t, resultDistro, cdir)
 
-		err = fout.Close()
-		if err != nil {
-			t.Fatalf("can't close fout: %v", err)
+			if hash != resultDistro.sha256sum {
+				t.Fatalf("hash mismatch, want: %s, got: %s", resultDistro.sha256sum, hash)
+			}
 		}
-
-		fin, err := os.Open(qcowPath)
-		if err != nil {
-			t.Fatal(err)
-		}
-
-		hasher := sha256.New()
-		if _, err := io.Copy(hasher, fin); err != nil {
-			t.Fatal(err)
-		}
-		hash := hex.EncodeToString(hasher.Sum(nil))
-
-		if hash != resultDistro.sha256sum {
-			t.Logf("got:  %q", hash)
-			t.Logf("want: %q", resultDistro.sha256sum)
-			t.Fatal("hash mismatch, someone is doing something nasty")
-		}
-
-		t.Logf("hash check passed (%s)", resultDistro.sha256sum)
 	}
 }
 
+func checkCachedImageHash(t *testing.T, d Distro, cacheDir string) (gotHash string) {
+	t.Helper()
+
+	qcowPath := filepath.Join(cacheDir, "qcow2", d.sha256sum)
+
+	fin, err := os.Open(qcowPath)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	hasher := sha256.New()
+	if _, err := io.Copy(hasher, fin); err != nil {
+		t.Fatal(err)
+	}
+	hash := hex.EncodeToString(hasher.Sum(nil))
+
+	if hash != d.sha256sum {
+		t.Fatalf("hash mismatch, got: %q, want: %q", hash, d.sha256sum)
+	}
+
+	gotHash = hash
+
+	return
+}
+
 // run runs a command or fails the test.
 func run(t *testing.T, dir, prog string, args ...string) {
 	t.Helper()
 	t.Logf("running: %s %s", prog, strings.Join(args, " "))
+	tstest.FixLogs(t)
 
 	cmd := exec.Command(prog, args...)
-	cmd.Stdout = os.Stdout
-	cmd.Stderr = os.Stderr
+	cmd.Stdout = logger.FuncWriter(t.Logf)
+	cmd.Stderr = logger.FuncWriter(t.Logf)
 	cmd.Dir = dir
 	if err := cmd.Run(); err != nil {
 		t.Fatal(err)
 	}
 }
 
-// mkLayeredQcow makes a layered qcow image that allows us to keep the upstream VM images
-// pristine and only do our changes on an overlay.
+// mkLayeredQcow makes a layered qcow image that allows us to keep the upstream
+// VM images pristine and only do our changes on an overlay.
 func mkLayeredQcow(t *testing.T, tdir string, d Distro) {
 	t.Helper()
 
@@ -150,7 +326,13 @@ func mkLayeredQcow(t *testing.T, tdir string, d Distro) {
 	)
 }
 
-// mkSeed makes the cloud-init seed ISO that is used to configure a VM with tailscale.
+var (
+	metaDataTempl = template.Must(template.New("meta-data.yaml").Parse(metaDataTemplate))
+	userDataTempl = template.Must(template.New("user-data.yaml").Parse(userDataTemplate))
+)
+
+// mkSeed makes the cloud-init seed ISO that is used to configure a VM with
+// tailscale.
 func mkSeed(t *testing.T, d Distro, sshKey, hostURL, tdir string, port int) {
 	t.Helper()
 
@@ -164,7 +346,7 @@ func mkSeed(t *testing.T, d Distro, sshKey, hostURL, tdir string, port int) {
 			t.Fatal(err)
 		}
 
-		err = template.Must(template.New("meta-data.yaml").Parse(metaDataTemplate)).Execute(fout, struct {
+		err = metaDataTempl.Execute(fout, struct {
 			ID       string
 			Hostname string
 		}{
@@ -188,18 +370,20 @@ func mkSeed(t *testing.T, d Distro, sshKey, hostURL, tdir string, port int) {
 			t.Fatal(err)
 		}
 
-		err = template.Must(template.New("user-data.yaml").Parse(userDataTemplate)).Execute(fout, struct {
+		err = userDataTempl.Execute(fout, struct {
 			SSHKey     string
 			HostURL    string
 			Hostname   string
 			Port       int
 			InstallPre string
+			Password   string
 		}{
 			SSHKey:     strings.TrimSpace(sshKey),
 			HostURL:    hostURL,
 			Hostname:   d.name,
 			Port:       port,
 			InstallPre: d.InstallPre(),
+			Password:   securePassword,
 		})
 		if err != nil {
 			t.Fatal(err)
@@ -211,17 +395,23 @@ func mkSeed(t *testing.T, d Distro, sshKey, hostURL, tdir string, port int) {
 		}
 	}
 
-	run(t, tdir, "genisoimage",
+	args := []string{
 		"-output", filepath.Join(dir, "seed.iso"),
 		"-volid", "cidata", "-joliet", "-rock",
 		filepath.Join(dir, "meta-data"),
 		filepath.Join(dir, "user-data"),
-	)
+	}
+
+	if hackOpenSUSE151UserData(t, d, dir) {
+		args = append(args, filepath.Join(dir, "openstack"))
+	}
+
+	run(t, tdir, "genisoimage", args...)
 }
 
-// mkVM makes a KVM-accelerated virtual machine and prepares it for introduction to the
-// testcontrol server. The function it returns is for killing the virtual machine when it
-// is time for it to die.
+// mkVM makes a KVM-accelerated virtual machine and prepares it for introduction
+// to the testcontrol server. The function it returns is for killing the virtual
+// machine when it is time for it to die.
 func mkVM(t *testing.T, n int, d Distro, sshKey, hostURL, tdir string) func() {
 	t.Helper()
 
@@ -229,9 +419,8 @@ func mkVM(t *testing.T, n int, d Distro, sshKey, hostURL, tdir string) func() {
 	if err != nil {
 		t.Fatalf("can't find cache dir: %v", err)
 	}
-	cdir = filepath.Join(cdir, "within", "mkvm")
+	cdir = filepath.Join(cdir, "tailscale", "vm-test")
 	os.MkdirAll(filepath.Join(cdir, "qcow2"), 0755)
-	os.MkdirAll(filepath.Join(cdir, "seed"), 0755)
 
 	port := 23100 + n
 
@@ -250,12 +439,16 @@ func mkVM(t *testing.T, n int, d Distro, sshKey, hostURL, tdir string) func() {
 		"-drive", driveArg,
 		"-cdrom", filepath.Join(tdir, d.name, "seed", "seed.iso"),
 		"-vnc", fmt.Sprintf(":%d", n),
+		"-smbios", "type=1,serial=ds=nocloud;h=" + d.name,
 	}
 
 	t.Logf("running: qemu-system-x86_64 %s", strings.Join(args, " "))
 
 	cmd := exec.Command("qemu-system-x86_64", args...)
+	cmd.Stdout = logger.FuncWriter(t.Logf)
+	cmd.Stderr = logger.FuncWriter(t.Logf)
 	err = cmd.Start()
+
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -271,44 +464,42 @@ func mkVM(t *testing.T, n int, d Distro, sshKey, hostURL, tdir string) func() {
 		if err != nil {
 			t.Errorf("can't kill %s (%d): %v", d.name, cmd.Process.Pid, err)
 		}
+
+		cmd.Wait()
 	}
 }
 
-// TestVMIntegrationEndToEnd creates a virtual machine with mkvm(1X), installs tailscale on it and then ensures that it connects to the network successfully.
+// ipMapping maps a hostname, SSH port and SSH IP together
+type ipMapping struct {
+	name string
+	port int
+	ip   string
+}
+
+// TestVMIntegrationEndToEnd creates a virtual machine with qemu, installs
+// tailscale on it and then ensures that it connects to the network
+// successfully.
 func TestVMIntegrationEndToEnd(t *testing.T) {
 	if !*runVMTests {
-		t.Skip("not running integration tests (need -run-vm-tests)")
+		t.Skip("not running integration tests (need --run-vm-tests)")
 	}
 
+	os.Setenv("CGO_ENABLED", "0")
+
 	if _, err := exec.LookPath("qemu-system-x86_64"); err != nil {
-		t.Logf("hint: nix-shell -p go -p qemu -p cdrkit --run 'go test -v -timeout=60m -run-vm-tests'")
+		t.Logf("hint: nix-shell -p go -p qemu -p cdrkit --run 'go test --v --timeout=60m --run-vm-tests'")
 		t.Fatalf("missing dependency: %v", err)
 	}
 
 	if _, err := exec.LookPath("genisoimage"); err != nil {
-		t.Logf("hint: nix-shell -p go -p qemu -p cdrkit --run 'go test -v -timeout=60m -run-vm-tests'")
+		t.Logf("hint: nix-shell -p go -p qemu -p cdrkit --run 'go test --v --timeout=60m --run-vm-tests'")
 		t.Fatalf("missing dependency: %v", err)
 	}
 
-	distros := []Distro{
-		{"amazon-linux", "https://cdn.amazonlinux.com/os-images/2.0.20210427.0/kvm/amzn2-kvm-2.0.20210427.0-x86_64.xfs.gpt.qcow2", "6ef9daef32cec69b2d0088626ec96410cd24afc504d57278bbf2f2ba2b7e529b", 512, "yum"},
-		{"centos-7", "https://cloud.centos.org/centos/7/images/CentOS-7-x86_64-GenericCloud.qcow2", "1db30c9c272fb37b00111b93dcebff16c278384755bdbe158559e9c240b73b80", 512, "yum"},
-		{"centos-8", "https://cloud.centos.org/centos/8/x86_64/images/CentOS-8-GenericCloud-8.3.2011-20201204.2.x86_64.qcow2", "7ec97062618dc0a7ebf211864abf63629da1f325578868579ee70c495bed3ba0", 768, "dnf"},
-		{"debian-9", "https://cdimage.debian.org/cdimage/openstack/9.13.21-20210511/debian-9.13.21-20210511-openstack-amd64.qcow2", "0667a08e2d947b331aee068db4bbf3a703e03edaf5afa52e23d534adff44b62a", 512, "apt"},
-		{"debian-10", "https://cdimage.debian.org/images/cloud/buster/20210329-591/debian-10-generic-amd64-20210329-591.qcow2", "70c61956095870c4082103d1a7a1cb5925293f8405fc6cb348588ec97e8611b0", 768, "apt"},
-		{"fedora-34", "https://download.fedoraproject.org/pub/fedora/linux/releases/34/Cloud/x86_64/images/Fedora-Cloud-Base-34-1.2.x86_64.qcow2", "b9b621b26725ba95442d9a56cbaa054784e0779a9522ec6eafff07c6e6f717ea", 768, "dnf"},
-		{"opensuse-leap-15.1", "https://download.opensuse.org/repositories/Cloud:/Images:/Leap_15.1/images/openSUSE-Leap-15.1-OpenStack.x86_64.qcow2", "3203e256dab5981ca3301408574b63bc522a69972fbe9850b65b54ff44a96e0a", 512, "zypper"},
-		{"opensuse-leap-15.2", "https://download.opensuse.org/repositories/Cloud:/Images:/Leap_15.2/images/openSUSE-Leap-15.2-OpenStack.x86_64.qcow2", "4df9cee9281d1f57d20f79dc65d76e255592b904760e73c0dd44ac753a54330f", 512, "zypper"},
-		{"opensuse-tumbleweed", "https://download.opensuse.org/tumbleweed/appliances/openSUSE-Tumbleweed-JeOS.x86_64-OpenStack-Cloud.qcow2", "ba3ecd281045b5019f0fb11378329a644a41870b77631ea647b128cd07eb804b", 512, "zypper"},
-		{"ubuntu-16-04", "https://cloud-images.ubuntu.com/xenial/current/xenial-server-cloudimg-amd64-disk1.img", "50a21bc067c05e0c73bf5d8727ab61152340d93073b3dc32eff18b626f7d813b", 512, "apt"},
-		{"ubuntu-18-04", "https://cloud-images.ubuntu.com/bionic/current/bionic-server-cloudimg-amd64.img", "08396cf95c18534a2e3f88289bd92d18eee76f0e75813636b3ab9f1e603816d7", 512, "apt"},
-		{"ubuntu-20-04", "https://cloud-images.ubuntu.com/focal/current/focal-server-cloudimg-amd64.img", "513158b22ff0f08d0a078d8d60293bcddffdb17094a7809c76c52aba415ecc54", 512, "apt"},
-		{"ubuntu-20-10", "https://cloud-images.ubuntu.com/groovy/current/groovy-server-cloudimg-amd64.img", "e470df72fce4fb8d0ee4ef8af8eed740ee3bf51290515eb42e5c747725e98b6d", 512, "apt"},
-		{"ubuntu-21-04", "https://cloud-images.ubuntu.com/hirsute/current/hirsute-server-cloudimg-amd64.img", "7fab8eda0bcf6f8f6e63845ccf1e29de4706e3359c82d3888835093020fe6f05", 512, "apt"},
-	}
-
 	dir := t.TempDir()
 
+	rex := distroRex.Unwrap()
+
 	ln, err := net.Listen("tcp", deriveBindhost(t)+":0")
 	if err != nil {
 		t.Fatalf("can't make TCP listener: %v", err)
@@ -320,7 +511,7 @@ func TestVMIntegrationEndToEnd(t *testing.T) {
 
 	var (
 		ipMu  sync.Mutex
-		ipMap = map[string]string{} // SSH port => IP address
+		ipMap = map[string]ipMapping{}
 	)
 
 	mux := http.NewServeMux()
@@ -336,7 +527,12 @@ func TestVMIntegrationEndToEnd(t *testing.T) {
 
 		name := path.Base(r.URL.Path)
 		host, _, _ := net.SplitHostPort(r.RemoteAddr)
-		ipMap[name] = host
+		port, err := strconv.Atoi(name)
+		if err != nil {
+			log.Panicf("bad port: %v", port)
+		}
+		distro := r.UserAgent()
+		ipMap[distro] = ipMapping{distro, port, host}
 		t.Logf("%s: %v", name, host)
 	})
 
@@ -362,139 +558,257 @@ func TestVMIntegrationEndToEnd(t *testing.T) {
 	loginServer := fmt.Sprintf("http://%s", ln.Addr())
 	t.Logf("loginServer: %s", loginServer)
 
-	cancels := make(chan func(), len(distros))
+	ramsem := semaphore.NewWeighted(int64(*vmRamLimit))
+	bins := integration.BuildTestBinaries(t)
 
-	t.Run("mkvm", func(t *testing.T) {
+	t.Run("do", func(t *testing.T) {
 		for n, distro := range distros {
 			n, distro := n, distro
+			if rex.MatchString(distro.name) {
+				t.Logf("%s matches %s", distro.name, rex)
+			} else {
+				continue
+			}
+
 			t.Run(distro.name, func(t *testing.T) {
+				ctx, done := context.WithCancel(context.Background())
+				defer done()
+
 				t.Parallel()
 
+				err := ramsem.Acquire(ctx, int64(distro.mem))
+				if err != nil {
+					t.Fatalf("can't acquire ram semaphore: %v", err)
+				}
+				defer ramsem.Release(int64(distro.mem))
+
 				cancel := mkVM(t, n, distro, string(pubkey), loginServer, dir)
-				cancels <- cancel
+				defer cancel()
+				var ipm ipMapping
+
+				t.Run("wait-for-start", func(t *testing.T) {
+					waiter := time.NewTicker(time.Second)
+					defer waiter.Stop()
+					var ok bool
+					for {
+						<-waiter.C
+						ipMu.Lock()
+						if ipm, ok = ipMap[distro.name]; ok {
+							ipMu.Unlock()
+							break
+						}
+						ipMu.Unlock()
+					}
+				})
+
+				testDistro(t, loginServer, distro, signer, ipm, bins)
 			})
 		}
 	})
+}
 
-	close(cancels)
-	for cancel := range cancels {
-		//lint:ignore SA9001 They do actually get ran
-		defer cancel()
-
-		if len(cancels) == 0 {
-			t.Log("all VMs started")
-			break
-		}
+func testDistro(t *testing.T, loginServer string, d Distro, signer ssh.Signer, ipm ipMapping, bins *integration.Binaries) {
+	t.Helper()
+	port := ipm.port
+	hostport := fmt.Sprintf("127.0.0.1:%d", port)
+	ccfg := &ssh.ClientConfig{
+		User:            "root",
+		Auth:            []ssh.AuthMethod{ssh.PublicKeys(signer), ssh.Password(securePassword)},
+		HostKeyCallback: ssh.InsecureIgnoreHostKey(),
 	}
 
-	t.Run("wait-for-vms", func(t *testing.T) {
-		t.Log("waiting for VMs to register")
-		waiter := time.NewTicker(time.Second)
-		defer waiter.Stop()
-		n := 0
-		for {
-			<-waiter.C
-			ipMu.Lock()
-			if len(ipMap) == len(distros) {
-				ipMu.Unlock()
-				break
-			} else {
-				if n%30 == 0 {
-					t.Logf("ipMap:   %d", len(ipMap))
-					t.Logf("distros: %d", len(distros))
-				}
-			}
-			n++
-			ipMu.Unlock()
+	// NOTE(Xe): This deadline loop helps to make things a bit faster, centos
+	// sometimes is slow at starting its sshd and will sometimes randomly kill
+	// SSH sessions on transition to multi-user.target. I don't know why they
+	// don't use socket activation.
+	const maxRetries = 5
+	var working bool
+	for i := 0; i < maxRetries; i++ {
+		cli, err := ssh.Dial("tcp", hostport, ccfg)
+		if err == nil {
+			working = true
+			cli.Close()
+			break
 		}
-	})
 
-	ipMu.Lock()
-	defer ipMu.Unlock()
-	t.Run("join-net", func(t *testing.T) {
-		for port := range ipMap {
-			port := port
-			t.Run(port, func(t *testing.T) {
-				config := &ssh.ClientConfig{
-					User:            "ts",
-					Auth:            []ssh.AuthMethod{ssh.PublicKeys(signer), ssh.Password("hunter2")},
-					HostKeyCallback: ssh.InsecureIgnoreHostKey(),
-				}
+		time.Sleep(10 * time.Second)
+	}
 
-				cli, err := ssh.Dial("tcp", fmt.Sprintf("127.0.0.1:%s", port), config)
-				if err != nil {
-					t.Fatalf("can't dial 127.0.0.1:%s: %v", port, err)
-				}
-				defer cli.Close()
+	if !working {
+		t.Fatalf("can't connect to %s, tried %d times", hostport, maxRetries)
+	}
 
-				t.Parallel()
-				t.Logf("about to ssh into 127.0.0.1:%s", port)
-				timeout := 5 * time.Minute
+	t.Logf("about to ssh into 127.0.0.1:%d", port)
+	cli, err := ssh.Dial("tcp", hostport, ccfg)
+	if err != nil {
+		t.Fatal(err)
+	}
+	copyBinaries(t, d, cli, bins)
 
-				e, _, err := expect.SpawnSSH(cli, timeout, expect.Verbose(true), expect.VerboseWriter(os.Stdout))
-				if err != nil {
-					t.Fatalf("%s: can't register a shell session: %v", port, err)
-				}
-				defer e.Close()
+	timeout := 30 * time.Second
 
-				_, _, err = e.Expect(regexp.MustCompile(`(\$|\>)`), timeout)
-				if err != nil {
-					t.Fatalf("%s: can't get a shell: %v", port, err)
-				}
-				t.Logf("got shell for %s", port)
-				err = e.Send(fmt.Sprintf("sudo tailscale up --login-server %s\n", loginServer))
-				if err != nil {
-					t.Fatalf("%s: can't send tailscale up command: %v", port, err)
-				}
-				_, _, err = e.Expect(regexp.MustCompile(`Success.`), timeout)
-				if err != nil {
-					t.Fatalf("can't extract URL: %v", err)
-				}
-			})
-		}
-	})
+	e, _, err := expect.SpawnSSH(cli, timeout,
+		expect.Verbose(true),
+		expect.VerboseWriter(logger.FuncWriter(t.Logf)),
 
-	if numNodes := cs.NumNodes(); numNodes != len(ipMap) {
-		t.Errorf("wanted %d nodes, got: %d", len(ipMap), numNodes)
+		// // NOTE(Xe): if you get a timeout, uncomment this line to have the raw
+		// output be sent to the test log quicker.
+		//expect.Tee(nopWriteCloser{logger.FuncWriter(t.Logf)}),
+	)
+	if err != nil {
+		t.Fatalf("%d: can't register a shell session: %v", port, err)
+	}
+	defer e.Close()
+
+	t.Log("opened session")
+
+	_, _, err = e.Expect(regexp.MustCompile(`(\#)`), timeout)
+	if err != nil {
+		t.Fatalf("%d: can't get a shell: %v", port, err)
+	}
+	t.Logf("got shell for %d", port)
+	switch d.initSystem {
+	case "openrc":
+		// NOTE(Xe): this is a sin, however openrc doesn't really have the concept
+		// of service readiness. If this sleep is removed then tailscale will not be
+		// ready once the `tailscale up` command is sent. This is not ideal, but I
+		// am not really sure there is a good way around this without a delay of
+		// some kind.
+		err = e.Send("rc-service tailscaled start && sleep 2\n")
+	case "systemd":
+		err = e.Send("systemctl start tailscaled.service\n")
+	}
+	if err != nil {
+		t.Fatalf("can't send command to start tailscaled: %v", err)
+	}
+	_, _, err = e.Expect(regexp.MustCompile(`(\#)`), timeout)
+	if err != nil {
+		t.Fatalf("%d: can't get a shell: %v", port, err)
+	}
+	err = e.Send(fmt.Sprintf("tailscale up --login-server %s\n", loginServer))
+	if err != nil {
+		t.Fatalf("%d: can't send tailscale up command: %v", port, err)
+	}
+	_, _, err = e.Expect(regexp.MustCompile(`Success.`), timeout)
+	if err != nil {
+		t.Fatalf("not successful: %v", err)
+	}
+}
+
+func copyBinaries(t *testing.T, d Distro, conn *ssh.Client, bins *integration.Binaries) {
+	cli, err := sftp.NewClient(conn)
+	if err != nil {
+		t.Fatalf("can't connect over sftp to copy binaries: %v", err)
+	}
+
+	mkdir(t, cli, "/usr/bin")
+	mkdir(t, cli, "/usr/sbin")
+	mkdir(t, cli, "/etc/default")
+	mkdir(t, cli, "/var/lib/tailscale")
+
+	copyFile(t, cli, bins.Daemon, "/usr/sbin/tailscaled")
+	copyFile(t, cli, bins.CLI, "/usr/bin/tailscale")
+
+	// TODO(Xe): revisit this assumption before it breaks the test.
+	copyFile(t, cli, "../../../cmd/tailscaled/tailscaled.defaults", "/etc/default/tailscaled")
+
+	switch d.initSystem {
+	case "openrc":
+		mkdir(t, cli, "/etc/init.d")
+		copyFile(t, cli, "../../../cmd/tailscaled/tailscaled.openrc", "/etc/init.d/tailscaled")
+	case "systemd":
+		mkdir(t, cli, "/etc/systemd/system")
+		copyFile(t, cli, "../../../cmd/tailscaled/tailscaled.service", "/etc/systemd/system/tailscaled.service")
+	}
+
+	t.Log("tailscale installed!")
+}
+
+func mkdir(t *testing.T, cli *sftp.Client, name string) {
+	t.Helper()
+
+	err := cli.MkdirAll(name)
+	if err != nil {
+		t.Fatalf("can't make %s: %v", name, err)
+	}
+}
+
+func copyFile(t *testing.T, cli *sftp.Client, localSrc, remoteDest string) {
+	t.Helper()
+
+	fin, err := os.Open(localSrc)
+	if err != nil {
+		t.Fatalf("can't open: %v", err)
+	}
+	defer fin.Close()
+
+	fi, err := fin.Stat()
+	if err != nil {
+		t.Fatalf("can't stat: %v", err)
+	}
+
+	fout, err := cli.Create(remoteDest)
+	if err != nil {
+		t.Fatalf("can't create output file: %v", err)
+	}
+
+	err = fout.Chmod(fi.Mode())
+	if err != nil {
+		fout.Close()
+		t.Fatalf("can't chmod fout: %v", err)
+	}
+
+	n, err := io.Copy(fout, fin)
+	if err != nil {
+		fout.Close()
+		t.Fatalf("copy failed: %v", err)
+	}
+
+	if fi.Size() != n {
+		t.Fatalf("incorrect number of bytes copied: wanted: %d, got: %d", fi.Size(), n)
+	}
+
+	err = fout.Close()
+	if err != nil {
+		t.Fatalf("can't close fout on remote host: %v", err)
 	}
 }
 
 func deriveBindhost(t *testing.T) string {
 	t.Helper()
 
-	ifaces, err := net.Interfaces()
+	ifName, err := interfaces.DefaultRouteInterface()
 	if err != nil {
 		t.Fatal(err)
 	}
 
-	rex := regexp.MustCompile(`^(eth|enp|wlp|wlan)`)
-
-	for _, iface := range ifaces {
-		t.Logf("found interface %s: %d", iface.Name, iface.Flags&net.FlagUp)
-		if (iface.Flags & net.FlagUp) == 0 {
-			continue
-		}
-
-		if rex.MatchString(iface.Name) {
-			addrs, err := iface.Addrs()
-			if err != nil {
-				t.Fatalf("can't get address for %s: %v", iface.Name, err)
-			}
-
-			for _, addr := range addrs {
-				return netaddr.MustParseIPPrefix(addr.String()).IP().String()
-			}
+	var ret string
+	err = interfaces.ForeachInterfaceAddress(func(i interfaces.Interface, prefix netaddr.IPPrefix) {
+		if ret != "" || i.Name != ifName {
+			return
 		}
+		ret = prefix.IP().String()
+	})
+	if ret != "" {
+		return ret
+	}
+	if err != nil {
+		t.Fatal(err)
 	}
-
 	t.Fatal("can't find a bindhost")
-	return "invalid"
+	return "unreachable"
 }
 
 func TestDeriveBindhost(t *testing.T) {
 	t.Log(deriveBindhost(t))
 }
 
+type nopWriteCloser struct {
+	io.Writer
+}
+
+func (nwc nopWriteCloser) Close() error { return nil }
+
 const metaDataTemplate = `instance-id: {{.ID}}
 local-hostname: {{.Hostname}}`
 
@@ -509,12 +823,15 @@ cloud_final_modules:
  - [scripts-user, once-per-instance]
 
 users:
-  - name: ts
-    plain_text_passwd: hunter2
-    groups: [ wheel ]
-    sudo: [ "ALL=(ALL) NOPASSWD:ALL" ]
-    shell: /bin/sh
-    ssh-authorized-keys:
+ - name: root
+   ssh-authorized-keys:
+    - {{.SSHKey}}
+ - name: ts
+   plain_text_passwd: {{.Password}}
+   groups: [ wheel ]
+   sudo: [ "ALL=(ALL) NOPASSWD:ALL" ]
+   shell: /bin/sh
+   ssh-authorized-keys:
     - {{.SSHKey}}
 
 write_files:
@@ -526,7 +843,5 @@ write_files:
 
 runcmd:
 {{.InstallPre}}
- - [ "sh", "-c", "curl https://raw.githubusercontent.com/tailscale/tailscale/Xe/test-install-script-libvirtd/scripts/installer.sh | sh" ]
- - [ systemctl, enable, --now, tailscaled.service ]
  - [ curl, "{{.HostURL}}/myip/{{.Port}}", "-H", "User-Agent: {{.Hostname}}" ]
 `

tstest/natlab/natlab.go

@@ -2,9 +2,6 @@
 // Use of this source code is governed by a BSD-style
 // license that can be found in the LICENSE file.
 
-//lint:file-ignore U1000 in development
-//lint:file-ignore S1000 in development
-
 // Package natlab lets us simulate different types of networks all
 // in-memory without running VMs or requiring root, etc. Despite the
 // name, it does more than just NATs. But NATs are the most

tstest/resource.go

@@ -15,19 +15,19 @@ import (
 )
 
 func ResourceCheck(tb testing.TB) {
+	tb.Helper()
 	startN, startStacks := goroutines()
 	tb.Cleanup(func() {
 		if tb.Failed() {
 			// Something else went wrong.
 			return
 		}
-		tb.Helper()
 		// Goroutines might be still exiting.
 		for i := 0; i < 100; i++ {
 			if runtime.NumGoroutine() <= startN {
 				return
 			}
-			time.Sleep(1 * time.Millisecond)
+			time.Sleep(5 * time.Millisecond)
 		}
 		endN, endStacks := goroutines()
 		tb.Logf("goroutine diff:\n%v\n", cmp.Diff(startStacks, endStacks))

tsweb/debug.go

@@ -0,0 +1,136 @@
+// Copyright (c) 2021 Tailscale Inc & AUTHORS All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package tsweb
+
+import (
+	"expvar"
+	"fmt"
+	"html"
+	"io"
+	"net/http"
+	"net/http/pprof"
+	"net/url"
+	"os"
+	"runtime"
+
+	"tailscale.com/version"
+)
+
+// DebugHandler is an http.Handler that serves a debugging "homepage",
+// and provides helpers to register more debug endpoints and reports.
+//
+// The rendered page consists of three sections: informational
+// key/value pairs, links to other pages, and additional
+// program-specific HTML. Callers can add to these sections using the
+// KV, URL and Section helpers respectively.
+//
+// Additionally, the Handle method offers a shorthand for correctly
+// registering debug handlers and cross-linking them from /debug/.
+type DebugHandler struct {
+	mux      *http.ServeMux                   // where this handler is registered
+	kvs      []func(io.Writer)                // output one <li>...</li> each, see KV()
+	urls     []string                         // one <li>...</li> block with link each
+	sections []func(io.Writer, *http.Request) // invoked in registration order prior to outputting </body>
+}
+
+// Debugger returns the DebugHandler registered on mux at /debug/,
+// creating it if necessary.
+func Debugger(mux *http.ServeMux) *DebugHandler {
+	h, pat := mux.Handler(&http.Request{URL: &url.URL{Path: "/debug/"}})
+	if d, ok := h.(*DebugHandler); ok && pat == "/debug/" {
+		return d
+	}
+	ret := &DebugHandler{
+		mux: mux,
+	}
+	mux.Handle("/debug/", ret)
+
+	ret.KVFunc("Uptime", func() interface{} { return Uptime() })
+	ret.KV("Version", version.Long)
+	ret.Handle("vars", "Metrics (Go)", expvar.Handler())
+	ret.Handle("varz", "Metrics (Prometheus)", http.HandlerFunc(VarzHandler))
+	ret.Handle("pprof/", "pprof", http.HandlerFunc(pprof.Index))
+	ret.URL("/debug/pprof/goroutine?debug=1", "Goroutines (collapsed)")
+	ret.URL("/debug/pprof/goroutine?debug=2", "Goroutines (full)")
+	ret.Handle("gc", "force GC", http.HandlerFunc(gcHandler))
+	hostname, err := os.Hostname()
+	if err == nil {
+		ret.KV("Machine", hostname)
+	}
+	return ret
+}
+
+// ServeHTTP implements http.Handler.
+func (d *DebugHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
+	if !AllowDebugAccess(r) {
+		http.Error(w, "debug access denied", http.StatusForbidden)
+		return
+	}
+	if r.URL.Path != "/debug/" {
+		// Sub-handlers are handled by the parent mux directly.
+		http.NotFound(w, r)
+		return
+	}
+
+	f := func(format string, args ...interface{}) { fmt.Fprintf(w, format, args...) }
+	f("<html><body><h1>%s debug</h1><ul>", version.CmdName())
+	for _, kv := range d.kvs {
+		kv(w)
+	}
+	for _, url := range d.urls {
+		io.WriteString(w, url)
+	}
+	for _, section := range d.sections {
+		section(w, r)
+	}
+}
+
+// Handle registers handler at /debug/<slug> and creates a descriptive
+// entry in /debug/ for it.
+func (d *DebugHandler) Handle(slug, desc string, handler http.Handler) {
+	href := "/debug/" + slug
+	d.mux.Handle(href, Protected(handler))
+	d.URL(href, desc)
+}
+
+// KV adds a key/value list item to /debug/.
+func (d *DebugHandler) KV(k string, v interface{}) {
+	val := html.EscapeString(fmt.Sprintf("%v", v))
+	d.kvs = append(d.kvs, func(w io.Writer) {
+		fmt.Fprintf(w, "<li><b>%s:</b> %s</li>", k, val)
+	})
+}
+
+// KVFunc adds a key/value list item to /debug/. v is called on every
+// render of /debug/.
+func (d *DebugHandler) KVFunc(k string, v func() interface{}) {
+	d.kvs = append(d.kvs, func(w io.Writer) {
+		val := html.EscapeString(fmt.Sprintf("%v", v()))
+		fmt.Fprintf(w, "<li><b>%s:</b> %s</li>", k, val)
+	})
+}
+
+// URL adds a URL and description list item to /debug/.
+func (d *DebugHandler) URL(url, desc string) {
+	if desc != "" {
+		desc = " (" + desc + ")"
+	}
+	d.urls = append(d.urls, fmt.Sprintf(`<li><a href="%s">%s</a>%s</li>`, url, url, html.EscapeString(desc)))
+}
+
+// Section invokes f on every render of /debug/ to add supplemental
+// HTML to the page body.
+func (d *DebugHandler) Section(f func(w io.Writer, r *http.Request)) {
+	d.sections = append(d.sections, f)
+}
+
+func gcHandler(w http.ResponseWriter, r *http.Request) {
+	w.Write([]byte("running GC...\n"))
+	if f, ok := w.(http.Flusher); ok {
+		f.Flush()
+	}
+	runtime.GC()
+	w.Write([]byte("Done.\n"))
+}

tsweb/debug_test.go

@@ -0,0 +1,189 @@
+// Copyright (c) 2021 Tailscale Inc & AUTHORS All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package tsweb
+
+import (
+	"fmt"
+	"io"
+	"net/http"
+	"net/http/httptest"
+	"strings"
+	"testing"
+)
+
+func TestDebugger(t *testing.T) {
+	mux := http.NewServeMux()
+
+	dbg1 := Debugger(mux)
+	if dbg1 == nil {
+		t.Fatal("didn't get a debugger from mux")
+	}
+
+	dbg2 := Debugger(mux)
+	if dbg2 != dbg1 {
+		t.Fatal("Debugger returned different debuggers for the same mux")
+	}
+}
+
+func get(m http.Handler, path, srcIP string) (int, string) {
+	req := httptest.NewRequest("GET", path, nil)
+	req.RemoteAddr = srcIP + ":1234"
+	rec := httptest.NewRecorder()
+	m.ServeHTTP(rec, req)
+	return rec.Result().StatusCode, rec.Body.String()
+}
+
+const (
+	tsIP  = "100.100.100.100"
+	pubIP = "8.8.8.8"
+)
+
+func TestDebuggerKV(t *testing.T) {
+	mux := http.NewServeMux()
+	dbg := Debugger(mux)
+	dbg.KV("Donuts", 42)
+	dbg.KV("Secret code", "hunter2")
+	val := "red"
+	dbg.KVFunc("Condition", func() interface{} { return val })
+
+	code, _ := get(mux, "/debug/", pubIP)
+	if code != 403 {
+		t.Fatalf("debug access wasn't denied, got %v", code)
+	}
+
+	code, body := get(mux, "/debug/", tsIP)
+	if code != 200 {
+		t.Fatalf("debug access failed, got %v", code)
+	}
+	for _, want := range []string{"Donuts", "42", "Secret code", "hunter2", "Condition", "red"} {
+		if !strings.Contains(body, want) {
+			t.Errorf("want %q in output, not found", want)
+		}
+	}
+
+	val = "green"
+	code, body = get(mux, "/debug/", tsIP)
+	if code != 200 {
+		t.Fatalf("debug access failed, got %v", code)
+	}
+	for _, want := range []string{"Condition", "green"} {
+		if !strings.Contains(body, want) {
+			t.Errorf("want %q in output, not found", want)
+		}
+	}
+}
+
+func TestDebuggerURL(t *testing.T) {
+	mux := http.NewServeMux()
+	dbg := Debugger(mux)
+	dbg.URL("https://www.tailscale.com", "Homepage")
+
+	code, body := get(mux, "/debug/", tsIP)
+	if code != 200 {
+		t.Fatalf("debug access failed, got %v", code)
+	}
+	for _, want := range []string{"https://www.tailscale.com", "Homepage"} {
+		if !strings.Contains(body, want) {
+			t.Errorf("want %q in output, not found", want)
+		}
+	}
+}
+
+func TestDebuggerSection(t *testing.T) {
+	mux := http.NewServeMux()
+	dbg := Debugger(mux)
+	dbg.Section(func(w io.Writer, r *http.Request) {
+		fmt.Fprintf(w, "Test output %v", r.RemoteAddr)
+	})
+
+	code, body := get(mux, "/debug/", tsIP)
+	if code != 200 {
+		t.Fatalf("debug access failed, got %v", code)
+	}
+	want := `Test output 100.100.100.100:1234`
+	if !strings.Contains(body, want) {
+		t.Errorf("want %q in output, not found", want)
+	}
+}
+
+func TestDebuggerHandle(t *testing.T) {
+	mux := http.NewServeMux()
+	dbg := Debugger(mux)
+	dbg.Handle("check", "Consistency check", http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		fmt.Fprintf(w, "Test output %v", r.RemoteAddr)
+	}))
+
+	code, body := get(mux, "/debug/", tsIP)
+	if code != 200 {
+		t.Fatalf("debug access failed, got %v", code)
+	}
+	for _, want := range []string{"/debug/check", "Consistency check"} {
+		if !strings.Contains(body, want) {
+			t.Errorf("want %q in output, not found", want)
+		}
+	}
+
+	code, _ = get(mux, "/debug/check", pubIP)
+	if code != 403 {
+		t.Fatal("/debug/check should be protected, but isn't")
+	}
+
+	code, body = get(mux, "/debug/check", tsIP)
+	if code != 200 {
+		t.Fatal("/debug/check denied debug access")
+	}
+	want := "Test output " + tsIP
+	if !strings.Contains(body, want) {
+		t.Errorf("want %q in output, not found", want)
+	}
+}
+
+func ExampleDebugHandler_Handle() {
+	mux := http.NewServeMux()
+	dbg := Debugger(mux)
+	// Registers /debug/flushcache with the given handler, and adds a
+	// link to /debug/ with the description "Flush caches".
+	dbg.Handle("flushcache", "Flush caches", http.HandlerFunc(http.NotFound))
+}
+
+func ExampleDebugHandler_KV() {
+	mux := http.NewServeMux()
+	dbg := Debugger(mux)
+	// Adds two list items to /debug/, showing that the condition is
+	// red and there are 42 donuts.
+	dbg.KV("Conditon", "red")
+	dbg.KV("Donuts", 42)
+}
+
+func ExampleDebugHandler_KVFunc() {
+	mux := http.NewServeMux()
+	dbg := Debugger(mux)
+	// Adds an count of page renders to /debug/. Note this example
+	// isn't concurrency-safe.
+	views := 0
+	dbg.KVFunc("Debug pageviews", func() interface{} {
+		views = views + 1
+		return views
+	})
+	dbg.KV("Donuts", 42)
+}
+
+func ExampleDebugHandler_URL() {
+	mux := http.NewServeMux()
+	dbg := Debugger(mux)
+	// Links to the Tailscale website from /debug/.
+	dbg.URL("https://www.tailscale.com", "Homepage")
+}
+
+func ExampleDebugHandler_Section() {
+	mux := http.NewServeMux()
+	dbg := Debugger(mux)
+	// Adds a section to /debug/ that dumps the HTTP request of the
+	// visitor.
+	dbg.Section(func(w io.Writer, r *http.Request) {
+		io.WriteString(w, "<h3>Dump of your HTTP request</h3>")
+		fmt.Fprintf(w, "<code>%#v</code>", r)
+	})
+}

tsweb/tsweb.go

@@ -29,34 +29,13 @@ import (
 	"tailscale.com/types/logger"
 )
 
-// DevMode controls whether extra output in shown, for when the binary is being run in dev mode.
-var DevMode bool
-
-// NewMux returns a new ServeMux with debugHandler registered (and protected) at /debug/.
-func NewMux(debugHandler http.Handler) *http.ServeMux {
-	mux := http.NewServeMux()
-	registerCommonDebug(mux)
-	mux.Handle("/debug/", Protected(debugHandler))
-	return mux
-}
-
-func registerCommonDebug(mux *http.ServeMux) {
+func init() {
 	expvar.Publish("counter_uptime_sec", expvar.Func(func() interface{} { return int64(Uptime().Seconds()) }))
 	expvar.Publish("gauge_goroutines", expvar.Func(func() interface{} { return runtime.NumGoroutine() }))
-	mux.Handle("/debug/pprof/", Protected(http.DefaultServeMux)) // to net/http/pprof
-	mux.Handle("/debug/vars", Protected(http.DefaultServeMux))   // to expvar
-	mux.Handle("/debug/varz", Protected(http.HandlerFunc(VarzHandler)))
-	mux.Handle("/debug/gc", Protected(http.HandlerFunc(gcHandler)))
 }
 
-func gcHandler(w http.ResponseWriter, r *http.Request) {
-	w.Write([]byte("running GC...\n"))
-	if f, ok := w.(http.Flusher); ok {
-		f.Flush()
-	}
-	runtime.GC()
-	w.Write([]byte("Done.\n"))
-}
+// DevMode controls whether extra output in shown, for when the binary is being run in dev mode.
+var DevMode bool
 
 func DefaultCertDir(leafDir string) string {
 	cacheDir, err := os.UserCacheDir()
@@ -183,13 +162,6 @@ type HandlerOptions struct {
 	StatusCodeCounters *expvar.Map
 }
 
-// StdHandler converts a ReturnHandler into a standard http.Handler.
-// Handled requests are logged using logf, as are any errors. Errors
-// are handled as specified by the Handler interface.
-func StdHandler(h ReturnHandler, logf logger.Logf) http.Handler {
-	return StdHandlerOpts(h, HandlerOptions{Logf: logf, Now: time.Now})
-}
-
 // ReturnHandlerFunc is an adapter to allow the use of ordinary
 // functions as ReturnHandlers. If f is a function with the
 // appropriate signature, ReturnHandlerFunc(f) is a ReturnHandler that
@@ -201,22 +173,16 @@ func (f ReturnHandlerFunc) ServeHTTPReturn(w http.ResponseWriter, r *http.Reques
 	return f(w, r)
 }
 
-// StdHandlerNo200s is like StdHandler, but successfully handled HTTP
-// requests don't write an access log entry to logf.
-//
-// TODO(josharian): eliminate this and StdHandler in favor of StdHandlerOpts,
-// rename StdHandlerOpts to StdHandler. Will be a breaking API change.
-func StdHandlerNo200s(h ReturnHandler, logf logger.Logf) http.Handler {
-	return StdHandlerOpts(h, HandlerOptions{Logf: logf, Now: time.Now, Quiet200s: true})
-}
-
-// StdHandlerOpts converts a ReturnHandler into a standard http.Handler.
+// StdHandler converts a ReturnHandler into a standard http.Handler.
 // Handled requests are logged using opts.Logf, as are any errors.
 // Errors are handled as specified by the Handler interface.
-func StdHandlerOpts(h ReturnHandler, opts HandlerOptions) http.Handler {
+func StdHandler(h ReturnHandler, opts HandlerOptions) http.Handler {
 	if opts.Now == nil {
 		opts.Now = time.Now
 	}
+	if opts.Logf == nil {
+		opts.Logf = logger.Discard
+	}
 	return retHandler{h, opts}
 }
 

tsweb/tsweb_test.go

@@ -248,7 +248,7 @@ func TestStdHandler(t *testing.T) {
 			clock.Reset()
 
 			rec := noopHijacker{httptest.NewRecorder(), false}
-			h := StdHandlerOpts(test.rh, HandlerOptions{Logf: logf, Now: clock.Now})
+			h := StdHandler(test.rh, HandlerOptions{Logf: logf, Now: clock.Now})
 			h.ServeHTTP(&rec, test.r)
 			res := rec.Result()
 			if res.StatusCode != test.wantCode {
@@ -277,8 +277,7 @@ func BenchmarkLogNot200(b *testing.B) {
 		// Implicit 200 OK.
 		return nil
 	})
-	discardLogger := func(string, ...interface{}) {}
-	h := StdHandlerNo200s(rh, discardLogger)
+	h := StdHandler(rh, HandlerOptions{Quiet200s: true})
 	req := httptest.NewRequest("GET", "/", nil)
 	rw := new(httptest.ResponseRecorder)
 	for i := 0; i < b.N; i++ {
@@ -293,8 +292,7 @@ func BenchmarkLog(b *testing.B) {
 		// Implicit 200 OK.
 		return nil
 	})
-	discardLogger := func(string, ...interface{}) {}
-	h := StdHandler(rh, discardLogger)
+	h := StdHandler(rh, HandlerOptions{})
 	req := httptest.NewRequest("GET", "/", nil)
 	rw := new(httptest.ResponseRecorder)
 	for i := 0; i < b.N; i++ {

types/key/key.go

@@ -85,8 +85,6 @@ func (k Private) Public() Public {
 func (k Private) SharedSecret(pub Public) (ss [32]byte) {
 	apk := (*[32]byte)(&pub)
 	ask := (*[32]byte)(&k)
-	//lint:ignore SA1019 Code copied from wireguard-go, we aim for
-	//minimal changes from it.
 	curve25519.ScalarMult(&ss, ask, apk)
 	return ss
 }

util/dnsname/dnsname.go

@@ -21,46 +21,48 @@ const (
 type FQDN string
 
 func ToFQDN(s string) (FQDN, error) {
-	if isValidFQDN(s) {
-		return FQDN(s), nil
-	}
 	if len(s) == 0 || s == "." {
 		return FQDN("."), nil
 	}
 
-	if s[len(s)-1] == '.' {
-		s = s[:len(s)-1]
-	}
 	if s[0] == '.' {
 		s = s[1:]
 	}
-	if len(s) > maxNameLength {
+	raw := s
+	totalLen := len(s)
+	if s[len(s)-1] == '.' {
+		s = s[:len(s)-1]
+	} else {
+		totalLen += 1 // account for missing dot
+	}
+	if totalLen > maxNameLength {
 		return "", fmt.Errorf("%q is too long to be a DNS name", s)
 	}
 
-	fs := strings.Split(s, ".")
-	for _, f := range fs {
-		if !validLabel(f) {
-			return "", fmt.Errorf("%q is not a valid DNS label", f)
+	st := 0
+	for i := 0; i < len(s); i++ {
+		if s[i] != '.' {
+			continue
 		}
+		label := s[st:i]
+		// You might be tempted to do further validation of the
+		// contents of labels here, based on the hostname rules in RFC
+		// 1123. However, DNS labels are not always subject to
+		// hostname rules. In general, they can contain any non-zero
+		// byte sequence, even though in practice a more restricted
+		// set is used.
+		//
+		// See https://github.com/tailscale/tailscale/issues/2024 for more.
+		if len(label) == 0 || len(label) > maxLabelLength {
+			return "", fmt.Errorf("%q is not a valid DNS label", label)
+		}
+		st = i + 1
 	}
 
-	return FQDN(s + "."), nil
-}
-
-func validLabel(s string) bool {
-	if len(s) == 0 || len(s) > maxLabelLength {
-		return false
+	if raw[len(raw)-1] != '.' {
+		raw = raw + "."
 	}
-	if !isalphanum(s[0]) || !isalphanum(s[len(s)-1]) {
-		return false
-	}
-	for i := 1; i < len(s)-1; i++ {
-		if !isalphanum(s[i]) && s[i] != '-' {
-			return false
-		}
-	}
-	return true
+	return FQDN(raw), nil
 }
 
 // WithTrailingDot returns f as a string, with a trailing dot.
@@ -92,51 +94,6 @@ func (f FQDN) Contains(other FQDN) bool {
 	return strings.HasSuffix(other.WithTrailingDot(), cmp)
 }
 
-// isValidFQDN reports whether s is already a valid FQDN, without
-// allocating.
-func isValidFQDN(s string) bool {
-	if len(s) == 0 {
-		return false
-	}
-	if len(s) > maxNameLength {
-		return false
-	}
-	// DNS root name.
-	if s == "." {
-		return true
-	}
-	// Missing trailing dot.
-	if s[len(s)-1] != '.' {
-		return false
-	}
-	// Leading dots not allowed.
-	if s[0] == '.' {
-		return false
-	}
-
-	st := 0
-	for i := 0; i < len(s); i++ {
-		if s[i] != '.' {
-			continue
-		}
-		label := s[st:i]
-		if len(label) == 0 || len(label) > maxLabelLength {
-			return false
-		}
-		if !isalphanum(label[0]) || !isalphanum(label[len(label)-1]) {
-			return false
-		}
-		for j := 1; j < len(label)-1; j++ {
-			if !isalphanum(label[j]) && label[j] != '-' {
-				return false
-			}
-		}
-		st = i + 1
-	}
-
-	return true
-}
-
 // SanitizeLabel takes a string intended to be a DNS name label
 // and turns it into a valid name label according to RFC 1035.
 func SanitizeLabel(label string) string {

util/dnsname/dnsname_test.go

@@ -24,6 +24,7 @@ func TestFQDN(t *testing.T) {
 		{".foo.com", "foo.com.", false, 2},
 		{"com", "com.", false, 1},
 		{"www.tailscale.com", "www.tailscale.com.", false, 3},
+		{"_ssh._tcp.tailscale.com", "_ssh._tcp.tailscale.com.", false, 4},
 		{"aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa.com", "", true, 0},
 		{strings.Repeat("aaaaa.", 60) + "com", "", true, 0},
 		{"foo..com", "", true, 0},
@@ -184,3 +185,24 @@ func TestTrimSuffix(t *testing.T) {
 		}
 	}
 }
+
+var sinkFQDN FQDN
+
+func BenchmarkToFQDN(b *testing.B) {
+	tests := []string{
+		"www.tailscale.com.",
+		"www.tailscale.com",
+		".www.tailscale.com",
+		"_ssh._tcp.www.tailscale.com.",
+		"_ssh._tcp.www.tailscale.com",
+	}
+
+	for _, test := range tests {
+		b.Run(test, func(b *testing.B) {
+			b.ReportAllocs()
+			for i := 0; i < b.N; i++ {
+				sinkFQDN, _ = ToFQDN(test)
+			}
+		})
+	}
+}

util/groupmember/groupmember.go

@@ -0,0 +1,21 @@
+// Copyright (c) 2021 Tailscale Inc & AUTHORS All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+// Package groupmemeber verifies group membership of the provided user on the
+// local system.
+package groupmember
+
+import (
+	"errors"
+	"runtime"
+)
+
+var ErrNotImplemented = errors.New("not implemented for GOOS=" + runtime.GOOS)
+
+// IsMemberOfGroup verifies if the provided user is member of the provided
+// system group.
+// If verfication fails, an error is returned.
+func IsMemberOfGroup(group, userName string) (bool, error) {
+	return isMemberOfGroup(group, userName)
+}

util/groupmember/groupmember_cgo.go

@@ -0,0 +1,48 @@
+// Copyright (c) 2021 Tailscale Inc & AUTHORS All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+// +build cgo
+
+package groupmember
+
+import (
+	"os/user"
+	"sync"
+)
+
+func isMemberOfGroup(group, name string) (bool, error) {
+	u, err := user.Lookup(name)
+	if err != nil {
+		return false, err
+	}
+	ugids, err := u.GroupIds()
+	if err != nil {
+		return false, err
+	}
+	gid, err := getGroupID(group)
+	if err != nil {
+		return false, err
+	}
+	for _, ugid := range ugids {
+		if gid == ugid {
+			return true, nil
+		}
+	}
+	return false, nil
+}
+
+var groupIDCache sync.Map // of string
+
+func getGroupID(groupName string) (string, error) {
+	s, ok := groupIDCache.Load(groupName)
+	if ok {
+		return s.(string), nil
+	}
+	g, err := user.LookupGroup(groupName)
+	if err != nil {
+		return "", err
+	}
+	groupIDCache.Store(groupName, g.Gid)
+	return g.Gid, nil
+}

util/groupmember/groupmember_noimpl.go

@@ -2,8 +2,8 @@
 // Use of this source code is governed by a BSD-style
 // license that can be found in the LICENSE file.
 
-// Placeholder that indicates this directory is a valid go package,
-// but that redo must 'redo all' in this directory before it can
-// be imported.
+// +build !cgo,!linux,!darwin
 
-package version
+package groupmember
+
+func isMemberOfGroup(group, name string) (bool, error) { return false, ErrNotImplemented }

util/groupmember/groupmember_notcgo.go

@@ -0,0 +1,71 @@
+// Copyright (c) 2021 Tailscale Inc & AUTHORS All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+// +build !cgo
+// +build linux darwin
+
+package groupmember
+
+import (
+	"bufio"
+	"fmt"
+	"os"
+	"os/exec"
+	"strings"
+
+	"go4.org/mem"
+	"tailscale.com/version/distro"
+)
+
+func isMemberOfGroup(group, name string) (bool, error) {
+	if distro.Get() == distro.Synology {
+		return isMemberOfGroupEtcGroup(group, name)
+	}
+	cmd := exec.Command("/usr/bin/env", "groups", name)
+	out, err := cmd.CombinedOutput()
+	if err != nil {
+		return false, err
+	}
+	groups := strings.Split(strings.TrimSpace(string(out)), " ")
+	for _, g := range groups {
+		if g == group {
+			return true, nil
+		}
+	}
+	return false, nil
+}
+
+func isMemberOfGroupEtcGroup(group, name string) (bool, error) {
+	f, err := os.Open("/etc/group")
+	if err != nil {
+		return false, err
+	}
+	defer f.Close()
+	s := bufio.NewScanner(f)
+	var agLine string
+	for s.Scan() {
+		if !mem.HasPrefix(mem.B(s.Bytes()), mem.S(fmt.Sprintf("%s:", group))) {
+			continue
+		}
+		agLine = s.Text()
+		break
+	}
+	if err := s.Err(); err != nil {
+		return false, err
+	}
+	if agLine == "" {
+		return false, fmt.Errorf("admin group not defined")
+	}
+	agEntry := strings.Split(agLine, ":")
+	if len(agEntry) < 4 {
+		return false, fmt.Errorf("malformed admin group entry")
+	}
+	agMembers := agEntry[3]
+	for _, m := range strings.Split(agMembers, ",") {
+		if m == name {
+			return true, nil
+		}
+	}
+	return false, nil
+}