ipn/localapi: 404 on bad endpoints

Confused us for a while!

Signed-off-by: David Crawshaw <crawshaw@tailscale.com>

client/tailscale/tailscale.go

@@ -8,18 +8,25 @@ package tailscale
 import (
 	"context"
 	"encoding/json"
+	"errors"
 	"fmt"
+	"io"
 	"io/ioutil"
 	"net"
 	"net/http"
 	"net/url"
 	"strconv"
+	"strings"
 
 	"tailscale.com/ipn/ipnstate"
+	"tailscale.com/paths"
 	"tailscale.com/safesocket"
 	"tailscale.com/tailcfg"
 )
 
+// TailscaledSocket is the tailscaled Unix socket.
+var TailscaledSocket = paths.DefaultTailscaledSocket()
+
 // tsClient does HTTP requests to the local Tailscale daemon.
 var tsClient = &http.Client{
 	Transport: &http.Transport{
@@ -27,14 +34,16 @@ var tsClient = &http.Client{
 			if addr != "local-tailscaled.sock:80" {
 				return nil, fmt.Errorf("unexpected URL address %q", addr)
 			}
-			// On macOS, when dialing from non-sandboxed program to sandboxed GUI running
-			// a TCP server on a random port, find the random port. For HTTP connections,
-			// we don't send the token. It gets added in an HTTP Basic-Auth header.
-			if port, _, err := safesocket.LocalTCPPortAndToken(); err == nil {
-				var d net.Dialer
-				return d.DialContext(ctx, "tcp", "localhost:"+strconv.Itoa(port))
+			if TailscaledSocket == paths.DefaultTailscaledSocket() {
+				// On macOS, when dialing from non-sandboxed program to sandboxed GUI running
+				// a TCP server on a random port, find the random port. For HTTP connections,
+				// we don't send the token. It gets added in an HTTP Basic-Auth header.
+				if port, _, err := safesocket.LocalTCPPortAndToken(); err == nil {
+					var d net.Dialer
+					return d.DialContext(ctx, "tcp", "localhost:"+strconv.Itoa(port))
+				}
 			}
-			return safesocket.ConnectDefault()
+			return safesocket.Connect(TailscaledSocket, 41112)
 		},
 	},
 }
@@ -101,6 +110,28 @@ func Goroutines(ctx context.Context) ([]byte, error) {
 	return body, nil
 }
 
+// BugReport logs and returns a log marker that can be shared by the user with support.
+func BugReport(ctx context.Context, note string) (string, error) {
+	u := fmt.Sprintf("http://local-tailscaled.sock/localapi/v0/bugreport?note=%s", url.QueryEscape(note))
+	req, err := http.NewRequestWithContext(ctx, "POST", u, nil)
+	if err != nil {
+		return "", err
+	}
+	res, err := DoLocalRequest(req)
+	if err != nil {
+		return "", err
+	}
+	defer res.Body.Close()
+	body, err := ioutil.ReadAll(res.Body)
+	if err != nil {
+		return "", err
+	}
+	if res.StatusCode != 200 {
+		return "", fmt.Errorf("HTTP %s: %s", res.Status, body)
+	}
+	return strings.TrimSpace(string(body)), nil
+}
+
 // Status returns the Tailscale daemon's status.
 func Status(ctx context.Context) (*ipnstate.Status, error) {
 	return status(ctx, "")
@@ -131,3 +162,94 @@ func status(ctx context.Context, queryString string) (*ipnstate.Status, error) {
 	}
 	return st, nil
 }
+
+type WaitingFile struct {
+	Name string
+	Size int64
+}
+
+func WaitingFiles(ctx context.Context) ([]WaitingFile, error) {
+	req, err := http.NewRequestWithContext(ctx, "GET", "http://local-tailscaled.sock/localapi/v0/files/", nil)
+	if err != nil {
+		return nil, err
+	}
+	res, err := DoLocalRequest(req)
+	if err != nil {
+		return nil, err
+	}
+	defer res.Body.Close()
+	if res.StatusCode != 200 {
+		body, _ := ioutil.ReadAll(res.Body)
+		return nil, fmt.Errorf("HTTP %s: %s", res.Status, body)
+	}
+	var wfs []WaitingFile
+	if err := json.NewDecoder(res.Body).Decode(&wfs); err != nil {
+		return nil, err
+	}
+	return wfs, nil
+}
+
+func DeleteWaitingFile(ctx context.Context, baseName string) error {
+	req, err := http.NewRequestWithContext(ctx, "DELETE", "http://local-tailscaled.sock/localapi/v0/files/"+url.PathEscape(baseName), nil)
+	if err != nil {
+		return err
+	}
+	res, err := DoLocalRequest(req)
+	if err != nil {
+		return err
+	}
+	defer res.Body.Close()
+	if res.StatusCode != http.StatusNoContent {
+		body, _ := ioutil.ReadAll(res.Body)
+		return fmt.Errorf("expected 204 No Content; got HTTP %s: %s", res.Status, body)
+	}
+	return nil
+}
+
+func GetWaitingFile(ctx context.Context, baseName string) (rc io.ReadCloser, size int64, err error) {
+	req, err := http.NewRequestWithContext(ctx, "GET", "http://local-tailscaled.sock/localapi/v0/files/"+url.PathEscape(baseName), nil)
+	if err != nil {
+		return nil, 0, err
+	}
+	res, err := DoLocalRequest(req)
+	if err != nil {
+		return nil, 0, err
+	}
+	if res.ContentLength == -1 {
+		res.Body.Close()
+		return nil, 0, fmt.Errorf("unexpected chunking")
+	}
+	if res.StatusCode != 200 {
+		body, _ := ioutil.ReadAll(res.Body)
+		res.Body.Close()
+		return nil, 0, fmt.Errorf("HTTP %s: %s", res.Status, body)
+	}
+	return res.Body, res.ContentLength, nil
+}
+
+func CheckIPForwarding(ctx context.Context) error {
+	req, err := http.NewRequestWithContext(ctx, "GET", "http://local-tailscaled.sock/localapi/v0/check-ip-forwarding", nil)
+	if err != nil {
+		return err
+	}
+	res, err := DoLocalRequest(req)
+	if err != nil {
+		return err
+	}
+	defer res.Body.Close()
+	if res.StatusCode != 200 {
+		body, _ := ioutil.ReadAll(res.Body)
+		res.Body.Close()
+		return fmt.Errorf("HTTP %s: %s", res.Status, body)
+	}
+	var jres struct {
+		Warning string
+	}
+	if err := json.NewDecoder(res.Body).Decode(&jres); err != nil {
+		return fmt.Errorf("invalid JSON from check-ip-forwarding: %w", err)
+	}
+	if jres.Warning != "" {
+		return errors.New(jres.Warning)
+	}
+	return nil
+}

cmd/tailscale/cli/bugreport.go

@@ -0,0 +1,38 @@
+// Copyright (c) 2021 Tailscale Inc & AUTHORS All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package cli
+
+import (
+	"context"
+	"errors"
+	"fmt"
+
+	"github.com/peterbourgon/ff/v2/ffcli"
+	"tailscale.com/client/tailscale"
+)
+
+var bugReportCmd = &ffcli.Command{
+	Name:       "bugreport",
+	Exec:       runBugReport,
+	ShortHelp:  "Print a shareable identifier to help diagnose issues",
+	ShortUsage: "bugreport [note]",
+}
+
+func runBugReport(ctx context.Context, args []string) error {
+	var note string
+	switch len(args) {
+	case 0:
+	case 1:
+		note = args[0]
+	default:
+		return errors.New("unknown argumets")
+	}
+	logMarker, err := tailscale.BugReport(ctx, note)
+	if err != nil {
+		return err
+	}
+	fmt.Println(logMarker)
+	return nil
+}

cmd/tailscale/cli/cli.go

@@ -20,6 +20,7 @@ import (
 	"text/tabwriter"
 
 	"github.com/peterbourgon/ff/v2/ffcli"
+	"tailscale.com/client/tailscale"
 	"tailscale.com/ipn"
 	"tailscale.com/paths"
 	"tailscale.com/safesocket"
@@ -70,6 +71,7 @@ change in the future.
 			versionCmd,
 			webCmd,
 			pushCmd,
+			bugReportCmd,
 		},
 		FlagSet:   rootfs,
 		Exec:      func(context.Context, []string) error { return flag.ErrHelp },
@@ -88,6 +90,8 @@ change in the future.
 		return err
 	}
 
+	tailscale.TailscaledSocket = rootArgs.socket
+
 	err := rootCmd.Run(context.Background())
 	if err == flag.ErrHelp {
 		return nil

cmd/tailscale/cli/debug.go

@@ -6,12 +6,18 @@ package cli
 
 import (
 	"context"
+	"encoding/json"
 	"errors"
 	"flag"
+	"fmt"
+	"io"
+	"log"
 	"os"
+	"strings"
 
 	"github.com/peterbourgon/ff/v2/ffcli"
 	"tailscale.com/client/tailscale"
+	"tailscale.com/ipn"
 )
 
 var debugCmd = &ffcli.Command{
@@ -20,12 +26,18 @@ var debugCmd = &ffcli.Command{
 	FlagSet: (func() *flag.FlagSet {
 		fs := flag.NewFlagSet("debug", flag.ExitOnError)
 		fs.BoolVar(&debugArgs.goroutines, "daemon-goroutines", false, "If true, dump the tailscaled daemon's goroutines")
+		fs.BoolVar(&debugArgs.ipn, "ipn", false, "If true, subscribe to IPN notifications")
+		fs.BoolVar(&debugArgs.netMap, "netmap", true, "whether to include netmap in --ipn mode")
+		fs.StringVar(&debugArgs.file, "file", "", "get, delete:NAME, or NAME")
 		return fs
 	})(),
 }
 
 var debugArgs struct {
 	goroutines bool
+	ipn        bool
+	netMap     bool
+	file       string
 }
 
 func runDebug(ctx context.Context, args []string) error {
@@ -38,6 +50,45 @@ func runDebug(ctx context.Context, args []string) error {
 			return err
 		}
 		os.Stdout.Write(goroutines)
+		return nil
+	}
+	if debugArgs.ipn {
+		c, bc, ctx, cancel := connect(ctx)
+		defer cancel()
+
+		bc.SetNotifyCallback(func(n ipn.Notify) {
+			if !debugArgs.netMap {
+				n.NetMap = nil
+			}
+			j, _ := json.MarshalIndent(n, "", "\t")
+			fmt.Printf("%s\n", j)
+		})
+		bc.RequestEngineStatus()
+		pump(ctx, bc, c)
+		return errors.New("exit")
+	}
+	if debugArgs.file != "" {
+		if debugArgs.file == "get" {
+			wfs, err := tailscale.WaitingFiles(ctx)
+			if err != nil {
+				log.Fatal(err)
+			}
+			e := json.NewEncoder(os.Stdout)
+			e.SetIndent("", "\t")
+			e.Encode(wfs)
+			return nil
+		}
+		delete := strings.HasPrefix(debugArgs.file, "delete:")
+		if delete {
+			return tailscale.DeleteWaitingFile(ctx, strings.TrimPrefix(debugArgs.file, "delete:"))
+		}
+		rc, size, err := tailscale.GetWaitingFile(ctx, debugArgs.file)
+		if err != nil {
+			return err
+		}
+		log.Printf("Size: %v\n", size)
+		io.Copy(os.Stdout, rc)
+		return nil
 	}
 	return nil
 }

cmd/tailscale/cli/down.go

@@ -59,7 +59,12 @@ func runDown(ctx context.Context, args []string) error {
 		}
 	})
 
-	bc.SetWantRunning(false)
+	bc.EditPrefs(&ipn.MaskedPrefs{
+		Prefs: ipn.Prefs{
+			WantRunning: false,
+		},
+		WantRunningSet: true,
+	})
 	pump(ctx, bc, c)
 
 	return nil

cmd/tailscale/cli/up.go

@@ -5,17 +5,14 @@
 package cli
 
 import (
-	"bytes"
 	"context"
 	"errors"
 	"flag"
 	"fmt"
 	"log"
 	"os"
-	"os/exec"
 	"runtime"
 	"sort"
-	"strconv"
 	"strings"
 	"sync"
 
@@ -97,34 +94,6 @@ func warnf(format string, args ...interface{}) {
 	fmt.Printf("Warning: "+format+"\n", args...)
 }
 
-// checkIPForwarding prints warnings on linux if IP forwarding is not
-// enabled, or if we were unable to verify the state of IP forwarding.
-func checkIPForwarding() {
-	var key string
-
-	if runtime.GOOS == "linux" {
-		key = "net.ipv4.ip_forward"
-	} else if isBSD(runtime.GOOS) {
-		key = "net.inet.ip.forwarding"
-	} else {
-		return
-	}
-
-	bs, err := exec.Command("sysctl", "-n", key).Output()
-	if err != nil {
-		warnf("couldn't check %s (%v).\nSubnet routes won't work without IP forwarding.", key, err)
-		return
-	}
-	on, err := strconv.ParseBool(string(bytes.TrimSpace(bs)))
-	if err != nil {
-		warnf("couldn't parse %s (%v).\nSubnet routes won't work without IP forwarding.", key, err)
-		return
-	}
-	if !on {
-		warnf("%s is disabled. Subnet routes won't work.", key)
-	}
-}
-
 var (
 	ipv4default = netaddr.MustParseIPPrefix("0.0.0.0/0")
 	ipv6default = netaddr.MustParseIPPrefix("::/0")
@@ -181,9 +150,8 @@ func runUp(ctx context.Context, args []string) error {
 		routeMap[netaddr.MustParseIPPrefix("::/0")] = true
 	}
 	if len(routeMap) > 0 {
-		checkIPForwarding()
-		if isBSD(runtime.GOOS) {
-			warnf("Subnet routing and exit nodes only work with additional manual configuration on %v, and is not currently officially supported.", runtime.GOOS)
+		if err := tailscale.CheckIPForwarding(context.Background()); err != nil {
+			warnf("%v", err)
 		}
 	}
 	routes := make([]netaddr.IPPrefix, 0, len(routeMap))

cmd/tailscale/cli/web.go

@@ -17,10 +17,12 @@ import (
 	"net/http/cgi"
 	"os/exec"
 	"runtime"
+	"strings"
 
 	"github.com/peterbourgon/ff/v2/ffcli"
 	"tailscale.com/client/tailscale"
 	"tailscale.com/ipn"
+	"tailscale.com/tailcfg"
 	"tailscale.com/types/preftype"
 	"tailscale.com/version/distro"
 )
@@ -28,9 +30,18 @@ import (
 //go:embed web.html
 var webHTML string
 
-var tmpl = template.Must(template.New("html").Parse(webHTML))
+//go:embed web.css
+var webCSS string
+
+var tmpl *template.Template
+
+func init() {
+	tmpl = template.Must(template.New("web.html").Parse(webHTML))
+	template.Must(tmpl.New("web.css").Parse(webCSS))
+}
 
 type tmplData struct {
+	Profile      tailcfg.UserProfile
 	SynologyUser string
 	Status       string
 	DeviceName   string
@@ -117,6 +128,67 @@ req.send(null);
 </body></html>
 `
 
+const authenticationRedirectHTML = `
+<html>
+<head>
+	<title>Redirecting...</title>
+	<style>
+		html,
+		body {
+			height: 100%;
+		}
+
+		html {
+			background-color: rgb(249, 247, 246);
+			font-family: ui-sans-serif, system-ui, -apple-system, BlinkMacSystemFont, "Segoe UI", Roboto, "Helvetica Neue", Arial, "Noto Sans", sans-serif, "Apple Color Emoji", "Segoe UI Emoji", "Segoe UI Symbol", "Noto Color Emoji";
+			line-height: 1.5;
+			-webkit-text-size-adjust: 100%;
+			-webkit-font-smoothing: antialiased;
+			-moz-osx-font-smoothing: grayscale;
+		}
+
+		body {
+			display: flex;
+			flex-direction: column;
+			align-items: center;
+			justify-content: center;
+		}
+
+		.spinner {
+			margin-bottom: 2rem;
+			border: 4px rgba(112, 110, 109, 0.5) solid;
+			border-left-color: transparent;
+			border-radius: 9999px;
+			width: 4rem;
+			height: 4rem;
+			-webkit-animation: spin 700ms linear infinite;
+      animation: spin 800ms linear infinite;
+		}
+
+		.label {
+			color: rgb(112, 110, 109);
+			padding-left: 0.4rem;
+		}
+
+		@-webkit-keyframes spin {
+			to {
+				transform: rotate(360deg);
+			}
+		}
+
+		@keyframes spin {
+			to {
+				transform: rotate(360deg);
+			}
+		}
+	</style>
+</head>
+<body>
+	<div class="spinner"></div>
+	<div class="label">Redirecting...</div>
+</body>
+`
+
 func webHandler(w http.ResponseWriter, r *http.Request) {
 	if synoTokenRedirect(w, r) {
 		return
@@ -128,6 +200,11 @@ func webHandler(w http.ResponseWriter, r *http.Request) {
 		return
 	}
 
+	if r.URL.Path == "/redirect" || r.URL.Path == "/redirect/" {
+		w.Write([]byte(authenticationRedirectHTML))
+		return
+	}
+
 	if r.Method == "POST" {
 		type mi map[string]interface{}
 		w.Header().Set("Content-Type", "application/json")
@@ -143,12 +220,16 @@ func webHandler(w http.ResponseWriter, r *http.Request) {
 	st, err := tailscale.Status(r.Context())
 	if err != nil {
 		http.Error(w, err.Error(), 500)
+		return
 	}
 
+	profile := st.User[st.Self.UserID]
+	deviceName := strings.Split(st.Self.DNSName, ".")[0]
 	data := tmplData{
 		SynologyUser: user,
+		Profile:      profile,
 		Status:       st.BackendState,
-		DeviceName:   st.Self.DNSName,
+		DeviceName:   deviceName,
 	}
 	if len(st.TailscaleIPs) != 0 {
 		data.IP = st.TailscaleIPs[0].String()

cmd/tailscale/cli/web.html

@@ -1,47 +1,150 @@
 <!doctype html>
-<html><title>Tailscale Client</title><body>
-<h1>Tailscale</h1>
-<div style="float:right;">{{.SynologyUser}}</div>
-<table>
-<tr><th>Status:</th><td>{{.Status}}</td></tr>
-<tr><th>Device Name:</th><td>{{.DeviceName}}</td></tr>
-<tr><th>Tailscale IP:</th><td>{{.IP}}</td></tr>
-</table>
+<html class="bg-gray-50">
 
-<p><input id="login" type="button" value="Log in…"></p>
+<head>
+	<meta charset="utf-8" />
+	<meta name="viewport" content="width=device-width, initial-scale=1" />
+	<link rel="shortcut icon"
+		href="data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAACAAAAAgCAQAAADZc7J/AAAABGdBTUEAALGPC/xhBQAAACBjSFJNAAB6JgAAgIQAAPoAAACA6AAAdTAAAOpgAAA6mAAAF3CculE8AAAAAmJLR0QA/4ePzL8AAAAHdElNRQflAx4QGA4EvmzDAAAA30lEQVRIx2NgGAWMCKa8JKM4A8Ovt88ekyLCDGOoyDBJMjExMbFy8zF8/EKsCAMDE8yAPyIwFps48SJIBpAL4AZwvoSx/r0lXgQpDN58EWL5x/7/H+vL20+JFxluQKVe5b3Ke5V+0kQQCamfoYKBg4GDwUKI8d0BYkWQkrLKewYBKPPDHUFiRaiZkBgmwhj/F5IgggyUJ6i8V3mv0kCayDAAeEsklXqGAgYGhgV3CnGrwVciYSYk0kokhgS44/JxqqFpiYSZbEgskd4dEBRk1GD4wdB5twKXmlHAwMDAAACdEZau06NQUwAAACV0RVh0ZGF0ZTpjcmVhdGUAMjAyMC0wNy0xNVQxNTo1Mzo0MCswMDowMCVXsDIAAAAldEVYdGRhdGU6bW9kaWZ5ADIwMjAtMDctMTVUMTU6NTM6NDArMDA6MDBUCgiOAAAAAElFTkSuQmCC" />
+	<title>Tailscale</title>
+	<style>{{template "web.css"}}</style>
+</head>
 
-<script>
-login.onclick = function() {
-	const urlParams = new URLSearchParams(window.location.search);
-	const token = urlParams.get("SynoToken");
+<body class="py-14">
+	<main class="container max-w-lg mx-auto py-6 px-8 bg-white rounded-md shadow-2xl" style="width: 95%">
+		<header class="flex justify-between items-center min-width-0 py-2 mb-8">
+			<svg width="26" height="26" viewBox="0 0 23 23" title="Tailscale" fill="none" xmlns="http://www.w3.org/2000/svg"
+				class="flex-shrink-0 mr-4">
+				<circle opacity="0.2" cx="3.4" cy="3.25" r="2.7" fill="currentColor"></circle>
+				<circle cx="3.4" cy="11.3" r="2.7" fill="currentColor"></circle>
+				<circle opacity="0.2" cx="3.4" cy="19.5" r="2.7" fill="currentColor"></circle>
+				<circle cx="11.5" cy="11.3" r="2.7" fill="currentColor"></circle>
+				<circle cx="11.5" cy="19.5" r="2.7" fill="currentColor"></circle>
+				<circle opacity="0.2" cx="11.5" cy="3.25" r="2.7" fill="currentColor"></circle>
+				<circle opacity="0.2" cx="19.5" cy="3.25" r="2.7" fill="currentColor"></circle>
+				<circle cx="19.5" cy="11.3" r="2.7" fill="currentColor"></circle>
+				<circle opacity="0.2" cx="19.5" cy="19.5" r="2.7" fill="currentColor"></circle>
+			</svg>
+			<div class="flex items-center justify-end space-x-2 w-2/3">
+				{{ with .Profile.LoginName }}
+				<div class="text-right truncate leading-4">
+					<h4 class="truncate">{{.}}</h4>
+					<a href="#" class="text-xs text-gray-500 hover:text-gray-700 js-loginButton">Switch account</a>
+				</div>
+				{{ end }}
+				<div class="relative flex-shrink-0 w-8 h-8 rounded-full overflow-hidden">
+					{{ with .Profile.ProfilePicURL }}
+					<div class="w-8 h-8 flex pointer-events-none rounded-full bg-gray-200"
+						style="background-image: url('{{.}}'); background-size: cover;"></div>
+					{{ else }}
+					<div class="w-8 h-8 flex pointer-events-none rounded-full border border-gray-400 border-dashed"></div>
+					{{ end }}
+				</div>
+			</div>
+		</header>
+		{{ if .IP }}
+		<div
+			class="border border-gray-200 bg-gray-0 rounded-lg p-2 pl-3 pr-3 mb-8 width-full flex items-center justify-between">
+			<div class="flex items-center min-width-0">
+				<svg class="flex-shrink-0 text-gray-600 mr-3 ml-1" xmlns="http://www.w3.org/2000/svg" width="20" height="20"
+					viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round"
+					stroke-linejoin="round">
+					<rect x="2" y="2" width="20" height="8" rx="2" ry="2"></rect>
+					<rect x="2" y="14" width="20" height="8" rx="2" ry="2"></rect>
+					<line x1="6" y1="6" x2="6.01" y2="6"></line>
+					<line x1="6" y1="18" x2="6.01" y2="18"></line>
+				</svg>
+				<h4 class="font-semibold truncate mr-2">{{.DeviceName}}</h4>
+			</div>
+			<h5>{{.IP}}</h5>
+		</div>
+		{{ end }}
+		{{ if or (eq .Status "NeedsLogin") (eq .Status "NoState") }}
+		{{ if .IP }}
+		<div class="mb-6">
+			<p class="text-gray-700">Your device's key has expired. Reauthenticate this device by logging in again, or <a
+					href="https://tailscale.com/kb/1028/key-expiry" class="link" target="_blank">learn more</a>.</p>
+		</div>
+		<a href="#" class="mb-4 js-loginButton" target="_blank">
+			<button class="button button-blue w-full">Reauthenticate</button>
+		</a>
+		{{ else }}
+		<div class="mb-6">
+			<h3 class="text-3xl font-semibold mb-3">Log in</h3>
+			<p class="text-gray-700">Get started by logging in to your Tailscale network. Or,&nbsp;learn&nbsp;more at <a
+					href="https://tailscale.com/" class="link" target="_blank">tailscale.com</a>.</p>
+		</div>
+		<a href="#" class="mb-4 js-loginButton" target="_blank">
+			<button class="button button-blue w-full">Log In</button>
+		</a>
+		{{ end }}
+		{{ else if eq .Status "NeedsMachineAuth" }}
+		<div class="mb-4">
+			This device is authorized, but needs approval from a network admin before it can connect to the network.
+		</div>
+		{{ else }}
+		<div class="mb-4">
+			<p>You are connected! Access this device over Tailscale using the device name or IP address above.</p>
+		</div>
+		<a href="#" class="mb-4 link font-medium js-loginButton" target="_blank">Reauthenticate</a>
+		{{ end }}
+	</main>
+	<script>
+		(function () {
+			let loginButtons = document.querySelectorAll(".js-loginButton");
+			let fetchingUrl = false;
 
-	var params = new URLSearchParams("up=true");
-	if (token) {
-		params.set("SynoToken", token)
-	}
+			function handleClick(e) {
+				e.preventDefault();
 
-	var req = new XMLHttpRequest();
-	const url = [location.protocol, '//', location.host, location.pathname, "?", params.toString()].join('');
-	req.overrideMimeType("application/json");
-	req.open("POST", url, true);
-	req.onload = function() {
-		var jsonResponse = JSON.parse(req.responseText);
-		const err = jsonResponse["error"];
-		if (err) {
-			document.body.innerText = err;
-			return
-		}
-		var url = jsonResponse["url"];
-		console.log("jsonResponse: ", jsonResponse);
-		if (url) {
-			document.location.href = url;
-		} else {
-			//location.reload();
-		}
-	};
-	req.send(null);
-}
-</script>
+				if (fetchingUrl) {
+					return;
+				}
 
+				fetchingUrl = true;
+				const urlParams = new URLSearchParams(window.location.search);
+				const token = urlParams.get("SynoToken");
+				const nextParams = new URLSearchParams({ up: true });
+				if (token) {
+					nextParams.set("SynoToken", token)
+				}
+				const nextUrl = new URL(window.location);
+				nextUrl.search = nextParams.toString()
+				const url = nextUrl.toString();
+
+				const tab = window.open("/redirect", "_blank");
+
+				fetch(url, {
+					method: "POST",
+					headers: {
+						"Accept": "application/json",
+						"Content-Type": "application/json",
+					}
+				}).then(res => res.json()).then(res => {
+					fetchingUrl = false;
+					const err = res["error"];
+					if (err) {
+						throw new Error(err);
+					}
+					const url = res["url"];
+					if (url) {
+						authUrl = url;
+						tab.location = url;
+						tab.focus();
+					} else {
+						location.reload();
+					}
+				}).catch(err => {
+					tab.close();
+					alert("Failed to log in: " + err.message);
+				});
+			}
+
+			Array.from(loginButtons).forEach(el => {
+				el.addEventListener("click", handleClick);
+			})
+		})();
+	</script>
 </body>
+
 </html>

cmd/tailscaled/depaware.txt

@@ -22,6 +22,7 @@ tailscale.com/cmd/tailscaled dependencies: (generated by github.com/tailscale/de
    L    github.com/mdlayher/sdnotify                                 from tailscale.com/util/systemd
    W    github.com/pkg/errors                                        from github.com/github/certstore
      💣 github.com/tailscale/wireguard-go/conn                       from github.com/tailscale/wireguard-go/device+
+   W 💣 github.com/tailscale/wireguard-go/conn/winrio                from github.com/tailscale/wireguard-go/conn
      💣 github.com/tailscale/wireguard-go/device                     from tailscale.com/wgengine+
      💣 github.com/tailscale/wireguard-go/ipc                        from github.com/tailscale/wireguard-go/device
    W 💣 github.com/tailscale/wireguard-go/ipc/winpipe                from github.com/tailscale/wireguard-go/ipc
@@ -91,6 +92,7 @@ tailscale.com/cmd/tailscaled dependencies: (generated by github.com/tailscale/de
         tailscale.com/logtail/filch                                  from tailscale.com/logpolicy
         tailscale.com/metrics                                        from tailscale.com/derp
         tailscale.com/net/dns                                        from tailscale.com/ipn/ipnlocal+
+        tailscale.com/net/dns/resolver                               from tailscale.com/wgengine+
         tailscale.com/net/dnscache                                   from tailscale.com/control/controlclient+
         tailscale.com/net/dnsfallback                                from tailscale.com/control/controlclient
         tailscale.com/net/flowtrack                                  from tailscale.com/wgengine/filter+
@@ -123,7 +125,6 @@ tailscale.com/cmd/tailscaled dependencies: (generated by github.com/tailscale/de
         tailscale.com/types/netmap                                   from tailscale.com/control/controlclient+
         tailscale.com/types/nettype                                  from tailscale.com/wgengine/magicsock
         tailscale.com/types/opt                                      from tailscale.com/control/controlclient+
-        tailscale.com/types/pad32                                    from tailscale.com/wgengine/magicsock
         tailscale.com/types/persist                                  from tailscale.com/control/controlclient+
         tailscale.com/types/preftype                                 from tailscale.com/ipn+
         tailscale.com/types/strbuilder                               from tailscale.com/net/packet

cmd/tailscaled/install_darwin.go

@@ -73,9 +73,18 @@ func uninstallSystemDaemonDarwin(args []string) (ret error) {
 		}
 	}
 
-	err = os.Remove(sysPlist)
-	if os.IsNotExist(err) {
-		err = nil
+	if err := os.Remove(sysPlist); err != nil {
+		if os.IsNotExist(err) {
+			err = nil
+		}
+		if ret == nil {
+			ret = err
+		}
+	}
+	if err := os.Remove(targetBin); err != nil {
+		if os.IsNotExist(err) {
+			err = nil
+		}
 		if ret == nil {
 			ret = err
 		}
@@ -93,6 +102,9 @@ func installSystemDaemonDarwin(args []string) (err error) {
 		}
 	}()
 
+	// Best effort:
+	uninstallSystemDaemonDarwin(nil)
+
 	// Copy ourselves to /usr/local/bin/tailscaled.
 	if err := os.MkdirAll(filepath.Dir(targetBin), 0755); err != nil {
 		return err
@@ -127,9 +139,6 @@ func installSystemDaemonDarwin(args []string) (err error) {
 		return err
 	}
 
-	// Best effort:
-	uninstallSystemDaemonDarwin(nil)
-
 	if err := ioutil.WriteFile(sysPlist, []byte(darwinLaunchdPlist), 0700); err != nil {
 		return err
 	}

cmd/tailscaled/tailscaled.go

@@ -31,6 +31,7 @@ import (
 	"github.com/go-multierror/multierror"
 	"tailscale.com/ipn/ipnserver"
 	"tailscale.com/logpolicy"
+	"tailscale.com/net/dns"
 	"tailscale.com/net/socks5"
 	"tailscale.com/net/tstun"
 	"tailscale.com/paths"
@@ -192,6 +193,7 @@ func run() error {
 	logf = logger.RateLimitedFn(logf, 5*time.Second, 5, 100)
 
 	if args.cleanup {
+		dns.Cleanup(logf, args.tunname)
 		router.Cleanup(logf, args.tunname)
 		return nil
 	}
@@ -312,16 +314,16 @@ func run() error {
 	return nil
 }
 
-func createEngine(logf logger.Logf, linkMon *monitor.Mon) (e wgengine.Engine, isUserspace bool, err error) {
+func createEngine(logf logger.Logf, linkMon *monitor.Mon) (e wgengine.Engine, useNetstack bool, err error) {
 	if args.tunname == "" {
 		return nil, false, errors.New("no --tun value specified")
 	}
 	var errs []error
 	for _, name := range strings.Split(args.tunname, ",") {
 		logf("wgengine.NewUserspaceEngine(tun %q) ...", name)
-		e, isUserspace, err = tryEngine(logf, linkMon, name)
+		e, useNetstack, err = tryEngine(logf, linkMon, name)
 		if err == nil {
-			return e, isUserspace, nil
+			return e, useNetstack, nil
 		}
 		logf("wgengine.NewUserspaceEngine(tun %q) error: %v", name, err)
 		errs = append(errs, err)
@@ -329,13 +331,13 @@ func createEngine(logf logger.Logf, linkMon *monitor.Mon) (e wgengine.Engine, is
 	return nil, false, multierror.New(errs)
 }
 
-func tryEngine(logf logger.Logf, linkMon *monitor.Mon, name string) (e wgengine.Engine, isUserspace bool, err error) {
+func tryEngine(logf logger.Logf, linkMon *monitor.Mon, name string) (e wgengine.Engine, useNetstack bool, err error) {
 	conf := wgengine.Config{
 		ListenPort:  args.port,
 		LinkMonitor: linkMon,
 	}
-	isUserspace = name == "userspace-networking"
-	if !isUserspace {
+	useNetstack = name == "userspace-networking"
+	if !useNetstack {
 		dev, err := tstun.New(logf, name)
 		if err != nil {
 			tstun.Diagnose(logf, name)
@@ -348,12 +350,19 @@ func tryEngine(logf logger.Logf, linkMon *monitor.Mon, name string) (e wgengine.
 			return nil, false, err
 		}
 		conf.Router = r
+		tunname, err := dev.Name()
+		if err != nil {
+			r.Close()
+			dev.Close()
+			return nil, false, err
+		}
+		conf.DNS = dns.NewOSConfigurator(logf, tunname)
 	}
 	e, err = wgengine.NewUserspaceEngine(logf, conf)
 	if err != nil {
-		return nil, isUserspace, err
+		return nil, useNetstack, err
 	}
-	return e, isUserspace, nil
+	return e, useNetstack, nil
 }
 
 func newDebugMux() *http.ServeMux {

control/controlclient/auto.go

@@ -404,7 +404,7 @@ func (c *Client) authRoutine() {
 				continue
 			} else if url != "" {
 				if goal.url != "" {
-					err = fmt.Errorf("weird: server required a new url?")
+					err = fmt.Errorf("[unexpected] server required a new URL?")
 					report(err, "WaitLoginURL")
 				}
 
@@ -590,6 +590,7 @@ func (c *Client) AuthCantContinue() bool {
 	return !c.loggedIn && (c.loginGoal == nil || c.loginGoal.url != "")
 }
 
+// SetStatusFunc sets fn as the callback to run on any status change.
 func (c *Client) SetStatusFunc(fn func(Status)) {
 	c.mu.Lock()
 	c.statusFunc = fn
@@ -693,6 +694,13 @@ func (c *Client) Logout() {
 	c.cancelAuth()
 }
 
+// UpdateEndpoints sets the client's discovered endpoints and sends
+// them to the control server if they've changed.
+//
+// It does not retain the provided slice.
+//
+// The localPort field is unused except for integration tests in
+// another repo.
 func (c *Client) UpdateEndpoints(localPort uint16, endpoints []string) {
 	changed := c.direct.SetEndpoints(localPort, endpoints)
 	if changed {

control/controlclient/direct.go

@@ -63,9 +63,10 @@ type Direct struct {
 	logf                   logger.Logf
 	linkMon                *monitor.Mon // or nil
 	discoPubKey            tailcfg.DiscoKey
-	machinePrivKey         wgkey.Private
+	getMachinePrivKey      func() (wgkey.Private, error)
 	debugFlags             []string
 	keepSharerAndUserSplit bool
+	skipIPForwardingCheck  bool
 
 	mu           sync.Mutex // mutex guards the following fields
 	serverKey    wgkey.Key
@@ -81,23 +82,28 @@ type Direct struct {
 }
 
 type Options struct {
-	Persist           persist.Persist   // initial persistent data
-	MachinePrivateKey wgkey.Private     // the machine key to use
-	ServerURL         string            // URL of the tailcontrol server
-	AuthKey           string            // optional node auth key for auto registration
-	TimeNow           func() time.Time  // time.Now implementation used by Client
-	Hostinfo          *tailcfg.Hostinfo // non-nil passes ownership, nil means to use default using os.Hostname, etc
-	DiscoPublicKey    tailcfg.DiscoKey
-	NewDecompressor   func() (Decompressor, error)
-	KeepAlive         bool
-	Logf              logger.Logf
-	HTTPTestClient    *http.Client // optional HTTP client to use (for tests only)
-	DebugFlags        []string     // debug settings to send to control
-	LinkMonitor       *monitor.Mon // optional link monitor
+	Persist              persist.Persist               // initial persistent data
+	GetMachinePrivateKey func() (wgkey.Private, error) // returns the machine key to use
+	ServerURL            string                        // URL of the tailcontrol server
+	AuthKey              string                        // optional node auth key for auto registration
+	TimeNow              func() time.Time              // time.Now implementation used by Client
+	Hostinfo             *tailcfg.Hostinfo             // non-nil passes ownership, nil means to use default using os.Hostname, etc
+	DiscoPublicKey       tailcfg.DiscoKey
+	NewDecompressor      func() (Decompressor, error)
+	KeepAlive            bool
+	Logf                 logger.Logf
+	HTTPTestClient       *http.Client // optional HTTP client to use (for tests only)
+	DebugFlags           []string     // debug settings to send to control
+	LinkMonitor          *monitor.Mon // optional link monitor
 
 	// KeepSharerAndUserSplit controls whether the client
 	// understands Node.Sharer. If false, the Sharer is mapped to the User.
 	KeepSharerAndUserSplit bool
+
+	// SkipIPForwardingCheck declares that the host's IP
+	// forwarding works and should not be double-checked by the
+	// controlclient package.
+	SkipIPForwardingCheck bool
 }
 
 type Decompressor interface {
@@ -110,8 +116,8 @@ func NewDirect(opts Options) (*Direct, error) {
 	if opts.ServerURL == "" {
 		return nil, errors.New("controlclient.New: no server URL specified")
 	}
-	if opts.MachinePrivateKey.IsZero() {
-		return nil, errors.New("controlclient.New: no MachinePrivateKey specified")
+	if opts.GetMachinePrivateKey == nil {
+		return nil, errors.New("controlclient.New: no GetMachinePrivateKey specified")
 	}
 	opts.ServerURL = strings.TrimRight(opts.ServerURL, "/")
 	serverURL, err := url.Parse(opts.ServerURL)
@@ -147,7 +153,7 @@ func NewDirect(opts Options) (*Direct, error) {
 
 	c := &Direct{
 		httpc:                  httpc,
-		machinePrivKey:         opts.MachinePrivateKey,
+		getMachinePrivKey:      opts.GetMachinePrivateKey,
 		serverURL:              opts.ServerURL,
 		timeNow:                opts.TimeNow,
 		logf:                   opts.Logf,
@@ -159,6 +165,7 @@ func NewDirect(opts Options) (*Direct, error) {
 		debugFlags:             opts.DebugFlags,
 		keepSharerAndUserSplit: opts.KeepSharerAndUserSplit,
 		linkMon:                opts.LinkMonitor,
+		skipIPForwardingCheck:  opts.SkipIPForwardingCheck,
 	}
 	if opts.Hostinfo == nil {
 		c.SetHostinfo(NewHostinfo())
@@ -270,12 +277,15 @@ func (c *Direct) TryLogin(ctx context.Context, t *tailcfg.Oauth2Token, flags Log
 	return c.doLoginOrRegen(ctx, t, flags, false, "")
 }
 
-func (c *Direct) WaitLoginURL(ctx context.Context, url string) (newUrl string, err error) {
+// WaitLoginURL sits in a long poll waiting for the user to authenticate at url.
+//
+// On success, newURL and err will both be nil.
+func (c *Direct) WaitLoginURL(ctx context.Context, url string) (newURL string, err error) {
 	c.logf("direct.WaitLoginURL")
 	return c.doLoginOrRegen(ctx, nil, LoginDefault, false, url)
 }
 
-func (c *Direct) doLoginOrRegen(ctx context.Context, t *tailcfg.Oauth2Token, flags LoginFlags, regen bool, url string) (newUrl string, err error) {
+func (c *Direct) doLoginOrRegen(ctx context.Context, t *tailcfg.Oauth2Token, flags LoginFlags, regen bool, url string) (newURL string, err error) {
 	mustregen, url, err := c.doLogin(ctx, t, flags, regen, url)
 	if err != nil {
 		return url, err
@@ -298,8 +308,12 @@ func (c *Direct) doLogin(ctx context.Context, t *tailcfg.Oauth2Token, flags Logi
 	expired := c.expiry != nil && !c.expiry.IsZero() && c.expiry.Before(c.timeNow())
 	c.mu.Unlock()
 
-	if c.machinePrivKey.IsZero() {
-		return false, "", errors.New("controlclient.Direct requires a machine private key")
+	machinePrivKey, err := c.getMachinePrivKey()
+	if err != nil {
+		return false, "", fmt.Errorf("getMachinePrivKey: %w", err)
+	}
+	if machinePrivKey.IsZero() {
+		return false, "", errors.New("getMachinePrivKey returned zero key")
 	}
 
 	if expired {
@@ -367,7 +381,7 @@ func (c *Direct) doLogin(ctx context.Context, t *tailcfg.Oauth2Token, flags Logi
 	request.Auth.Provider = persist.Provider
 	request.Auth.LoginName = persist.LoginName
 	request.Auth.AuthKey = authKey
-	err = signRegisterRequest(&request, c.serverURL, c.serverKey, c.machinePrivKey.Public())
+	err = signRegisterRequest(&request, c.serverURL, c.serverKey, machinePrivKey.Public())
 	if err != nil {
 		// If signing failed, clear all related fields
 		request.SignatureType = tailcfg.SignatureNone
@@ -381,13 +395,13 @@ func (c *Direct) doLogin(ctx context.Context, t *tailcfg.Oauth2Token, flags Logi
 			c.logf("RegisterReq sign error: %v", err)
 		}
 	}
-	bodyData, err := encode(request, &serverKey, &c.machinePrivKey)
+	bodyData, err := encode(request, &serverKey, &machinePrivKey)
 	if err != nil {
 		return regen, url, err
 	}
 	body := bytes.NewReader(bodyData)
 
-	u := fmt.Sprintf("%s/machine/%s", c.serverURL, c.machinePrivKey.Public().HexString())
+	u := fmt.Sprintf("%s/machine/%s", c.serverURL, machinePrivKey.Public().HexString())
 	req, err := http.NewRequest("POST", u, body)
 	if err != nil {
 		return regen, url, err
@@ -405,8 +419,8 @@ func (c *Direct) doLogin(ctx context.Context, t *tailcfg.Oauth2Token, flags Logi
 			res.StatusCode, strings.TrimSpace(string(msg)))
 	}
 	resp := tailcfg.RegisterResponse{}
-	if err := decode(res, &resp, &serverKey, &c.machinePrivKey); err != nil {
-		c.logf("error decoding RegisterResponse with server key %s and machine key %s: %v", serverKey, c.machinePrivKey.Public(), err)
+	if err := decode(res, &resp, &serverKey, &machinePrivKey); err != nil {
+		c.logf("error decoding RegisterResponse with server key %s and machine key %s: %v", serverKey, machinePrivKey.Public(), err)
 		return regen, url, fmt.Errorf("register request: %v", err)
 	}
 	// Log without PII:
@@ -533,6 +547,14 @@ func (c *Direct) sendMapRequest(ctx context.Context, maxPolls int, cb func(*netm
 	everEndpoints := c.everEndpoints
 	c.mu.Unlock()
 
+	machinePrivKey, err := c.getMachinePrivKey()
+	if err != nil {
+		return fmt.Errorf("getMachinePrivKey: %w", err)
+	}
+	if machinePrivKey.IsZero() {
+		return errors.New("getMachinePrivKey returned zero key")
+	}
+
 	if persist.PrivateNodeKey.IsZero() {
 		return errors.New("privateNodeKey is zero")
 	}
@@ -562,7 +584,8 @@ func (c *Direct) sendMapRequest(ctx context.Context, maxPolls int, cb func(*netm
 		OmitPeers:  cb == nil,
 	}
 	var extraDebugFlags []string
-	if hostinfo != nil && c.linkMon != nil && ipForwardingBroken(hostinfo.RoutableIPs, c.linkMon.InterfaceState()) {
+	if hostinfo != nil && c.linkMon != nil && !c.skipIPForwardingCheck &&
+		ipForwardingBroken(hostinfo.RoutableIPs, c.linkMon.InterfaceState()) {
 		extraDebugFlags = append(extraDebugFlags, "warn-ip-forwarding-off")
 	}
 	if health.RouterHealth() != nil {
@@ -590,7 +613,7 @@ func (c *Direct) sendMapRequest(ctx context.Context, maxPolls int, cb func(*netm
 		request.ReadOnly = true
 	}
 
-	bodyData, err := encode(request, &serverKey, &c.machinePrivKey)
+	bodyData, err := encode(request, &serverKey, &machinePrivKey)
 	if err != nil {
 		vlogf("netmap: encode: %v", err)
 		return err
@@ -599,7 +622,7 @@ func (c *Direct) sendMapRequest(ctx context.Context, maxPolls int, cb func(*netm
 	ctx, cancel := context.WithCancel(ctx)
 	defer cancel()
 
-	machinePubKey := tailcfg.MachineKey(c.machinePrivKey.Public())
+	machinePubKey := tailcfg.MachineKey(machinePrivKey.Public())
 	t0 := time.Now()
 	u := fmt.Sprintf("%s/machine/%s/map", serverURL, machinePubKey.HexString())
 
@@ -692,7 +715,7 @@ func (c *Direct) sendMapRequest(ctx context.Context, maxPolls int, cb func(*netm
 		vlogf("netmap: read body after %v", time.Since(t0).Round(time.Millisecond))
 
 		var resp tailcfg.MapResponse
-		if err := c.decodeMsg(msg, &resp); err != nil {
+		if err := c.decodeMsg(msg, &resp, &machinePrivKey); err != nil {
 			vlogf("netmap: decode error: %v")
 			return err
 		}
@@ -875,12 +898,12 @@ var debugMap, _ = strconv.ParseBool(os.Getenv("TS_DEBUG_MAP"))
 
 var jsonEscapedZero = []byte(`\u0000`)
 
-func (c *Direct) decodeMsg(msg []byte, v interface{}) error {
+func (c *Direct) decodeMsg(msg []byte, v interface{}, machinePrivKey *wgkey.Private) error {
 	c.mu.Lock()
 	serverKey := c.serverKey
 	c.mu.Unlock()
 
-	decrypted, err := decryptMsg(msg, &serverKey, &c.machinePrivKey)
+	decrypted, err := decryptMsg(msg, &serverKey, machinePrivKey)
 	if err != nil {
 		return err
 	}
@@ -914,8 +937,8 @@ func (c *Direct) decodeMsg(msg []byte, v interface{}) error {
 
 }
 
-func decodeMsg(msg []byte, v interface{}, serverKey *wgkey.Key, mkey *wgkey.Private) error {
-	decrypted, err := decryptMsg(msg, serverKey, mkey)
+func decodeMsg(msg []byte, v interface{}, serverKey *wgkey.Key, machinePrivKey *wgkey.Private) error {
+	decrypted, err := decryptMsg(msg, serverKey, machinePrivKey)
 	if err != nil {
 		return err
 	}
@@ -1166,6 +1189,11 @@ func TrimWGConfig() opt.Bool {
 // and will definitely not work for the routes provided.
 //
 // It should not return false positives.
+//
+// TODO(bradfitz): merge this code into LocalBackend.CheckIPForwarding
+// and change controlclient.Options.SkipIPForwardingCheck into a
+// func([]netaddr.IPPrefix) error signature instead. Then we only have
+// one copy of this code.
 func ipForwardingBroken(routes []netaddr.IPPrefix, state *interfaces.State) bool {
 	if len(routes) == 0 {
 		// Nothing to route, so no need to warn.

control/controlclient/direct_test.go

@@ -103,7 +103,13 @@ func TestNewDirect(t *testing.T) {
 	if err != nil {
 		t.Error(err)
 	}
-	opts := Options{ServerURL: "https://example.com", MachinePrivateKey: key, Hostinfo: hi}
+	opts := Options{
+		ServerURL: "https://example.com",
+		Hostinfo:  hi,
+		GetMachinePrivateKey: func() (wgkey.Private, error) {
+			return key, nil
+		},
+	}
 	c, err := NewDirect(opts)
 	if err != nil {
 		t.Fatal(err)

go.mod

@@ -24,14 +24,14 @@ require (
 	github.com/peterbourgon/ff/v2 v2.0.0
 	github.com/pkg/errors v0.9.1 // indirect
 	github.com/tailscale/depaware v0.0.0-20201214215404-77d1e9757027
-	github.com/tailscale/wireguard-go v0.0.0-20210327173134-f6a42a1646a0
+	github.com/tailscale/wireguard-go v0.0.0-20210403171604-17614717a9b5
 	github.com/tcnksm/go-httpstat v0.2.0
 	github.com/toqueteos/webbrowser v1.2.0
 	go4.org/mem v0.0.0-20201119185036-c04c5a6ff174
 	golang.org/x/crypto v0.0.0-20210317152858-513c2a44f670
 	golang.org/x/net v0.0.0-20210226172049-e18ecbb05110
 	golang.org/x/sync v0.0.0-20210220032951-036812b2e83c
-	golang.org/x/sys v0.0.0-20210317225723-c4fcb01b228e
+	golang.org/x/sys v0.0.0-20210403161142-5e06dd20ab57
 	golang.org/x/term v0.0.0-20210317153231-de623e64d2a6
 	golang.org/x/time v0.0.0-20210220033141-f8bda1e9f3ba
 	golang.org/x/tools v0.0.0-20201211185031-d93e913c1a58

go.sum

@@ -123,6 +123,22 @@ github.com/tailscale/wireguard-go v0.0.0-20210324165952-2963b66bc23a h1:tQ7Y0ALS
 github.com/tailscale/wireguard-go v0.0.0-20210324165952-2963b66bc23a/go.mod h1:6t0OVdJwFOKFnvaHaVMKG6GznWaHqkmiR2n3kH0t924=
 github.com/tailscale/wireguard-go v0.0.0-20210327173134-f6a42a1646a0 h1:7KFBvUmm3TW/K+bAN22D7M6xSSoY/39s+PajaNBGrLw=
 github.com/tailscale/wireguard-go v0.0.0-20210327173134-f6a42a1646a0/go.mod h1:6t0OVdJwFOKFnvaHaVMKG6GznWaHqkmiR2n3kH0t924=
+github.com/tailscale/wireguard-go v0.0.0-20210330185929-1689f2635004 h1:GNEPNdNHsYe5zhoR/0z2Pl/a9zXbr0IySmHV6PhCrzI=
+github.com/tailscale/wireguard-go v0.0.0-20210330185929-1689f2635004/go.mod h1:6t0OVdJwFOKFnvaHaVMKG6GznWaHqkmiR2n3kH0t924=
+github.com/tailscale/wireguard-go v0.0.0-20210330200845-4914b4a944c4 h1:7Y0H5NzrV3fwHeDrUXDFcTy8QNbAEDwr+qHyOfX4VyE=
+github.com/tailscale/wireguard-go v0.0.0-20210330200845-4914b4a944c4/go.mod h1:6t0OVdJwFOKFnvaHaVMKG6GznWaHqkmiR2n3kH0t924=
+github.com/tailscale/wireguard-go v0.0.0-20210401164443-2d6878b6b30d h1:zbDBqtYvc492gcRL5BB7AO5Aed+aVht2jbYg8SKoMYs=
+github.com/tailscale/wireguard-go v0.0.0-20210401164443-2d6878b6b30d/go.mod h1:6t0OVdJwFOKFnvaHaVMKG6GznWaHqkmiR2n3kH0t924=
+github.com/tailscale/wireguard-go v0.0.0-20210401172819-1aca620a8afb h1:6TGRROCOrjTKbt1ucBTZaDMBeScG6yVEXEjuabOiBzU=
+github.com/tailscale/wireguard-go v0.0.0-20210401172819-1aca620a8afb/go.mod h1:jy12FSeiDLRvS7VQvSoiaqH9WtpapbrC8YSzyZ7fUAk=
+github.com/tailscale/wireguard-go v0.0.0-20210401194826-bb7bc2f24083 h1:e3k65apTVs7NM6mhQ1c94XISLe+2gdizPfRdsImNL8Y=
+github.com/tailscale/wireguard-go v0.0.0-20210401194826-bb7bc2f24083/go.mod h1:jy12FSeiDLRvS7VQvSoiaqH9WtpapbrC8YSzyZ7fUAk=
+github.com/tailscale/wireguard-go v0.0.0-20210402173217-0a47c6e64d15 h1:13GZsTKbCmPGwDBurcSXT+ssYID2IfcX0MfsvhaaagY=
+github.com/tailscale/wireguard-go v0.0.0-20210402173217-0a47c6e64d15/go.mod h1:jy12FSeiDLRvS7VQvSoiaqH9WtpapbrC8YSzyZ7fUAk=
+github.com/tailscale/wireguard-go v0.0.0-20210402193818-fc309421dd43 h1:SRUknVD6AHsxfghv0By9SFjQ8dhn8K8gIFwxf3OEPyU=
+github.com/tailscale/wireguard-go v0.0.0-20210402193818-fc309421dd43/go.mod h1:g3WdWX37upLnDT8STKFWhvA34Gwrt4hIpnWR3HGufpM=
+github.com/tailscale/wireguard-go v0.0.0-20210403171604-17614717a9b5 h1:FegsXWjtyhCxpB8bBSL1kLzagtV+e7BaX07phMM8uQM=
+github.com/tailscale/wireguard-go v0.0.0-20210403171604-17614717a9b5/go.mod h1:ys4yUmhKncXy1jWP34qUHKipRjl322VVhxoh1Rkfo7c=
 github.com/tcnksm/go-httpstat v0.2.0 h1:rP7T5e5U2HfmOBmZzGgGZjBQ5/GluWUylujl0tJ04I0=
 github.com/tcnksm/go-httpstat v0.2.0/go.mod h1:s3JVJFtQxtBEBC9dwcdTTXS9xFnM3SXAZwPG41aurT8=
 github.com/toqueteos/webbrowser v1.2.0 h1:tVP/gpK69Fx+qMJKsLE7TD8LuGWPnEV71wBN9rrstGQ=
@@ -206,6 +222,10 @@ golang.org/x/sys v0.0.0-20210309040221-94ec62e08169/go.mod h1:h1NjWce9XRLGQEsW7w
 golang.org/x/sys v0.0.0-20210316164454-77fc1eacc6aa/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20210317225723-c4fcb01b228e h1:XNp2Flc/1eWQGk5BLzqTAN7fQIwIbfyVTuVxXxZh73M=
 golang.org/x/sys v0.0.0-20210317225723-c4fcb01b228e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20210402192133-700132347e07 h1:4k6HsQjxj6hVMsI2Vf0yKlzt5lXxZsMW1q0zaq2k8zY=
+golang.org/x/sys v0.0.0-20210402192133-700132347e07/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20210403161142-5e06dd20ab57 h1:F5Gozwx4I1xtr/sr/8CFbb57iKi3297KFs0QDbGN60A=
+golang.org/x/sys v0.0.0-20210403161142-5e06dd20ab57/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw=
 golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
 golang.org/x/term v0.0.0-20210317153231-de623e64d2a6 h1:EC6+IGYTjPpRfv9a2b/6Puw0W+hLtAhkV1tPsXhutqs=

health/health.go

@@ -49,6 +49,9 @@ const (
 	// SysRouter is the name the wgengine/router subsystem.
 	SysRouter = Subsystem("router")
 
+	// SysDNS is the name of the net/dns subsystem.
+	SysDNS = Subsystem("dns")
+
 	// SysNetworkCategory is the name of the subsystem that sets
 	// the Windows network adapter's "category" (public, private, domain).
 	// If it's unhealthy, the Windows firewall rules won't match.
@@ -80,12 +83,18 @@ func RegisterWatcher(cb func(key Subsystem, err error)) (unregister func()) {
 	}
 }
 
-// SetRouter sets the state of the wgengine/router.Router.
+// SetRouterHealth sets the state of the wgengine/router.Router.
 func SetRouterHealth(err error) { set(SysRouter, err) }
 
 // RouterHealth returns the wgengine/router.Router error state.
 func RouterHealth() error { return get(SysRouter) }
 
+// SetDNSHealth sets the state of the net/dns.Manager
+func SetDNSHealth(err error) { set(SysDNS, err) }
+
+// DNSHealth returns the net/dns.Manager error state.
+func DNSHealth() error { return get(SysDNS) }
+
 // SetNetworkCategoryHealth sets the state of setting the network adaptor's category.
 // This only applies on Windows.
 func SetNetworkCategoryHealth(err error) { set(SysNetworkCategory, err) }

internal/deepprint/deepprint_test.go

@@ -9,7 +9,6 @@ import (
 	"testing"
 
 	"inet.af/netaddr"
-	"tailscale.com/net/dns"
 	"tailscale.com/wgengine/router"
 	"tailscale.com/wgengine/wgcfg"
 )
@@ -36,9 +35,8 @@ func TestDeepPrint(t *testing.T) {
 func getVal() []interface{} {
 	return []interface{}{
 		&wgcfg.Config{
-			Name:       "foo",
-			Addresses:  []netaddr.IPPrefix{{Bits: 5, IP: netaddr.IPFrom16([16]byte{3: 3})}},
-			ListenPort: 5,
+			Name:      "foo",
+			Addresses: []netaddr.IPPrefix{{Bits: 5, IP: netaddr.IPFrom16([16]byte{3: 3})}},
 			Peers: []wgcfg.Peer{
 				{
 					Endpoints: "foo:5",
@@ -46,9 +44,9 @@ func getVal() []interface{} {
 			},
 		},
 		&router.Config{
-			DNS: dns.Config{
-				Nameservers: []netaddr.IP{netaddr.IPv4(8, 8, 8, 8)},
-				Domains:     []string{"tailscale.net"},
+			Routes: []netaddr.IPPrefix{
+				netaddr.MustParseIPPrefix("1.2.3.0/24"),
+				netaddr.MustParseIPPrefix("1234::/64"),
 			},
 		},
 		map[string]string{

ipn/backend.go

@@ -68,6 +68,8 @@ type Notify struct {
 	BackendLogID  *string            // public logtail id used by backend
 	PingResult    *ipnstate.PingResult
 
+	FilesWaiting *empty.Message `json:",omitempty"`
+
 	// LocalTCPPort, if non-nil, informs the UI frontend which
 	// (non-zero) localhost TCP port it's listening on.
 	// This is currently only used by Tailscale when run in the
@@ -149,9 +151,8 @@ type Backend interface {
 	// WantRunning. This may cause the wireguard engine to
 	// reconfigure or stop.
 	SetPrefs(*Prefs)
-	// SetWantRunning is like SetPrefs but sets only the
-	// WantRunning field.
-	SetWantRunning(wantRunning bool)
+	// EditPrefs is like SetPrefs but only sets the specified fields.
+	EditPrefs(*MaskedPrefs)
 	// RequestEngineStatus polls for an update from the wireguard
 	// engine. Only needed if you want to display byte
 	// counts. Connection events are emitted automatically without

ipn/fake_test.go

@@ -79,8 +79,11 @@ func (b *FakeBackend) SetPrefs(new *Prefs) {
 	}
 }
 
-func (b *FakeBackend) SetWantRunning(v bool) {
-	b.SetPrefs(&Prefs{WantRunning: v})
+func (b *FakeBackend) EditPrefs(mp *MaskedPrefs) {
+	// This fake implementation only cares about this one pref.
+	if mp.WantRunningSet {
+		b.SetPrefs(&mp.Prefs)
+	}
 }
 
 func (b *FakeBackend) RequestEngineStatus() {

ipn/ipnlocal/local.go

@@ -9,13 +9,16 @@ import (
 	"context"
 	"errors"
 	"fmt"
+	"io"
 	"net"
 	"os"
+	"os/exec"
 	"path/filepath"
 	"runtime"
 	"strconv"
 	"strings"
 	"sync"
+	"sync/atomic"
 	"time"
 
 	"inet.af/netaddr"
@@ -107,6 +110,7 @@ type LocalBackend struct {
 	authURL          string
 	interact         bool
 	prevIfState      *interfaces.State
+	peerAPIServer    *peerAPIServer // or nil
 	peerAPIListeners []*peerAPIListener
 
 	// statusLock must be held before calling statusChanged.Wait() or
@@ -198,6 +202,14 @@ func (b *LocalBackend) linkChange(major bool, ifst *interfaces.State) {
 	// If the local network configuration has changed, our filter may
 	// need updating to tweak default routes.
 	b.updateFilter(b.netMap, b.prefs)
+
+	if runtime.GOOS == "windows" && b.netMap != nil {
+		want := len(b.netMap.Addresses)
+		b.logf("linkChange: peerAPIListeners too low; trying again")
+		if len(b.peerAPIListeners) < want {
+			go b.initPeerAPIListener()
+		}
+	}
 }
 
 func (b *LocalBackend) onHealthChange(sys health.Subsystem, err error) {
@@ -385,11 +397,6 @@ func (b *LocalBackend) setClientStatus(st controlclient.Status) {
 			b.prefs.Persist = st.Persist.Clone()
 		}
 	}
-	if temporarilySetMachineKeyInPersist() && b.prefs.Persist != nil &&
-		b.prefs.Persist.LegacyFrontendPrivateMachineKey.IsZero() {
-		b.prefs.Persist.LegacyFrontendPrivateMachineKey = b.machinePrivKey
-		prefsChanged = true
-	}
 	if st.NetMap != nil {
 		if b.findExitNodeIDLocked(st.NetMap) {
 			prefsChanged = true
@@ -433,9 +440,6 @@ func (b *LocalBackend) setClientStatus(st controlclient.Status) {
 
 		b.updateFilter(st.NetMap, prefs)
 		b.e.SetNetworkMap(st.NetMap)
-		if !dnsMapsEqual(st.NetMap, netMap) {
-			b.updateDNSMap(st.NetMap)
-		}
 		b.e.SetDERPMap(st.NetMap.DERPMap)
 
 		b.send(ipn.Notify{NetMap: st.NetMap})
@@ -564,6 +568,13 @@ func (b *LocalBackend) Start(opts ipn.Options) error {
 		return fmt.Errorf("loading requested state: %v", err)
 	}
 
+	wantRunning := b.prefs.WantRunning
+	if wantRunning {
+		if err := b.initMachineKeyLocked(); err != nil {
+			return fmt.Errorf("initMachineKeyLocked: %w", err)
+		}
+	}
+
 	b.inServerMode = b.prefs.ForceDaemon
 	b.serverURL = b.prefs.ControlURL
 	hostinfo.RoutableIPs = append(hostinfo.RoutableIPs, b.prefs.AdvertiseRoutes...)
@@ -576,7 +587,6 @@ func (b *LocalBackend) Start(opts ipn.Options) error {
 	b.notify = opts.Notify
 	b.setNetMapLocked(nil)
 	persistv := b.prefs.Persist
-	machinePrivKey := b.machinePrivKey
 	b.mu.Unlock()
 
 	b.updateFilter(nil, nil)
@@ -613,18 +623,22 @@ func (b *LocalBackend) Start(opts ipn.Options) error {
 		persistv = &persist.Persist{}
 	}
 	cli, err := controlclient.New(controlclient.Options{
-		MachinePrivateKey: machinePrivKey,
-		Logf:              logger.WithPrefix(b.logf, "control: "),
-		Persist:           *persistv,
-		ServerURL:         b.serverURL,
-		AuthKey:           opts.AuthKey,
-		Hostinfo:          hostinfo,
-		KeepAlive:         true,
-		NewDecompressor:   b.newDecompressor,
-		HTTPTestClient:    opts.HTTPTestClient,
-		DiscoPublicKey:    discoPublic,
-		DebugFlags:        controlDebugFlags,
-		LinkMonitor:       b.e.GetLinkMonitor(),
+		GetMachinePrivateKey: b.createGetMachinePrivateKeyFunc(),
+		Logf:                 logger.WithPrefix(b.logf, "control: "),
+		Persist:              *persistv,
+		ServerURL:            b.serverURL,
+		AuthKey:              opts.AuthKey,
+		Hostinfo:             hostinfo,
+		KeepAlive:            true,
+		NewDecompressor:      b.newDecompressor,
+		HTTPTestClient:       opts.HTTPTestClient,
+		DiscoPublicKey:       discoPublic,
+		DebugFlags:           controlDebugFlags,
+		LinkMonitor:          b.e.GetLinkMonitor(),
+
+		// Don't warn about broken Linux IP forwading when
+		// netstack is being used.
+		SkipIPForwardingCheck: wgengine.IsNetstack(b.e),
 	})
 	if err != nil {
 		return err
@@ -645,12 +659,6 @@ func (b *LocalBackend) Start(opts ipn.Options) error {
 
 	b.mu.Lock()
 	prefs := b.prefs.Clone()
-
-	if temporarilySetMachineKeyInPersist() && prefs.Persist != nil &&
-		prefs.Persist.LegacyFrontendPrivateMachineKey.IsZero() {
-		prefs.Persist.LegacyFrontendPrivateMachineKey = b.machinePrivKey
-	}
-
 	b.mu.Unlock()
 
 	blid := b.backendLogID
@@ -658,7 +666,9 @@ func (b *LocalBackend) Start(opts ipn.Options) error {
 	b.send(ipn.Notify{BackendLogID: &blid})
 	b.send(ipn.Notify{Prefs: prefs})
 
-	cli.Login(nil, controlclient.LoginDefault)
+	if wantRunning {
+		cli.Login(nil, controlclient.LoginDefault)
+	}
 	return nil
 }
 
@@ -841,32 +851,6 @@ func dnsMapsEqual(new, old *netmap.NetworkMap) bool {
 	return true
 }
 
-// updateDNSMap updates the domain map in the DNS resolver in wgengine
-// based on the given netMap and user preferences.
-func (b *LocalBackend) updateDNSMap(netMap *netmap.NetworkMap) {
-	if netMap == nil {
-		b.logf("dns map: (not ready)")
-		return
-	}
-
-	nameToIP := make(map[string]netaddr.IP)
-	set := func(name string, addrs []netaddr.IPPrefix) {
-		if len(addrs) == 0 || name == "" {
-			return
-		}
-		nameToIP[name] = addrs[0].IP
-	}
-
-	for _, peer := range netMap.Peers {
-		set(peer.Name, peer.Addresses)
-	}
-	set(netMap.Name, netMap.Addresses)
-
-	dnsMap := dns.NewMap(nameToIP, magicDNSRootDomains(netMap))
-	// map diff will be logged in dns.Resolver.SetMap.
-	b.e.SetDNSMap(dnsMap)
-}
-
 // readPoller is a goroutine that receives service lists from
 // b.portpoll and propagates them into the controlclient's HostInfo.
 func (b *LocalBackend) readPoller() {
@@ -909,15 +893,20 @@ func (b *LocalBackend) readPoller() {
 // connected, the notification is dropped without being delivered.
 func (b *LocalBackend) send(n ipn.Notify) {
 	b.mu.Lock()
-	notify := b.notify
+	notifyFunc := b.notify
+	apiSrv := b.peerAPIServer
 	b.mu.Unlock()
 
-	if notify != nil {
-		n.Version = version.Long
-		notify(n)
-	} else {
+	if notifyFunc == nil {
 		b.logf("nil notify callback; dropping %+v", n)
+		return
 	}
+
+	n.Version = version.Long
+	if apiSrv != nil && apiSrv.hasFilesWaiting() {
+		n.FilesWaiting = &empty.Message{}
+	}
+	notifyFunc(n)
 }
 
 // popBrowserAuthNow shuts down the data plane and sends an auth URL
@@ -939,23 +928,37 @@ func (b *LocalBackend) popBrowserAuthNow() {
 	}
 }
 
+// For testing lazy machine key generation.
+var panicOnMachineKeyGeneration, _ = strconv.ParseBool(os.Getenv("TS_DEBUG_PANIC_MACHINE_KEY"))
+
+func (b *LocalBackend) createGetMachinePrivateKeyFunc() func() (wgkey.Private, error) {
+	var cache atomic.Value
+	return func() (wgkey.Private, error) {
+		if panicOnMachineKeyGeneration {
+			panic("machine key generated")
+		}
+		if v, ok := cache.Load().(wgkey.Private); ok {
+			return v, nil
+		}
+		b.mu.Lock()
+		defer b.mu.Unlock()
+		if v, ok := cache.Load().(wgkey.Private); ok {
+			return v, nil
+		}
+		if err := b.initMachineKeyLocked(); err != nil {
+			return wgkey.Private{}, err
+		}
+		cache.Store(b.machinePrivKey)
+		return b.machinePrivKey, nil
+	}
+}
+
 // initMachineKeyLocked is called to initialize b.machinePrivKey.
 //
 // b.prefs must already be initialized.
 // b.stateKey should be set too, but just for nicer log messages.
 // b.mu must be held.
 func (b *LocalBackend) initMachineKeyLocked() (err error) {
-	if temporarilySetMachineKeyInPersist() {
-		defer func() {
-			if err != nil {
-				return
-			}
-			if b.prefs != nil && b.prefs.Persist != nil {
-				b.prefs.Persist.LegacyFrontendPrivateMachineKey = b.machinePrivKey
-			}
-		}()
-	}
-
 	if !b.machinePrivKey.IsZero() {
 		// Already set.
 		return nil
@@ -1067,9 +1070,6 @@ func (b *LocalBackend) loadStateLocked(key ipn.StateKey, prefs *ipn.Prefs, legac
 		// value instead of making up a new one.
 		b.logf("using frontend prefs: %s", prefs.Pretty())
 		b.prefs = prefs.Clone()
-		if err := b.initMachineKeyLocked(); err != nil {
-			return fmt.Errorf("initMachineKeyLocked: %w", err)
-		}
 		b.writeServerModeStartState(b.userID, b.prefs)
 		return nil
 	}
@@ -1085,27 +1085,28 @@ func (b *LocalBackend) loadStateLocked(key ipn.StateKey, prefs *ipn.Prefs, legac
 
 	b.logf("using backend prefs")
 	bs, err := b.store.ReadState(key)
-	if err != nil {
-		if errors.Is(err, ipn.ErrStateNotExist) {
-			if legacyPath != "" {
-				b.prefs, err = ipn.LoadPrefs(legacyPath)
-				if err != nil {
-					if !errors.Is(err, os.ErrNotExist) {
-						b.logf("failed to load legacy prefs: %v", err)
-					}
-					b.prefs = ipn.NewPrefs()
-				} else {
-					b.logf("imported prefs from relaynode for %q: %v", key, b.prefs.Pretty())
-				}
-			} else {
-				b.prefs = ipn.NewPrefs()
-				b.logf("created empty state for %q: %s", key, b.prefs.Pretty())
+	switch {
+	case errors.Is(err, ipn.ErrStateNotExist):
+		loaded := false
+		if legacyPath != "" {
+			b.prefs, err = ipn.LoadPrefs(legacyPath)
+			switch {
+			case errors.Is(err, os.ErrNotExist):
+				// Quiet. Normal case.
+			case err != nil:
+				b.logf("failed to load legacy prefs: %v", err)
+			default:
+				loaded = true
+				b.logf("imported prefs from relaynode for %q: %v", key, b.prefs.Pretty())
 			}
-			if err := b.initMachineKeyLocked(); err != nil {
-				return fmt.Errorf("initMachineKeyLocked: %w", err)
-			}
-			return nil
 		}
+		if !loaded {
+			b.prefs = ipn.NewPrefs()
+			b.prefs.WantRunning = false
+			b.logf("created empty state for %q: %s", key, b.prefs.Pretty())
+		}
+		return nil
+	case err != nil:
 		return fmt.Errorf("store.ReadState(%q): %v", key, err)
 	}
 	b.prefs, err = ipn.PrefsFromBytes(bs, false)
@@ -1113,9 +1114,6 @@ func (b *LocalBackend) loadStateLocked(key ipn.StateKey, prefs *ipn.Prefs, legac
 		return fmt.Errorf("PrefsFromBytes: %v", err)
 	}
 	b.logf("backend prefs for %q: %s", key, b.prefs.Pretty())
-	if err := b.initMachineKeyLocked(); err != nil {
-		return fmt.Errorf("initMachineKeyLocked: %w", err)
-	}
 	return nil
 }
 
@@ -1257,16 +1255,17 @@ func (b *LocalBackend) SetCurrentUserID(uid string) {
 	b.mu.Unlock()
 }
 
-func (b *LocalBackend) SetWantRunning(wantRunning bool) {
+func (b *LocalBackend) EditPrefs(mp *ipn.MaskedPrefs) {
 	b.mu.Lock()
-	new := b.prefs.Clone()
-	b.mu.Unlock()
-	if new.WantRunning == wantRunning {
+	p0 := b.prefs.Clone()
+	p1 := b.prefs.Clone()
+	p1.ApplyEdits(mp)
+	if p1.Equals(p0) {
+		b.mu.Unlock()
 		return
 	}
-	new.WantRunning = wantRunning
-	b.logf("SetWantRunning: %v", wantRunning)
-	b.SetPrefs(new)
+	b.logf("EditPrefs: %v", mp.Pretty())
+	b.setPrefsLockedOnEntry("EditPrefs", p1)
 }
 
 // SetPrefs saves new user preferences and propagates them throughout
@@ -1275,9 +1274,13 @@ func (b *LocalBackend) SetPrefs(newp *ipn.Prefs) {
 	if newp == nil {
 		panic("SetPrefs got nil prefs")
 	}
-
 	b.mu.Lock()
+	b.setPrefsLockedOnEntry("SetPrefs", newp)
+}
 
+// setPrefsLockedOnEntry requires b.mu be held to call it, but it
+// unlocks b.mu when done.
+func (b *LocalBackend) setPrefsLockedOnEntry(caller string, newp *ipn.Prefs) {
 	netMap := b.netMap
 	stateKey := b.stateKey
 
@@ -1300,13 +1303,15 @@ func (b *LocalBackend) SetPrefs(newp *ipn.Prefs) {
 
 	if stateKey != "" {
 		if err := b.store.WriteState(stateKey, newp.ToBytes()); err != nil {
-			b.logf("Failed to save new controlclient state: %v", err)
+			b.logf("failed to save new controlclient state: %v", err)
 		}
 	}
 	b.writeServerModeStartState(userID, newp)
 
 	// [GRINDER STATS LINE] - please don't remove (used for log parsing)
-	b.logf("SetPrefs: %v", newp.Pretty())
+	if caller == "SetPrefs" {
+		b.logf("SetPrefs: %v", newp.Pretty())
+	}
 	if netMap != nil {
 		if login := netMap.UserProfiles[netMap.User].LoginName; login != "" {
 			if newp.Persist == nil {
@@ -1346,7 +1351,7 @@ func (b *LocalBackend) getPeerAPIPortForTSMPPing(ip netaddr.IP) (port uint16, ok
 	defer b.mu.Unlock()
 	for _, pln := range b.peerAPIListeners {
 		if pln.ip.BitLen() == ip.BitLen() {
-			return uint16(pln.Port()), true
+			return uint16(pln.port), true
 		}
 	}
 	return 0, false
@@ -1360,7 +1365,7 @@ func (b *LocalBackend) peerAPIServicesLocked() (ret []tailcfg.Service) {
 		}
 		ret = append(ret, tailcfg.Service{
 			Proto: proto,
-			Port:  uint16(pln.Port()),
+			Port:  uint16(pln.port),
 		})
 	}
 	return ret
@@ -1461,22 +1466,45 @@ func (b *LocalBackend) authReconfig() {
 
 	rcfg := routerConfig(cfg, uc)
 
-	// If CorpDNS is false, rcfg.DNS remains the zero value.
+	var dcfg dns.Config
+
+	// If CorpDNS is false, dcfg remains the zero value.
 	if uc.CorpDNS {
 		proxied := nm.DNS.Proxied
 		if proxied && len(nm.DNS.Nameservers) == 0 {
 			b.logf("[unexpected] dns proxied but no nameservers")
 			proxied = false
 		}
-		rcfg.DNS = dns.Config{
-			Nameservers: nm.DNS.Nameservers,
-			Domains:     nm.DNS.Domains,
-			PerDomain:   nm.DNS.PerDomain,
-			Proxied:     proxied,
+		for _, ip := range nm.DNS.Nameservers {
+			dcfg.DefaultResolvers = append(dcfg.DefaultResolvers, netaddr.IPPort{
+				IP:   ip,
+				Port: 53,
+			})
+		}
+		dcfg.SearchDomains = nm.DNS.Domains
+		dcfg.AuthoritativeSuffixes = magicDNSRootDomains(nm)
+		set := func(name string, addrs []netaddr.IPPrefix) {
+			if len(addrs) == 0 || name == "" {
+				return
+			}
+			var ips []netaddr.IP
+			for _, addr := range addrs {
+				ips = append(ips, addr.IP)
+			}
+			dcfg.Hosts[name] = ips
+		}
+		// TODO: hack to make the current code continue to work while
+		// refactoring happens.
+		if proxied {
+			dcfg.Hosts = map[string][]netaddr.IP{}
+			set(nm.Name, nm.Addresses)
+			for _, peer := range nm.Peers {
+				set(peer.Name, peer.Addresses)
+			}
 		}
 	}
 
-	err = b.e.Reconfig(cfg, rcfg)
+	err = b.e.Reconfig(cfg, rcfg, &dcfg)
 	if err == wgengine.ErrNoChanges {
 		return
 	}
@@ -1485,12 +1513,27 @@ func (b *LocalBackend) authReconfig() {
 	b.initPeerAPIListener()
 }
 
+// tailscaleVarRoot returns the root directory of Tailscale's writable
+// storage area. (e.g. "/var/lib/tailscale")
+func tailscaleVarRoot() string {
+	if runtime.GOOS == "ios" {
+		dir, _ := paths.IOSSharedDir.Load().(string)
+		return dir
+	}
+	stateFile := paths.DefaultTailscaledStateFile()
+	if stateFile == "" {
+		return ""
+	}
+	return filepath.Dir(stateFile)
+}
+
 func (b *LocalBackend) initPeerAPIListener() {
 	b.mu.Lock()
 	defer b.mu.Unlock()
 
+	b.peerAPIServer = nil
 	for _, pln := range b.peerAPIListeners {
-		pln.ln.Close()
+		pln.Close()
 	}
 	b.peerAPIListeners = nil
 
@@ -1499,15 +1542,15 @@ func (b *LocalBackend) initPeerAPIListener() {
 		return
 	}
 
-	stateFile := paths.DefaultTailscaledStateFile()
-	if stateFile == "" {
+	varRoot := tailscaleVarRoot()
+	if varRoot == "" {
 		b.logf("peerapi disabled; no state directory")
 		return
 	}
 	baseDir := fmt.Sprintf("%s-uid-%d",
 		strings.ReplaceAll(b.activeLogin, "@", "-"),
 		selfNode.User)
-	dir := filepath.Join(filepath.Dir(stateFile), "files", baseDir)
+	dir := filepath.Join(varRoot, "files", baseDir)
 	if err := os.MkdirAll(dir, 0700); err != nil {
 		b.logf("peerapi disabled; error making directory: %v", err)
 		return
@@ -1526,21 +1569,33 @@ func (b *LocalBackend) initPeerAPIListener() {
 		tunName:  tunName,
 		selfNode: selfNode,
 	}
+	b.peerAPIServer = ps
 
-	for _, a := range b.netMap.Addresses {
-		ln, err := ps.listen(a.IP, b.prevIfState)
-		if err != nil {
-			b.logf("[unexpected] peerAPI listen(%q) error: %v", a.IP, err)
-			continue
+	isNetstack := wgengine.IsNetstack(b.e)
+	for i, a := range b.netMap.Addresses {
+		var ln net.Listener
+		var err error
+		skipListen := i > 0 && isNetstack
+		if !skipListen {
+			ln, err = ps.listen(a.IP, b.prevIfState)
+			if err != nil {
+				b.logf("[unexpected] peerapi listen(%q) error: %v", a.IP, err)
+				continue
+			}
 		}
 		pln := &peerAPIListener{
 			ps: ps,
 			ip: a.IP,
-			ln: ln,
+			ln: ln, // nil for 2nd+ on netstack
 			lb: b,
 		}
-		pln.urlStr = "http://" + net.JoinHostPort(a.IP.String(), strconv.Itoa(pln.Port()))
-
+		if skipListen {
+			pln.port = b.peerAPIListeners[0].port
+		} else {
+			pln.port = ln.Addr().(*net.TCPAddr).Port
+		}
+		pln.urlStr = "http://" + net.JoinHostPort(a.IP.String(), strconv.Itoa(pln.port))
+		b.logf("peerapi: serving on %s", pln.urlStr)
 		go pln.serve()
 		b.peerAPIListeners = append(b.peerAPIListeners, pln)
 	}
@@ -1704,7 +1759,7 @@ func (b *LocalBackend) enterState(newState ipn.State) {
 		b.blockEngineUpdates(true)
 		fallthrough
 	case ipn.Stopped:
-		err := b.e.Reconfig(&wgcfg.Config{}, &router.Config{})
+		err := b.e.Reconfig(&wgcfg.Config{}, &router.Config{}, &dns.Config{})
 		if err != nil {
 			b.logf("Reconfig(down): %v", err)
 		}
@@ -1796,7 +1851,7 @@ func (b *LocalBackend) stateMachine() {
 // a status update that predates the "I've shut down" update.
 func (b *LocalBackend) stopEngineAndWait() {
 	b.logf("stopEngineAndWait...")
-	b.e.Reconfig(&wgcfg.Config{}, &router.Config{})
+	b.e.Reconfig(&wgcfg.Config{}, &router.Config{}, &dns.Config{})
 	b.requestEngineStatusAndWait()
 	b.logf("stopEngineAndWait: done.")
 }
@@ -1938,19 +1993,74 @@ func (b *LocalBackend) TestOnlyPublicKeys() (machineKey tailcfg.MachineKey, node
 	return tailcfg.MachineKey(mk), tailcfg.NodeKey(nk)
 }
 
-// temporarilySetMachineKeyInPersist reports whether we should set
-// the machine key in Prefs.Persist.LegacyFrontendPrivateMachineKey
-// for the frontend to write out to its preferences for use later.
-//
-// TODO: remove this in Tailscale 1.3.x (so it effectively always
-// returns false). It just exists so users can downgrade from 1.2.x to
-// 1.0.x.  But eventually we want to stop sending the machine key to
-// clients. We can't do that until 1.0.x is no longer supported.
-func temporarilySetMachineKeyInPersist() bool {
-	switch runtime.GOOS {
-	case "darwin", "ios", "android":
-		// iOS, macOS, Android users can't downgrade anyway.
-		return false
+func (b *LocalBackend) WaitingFiles() ([]WaitingFile, error) {
+	b.mu.Lock()
+	apiSrv := b.peerAPIServer
+	b.mu.Unlock()
+	if apiSrv == nil {
+		return nil, errors.New("peerapi disabled")
 	}
-	return true
+	return apiSrv.WaitingFiles()
+}
+
+func (b *LocalBackend) DeleteFile(name string) error {
+	b.mu.Lock()
+	apiSrv := b.peerAPIServer
+	b.mu.Unlock()
+	if apiSrv == nil {
+		return errors.New("peerapi disabled")
+	}
+	return apiSrv.DeleteFile(name)
+}
+
+func (b *LocalBackend) OpenFile(name string) (rc io.ReadCloser, size int64, err error) {
+	b.mu.Lock()
+	apiSrv := b.peerAPIServer
+	b.mu.Unlock()
+	if apiSrv == nil {
+		return nil, 0, errors.New("peerapi disabled")
+	}
+	return apiSrv.OpenFile(name)
+}
+
+func isBSD(s string) bool {
+	return s == "dragonfly" || s == "freebsd" || s == "netbsd" || s == "openbsd"
+}
+
+func (b *LocalBackend) CheckIPForwarding() error {
+	if wgengine.IsNetstack(b.e) {
+		return nil
+	}
+	if isBSD(runtime.GOOS) {
+		//lint:ignore ST1005 output to users as is
+		return fmt.Errorf("Subnet routing and exit nodes only work with additional manual configuration on %v, and is not currently officially supported.", runtime.GOOS)
+	}
+
+	var keys []string
+
+	if runtime.GOOS == "linux" {
+		keys = append(keys, "net.ipv4.ip_forward", "net.ipv6.conf.all.forwarding")
+	} else if isBSD(runtime.GOOS) {
+		keys = append(keys, "net.inet.ip.forwarding")
+	} else {
+		return nil
+	}
+
+	for _, key := range keys {
+		bs, err := exec.Command("sysctl", "-n", key).Output()
+		if err != nil {
+			//lint:ignore ST1005 output to users as is
+			return fmt.Errorf("couldn't check %s (%v).\nSubnet routes won't work without IP forwarding.", key, err)
+		}
+		on, err := strconv.ParseBool(string(bytes.TrimSpace(bs)))
+		if err != nil {
+			//lint:ignore ST1005 output to users as is
+			return fmt.Errorf("couldn't parse %s (%v).\nSubnet routes won't work without IP forwarding.", key, err)
+		}
+		if !on {
+			//lint:ignore ST1005 output to users as is
+			return fmt.Errorf("%s is disabled. Subnet routes won't work.", key)
+		}
+	}
+	return nil
 }

ipn/ipnlocal/peerapi.go

@@ -22,17 +22,152 @@ import (
 	"strings"
 
 	"inet.af/netaddr"
+	"tailscale.com/ipn"
 	"tailscale.com/net/interfaces"
+	"tailscale.com/syncs"
 	"tailscale.com/tailcfg"
+	"tailscale.com/wgengine"
 )
 
 var initListenConfig func(*net.ListenConfig, netaddr.IP, *interfaces.State, string) error
 
 type peerAPIServer struct {
-	b        *LocalBackend
-	rootDir  string
-	tunName  string
-	selfNode *tailcfg.Node
+	b          *LocalBackend
+	rootDir    string
+	tunName    string
+	selfNode   *tailcfg.Node
+	knownEmpty syncs.AtomicBool
+}
+
+const partialSuffix = ".tspartial"
+
+func (s *peerAPIServer) diskPath(baseName string) (fullPath string, ok bool) {
+	clean := path.Clean(baseName)
+	if clean != baseName ||
+		clean == "." ||
+		strings.ContainsAny(clean, `/\`) ||
+		strings.HasSuffix(clean, partialSuffix) {
+		return "", false
+	}
+	return filepath.Join(s.rootDir, strings.ReplaceAll(url.PathEscape(baseName), ":", "%3a")), true
+}
+
+// hasFilesWaiting reports whether any files are buffered in the
+// tailscaled daemon storage.
+func (s *peerAPIServer) hasFilesWaiting() bool {
+	if s.rootDir == "" {
+		return false
+	}
+	if s.knownEmpty.Get() {
+		// Optimization: this is usually empty, so avoid opening
+		// the directory and checking. We can't cache the actual
+		// has-files-or-not values as the macOS/iOS client might
+		// in the future use+delete the files directly. So only
+		// keep this negative cache.
+		return false
+	}
+	f, err := os.Open(s.rootDir)
+	if err != nil {
+		return false
+	}
+	defer f.Close()
+	for {
+		des, err := f.ReadDir(10)
+		for _, de := range des {
+			if strings.HasSuffix(de.Name(), partialSuffix) {
+				continue
+			}
+			if de.Type().IsRegular() {
+				return true
+			}
+		}
+		if err == io.EOF {
+			s.knownEmpty.Set(true)
+		}
+		if err != nil {
+			break
+		}
+	}
+	return false
+}
+
+// WaitingFile is a JSON-marshaled struct sent by the localapi to pick
+// up queued files.
+type WaitingFile struct {
+	Name string
+	Size int64
+}
+
+func (s *peerAPIServer) WaitingFiles() (ret []WaitingFile, err error) {
+	if s.rootDir == "" {
+		return nil, errors.New("peerapi disabled; no storage configured")
+	}
+	f, err := os.Open(s.rootDir)
+	if err != nil {
+		return nil, err
+	}
+	defer f.Close()
+	for {
+		des, err := f.ReadDir(10)
+		for _, de := range des {
+			name := de.Name()
+			if strings.HasSuffix(name, partialSuffix) {
+				continue
+			}
+			if de.Type().IsRegular() {
+				fi, err := de.Info()
+				if err != nil {
+					continue
+				}
+				ret = append(ret, WaitingFile{
+					Name: filepath.Base(name),
+					Size: fi.Size(),
+				})
+			}
+		}
+		if err == io.EOF {
+			break
+		}
+		if err != nil {
+			return nil, err
+		}
+	}
+	return ret, nil
+}
+
+func (s *peerAPIServer) DeleteFile(baseName string) error {
+	if s.rootDir == "" {
+		return errors.New("peerapi disabled; no storage configured")
+	}
+	path, ok := s.diskPath(baseName)
+	if !ok {
+		return errors.New("bad filename")
+	}
+	err := os.Remove(path)
+	if err != nil && !os.IsNotExist(err) {
+		return err
+	}
+	return nil
+}
+
+func (s *peerAPIServer) OpenFile(baseName string) (rc io.ReadCloser, size int64, err error) {
+	if s.rootDir == "" {
+		return nil, 0, errors.New("peerapi disabled; no storage configured")
+	}
+	path, ok := s.diskPath(baseName)
+	if !ok {
+		return nil, 0, errors.New("bad filename")
+	}
+	f, err := os.Open(path)
+	if err != nil {
+		return nil, 0, err
+	}
+	fi, err := f.Stat()
+	if err != nil {
+		f.Close()
+		return nil, 0, err
+	}
+	return f, fi.Size(), nil
 }
 
 func (s *peerAPIServer) listen(ip netaddr.IP, ifState *interfaces.State) (ln net.Listener, err error) {
@@ -51,6 +186,10 @@ func (s *peerAPIServer) listen(ip netaddr.IP, ifState *interfaces.State) (ln net
 		}
 	}
 
+	if wgengine.IsNetstack(s.b.e) {
+		ipStr = ""
+	}
+
 	tcp4or6 := "tcp4"
 	if ip.Is6() {
 		tcp4or6 = "tcp6"
@@ -79,22 +218,32 @@ func (s *peerAPIServer) listen(ip netaddr.IP, ifState *interfaces.State) (ln net
 }
 
 type peerAPIListener struct {
-	ps     *peerAPIServer
-	ip     netaddr.IP
-	ln     net.Listener
-	lb     *LocalBackend
+	ps *peerAPIServer
+	ip netaddr.IP
+	lb *LocalBackend
+
+	// ln is the Listener. It can be nil in netstack mode if there are more than
+	// 1 local addresses (e.g. both an IPv4 and IPv6). When it's nil, port
+	// and urlStr are still populated.
+	ln net.Listener
+
+	// urlStr is the base URL to access the peer API (http://ip:port/).
 	urlStr string
+	// port is just the port of urlStr.
+	port int
 }
 
-func (pln *peerAPIListener) Port() int {
-	ta, ok := pln.ln.Addr().(*net.TCPAddr)
-	if !ok {
-		return 0
+func (pln *peerAPIListener) Close() error {
+	if pln.ln != nil {
+		return pln.ln.Close()
 	}
-	return ta.Port
+	return nil
 }
 
 func (pln *peerAPIListener) serve() {
+	if pln.ln == nil {
+		return
+	}
 	defer pln.ln.Close()
 	logf := pln.lb.logf
 	for {
@@ -202,13 +351,12 @@ func (h *peerAPIHandler) put(w http.ResponseWriter, r *http.Request) {
 		http.Error(w, "no rootdir", http.StatusInternalServerError)
 		return
 	}
-	name := path.Base(r.URL.Path)
-	if name == "." || name == "/" {
-		http.Error(w, "bad filename", http.StatusForbidden)
+	baseName := path.Base(r.URL.Path)
+	dstFile, ok := h.ps.diskPath(baseName)
+	if !ok {
+		http.Error(w, "bad filename", 400)
 		return
 	}
-	fileBase := strings.ReplaceAll(url.PathEscape(name), ":", "%3a")
-	dstFile := filepath.Join(h.ps.rootDir, fileBase)
 	f, err := os.Create(dstFile)
 	if err != nil {
 		h.logf("put Create error: %v", err)
@@ -234,10 +382,22 @@ func (h *peerAPIHandler) put(w http.ResponseWriter, r *http.Request) {
 		return
 	}
 
-	h.logf("put(%q): %d bytes from %v/%v", name, n, h.remoteAddr.IP, h.peerNode.ComputedName)
+	h.logf("put of %s from %v/%v", baseName, approxSize(n), h.remoteAddr.IP, h.peerNode.ComputedName)
 
 	// TODO: set modtime
 	// TODO: some real response
 	success = true
 	io.WriteString(w, "{}\n")
+	h.ps.knownEmpty.Set(false)
+	h.ps.b.send(ipn.Notify{}) // it will set FilesWaiting
+}
+
+func approxSize(n int64) string {
+	if n <= 1<<10 {
+		return "<=1KB"
+	}
+	if n <= 1<<20 {
+		return "<=1MB"
+	}
+	return fmt.Sprintf("~%dMB", n/1<<20)
 }

ipn/ipnserver/server.go

@@ -97,8 +97,9 @@ type Options struct {
 // server is an IPN backend and its set of 0 or more active connections
 // talking to an IPN backend.
 type server struct {
-	b    *ipnlocal.LocalBackend
-	logf logger.Logf
+	b            *ipnlocal.LocalBackend
+	logf         logger.Logf
+	backendLogID string
 	// resetOnZero is whether to call bs.Reset on transition from
 	// 1->0 connections.  That is, this is whether the backend is
 	// being run in "client mode" that requires an active GUI
@@ -610,8 +611,9 @@ func Run(ctx context.Context, logf logger.Logf, logid string, getEngine func() (
 	}
 
 	server := &server{
-		logf:        logf,
-		resetOnZero: !opts.SurviveDisconnects,
+		backendLogID: logid,
+		logf:         logf,
+		resetOnZero:  !opts.SurviveDisconnects,
 	}
 
 	// When the context is closed or when we return, whichever is first, close our listner
@@ -982,7 +984,7 @@ func (psc *protoSwitchConn) Close() error {
 }
 
 func (s *server) localhostHandler(ci connIdentity) http.Handler {
-	lah := localapi.NewHandler(s.b)
+	lah := localapi.NewHandler(s.b, s.logf, s.backendLogID)
 	lah.PermitRead, lah.PermitWrite = s.localAPIPermissions(ci)
 
 	return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {

ipn/localapi/localapi.go

@@ -6,20 +6,33 @@
 package localapi
 
 import (
+	"crypto/rand"
+	"encoding/hex"
 	"encoding/json"
+	"fmt"
 	"io"
 	"net/http"
+	"net/url"
 	"runtime"
 	"strconv"
+	"strings"
+	"time"
 
 	"inet.af/netaddr"
 	"tailscale.com/ipn/ipnlocal"
 	"tailscale.com/ipn/ipnstate"
 	"tailscale.com/tailcfg"
+	"tailscale.com/types/logger"
 )
 
-func NewHandler(b *ipnlocal.LocalBackend) *Handler {
-	return &Handler{b: b}
+func randHex(n int) string {
+	b := make([]byte, n)
+	rand.Read(b)
+	return hex.EncodeToString(b)
+}
+
+func NewHandler(b *ipnlocal.LocalBackend, logf logger.Logf, logID string) *Handler {
+	return &Handler{b: b, logf: logf, backendLogID: logID}
 }
 
 type Handler struct {
@@ -34,7 +47,9 @@ type Handler struct {
 	// PermitWrite is whether mutating HTTP handlers are allowed.
 	PermitWrite bool
 
-	b *ipnlocal.LocalBackend
+	b            *ipnlocal.LocalBackend
+	logf         logger.Logf
+	backendLogID string
 }
 
 func (h *Handler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
@@ -53,6 +68,10 @@ func (h *Handler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
 			return
 		}
 	}
+	if strings.HasPrefix(r.URL.Path, "/localapi/v0/files/") {
+		h.serveFiles(w, r)
+		return
+	}
 	switch r.URL.Path {
 	case "/localapi/v0/whois":
 		h.serveWhoIs(w, r)
@@ -60,11 +79,32 @@ func (h *Handler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
 		h.serveGoroutines(w, r)
 	case "/localapi/v0/status":
 		h.serveStatus(w, r)
-	default:
+	case "/localapi/v0/check-ip-forwarding":
+		h.serveCheckIPForwarding(w, r)
+	case "/localapi/v0/bugreport":
+		h.serveBugReport(w, r)
+	case "/":
 		io.WriteString(w, "tailscaled\n")
+	default:
+		http.Error(w, "404 not found", 404)
 	}
 }
 
+func (h *Handler) serveBugReport(w http.ResponseWriter, r *http.Request) {
+	if !h.PermitRead {
+		http.Error(w, "bugreport access denied", http.StatusForbidden)
+		return
+	}
+
+	logMarker := fmt.Sprintf("BUG-%v-%v-%v", h.backendLogID, time.Now().UTC().Format("20060102150405Z"), randHex(8))
+	h.logf("user bugreport: %s", logMarker)
+	if note := r.FormValue("note"); len(note) > 0 {
+		h.logf("user bugreport note: %s", note)
+	}
+	w.Header().Set("Content-Type", "text/plain")
+	fmt.Fprintln(w, logMarker)
+}
+
 func (h *Handler) serveWhoIs(w http.ResponseWriter, r *http.Request) {
 	if !h.PermitRead {
 		http.Error(w, "whois access denied", http.StatusForbidden)
@@ -114,6 +154,23 @@ func (h *Handler) serveGoroutines(w http.ResponseWriter, r *http.Request) {
 	w.Write(buf)
 }
 
+func (h *Handler) serveCheckIPForwarding(w http.ResponseWriter, r *http.Request) {
+	if !h.PermitRead {
+		http.Error(w, "IP forwarding check access denied", http.StatusForbidden)
+		return
+	}
+	var warning string
+	if err := h.b.CheckIPForwarding(); err != nil {
+		warning = err.Error()
+	}
+	w.Header().Set("Content-Type", "application/json")
+	json.NewEncoder(w).Encode(struct {
+		Warning string
+	}{
+		Warning: warning,
+	})
+}
+
 func (h *Handler) serveStatus(w http.ResponseWriter, r *http.Request) {
 	if !h.PermitRead {
 		http.Error(w, "status access denied", http.StatusForbidden)
@@ -131,6 +188,49 @@ func (h *Handler) serveStatus(w http.ResponseWriter, r *http.Request) {
 	e.Encode(st)
 }
 
+func (h *Handler) serveFiles(w http.ResponseWriter, r *http.Request) {
+	if !h.PermitWrite {
+		http.Error(w, "file access denied", http.StatusForbidden)
+		return
+	}
+	suffix := strings.TrimPrefix(r.URL.Path, "/localapi/v0/files/")
+	if suffix == "" {
+		if r.Method != "GET" {
+			http.Error(w, "want GET to list files", 400)
+			return
+		}
+		wfs, err := h.b.WaitingFiles()
+		if err != nil {
+			http.Error(w, err.Error(), 500)
+			return
+		}
+		w.Header().Set("Content-Type", "application/json")
+		json.NewEncoder(w).Encode(wfs)
+		return
+	}
+	name, err := url.PathUnescape(suffix)
+	if err != nil {
+		http.Error(w, "bad filename", 400)
+		return
+	}
+	if r.Method == "DELETE" {
+		if err := h.b.DeleteFile(name); err != nil {
+			http.Error(w, err.Error(), 500)
+			return
+		}
+		w.WriteHeader(http.StatusNoContent)
+		return
+	}
+	rc, size, err := h.b.OpenFile(name)
+	if err != nil {
+		http.Error(w, err.Error(), 500)
+		return
+	}
+	defer rc.Close()
+	w.Header().Set("Content-Length", fmt.Sprint(size))
+	io.Copy(w, rc)
+}
+
 func defBool(a string, def bool) bool {
 	if a == "" {
 		return def

ipn/message.go

@@ -80,7 +80,7 @@ type Command struct {
 	Login                 *tailcfg.Oauth2Token
 	Logout                *NoArgs
 	SetPrefs              *SetPrefsArgs
-	SetWantRunning        *bool
+	EditPrefs             *MaskedPrefs
 	RequestEngineStatus   *NoArgs
 	RequestStatus         *NoArgs
 	FakeExpireAfter       *FakeExpireAfterArgs
@@ -204,8 +204,8 @@ func (bs *BackendServer) GotCommand(ctx context.Context, cmd *Command) error {
 	} else if c := cmd.SetPrefs; c != nil {
 		bs.b.SetPrefs(c.New)
 		return nil
-	} else if c := cmd.SetWantRunning; c != nil {
-		bs.b.SetWantRunning(*c)
+	} else if c := cmd.EditPrefs; c != nil {
+		bs.b.EditPrefs(c)
 		return nil
 	} else if c := cmd.FakeExpireAfter; c != nil {
 		bs.b.FakeExpireAfter(c.Duration)
@@ -309,6 +309,10 @@ func (bc *BackendClient) SetPrefs(new *Prefs) {
 	bc.send(Command{SetPrefs: &SetPrefsArgs{New: new}})
 }
 
+func (bc *BackendClient) EditPrefs(mp *MaskedPrefs) {
+	bc.send(Command{EditPrefs: mp})
+}
+
 func (bc *BackendClient) RequestEngineStatus() {
 	bc.send(Command{RequestEngineStatus: &NoArgs{}})
 }
@@ -328,10 +332,6 @@ func (bc *BackendClient) Ping(ip string, useTSMP bool) {
 	}})
 }
 
-func (bc *BackendClient) SetWantRunning(v bool) {
-	bc.send(Command{SetWantRunning: &v})
-}
-
 // MaxMessageSize is the maximum message size, in bytes.
 const MaxMessageSize = 10 << 20
 

ipn/policy/policy.go

@@ -6,12 +6,17 @@
 // shared between the node client & control server.
 package policy
 
-import "tailscale.com/tailcfg"
+import (
+	"tailscale.com/tailcfg"
+)
 
 // IsInterestingService reports whether service s on the given operating
 // system (a version.OS value) is an interesting enough port to report
 // to our peer nodes for discovery purposes.
 func IsInterestingService(s tailcfg.Service, os string) bool {
+	if s.Proto == "peerapi4" || s.Proto == "peerapi6" {
+		return true
+	}
 	if s.Proto != tailcfg.TCP {
 		return false
 	}

ipn/prefs.go

@@ -12,6 +12,7 @@ import (
 	"log"
 	"os"
 	"path/filepath"
+	"reflect"
 	"runtime"
 	"strings"
 
@@ -147,6 +148,73 @@ type Prefs struct {
 	Persist *persist.Persist `json:"Config"`
 }
 
+// MaskedPrefs is a Prefs with an associated bitmask of which fields are set.
+type MaskedPrefs struct {
+	Prefs
+
+	ControlURLSet       bool `json:",omitempty"`
+	RouteAllSet         bool `json:",omitempty"`
+	AllowSingleHostsSet bool `json:",omitempty"`
+	ExitNodeIDSet       bool `json:",omitempty"`
+	ExitNodeIPSet       bool `json:",omitempty"`
+	CorpDNSSet          bool `json:",omitempty"`
+	WantRunningSet      bool `json:",omitempty"`
+	ShieldsUpSet        bool `json:",omitempty"`
+	AdvertiseTagsSet    bool `json:",omitempty"`
+	HostnameSet         bool `json:",omitempty"`
+	OSVersionSet        bool `json:",omitempty"`
+	DeviceModelSet      bool `json:",omitempty"`
+	NotepadURLsSet      bool `json:",omitempty"`
+	ForceDaemonSet      bool `json:",omitempty"`
+	AdvertiseRoutesSet  bool `json:",omitempty"`
+	NoSNATSet           bool `json:",omitempty"`
+	NetfilterModeSet    bool `json:",omitempty"`
+}
+
+// ApplyEdits mutates p, assigning fields from m.Prefs for each MaskedPrefs
+// Set field that's true.
+func (p *Prefs) ApplyEdits(m *MaskedPrefs) {
+	if p == nil {
+		panic("can't edit nil Prefs")
+	}
+	pv := reflect.ValueOf(p).Elem()
+	mv := reflect.ValueOf(m).Elem()
+	mpv := reflect.ValueOf(&m.Prefs).Elem()
+	fields := mv.NumField()
+	for i := 1; i < fields; i++ {
+		if mv.Field(i).Bool() {
+			newFieldValue := mpv.Field(i - 1)
+			pv.Field(i - 1).Set(newFieldValue)
+		}
+	}
+}
+
+func (m *MaskedPrefs) Pretty() string {
+	if m == nil {
+		return "MaskedPrefs{<nil>}"
+	}
+	var sb strings.Builder
+	sb.WriteString("MaskedPrefs{")
+	mv := reflect.ValueOf(m).Elem()
+	mt := mv.Type()
+	mpv := reflect.ValueOf(&m.Prefs).Elem()
+	first := true
+	for i := 1; i < mt.NumField(); i++ {
+		name := mt.Field(i).Name
+		if mv.Field(i).Bool() {
+			if !first {
+				sb.WriteString(" ")
+			}
+			first = false
+			fmt.Fprintf(&sb, "%s=%#v",
+				strings.TrimSuffix(name, "Set"),
+				mpv.Field(i-1).Interface())
+		}
+	}
+	sb.WriteString("}")
+	return sb.String()
+}
+
 // IsEmpty reports whether p is nil or pointing to a Prefs zero value.
 func (p *Prefs) IsEmpty() bool { return p == nil || p.Equals(&Prefs{}) }
 
@@ -267,7 +335,7 @@ func NewPrefs() *Prefs {
 		RouteAll:         true,
 		AllowSingleHosts: true,
 		CorpDNS:          true,
-		WantRunning:      true,
+		WantRunning:      false,
 		NetfilterMode:    preftype.NetfilterOn,
 	}
 }

ipn/prefs_test.go

@@ -5,11 +5,13 @@
 package ipn
 
 import (
+	"encoding/json"
 	"errors"
 	"fmt"
 	"io/ioutil"
 	"os"
 	"reflect"
+	"strings"
 	"testing"
 	"time"
 
@@ -432,3 +434,146 @@ func TestLoadPrefsFileWithZeroInIt(t *testing.T) {
 	}
 	t.Fatalf("unexpected prefs=%#v, err=%v", p, err)
 }
+
+func TestMaskedPrefsFields(t *testing.T) {
+	have := map[string]bool{}
+	for _, f := range fieldsOf(reflect.TypeOf(Prefs{})) {
+		if f == "Persist" {
+			// This one can't be edited.
+			continue
+		}
+		have[f] = true
+	}
+	for _, f := range fieldsOf(reflect.TypeOf(MaskedPrefs{})) {
+		if f == "Prefs" {
+			continue
+		}
+		if !strings.HasSuffix(f, "Set") {
+			t.Errorf("unexpected non-/Set$/ field %q", f)
+			continue
+		}
+		bare := strings.TrimSuffix(f, "Set")
+		_, ok := have[bare]
+		if !ok {
+			t.Errorf("no corresponding Prefs.%s field for MaskedPrefs.%s", bare, f)
+			continue
+		}
+		delete(have, bare)
+	}
+	for f := range have {
+		t.Errorf("missing MaskedPrefs.%sSet for Prefs.%s", f, f)
+	}
+
+	// And also make sure they line up in the right order, which
+	// ApplyEdits assumes.
+	pt := reflect.TypeOf(Prefs{})
+	mt := reflect.TypeOf(MaskedPrefs{})
+	for i := 0; i < mt.NumField(); i++ {
+		name := mt.Field(i).Name
+		if i == 0 {
+			if name != "Prefs" {
+				t.Errorf("first field of MaskedPrefs should be Prefs")
+			}
+			continue
+		}
+		prefName := pt.Field(i - 1).Name
+		if prefName+"Set" != name {
+			t.Errorf("MaskedField[%d] = %s; want %sSet", i-1, name, prefName)
+		}
+	}
+}
+
+func TestPrefsApplyEdits(t *testing.T) {
+	tests := []struct {
+		name  string
+		prefs *Prefs
+		edit  *MaskedPrefs
+		want  *Prefs
+	}{
+		{
+			name: "no_change",
+			prefs: &Prefs{
+				Hostname: "foo",
+			},
+			edit: &MaskedPrefs{},
+			want: &Prefs{
+				Hostname: "foo",
+			},
+		},
+		{
+			name: "set1_decoy1",
+			prefs: &Prefs{
+				Hostname: "foo",
+			},
+			edit: &MaskedPrefs{
+				Prefs: Prefs{
+					Hostname:    "bar",
+					DeviceModel: "ignore-this", // not set
+				},
+				HostnameSet: true,
+			},
+			want: &Prefs{
+				Hostname: "bar",
+			},
+		},
+		{
+			name:  "set_several",
+			prefs: &Prefs{},
+			edit: &MaskedPrefs{
+				Prefs: Prefs{
+					Hostname:    "bar",
+					DeviceModel: "galaxybrain",
+				},
+				HostnameSet:    true,
+				DeviceModelSet: true,
+			},
+			want: &Prefs{
+				Hostname:    "bar",
+				DeviceModel: "galaxybrain",
+			},
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			got := tt.prefs.Clone()
+			got.ApplyEdits(tt.edit)
+			if !got.Equals(tt.want) {
+				gotj, _ := json.Marshal(got)
+				wantj, _ := json.Marshal(tt.want)
+				t.Errorf("fail.\n got: %s\nwant: %s\n", gotj, wantj)
+			}
+		})
+	}
+}
+
+func TestMaskedPrefsPretty(t *testing.T) {
+	tests := []struct {
+		m    *MaskedPrefs
+		want string
+	}{
+		{
+			m:    &MaskedPrefs{},
+			want: "MaskedPrefs{}",
+		},
+		{
+			m: &MaskedPrefs{
+				Prefs: Prefs{
+					Hostname:         "bar",
+					DeviceModel:      "galaxybrain",
+					AllowSingleHosts: true,
+					RouteAll:         false,
+				},
+				RouteAllSet:    true,
+				HostnameSet:    true,
+				DeviceModelSet: true,
+			},
+			want: `MaskedPrefs{RouteAll=false Hostname="bar" DeviceModel="galaxybrain"}`,
+		},
+	}
+	for i, tt := range tests {
+		got := tt.m.Pretty()
+		if got != tt.want {
+			t.Errorf("%d.\n got: %#q\nwant: %#q\n", i, got, tt.want)
+		}
+	}
+}

net/dns/config.go

@@ -6,72 +6,40 @@ package dns
 
 import (
 	"inet.af/netaddr"
-
-	"tailscale.com/types/logger"
 )
 
-// Config is the set of parameters that uniquely determine
-// the state to which a manager should bring system DNS settings.
+// Config is a DNS configuration.
 type Config struct {
+	// DefaultResolvers are the DNS resolvers to use for DNS names
+	// which aren't covered by more specific per-domain routes below.
+	// If empty, the OS's default resolvers (the ones that predate
+	// Tailscale altering the configuration) are used.
+	DefaultResolvers []netaddr.IPPort
+	// Routes maps a DNS suffix to the resolvers that should be used
+	// for queries that fall within that suffix.
+	// If a query doesn't match any entry in Routes, the
+	// DefaultResolvers are used.
+	Routes map[string][]netaddr.IPPort
+	// SearchDomains are DNS suffixes to try when expanding
+	// single-label queries.
+	SearchDomains []string
+	// Hosts maps DNS FQDNs to their IPs, which can be a mix of IPv4
+	// and IPv6.
+	// Queries matching entries in Hosts are resolved locally without
+	// recursing off-machine.
+	Hosts map[string][]netaddr.IP
+	// AuthoritativeSuffixes is a list of fully-qualified DNS suffixes
+	// for which the in-process Tailscale resolver is authoritative.
+	// Queries for names within AuthoritativeSuffixes can only be
+	// fulfilled by entries in Hosts. Queries with no match in Hosts
+	// return NXDOMAIN.
+	AuthoritativeSuffixes []string
+}
+
+// OSConfig is an OS DNS configuration.
+type OSConfig struct {
 	// Nameservers are the IP addresses of the nameservers to use.
 	Nameservers []netaddr.IP
 	// Domains are the search domains to use.
 	Domains []string
-	// PerDomain indicates whether it is preferred to use Nameservers
-	// only for DNS queries for subdomains of Domains.
-	// Note that Nameservers may still be applied to all queries
-	// if the manager does not support per-domain settings.
-	PerDomain bool
-	// Proxied indicates whether DNS requests are proxied through a dns.Resolver.
-	// This enables MagicDNS.
-	Proxied bool
-}
-
-// Equal determines whether its argument and receiver
-// represent equivalent DNS configurations (then DNS reconfig is a no-op).
-func (lhs Config) Equal(rhs Config) bool {
-	if lhs.Proxied != rhs.Proxied || lhs.PerDomain != rhs.PerDomain {
-		return false
-	}
-
-	if len(lhs.Nameservers) != len(rhs.Nameservers) {
-		return false
-	}
-
-	if len(lhs.Domains) != len(rhs.Domains) {
-		return false
-	}
-
-	// With how we perform resolution order shouldn't matter,
-	// but it is unlikely that we will encounter different orders.
-	for i, server := range lhs.Nameservers {
-		if rhs.Nameservers[i] != server {
-			return false
-		}
-	}
-
-	// The order of domains, on the other hand, is significant.
-	for i, domain := range lhs.Domains {
-		if rhs.Domains[i] != domain {
-			return false
-		}
-	}
-
-	return true
-}
-
-// ManagerConfig is the set of parameters from which
-// a manager implementation is chosen and initialized.
-type ManagerConfig struct {
-	// Logf is the logger for the manager to use.
-	// It is wrapped with a "dns: " prefix.
-	Logf logger.Logf
-	// InterfaceName is the name of the interface with which DNS settings should be associated.
-	InterfaceName string
-	// Cleanup indicates that the manager is created for cleanup only.
-	// A no-op manager will be instantiated if the system needs no cleanup.
-	Cleanup bool
-	// PerDomain indicates that a manager capable of per-domain configuration is preferred.
-	// Certain managers are per-domain only; they will not be considered if this is false.
-	PerDomain bool
 }

net/dns/direct.go

@@ -48,8 +48,8 @@ func writeResolvConf(w io.Writer, servers []netaddr.IP, domains []string) {
 }
 
 // readResolvConf reads DNS configuration from /etc/resolv.conf.
-func readResolvConf() (Config, error) {
-	var config Config
+func readResolvConf() (OSConfig, error) {
+	var config OSConfig
 
 	f, err := os.Open("/etc/resolv.conf")
 	if err != nil {
@@ -110,12 +110,11 @@ func isResolvedRunning() bool {
 // or as cleanup if the program terminates unexpectedly.
 type directManager struct{}
 
-func newDirectManager(mconfig ManagerConfig) managerImpl {
+func newDirectManager() directManager {
 	return directManager{}
 }
 
-// Up implements managerImpl.
-func (m directManager) Up(config Config) error {
+func (m directManager) SetDNS(config OSConfig) error {
 	// Write the tsConf file.
 	buf := new(bytes.Buffer)
 	writeResolvConf(buf, config.Nameservers, config.Domains)
@@ -160,8 +159,11 @@ func (m directManager) Up(config Config) error {
 	return nil
 }
 
-// Down implements managerImpl.
-func (m directManager) Down() error {
+func (m directManager) RoutingMode() RoutingMode {
+	return RoutingModeNone
+}
+
+func (m directManager) Close() error {
 	if _, err := os.Stat(backupConf); err != nil {
 		// If the backup file does not exist, then Up never ran successfully.
 		if os.IsNotExist(err) {

net/dns/manager.go

@@ -7,7 +7,11 @@ package dns
 import (
 	"time"
 
+	"inet.af/netaddr"
+	"tailscale.com/net/dns/resolver"
+	"tailscale.com/net/tsaddr"
 	"tailscale.com/types/logger"
+	"tailscale.com/wgengine/monitor"
 )
 
 // We use file-ignore below instead of ignore because on some platforms,
@@ -23,78 +27,95 @@ import (
 // Such operations should be wrapped in a timeout context.
 const reconfigTimeout = time.Second
 
-type managerImpl interface {
-	// Up updates system DNS settings to match the given configuration.
-	Up(Config) error
-	// Down undoes the effects of Up.
-	// It is idempotent and performs no action if Up has never been called.
-	Down() error
-}
-
 // Manager manages system DNS settings.
 type Manager struct {
 	logf logger.Logf
 
-	impl managerImpl
+	resolver *resolver.Resolver
+	os       OSConfigurator
 
-	config  Config
-	mconfig ManagerConfig
+	config Config
 }
 
 // NewManagers created a new manager from the given config.
-func NewManager(mconfig ManagerConfig) *Manager {
-	mconfig.Logf = logger.WithPrefix(mconfig.Logf, "dns: ")
+func NewManager(logf logger.Logf, oscfg OSConfigurator, linkMon *monitor.Mon) *Manager {
+	logf = logger.WithPrefix(logf, "dns: ")
 	m := &Manager{
-		logf: mconfig.Logf,
-		impl: newManager(mconfig),
-
-		config:  Config{PerDomain: mconfig.PerDomain},
-		mconfig: mconfig,
+		logf:     logf,
+		resolver: resolver.New(logf, linkMon),
+		os:       oscfg,
 	}
 
-	m.logf("using %T", m.impl)
+	m.logf("using %T", m.os)
 	return m
 }
 
-func (m *Manager) Set(config Config) error {
-	if config.Equal(m.config) {
-		return nil
+func (m *Manager) Set(cfg Config) error {
+	m.logf("Set: %+v", cfg)
+
+	if len(cfg.DefaultResolvers) == 0 {
+		// TODO: make other settings work even if you didn't set a
+		// default resolver. For now, no default resolvers == no
+		// managed DNS config.
+		cfg = Config{}
 	}
 
-	m.logf("Set: %+v", config)
-
-	if len(config.Nameservers) == 0 {
-		err := m.impl.Down()
-		// If we save the config, we will not retry next time. Only do this on success.
-		if err == nil {
-			m.config = config
+	resolverCfg := resolver.Config{
+		Hosts:        cfg.Hosts,
+		LocalDomains: cfg.AuthoritativeSuffixes,
+		Routes:       map[string][]netaddr.IPPort{},
+	}
+	osCfg := OSConfig{
+		Domains: cfg.SearchDomains,
+	}
+	// We must proxy through quad-100 if MagicDNS hosts are in
+	// use, or there are any per-domain routes.
+	mustProxy := len(cfg.Hosts) > 0 || len(cfg.Routes) > 0
+	if mustProxy {
+		osCfg.Nameservers = []netaddr.IP{tsaddr.TailscaleServiceIP()}
+		resolverCfg.Routes["."] = cfg.DefaultResolvers
+		for suffix, resolvers := range cfg.Routes {
+			resolverCfg.Routes[suffix] = resolvers
 		}
+	} else {
+		for _, resolver := range cfg.DefaultResolvers {
+			osCfg.Nameservers = append(osCfg.Nameservers, resolver.IP)
+		}
+	}
+
+	if err := m.resolver.SetConfig(resolverCfg); err != nil {
+		return err
+	}
+	if err := m.os.SetDNS(osCfg); err != nil {
 		return err
 	}
 
-	// Switching to and from per-domain mode may require a change of manager.
-	if config.PerDomain != m.config.PerDomain {
-		if err := m.impl.Down(); err != nil {
-			return err
-		}
-		m.mconfig.PerDomain = config.PerDomain
-		m.impl = newManager(m.mconfig)
-		m.logf("switched to %T", m.impl)
-	}
-
-	err := m.impl.Up(config)
-	// If we save the config, we will not retry next time. Only do this on success.
-	if err == nil {
-		m.config = config
-	}
-
-	return err
+	return nil
 }
 
-func (m *Manager) Up() error {
-	return m.impl.Up(m.config)
+func (m *Manager) EnqueueRequest(bs []byte, from netaddr.IPPort) error {
+	return m.resolver.EnqueueRequest(bs, from)
+}
+
+func (m *Manager) NextResponse() ([]byte, netaddr.IPPort, error) {
+	return m.resolver.NextResponse()
 }
 
 func (m *Manager) Down() error {
-	return m.impl.Down()
+	if err := m.os.Close(); err != nil {
+		return err
+	}
+	m.resolver.Close()
+	return nil
+}
+
+// Cleanup restores the system DNS configuration to its original state
+// in case the Tailscale daemon terminated without closing the router.
+// No other state needs to be instantiated before this runs.
+func Cleanup(logf logger.Logf, interfaceName string) {
+	oscfg := NewOSConfigurator(logf, interfaceName)
+	dns := NewManager(logf, oscfg, nil)
+	if err := dns.Down(); err != nil {
+		logf("dns down: %v", err)
+	}
 }

net/dns/manager_default.go

@@ -6,9 +6,11 @@
 
 package dns
 
-func newManager(mconfig ManagerConfig) managerImpl {
+import "tailscale.com/types/logger"
+
+func NewOSConfigurator(logger.Logf, string) OSConfigurator {
 	// TODO(dmytro): on darwin, we should use a macOS-specific method such as scutil.
 	// This is currently not implemented. Editing /etc/resolv.conf does not work,
 	// as most applications use the system resolver, which disregards it.
-	return newNoopManager(mconfig)
+	return NewNoopManager()
 }

net/dns/manager_freebsd.go

@@ -4,11 +4,13 @@
 
 package dns
 
-func newManager(mconfig ManagerConfig) managerImpl {
+import "tailscale.com/types/logger"
+
+func NewOSConfigurator(logf logger.Logf, _ string) OSConfigurator {
 	switch {
 	case isResolvconfActive():
-		return newResolvconfManager(mconfig)
+		return newResolvconfManager(logf)
 	default:
-		return newDirectManager(mconfig)
+		return newDirectManager()
 	}
 }

net/dns/manager_linux.go

@@ -4,24 +4,18 @@
 
 package dns
 
-func newManager(mconfig ManagerConfig) managerImpl {
+import "tailscale.com/types/logger"
+
+func NewOSConfigurator(logf logger.Logf, interfaceName string) OSConfigurator {
 	switch {
-	// systemd-resolved should only activate per-domain.
-	case isResolvedActive() && mconfig.PerDomain:
-		if mconfig.Cleanup {
-			return newNoopManager(mconfig)
-		} else {
-			return newResolvedManager(mconfig)
-		}
-	case isNMActive():
-		if mconfig.Cleanup {
-			return newNoopManager(mconfig)
-		} else {
-			return newNMManager(mconfig)
-		}
+	// TODO: rework NetworkManager and resolved support.
+	// case isResolvedActive():
+	// 	return newResolvedManager()
+	// case isNMActive():
+	// 	return newNMManager(interfaceName)
 	case isResolvconfActive():
-		return newResolvconfManager(mconfig)
+		return newResolvconfManager(logf)
 	default:
-		return newDirectManager(mconfig)
+		return newDirectManager()
 	}
 }

net/dns/manager_openbsd.go

@@ -4,6 +4,8 @@
 
 package dns
 
-func newManager(mconfig ManagerConfig) managerImpl {
-	return newDirectManager(mconfig)
+import "tailscale.com/types/logger"
+
+func NewOSConfigurator(logger.Logf, string) OSConfigurator {
+	return newDirectManager()
 }

net/dns/manager_windows.go

@@ -25,10 +25,10 @@ type windowsManager struct {
 	guid string
 }
 
-func newManager(mconfig ManagerConfig) managerImpl {
+func NewOSConfigurator(logf logger.Logf, interfaceName string) OSConfigurator {
 	return windowsManager{
-		logf: mconfig.Logf,
-		guid: mconfig.InterfaceName,
+		logf: logf,
+		guid: interfaceName,
 	}
 }
 
@@ -64,7 +64,7 @@ func (m windowsManager) setDomains(basePath string, domains []string) error {
 	return setRegistryString(path, "SearchList", value)
 }
 
-func (m windowsManager) Up(config Config) error {
+func (m windowsManager) SetDNS(config OSConfig) error {
 	var ipsv4 []string
 	var ipsv6 []string
 
@@ -113,6 +113,10 @@ func (m windowsManager) Up(config Config) error {
 	return nil
 }
 
-func (m windowsManager) Down() error {
-	return m.Up(Config{Nameservers: nil, Domains: nil})
+func (m windowsManager) RoutingMode() RoutingMode {
+	return RoutingModeNone
+}
+
+func (m windowsManager) Close() error {
+	return m.SetDNS(OSConfig{})
 }

net/dns/map.go

@@ -1,160 +0,0 @@
-// Copyright (c) 2020 Tailscale Inc & AUTHORS All rights reserved.
-// Use of this source code is governed by a BSD-style
-// license that can be found in the LICENSE file.
-
-package dns
-
-import (
-	"sort"
-	"strings"
-
-	"inet.af/netaddr"
-)
-
-// Map is all the data Resolver needs to resolve DNS queries within the Tailscale network.
-type Map struct {
-	// nameToIP is a mapping of Tailscale domain names to their IP addresses.
-	// For example, monitoring.tailscale.us -> 100.64.0.1.
-	nameToIP map[string]netaddr.IP
-	// ipToName is the inverse of nameToIP.
-	ipToName map[netaddr.IP]string
-	// names are the keys of nameToIP in sorted order.
-	names []string
-	// rootDomains are the domains whose subdomains should always
-	// be resolved locally to prevent leakage of sensitive names.
-	rootDomains []string // e.g. "user.provider.beta.tailscale.net."
-}
-
-// NewMap returns a new Map with name to address mapping given by nameToIP.
-//
-// rootDomains are the domains whose subdomains should always be
-// resolved locally to prevent leakage of sensitive names. They should
-// end in a period ("user-foo.tailscale.net.").
-func NewMap(initNameToIP map[string]netaddr.IP, rootDomains []string) *Map {
-	// TODO(dmytro): we have to allocate names and ipToName, but nameToIP can be avoided.
-	// It is here because control sends us names not in canonical form. Change this.
-	names := make([]string, 0, len(initNameToIP))
-	nameToIP := make(map[string]netaddr.IP, len(initNameToIP))
-	ipToName := make(map[netaddr.IP]string, len(initNameToIP))
-
-	for name, ip := range initNameToIP {
-		if len(name) == 0 {
-			// Nothing useful can be done with empty names.
-			continue
-		}
-		if name[len(name)-1] != '.' {
-			name += "."
-		}
-		names = append(names, name)
-		nameToIP[name] = ip
-		ipToName[ip] = name
-	}
-	sort.Strings(names)
-
-	return &Map{
-		nameToIP: nameToIP,
-		ipToName: ipToName,
-		names:    names,
-
-		rootDomains: rootDomains,
-	}
-}
-
-func printSingleNameIP(buf *strings.Builder, name string, ip netaddr.IP) {
-	buf.WriteString(name)
-	buf.WriteByte('\t')
-	buf.WriteString(ip.String())
-	buf.WriteByte('\n')
-}
-
-func (m *Map) Pretty() string {
-	buf := new(strings.Builder)
-	for _, name := range m.names {
-		printSingleNameIP(buf, name, m.nameToIP[name])
-	}
-	return buf.String()
-}
-
-func (m *Map) PrettyDiffFrom(old *Map) string {
-	var (
-		oldNameToIP map[string]netaddr.IP
-		newNameToIP map[string]netaddr.IP
-		oldNames    []string
-		newNames    []string
-	)
-	if old != nil {
-		oldNameToIP = old.nameToIP
-		oldNames = old.names
-	}
-	if m != nil {
-		newNameToIP = m.nameToIP
-		newNames = m.names
-	}
-
-	buf := new(strings.Builder)
-	space := func() bool {
-		return buf.Len() < (1 << 10)
-	}
-
-	for len(oldNames) > 0 && len(newNames) > 0 {
-		var name string
-
-		newName, oldName := newNames[0], oldNames[0]
-		switch {
-		case oldName < newName:
-			name = oldName
-			oldNames = oldNames[1:]
-		case oldName > newName:
-			name = newName
-			newNames = newNames[1:]
-		case oldNames[0] == newNames[0]:
-			name = oldNames[0]
-			oldNames = oldNames[1:]
-			newNames = newNames[1:]
-		}
-		if !space() {
-			continue
-		}
-
-		ipOld, inOld := oldNameToIP[name]
-		ipNew, inNew := newNameToIP[name]
-		switch {
-		case !inOld:
-			buf.WriteByte('+')
-			printSingleNameIP(buf, name, ipNew)
-		case !inNew:
-			buf.WriteByte('-')
-			printSingleNameIP(buf, name, ipOld)
-		case ipOld != ipNew:
-			buf.WriteByte('-')
-			printSingleNameIP(buf, name, ipOld)
-			buf.WriteByte('+')
-			printSingleNameIP(buf, name, ipNew)
-		}
-	}
-
-	for _, name := range oldNames {
-		if !space() {
-			break
-		}
-		if _, ok := newNameToIP[name]; !ok {
-			buf.WriteByte('-')
-			printSingleNameIP(buf, name, oldNameToIP[name])
-		}
-	}
-
-	for _, name := range newNames {
-		if !space() {
-			break
-		}
-		if _, ok := oldNameToIP[name]; !ok {
-			buf.WriteByte('+')
-			printSingleNameIP(buf, name, newNameToIP[name])
-		}
-	}
-	if !space() {
-		buf.WriteString("... [truncated]\n")
-	}
-
-	return buf.String()
-}

net/dns/map_test.go

@@ -1,156 +0,0 @@
-// Copyright (c) 2020 Tailscale Inc & AUTHORS All rights reserved.
-// Use of this source code is governed by a BSD-style
-// license that can be found in the LICENSE file.
-
-package dns
-
-import (
-	"fmt"
-	"strings"
-	"testing"
-
-	"inet.af/netaddr"
-)
-
-func TestPretty(t *testing.T) {
-	tests := []struct {
-		name string
-		dmap *Map
-		want string
-	}{
-		{"empty", NewMap(nil, nil), ""},
-		{
-			"single",
-			NewMap(map[string]netaddr.IP{
-				"hello.ipn.dev.": netaddr.IPv4(100, 101, 102, 103),
-			}, nil),
-			"hello.ipn.dev.\t100.101.102.103\n",
-		},
-		{
-			"multiple",
-			NewMap(map[string]netaddr.IP{
-				"test1.domain.":     netaddr.IPv4(100, 101, 102, 103),
-				"test2.sub.domain.": netaddr.IPv4(100, 99, 9, 1),
-			}, nil),
-			"test1.domain.\t100.101.102.103\ntest2.sub.domain.\t100.99.9.1\n",
-		},
-	}
-
-	for _, tt := range tests {
-		t.Run(tt.name, func(t *testing.T) {
-			got := tt.dmap.Pretty()
-			if tt.want != got {
-				t.Errorf("want %v; got %v", tt.want, got)
-			}
-		})
-	}
-}
-
-func TestPrettyDiffFrom(t *testing.T) {
-	tests := []struct {
-		name string
-		map1 *Map
-		map2 *Map
-		want string
-	}{
-		{
-			"from_empty",
-			nil,
-			NewMap(map[string]netaddr.IP{
-				"test1.ipn.dev.": netaddr.IPv4(100, 101, 102, 103),
-				"test2.ipn.dev.": netaddr.IPv4(100, 103, 102, 101),
-			}, nil),
-			"+test1.ipn.dev.\t100.101.102.103\n+test2.ipn.dev.\t100.103.102.101\n",
-		},
-		{
-			"equal",
-			NewMap(map[string]netaddr.IP{
-				"test1.ipn.dev.": netaddr.IPv4(100, 101, 102, 103),
-				"test2.ipn.dev.": netaddr.IPv4(100, 103, 102, 101),
-			}, nil),
-			NewMap(map[string]netaddr.IP{
-				"test2.ipn.dev.": netaddr.IPv4(100, 103, 102, 101),
-				"test1.ipn.dev.": netaddr.IPv4(100, 101, 102, 103),
-			}, nil),
-			"",
-		},
-		{
-			"changed_ip",
-			NewMap(map[string]netaddr.IP{
-				"test1.ipn.dev.": netaddr.IPv4(100, 101, 102, 103),
-				"test2.ipn.dev.": netaddr.IPv4(100, 103, 102, 101),
-			}, nil),
-			NewMap(map[string]netaddr.IP{
-				"test2.ipn.dev.": netaddr.IPv4(100, 104, 102, 101),
-				"test1.ipn.dev.": netaddr.IPv4(100, 101, 102, 103),
-			}, nil),
-			"-test2.ipn.dev.\t100.103.102.101\n+test2.ipn.dev.\t100.104.102.101\n",
-		},
-		{
-			"new_domain",
-			NewMap(map[string]netaddr.IP{
-				"test1.ipn.dev.": netaddr.IPv4(100, 101, 102, 103),
-				"test2.ipn.dev.": netaddr.IPv4(100, 103, 102, 101),
-			}, nil),
-			NewMap(map[string]netaddr.IP{
-				"test3.ipn.dev.": netaddr.IPv4(100, 105, 106, 107),
-				"test2.ipn.dev.": netaddr.IPv4(100, 103, 102, 101),
-				"test1.ipn.dev.": netaddr.IPv4(100, 101, 102, 103),
-			}, nil),
-			"+test3.ipn.dev.\t100.105.106.107\n",
-		},
-		{
-			"gone_domain",
-			NewMap(map[string]netaddr.IP{
-				"test1.ipn.dev.": netaddr.IPv4(100, 101, 102, 103),
-				"test2.ipn.dev.": netaddr.IPv4(100, 103, 102, 101),
-			}, nil),
-			NewMap(map[string]netaddr.IP{
-				"test1.ipn.dev.": netaddr.IPv4(100, 101, 102, 103),
-			}, nil),
-			"-test2.ipn.dev.\t100.103.102.101\n",
-		},
-		{
-			"mixed",
-			NewMap(map[string]netaddr.IP{
-				"test1.ipn.dev.": netaddr.IPv4(100, 101, 102, 103),
-				"test4.ipn.dev.": netaddr.IPv4(100, 107, 106, 105),
-				"test5.ipn.dev.": netaddr.IPv4(100, 64, 1, 1),
-				"test2.ipn.dev.": netaddr.IPv4(100, 103, 102, 101),
-			}, nil),
-			NewMap(map[string]netaddr.IP{
-				"test2.ipn.dev.": netaddr.IPv4(100, 104, 102, 101),
-				"test1.ipn.dev.": netaddr.IPv4(100, 100, 101, 102),
-				"test3.ipn.dev.": netaddr.IPv4(100, 64, 1, 1),
-			}, nil),
-			"-test1.ipn.dev.\t100.101.102.103\n+test1.ipn.dev.\t100.100.101.102\n" +
-				"-test2.ipn.dev.\t100.103.102.101\n+test2.ipn.dev.\t100.104.102.101\n" +
-				"+test3.ipn.dev.\t100.64.1.1\n-test4.ipn.dev.\t100.107.106.105\n-test5.ipn.dev.\t100.64.1.1\n",
-		},
-	}
-
-	for _, tt := range tests {
-		t.Run(tt.name, func(t *testing.T) {
-			got := tt.map2.PrettyDiffFrom(tt.map1)
-			if tt.want != got {
-				t.Errorf("want %v; got %v", tt.want, got)
-			}
-		})
-	}
-
-	t.Run("truncated", func(t *testing.T) {
-		small := NewMap(nil, nil)
-		m := map[string]netaddr.IP{}
-		for i := 0; i < 5000; i++ {
-			m[fmt.Sprintf("host%d.ipn.dev.", i)] = netaddr.IPv4(100, 64, 1, 1)
-		}
-		veryBig := NewMap(m, nil)
-		diff := veryBig.PrettyDiffFrom(small)
-		if len(diff) > 3<<10 {
-			t.Errorf("pretty diff too large: %d bytes", len(diff))
-		}
-		if !strings.Contains(diff, "truncated") {
-			t.Errorf("big diff not truncated")
-		}
-	})
-}

net/dns/nm.go

@@ -4,6 +4,8 @@
 
 // +build linux
 
+//lint:file-ignore U1000 refactoring, temporarily unused code.
+
 package dns
 
 import (
@@ -53,16 +55,15 @@ type nmManager struct {
 	interfaceName string
 }
 
-func newNMManager(mconfig ManagerConfig) managerImpl {
+func newNMManager(interfaceName string) nmManager {
 	return nmManager{
-		interfaceName: mconfig.InterfaceName,
+		interfaceName: interfaceName,
 	}
 }
 
 type nmConnectionSettings map[string]map[string]dbus.Variant
 
-// Up implements managerImpl.
-func (m nmManager) Up(config Config) error {
+func (m nmManager) SetDNS(config OSConfig) error {
 	ctx, cancel := context.WithTimeout(context.Background(), reconfigTimeout)
 	defer cancel()
 
@@ -199,7 +200,8 @@ func (m nmManager) Up(config Config) error {
 	return nil
 }
 
-// Down implements managerImpl.
-func (m nmManager) Down() error {
-	return m.Up(Config{Nameservers: nil, Domains: nil})
+func (m nmManager) RoutingMode() RoutingMode { return RoutingModeNone }
+
+func (m nmManager) Close() error {
+	return m.SetDNS(OSConfig{})
 }

net/dns/noop.go

@@ -6,12 +6,10 @@ package dns
 
 type noopManager struct{}
 
-// Up implements managerImpl.
-func (m noopManager) Up(Config) error { return nil }
+func (m noopManager) SetDNS(OSConfig) error    { return nil }
+func (m noopManager) RoutingMode() RoutingMode { return RoutingModeNone }
+func (m noopManager) Close() error             { return nil }
 
-// Down implements managerImpl.
-func (m noopManager) Down() error { return nil }
-
-func newNoopManager(mconfig ManagerConfig) managerImpl {
+func NewNoopManager() noopManager {
 	return noopManager{}
 }

net/dns/osconfig.go

@@ -0,0 +1,36 @@
+// Copyright (c) 2021 Tailscale Inc & AUTHORS All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package dns
+
+// DNSRoutingMode describes the type of per-domain DNS routing that
+// the OS is capable of.
+type RoutingMode int
+
+const (
+	// RoutingModeNone means the OS only supports setting a single
+	// primary set of DNS resolvers.
+	RoutingModeNone RoutingMode = iota
+	// RoutingModeSingle means the OS supports a set of
+	// primary resolvers, as well as one set of additional per-suffix
+	// resolvers per network interface.
+	RoutingModeSingle
+	// RoutingModeMulti means the OS supports a set of primary
+	// resolvers, as well as an arbitrary overlay of DNS routes.
+	RoutingModeMulti
+)
+
+// An OSConfigurator applies DNS settings to the operating system.
+type OSConfigurator interface {
+	// SetDNS updates the OS's DNS configuration to match cfg.
+	// If cfg is the zero value, all Tailscale-related DNS
+	// configuration is removed.
+	// SetDNS must not be called after Close.
+	SetDNS(cfg OSConfig) error
+	// DNSRoutingMode reports the DNS routing capabilities of this OS
+	// configurator.
+	RoutingMode() RoutingMode
+	// Close removes Tailscale-related DNS configuration from the OS.
+	Close() error
+}

net/dns/resolvconf.go

@@ -12,6 +12,8 @@ import (
 	"fmt"
 	"os"
 	"os/exec"
+
+	"tailscale.com/types/logger"
 )
 
 // isResolvconfActive indicates whether the system appears to be using resolvconf.
@@ -99,9 +101,9 @@ type resolvconfManager struct {
 	impl resolvconfImpl
 }
 
-func newResolvconfManager(mconfig ManagerConfig) managerImpl {
+func newResolvconfManager(logf logger.Logf) resolvconfManager {
 	impl := getResolvconfImpl()
-	mconfig.Logf("resolvconf implementation is %s", impl)
+	logf("resolvconf implementation is %s", impl)
 
 	return resolvconfManager{
 		impl: impl,
@@ -113,8 +115,7 @@ func newResolvconfManager(mconfig ManagerConfig) managerImpl {
 // when running resolvconfLegacy, hopefully placing our config first.
 const resolvconfConfigName = "tun-tailscale.inet"
 
-// Up implements managerImpl.
-func (m resolvconfManager) Up(config Config) error {
+func (m resolvconfManager) SetDNS(config OSConfig) error {
 	stdin := new(bytes.Buffer)
 	writeResolvConf(stdin, config.Nameservers, config.Domains) // dns_direct.go
 
@@ -137,8 +138,11 @@ func (m resolvconfManager) Up(config Config) error {
 	return nil
 }
 
-// Down implements managerImpl.
-func (m resolvconfManager) Down() error {
+func (m resolvconfManager) RoutingMode() RoutingMode {
+	return RoutingModeNone
+}
+
+func (m resolvconfManager) Close() error {
 	var cmd *exec.Cmd
 	switch m.impl {
 	case resolvconfOpenresolv:

net/dns/resolved.go

@@ -4,6 +4,8 @@
 
 // +build linux
 
+//lint:file-ignore U1000 refactoring, temporarily unused code.
+
 package dns
 
 import (
@@ -77,12 +79,12 @@ func isResolvedActive() bool {
 // resolvedManager uses the systemd-resolved DBus API.
 type resolvedManager struct{}
 
-func newResolvedManager(mconfig ManagerConfig) managerImpl {
+func newResolvedManager() resolvedManager {
 	return resolvedManager{}
 }
 
 // Up implements managerImpl.
-func (m resolvedManager) Up(config Config) error {
+func (m resolvedManager) SetDNS(config OSConfig) error {
 	ctx, cancel := context.WithTimeout(context.Background(), reconfigTimeout)
 	defer cancel()
 
@@ -151,8 +153,11 @@ func (m resolvedManager) Up(config Config) error {
 	return nil
 }
 
-// Down implements managerImpl.
-func (m resolvedManager) Down() error {
+func (m resolvedManager) RoutingMode() RoutingMode {
+	return RoutingModeNone
+}
+
+func (m resolvedManager) Close() error {
 	ctx, cancel := context.WithTimeout(context.Background(), reconfigTimeout)
 	defer cancel()
 

net/dns/resolver/forwarder.go

@@ -2,7 +2,7 @@
 // Use of this source code is governed by a BSD-style
 // license that can be found in the LICENSE file.
 
-package dns
+package resolver
 
 import (
 	"bytes"
@@ -17,10 +17,12 @@ import (
 	"sync"
 	"time"
 
+	dns "golang.org/x/net/dns/dnsmessage"
 	"inet.af/netaddr"
 	"tailscale.com/logtail/backoff"
 	"tailscale.com/net/netns"
 	"tailscale.com/types/logger"
+	"tailscale.com/util/dnsname"
 )
 
 // headerBytes is the number of bytes in a DNS message header.
@@ -100,12 +102,17 @@ func getTxID(packet []byte) txid {
 	return (txid(hash) << 32) | txid(dnsid)
 }
 
+type route struct {
+	suffix    string
+	resolvers []netaddr.IPPort
+}
+
 // forwarder forwards DNS packets to a number of upstream nameservers.
 type forwarder struct {
 	logf logger.Logf
 
 	// responses is a channel by which responses are returned.
-	responses chan Packet
+	responses chan packet
 	// closed signals all goroutines to stop.
 	closed chan struct{}
 	// wg signals when all goroutines have stopped.
@@ -116,35 +123,32 @@ type forwarder struct {
 	conns []*fwdConn
 
 	mu sync.Mutex
-	// upstreams are the nameserver addresses that should be used for forwarding.
-	upstreams []net.Addr
-	// txMap maps DNS txids to active forwarding records.
-	txMap map[txid]forwardingRecord
+	// routes are per-suffix resolvers to use.
+	routes []route                   // most specific routes first
+	txMap  map[txid]forwardingRecord // txids to in-flight requests
 }
 
 func init() {
 	rand.Seed(time.Now().UnixNano())
 }
 
-func newForwarder(logf logger.Logf, responses chan Packet) *forwarder {
-	return &forwarder{
+func newForwarder(logf logger.Logf, responses chan packet) *forwarder {
+	ret := &forwarder{
 		logf:      logger.WithPrefix(logf, "forward: "),
 		responses: responses,
 		closed:    make(chan struct{}),
 		conns:     make([]*fwdConn, connCount),
 		txMap:     make(map[txid]forwardingRecord),
 	}
-}
 
-func (f *forwarder) Start() error {
-	f.wg.Add(connCount + 1)
-	for idx := range f.conns {
-		f.conns[idx] = newFwdConn(f.logf, idx)
-		go f.recv(f.conns[idx])
+	ret.wg.Add(connCount + 1)
+	for idx := range ret.conns {
+		ret.conns[idx] = newFwdConn(ret.logf, idx)
+		go ret.recv(ret.conns[idx])
 	}
-	go f.cleanMap()
+	go ret.cleanMap()
 
-	return nil
+	return ret
 }
 
 func (f *forwarder) Close() {
@@ -171,14 +175,14 @@ func (f *forwarder) rebindFromNetworkChange() {
 	}
 }
 
-func (f *forwarder) setUpstreams(upstreams []net.Addr) {
+func (f *forwarder) setRoutes(routes []route) {
 	f.mu.Lock()
-	f.upstreams = upstreams
+	f.routes = routes
 	f.mu.Unlock()
 }
 
 // send sends packet to dst. It is best effort.
-func (f *forwarder) send(packet []byte, dst net.Addr) {
+func (f *forwarder) send(packet []byte, dst netaddr.IPPort) {
 	connIdx := rand.Intn(connCount)
 	conn := f.conns[connIdx]
 	conn.send(packet, dst)
@@ -218,14 +222,11 @@ func (f *forwarder) recv(conn *fwdConn) {
 
 		f.mu.Unlock()
 
-		packet := Packet{
-			Payload: out,
-			Addr:    record.src,
-		}
+		pkt := packet{out, record.src}
 		select {
 		case <-f.closed:
 			return
-		case f.responses <- packet:
+		case f.responses <- pkt:
 			// continue
 		}
 	}
@@ -258,25 +259,39 @@ func (f *forwarder) cleanMap() {
 }
 
 // forward forwards the query to all upstream nameservers and returns the first response.
-func (f *forwarder) forward(query Packet) error {
-	txid := getTxID(query.Payload)
+func (f *forwarder) forward(query packet) error {
+	domain, err := nameFromQuery(query.bs)
+	if err != nil {
+		return err
+	}
+
+	txid := getTxID(query.bs)
 
 	f.mu.Lock()
-
-	upstreams := f.upstreams
-	if len(upstreams) == 0 {
-		f.mu.Unlock()
-		return errNoUpstreams
-	}
-	f.txMap[txid] = forwardingRecord{
-		src:       query.Addr,
-		createdAt: time.Now(),
-	}
-
+	routes := f.routes
 	f.mu.Unlock()
 
-	for _, upstream := range upstreams {
-		f.send(query.Payload, upstream)
+	var resolvers []netaddr.IPPort
+	for _, route := range routes {
+		if route.suffix != "." && !dnsname.HasSuffix(domain, route.suffix) {
+			continue
+		}
+		resolvers = route.resolvers
+		break
+	}
+	if len(resolvers) == 0 {
+		return errNoUpstreams
+	}
+
+	f.mu.Lock()
+	f.txMap[txid] = forwardingRecord{
+		src:       query.addr,
+		createdAt: time.Now(),
+	}
+	f.mu.Unlock()
+
+	for _, resolver := range resolvers {
+		f.send(query.bs, resolver)
 	}
 
 	return nil
@@ -312,7 +327,7 @@ func newFwdConn(logf logger.Logf, idx int) *fwdConn {
 
 // send sends packet to dst using c's connection.
 // It is best effort. It is UDP, after all. Failures are logged.
-func (c *fwdConn) send(packet []byte, dst net.Addr) {
+func (c *fwdConn) send(packet []byte, dst netaddr.IPPort) {
 	var b *backoff.Backoff // lazily initialized, since it is not needed in the common case
 	backOff := func(err error) {
 		if b == nil {
@@ -338,8 +353,9 @@ func (c *fwdConn) send(packet []byte, dst net.Addr) {
 		}
 		c.mu.Unlock()
 
+		a := dst.UDPAddr()
 		c.wg.Add(1)
-		_, err := conn.WriteTo(packet, dst)
+		_, err := conn.WriteTo(packet, a)
 		c.wg.Done()
 		if err == nil {
 			// Success
@@ -472,3 +488,24 @@ func (c *fwdConn) close() {
 	// Unblock any remaining readers.
 	c.change.Broadcast()
 }
+
+// nameFromQuery extracts the normalized query name from bs.
+func nameFromQuery(bs []byte) (string, error) {
+	var parser dns.Parser
+
+	hdr, err := parser.Start(bs)
+	if err != nil {
+		return "", err
+	}
+	if hdr.Response {
+		return "", errNotQuery
+	}
+
+	q, err := parser.Question()
+	if err != nil {
+		return "", err
+	}
+
+	n := q.Name.Data[:q.Name.Length]
+	return rawNameToLower(n), nil
+}

net/dns/resolver/neterr_darwin.go

@@ -2,7 +2,7 @@
 // Use of this source code is governed by a BSD-style
 // license that can be found in the LICENSE file.
 
-package dns
+package resolver
 
 import (
 	"errors"

net/dns/resolver/neterr_other.go

@@ -4,7 +4,7 @@
 
 // +build !darwin,!windows
 
-package dns
+package resolver
 
 func networkIsDown(err error) bool        { return false }
 func networkIsUnreachable(err error) bool { return false }

net/dns/resolver/neterr_windows.go

@@ -2,7 +2,7 @@
 // Use of this source code is governed by a BSD-style
 // license that can be found in the LICENSE file.
 
-package dns
+package resolver
 
 import (
 	"net"

net/dns/resolver/tsdns.go

@@ -2,14 +2,15 @@
 // Use of this source code is governed by a BSD-style
 // license that can be found in the LICENSE file.
 
-// Package dns provides a Resolver capable of resolving
-// domains on a Tailscale network.
-package dns
+// Package resolver implements a stub DNS resolver that can also serve
+// records out of an internal local zone.
+package resolver
 
 import (
 	"encoding/hex"
 	"errors"
-	"net"
+	"fmt"
+	"sort"
 	"strings"
 	"sync"
 	"time"
@@ -37,22 +38,33 @@ const defaultTTL = 600 * time.Second
 var ErrClosed = errors.New("closed")
 
 var (
-	errFullQueue      = errors.New("request queue full")
-	errMapNotSet      = errors.New("domain map not set")
-	errNotForwarding  = errors.New("forwarding disabled")
-	errNotImplemented = errors.New("query type not implemented")
-	errNotQuery       = errors.New("not a DNS query")
-	errNotOurName     = errors.New("not a Tailscale DNS name")
+	errFullQueue  = errors.New("request queue full")
+	errNotQuery   = errors.New("not a DNS query")
+	errNotOurName = errors.New("not a Tailscale DNS name")
 )
 
-// Packet represents a DNS payload together with the address of its origin.
-type Packet struct {
-	// Payload is the application layer DNS payload.
-	// Resolver assumes ownership of the request payload when it is enqueued
-	// and cedes ownership of the response payload when it is returned from NextResponse.
-	Payload []byte
-	// Addr is the source address for a request and the destination address for a response.
-	Addr netaddr.IPPort
+type packet struct {
+	bs   []byte
+	addr netaddr.IPPort // src for a request, dst for a response
+}
+
+// Config is a resolver configuration.
+// Given a Config, queries are resolved in the following order:
+// If the query is an exact match for an entry in LocalHosts, return that.
+// Else if the query suffix matches an entry in LocalDomains, return NXDOMAIN.
+// Else forward the query to the most specific matching entry in Routes.
+// Else return SERVFAIL.
+type Config struct {
+	// Routes is a map of DNS name suffix to the resolvers to use for
+	// queries within that suffix.
+	// Queries only match the most specific suffix.
+	// To register a "default route", add an entry for ".".
+	Routes map[string][]netaddr.IPPort
+	// LocalHosts is a map of FQDNs to corresponding IPs.
+	Hosts map[string][]netaddr.IP
+	// LocalDomains is a list of DNS name suffixes that should not be
+	// routed to upstream resolvers.
+	LocalDomains []string
 }
 
 // Resolver is a DNS resolver for nodes on the Tailscale network,
@@ -67,9 +79,9 @@ type Resolver struct {
 	forwarder *forwarder
 
 	// queue is a buffered channel holding DNS requests queued for resolution.
-	queue chan Packet
+	queue chan packet
 	// responses is an unbuffered channel to which responses are returned.
-	responses chan Packet
+	responses chan packet
 	// errors is an unbuffered channel to which errors are returned.
 	errors chan error
 	// closed signals all goroutines to stop.
@@ -78,56 +90,79 @@ type Resolver struct {
 	wg sync.WaitGroup
 
 	// mu guards the following fields from being updated while used.
-	mu sync.Mutex
-	// dnsMap is the map most recently received from the control server.
-	dnsMap *Map
+	mu           sync.Mutex
+	localDomains []string
+	hostToIP     map[string][]netaddr.IP
+	ipToHost     map[netaddr.IP]string
 }
 
-// ResolverConfig is the set of configuration options for a Resolver.
-type ResolverConfig struct {
-	// Logf is the logger to use throughout the Resolver.
-	Logf logger.Logf
-	// Forward determines whether the resolver will forward packets to
-	// nameservers set with SetUpstreams if the domain name is not of a Tailscale node.
-	Forward bool
-	// LinkMonitor optionally provides a link monitor to use to rebind
-	// connections on link changes.
-	// If nil, rebinds are not performend.
-	LinkMonitor *monitor.Mon
-}
-
-// NewResolver constructs a resolver associated with the given root domain.
-// The root domain must be in canonical form (with a trailing period).
-func NewResolver(config ResolverConfig) *Resolver {
+// New returns a new resolver.
+// linkMon optionally specifies a link monitor to use for socket rebinding.
+func New(logf logger.Logf, linkMon *monitor.Mon) *Resolver {
 	r := &Resolver{
-		logf:      logger.WithPrefix(config.Logf, "dns: "),
-		linkMon:   config.LinkMonitor,
-		queue:     make(chan Packet, queueSize),
-		responses: make(chan Packet),
+		logf:      logger.WithPrefix(logf, "dns: "),
+		linkMon:   linkMon,
+		queue:     make(chan packet, queueSize),
+		responses: make(chan packet),
 		errors:    make(chan error),
 		closed:    make(chan struct{}),
+		hostToIP:  map[string][]netaddr.IP{},
+		ipToHost:  map[netaddr.IP]string{},
 	}
-
-	if config.Forward {
-		r.forwarder = newForwarder(r.logf, r.responses)
-	}
+	r.forwarder = newForwarder(r.logf, r.responses)
 	if r.linkMon != nil {
 		r.unregLinkMon = r.linkMon.RegisterChangeCallback(r.onLinkMonitorChange)
 	}
 
-	return r
-}
-
-func (r *Resolver) Start() error {
-	if r.forwarder != nil {
-		if err := r.forwarder.Start(); err != nil {
-			return err
-		}
-	}
-
 	r.wg.Add(1)
 	go r.poll()
 
+	return r
+}
+
+func isFQDN(s string) bool {
+	return strings.HasSuffix(s, ".")
+}
+
+func (r *Resolver) SetConfig(cfg Config) error {
+	routes := make([]route, 0, len(cfg.Routes))
+	reverse := make(map[netaddr.IP]string, len(cfg.Hosts))
+
+	for host, ips := range cfg.Hosts {
+		if !isFQDN(host) {
+			return fmt.Errorf("host entry %q is not a FQDN", host)
+		}
+		for _, ip := range ips {
+			reverse[ip] = host
+		}
+	}
+	for _, domain := range cfg.LocalDomains {
+		if !isFQDN(domain) {
+			return fmt.Errorf("local domain %q is not a FQDN", domain)
+		}
+	}
+
+	for suffix, ips := range cfg.Routes {
+		if !strings.HasSuffix(suffix, ".") {
+			return fmt.Errorf("route suffix %q is not a FQDN", suffix)
+		}
+		routes = append(routes, route{
+			suffix:    suffix,
+			resolvers: ips,
+		})
+	}
+	// Sort from longest prefix to shortest.
+	sort.Slice(routes, func(i, j int) bool {
+		return dnsname.NumLabels(routes[i].suffix) > dnsname.NumLabels(routes[j].suffix)
+	})
+
+	r.forwarder.setRoutes(routes)
+
+	r.mu.Lock()
+	defer r.mu.Unlock()
+	r.localDomains = cfg.LocalDomains
+	r.hostToIP = cfg.Hosts
+	r.ipToHost = reverse
 	return nil
 }
 
@@ -146,10 +181,7 @@ func (r *Resolver) Close() {
 		r.unregLinkMon()
 	}
 
-	if r.forwarder != nil {
-		r.forwarder.Close()
-	}
-
+	r.forwarder.Close()
 	r.wg.Wait()
 }
 
@@ -157,37 +189,17 @@ func (r *Resolver) onLinkMonitorChange(changed bool, state *interfaces.State) {
 	if !changed {
 		return
 	}
-	if r.forwarder != nil {
-		r.forwarder.rebindFromNetworkChange()
-	}
-}
-
-// SetMap sets the resolver's DNS map, taking ownership of it.
-func (r *Resolver) SetMap(m *Map) {
-	r.mu.Lock()
-	oldMap := r.dnsMap
-	r.dnsMap = m
-	r.mu.Unlock()
-	r.logf("map diff:\n%s", m.PrettyDiffFrom(oldMap))
-}
-
-// SetUpstreams sets the addresses of the resolver's
-// upstream nameservers, taking ownership of the argument.
-func (r *Resolver) SetUpstreams(upstreams []net.Addr) {
-	if r.forwarder != nil {
-		r.forwarder.setUpstreams(upstreams)
-	}
-	r.logf("set upstreams: %v", upstreams)
+	r.forwarder.rebindFromNetworkChange()
 }
 
 // EnqueueRequest places the given DNS request in the resolver's queue.
 // It takes ownership of the payload and does not block.
 // If the queue is full, the request will be dropped and an error will be returned.
-func (r *Resolver) EnqueueRequest(request Packet) error {
+func (r *Resolver) EnqueueRequest(bs []byte, from netaddr.IPPort) error {
 	select {
 	case <-r.closed:
 		return ErrClosed
-	case r.queue <- request:
+	case r.queue <- packet{bs, from}:
 		return nil
 	default:
 		return errFullQueue
@@ -196,73 +208,81 @@ func (r *Resolver) EnqueueRequest(request Packet) error {
 
 // NextResponse returns a DNS response to a previously enqueued request.
 // It blocks until a response is available and gives up ownership of the response payload.
-func (r *Resolver) NextResponse() (Packet, error) {
+func (r *Resolver) NextResponse() (packet []byte, to netaddr.IPPort, err error) {
 	select {
 	case <-r.closed:
-		return Packet{}, ErrClosed
+		return nil, netaddr.IPPort{}, ErrClosed
 	case resp := <-r.responses:
-		return resp, nil
+		return resp.bs, resp.addr, nil
 	case err := <-r.errors:
-		return Packet{}, err
+		return nil, netaddr.IPPort{}, err
 	}
 }
 
-// Resolve maps a given domain name to the IP address of the host that owns it,
-// if the IP address conforms to the DNS resource type given by tp (one of A, AAAA, ALL).
+// resolveLocal returns an IP for the given domain, if domain is in
+// the local hosts map and has an IP corresponding to the requested
+// typ (A, AAAA, ALL).
 // The domain name must be in canonical form (with a trailing period).
-func (r *Resolver) Resolve(domain string, tp dns.Type) (netaddr.IP, dns.RCode, error) {
-	r.mu.Lock()
-	dnsMap := r.dnsMap
-	r.mu.Unlock()
-
-	if dnsMap == nil {
-		return netaddr.IP{}, dns.RCodeServerFailure, errMapNotSet
-	}
-
+// Returns dns.RCodeRefused to indicate that the local map is not
+// authoritative for domain.
+func (r *Resolver) resolveLocal(domain string, typ dns.Type) (netaddr.IP, dns.RCode) {
 	// Reject .onion domains per RFC 7686.
 	if dnsname.HasSuffix(domain, ".onion") {
-		return netaddr.IP{}, dns.RCodeNameError, nil
+		return netaddr.IP{}, dns.RCodeNameError
 	}
 
-	anyHasSuffix := false
-	for _, suffix := range dnsMap.rootDomains {
-		if dnsname.HasSuffix(domain, suffix) {
-			anyHasSuffix = true
-			break
-		}
-	}
-	addr, found := dnsMap.nameToIP[domain]
+	r.mu.Lock()
+	hosts := r.hostToIP
+	localDomains := r.localDomains
+	r.mu.Unlock()
+
+	addrs, found := hosts[domain]
 	if !found {
-		if !anyHasSuffix {
-			return netaddr.IP{}, dns.RCodeRefused, nil
+		for _, suffix := range localDomains {
+			if dnsname.HasSuffix(domain, suffix) {
+				// We are authoritative for the queried domain.
+				return netaddr.IP{}, dns.RCodeNameError
+			}
 		}
-		return netaddr.IP{}, dns.RCodeNameError, nil
+		// Not authoritative, signal that forwarding is advisable.
+		return netaddr.IP{}, dns.RCodeRefused
 	}
 
 	// Refactoring note: this must happen after we check suffixes,
 	// otherwise we will respond with NOTIMP to requests that should be forwarded.
-	switch tp {
+	//
+	// DNS semantics subtlety: when a DNS name exists, but no records
+	// are available for the requested record type, we must return
+	// RCodeSuccess with no data, not NXDOMAIN.
+	switch typ {
 	case dns.TypeA:
-		if !addr.Is4() {
-			return netaddr.IP{}, dns.RCodeSuccess, nil
+		for _, ip := range addrs {
+			if ip.Is4() {
+				return ip, dns.RCodeSuccess
+			}
 		}
-		return addr, dns.RCodeSuccess, nil
+		return netaddr.IP{}, dns.RCodeSuccess
 	case dns.TypeAAAA:
-		if !addr.Is6() {
-			return netaddr.IP{}, dns.RCodeSuccess, nil
+		for _, ip := range addrs {
+			if ip.Is6() {
+				return ip, dns.RCodeSuccess
+			}
 		}
-		return addr, dns.RCodeSuccess, nil
+		return netaddr.IP{}, dns.RCodeSuccess
 	case dns.TypeALL:
 		// Answer with whatever we've got.
 		// It could be IPv4, IPv6, or a zero addr.
 		// TODO: Return all available resolutions (A and AAAA, if we have them).
-		return addr, dns.RCodeSuccess, nil
+		if len(addrs) == 0 {
+			return netaddr.IP{}, dns.RCodeSuccess
+		}
+		return addrs[0], dns.RCodeSuccess
 
 	// Leave some some record types explicitly unimplemented.
 	// These types relate to recursive resolution or special
-	// DNS sematics and might be implemented in the future.
+	// DNS semantics and might be implemented in the future.
 	case dns.TypeNS, dns.TypeSOA, dns.TypeAXFR, dns.TypeHINFO:
-		return netaddr.IP{}, dns.RCodeNotImplemented, errNotImplemented
+		return netaddr.IP{}, dns.RCodeNotImplemented
 
 	// For everything except for the few types above that are explictly not implemented, return no records.
 	// This is what other DNS systems do: always return NOERROR
@@ -271,51 +291,44 @@ func (r *Resolver) Resolve(domain string, tp dns.Type) (netaddr.IP, dns.RCode, e
 	//   dig -t TYPE9824 example.com
 	// and note that NOERROR is returned, despite that record type being made up.
 	default:
-		// no records exist of this type
-		return netaddr.IP{}, dns.RCodeSuccess, nil
+		// The name exists, but no records exist of the requested type.
+		return netaddr.IP{}, dns.RCodeSuccess
 	}
 }
 
-// ResolveReverse returns the unique domain name that maps to the given address.
+// resolveReverse returns the unique domain name that maps to the given address.
 // The returned domain name is in canonical form (with a trailing period).
-func (r *Resolver) ResolveReverse(ip netaddr.IP) (string, dns.RCode, error) {
+func (r *Resolver) resolveLocalReverse(ip netaddr.IP) (string, dns.RCode) {
 	r.mu.Lock()
-	dnsMap := r.dnsMap
+	ips := r.ipToHost
 	r.mu.Unlock()
 
-	if dnsMap == nil {
-		return "", dns.RCodeServerFailure, errMapNotSet
-	}
-	name, found := dnsMap.ipToName[ip]
+	name, found := ips[ip]
 	if !found {
-		return "", dns.RCodeNameError, nil
+		return "", dns.RCodeNameError
 	}
-	return name, dns.RCodeSuccess, nil
+	return name, dns.RCodeSuccess
 }
 
 func (r *Resolver) poll() {
 	defer r.wg.Done()
 
-	var packet Packet
+	var pkt packet
 	for {
 		select {
 		case <-r.closed:
 			return
-		case packet = <-r.queue:
+		case pkt = <-r.queue:
 			// continue
 		}
 
-		out, err := r.respond(packet.Payload)
+		out, err := r.respond(pkt.bs)
 
 		if err == errNotOurName {
-			if r.forwarder != nil {
-				err = r.forwarder.forward(packet)
-				if err == nil {
-					// forward will send response into r.responses, nothing to do.
-					continue
-				}
-			} else {
-				err = errNotForwarding
+			err = r.forwarder.forward(pkt)
+			if err == nil {
+				// forward will send response into r.responses, nothing to do.
+				continue
 			}
 		}
 
@@ -327,11 +340,11 @@ func (r *Resolver) poll() {
 				// continue
 			}
 		} else {
-			packet.Payload = out
+			pkt.bs = out
 			select {
 			case <-r.closed:
 				return
-			case r.responses <- packet:
+			case r.responses <- pkt:
 				// continue
 			}
 		}
@@ -348,6 +361,8 @@ type response struct {
 }
 
 // parseQuery parses the query in given packet into a response struct.
+// if the parse is successful, resp.Name contains the normalized name being queried.
+// TODO: stuffing the query name in resp.Name temporarily is a hack. Clean it up.
 func parseQuery(query []byte, resp *response) error {
 	var parser dns.Parser
 	var err error
@@ -606,11 +621,7 @@ func (r *Resolver) respondReverse(query []byte, name string, resp *response) ([]
 		return nil, errNotOurName
 	}
 
-	var err error
-	resp.Name, resp.Header.RCode, err = r.ResolveReverse(ip)
-	if err != nil {
-		r.logf("resolving rdns: %v", ip, err)
-	}
+	resp.Name, resp.Header.RCode = r.resolveLocalReverse(ip)
 	if resp.Header.RCode == dns.RCodeNameError {
 		return nil, errNotOurName
 	}
@@ -647,16 +658,11 @@ func (r *Resolver) respond(query []byte) ([]byte, error) {
 		return r.respondReverse(query, name, resp)
 	}
 
-	resp.IP, resp.Header.RCode, err = r.Resolve(name, resp.Question.Type)
+	resp.IP, resp.Header.RCode = r.resolveLocal(name, resp.Question.Type)
 	// This return code is special: it requests forwarding.
 	if resp.Header.RCode == dns.RCodeRefused {
 		return nil, errNotOurName
 	}
 
-	// We will not return this error: it is the sender's fault.
-	if err != nil {
-		r.logf("resolving: %v", err)
-	}
-
 	return marshalResponse(resp)
 }

net/dns/resolver/tsdns_server_test.go

@@ -2,10 +2,10 @@
 // Use of this source code is governed by a BSD-style
 // license that can be found in the LICENSE file.
 
-package dns
+package resolver
 
 import (
-	"log"
+	"fmt"
 	"testing"
 
 	"github.com/miekg/dns"
@@ -16,8 +16,6 @@ import (
 // that depends on github.com/miekg/dns
 // from the rest, which only depends on dnsmessage.
 
-var dnsHandleFunc = dns.HandleFunc
-
 // resolveToIP returns a handler function which responds
 // to queries of type A it receives with an A record containing ipv4,
 // to queries of type AAAA with an AAAA record containing ipv6,
@@ -68,28 +66,38 @@ func resolveToIP(ipv4, ipv6 netaddr.IP, ns string) dns.HandlerFunc {
 	}
 }
 
-func resolveToNXDOMAIN(w dns.ResponseWriter, req *dns.Msg) {
+var resolveToNXDOMAIN = dns.HandlerFunc(func(w dns.ResponseWriter, req *dns.Msg) {
 	m := new(dns.Msg)
 	m.SetRcode(req, dns.RcodeNameError)
 	w.WriteMsg(m)
-}
-
-func serveDNS(tb testing.TB, addr string) (*dns.Server, chan error) {
-	server := &dns.Server{Addr: addr, Net: "udp"}
+})
 
+func serveDNS(tb testing.TB, addr string, records ...interface{}) *dns.Server {
+	if len(records)%2 != 0 {
+		panic("must have an even number of record values")
+	}
+	mux := dns.NewServeMux()
+	for i := 0; i < len(records); i += 2 {
+		name := records[i].(string)
+		handler := records[i+1].(dns.Handler)
+		mux.Handle(name, handler)
+	}
 	waitch := make(chan struct{})
-	server.NotifyStartedFunc = func() { close(waitch) }
+	server := &dns.Server{
+		Addr:              addr,
+		Net:               "udp",
+		Handler:           mux,
+		NotifyStartedFunc: func() { close(waitch) },
+		ReusePort:         true,
+	}
 
-	errch := make(chan error, 1)
 	go func() {
 		err := server.ListenAndServe()
 		if err != nil {
-			log.Printf("ListenAndServe(%q): %v", addr, err)
+			panic(fmt.Sprintf("ListenAndServe(%q): %v", addr, err))
 		}
-		errch <- err
-		close(errch)
 	}()
 
 	<-waitch
-	return server, errch
+	return server
 }

net/dns/resolver/tsdns_test.go

@@ -2,13 +2,12 @@
 // Use of this source code is governed by a BSD-style
 // license that can be found in the LICENSE file.
 
-package dns
+package resolver
 
 import (
 	"bytes"
 	"errors"
 	"net"
-	"sync"
 	"testing"
 
 	dns "golang.org/x/net/dns/dnsmessage"
@@ -16,21 +15,16 @@ import (
 	"tailscale.com/tstest"
 )
 
-var testipv4 = netaddr.IPv4(1, 2, 3, 4)
-var testipv6 = netaddr.IPv6Raw([16]byte{
-	0x00, 0x01, 0x02, 0x03,
-	0x04, 0x05, 0x06, 0x07,
-	0x08, 0x09, 0x0a, 0x0b,
-	0x0c, 0x0d, 0x0e, 0x0f,
-})
+var testipv4 = netaddr.MustParseIP("1.2.3.4")
+var testipv6 = netaddr.MustParseIP("0001:0203:0405:0607:0809:0a0b:0c0d:0e0f")
 
-var dnsMap = NewMap(
-	map[string]netaddr.IP{
-		"test1.ipn.dev.": testipv4,
-		"test2.ipn.dev.": testipv6,
+var dnsCfg = Config{
+	Hosts: map[string][]netaddr.IP{
+		"test1.ipn.dev.": []netaddr.IP{testipv4},
+		"test2.ipn.dev.": []netaddr.IP{testipv6},
 	},
-	[]string{"ipn.dev."},
-)
+	LocalDomains: []string{"ipn.dev."},
+}
 
 func dnspacket(domain string, tp dns.Type) []byte {
 	var dnsHeader dns.Header
@@ -109,10 +103,9 @@ func unpackResponse(payload []byte) (dnsResponse, error) {
 }
 
 func syncRespond(r *Resolver, query []byte) ([]byte, error) {
-	request := Packet{Payload: query}
-	r.EnqueueRequest(request)
-	resp, err := r.NextResponse()
-	return resp.Payload, err
+	r.EnqueueRequest(query, netaddr.IPPort{})
+	payload, _, err := r.NextResponse()
+	return payload, err
 }
 
 func mustIP(str string) netaddr.IP {
@@ -193,15 +186,12 @@ func TestRDNSNameToIPv6(t *testing.T) {
 	}
 }
 
-func TestResolve(t *testing.T) {
-	r := NewResolver(ResolverConfig{Logf: t.Logf, Forward: false})
-	r.SetMap(dnsMap)
-
-	if err := r.Start(); err != nil {
-		t.Fatalf("start: %v", err)
-	}
+func TestResolveLocal(t *testing.T) {
+	r := New(t.Logf, nil)
 	defer r.Close()
 
+	r.SetConfig(dnsCfg)
+
 	tests := []struct {
 		name  string
 		qname string
@@ -224,10 +214,7 @@ func TestResolve(t *testing.T) {
 
 	for _, tt := range tests {
 		t.Run(tt.name, func(t *testing.T) {
-			ip, code, err := r.Resolve(tt.qname, tt.qtype)
-			if err != nil {
-				t.Errorf("err = %v; want nil", err)
-			}
+			ip, code := r.resolveLocal(tt.qname, tt.qtype)
 			if code != tt.code {
 				t.Errorf("code = %v; want %v", code, tt.code)
 			}
@@ -239,15 +226,12 @@ func TestResolve(t *testing.T) {
 	}
 }
 
-func TestResolveReverse(t *testing.T) {
-	r := NewResolver(ResolverConfig{Logf: t.Logf, Forward: false})
-	r.SetMap(dnsMap)
-
-	if err := r.Start(); err != nil {
-		t.Fatalf("start: %v", err)
-	}
+func TestResolveLocalReverse(t *testing.T) {
+	r := New(t.Logf, nil)
 	defer r.Close()
 
+	r.SetConfig(dnsCfg)
+
 	tests := []struct {
 		name string
 		ip   netaddr.IP
@@ -261,10 +245,7 @@ func TestResolveReverse(t *testing.T) {
 
 	for _, tt := range tests {
 		t.Run(tt.name, func(t *testing.T) {
-			name, code, err := r.ResolveReverse(tt.ip)
-			if err != nil {
-				t.Errorf("err = %v; want nil", err)
-			}
+			name, code := r.resolveLocalReverse(tt.ip)
 			if code != tt.code {
 				t.Errorf("code = %v; want %v", code, tt.code)
 			}
@@ -291,45 +272,27 @@ func TestDelegate(t *testing.T) {
 		t.Skip("skipping test that requires localhost IPv6")
 	}
 
-	dnsHandleFunc("test.site.", resolveToIP(testipv4, testipv6, "dns.test.site."))
-	dnsHandleFunc("nxdomain.site.", resolveToNXDOMAIN)
+	v4server := serveDNS(t, "127.0.0.1:0",
+		"test.site.", resolveToIP(testipv4, testipv6, "dns.test.site."),
+		"nxdomain.site.", resolveToNXDOMAIN)
+	defer v4server.Shutdown()
+	v6server := serveDNS(t, "[::1]:0",
+		"test.site.", resolveToIP(testipv4, testipv6, "dns.test.site."),
+		"nxdomain.site.", resolveToNXDOMAIN)
+	defer v6server.Shutdown()
 
-	v4server, v4errch := serveDNS(t, "127.0.0.1:0")
-	v6server, v6errch := serveDNS(t, "[::1]:0")
-
-	defer func() {
-		if err := <-v4errch; err != nil {
-			t.Errorf("v4 server error: %v", err)
-		}
-		if err := <-v6errch; err != nil {
-			t.Errorf("v6 server error: %v", err)
-		}
-	}()
-	if v4server != nil {
-		defer v4server.Shutdown()
-	}
-	if v6server != nil {
-		defer v6server.Shutdown()
-	}
-
-	if v4server == nil || v6server == nil {
-		// There is an error in at least one of the channels
-		// and we cannot proceed; return to see it.
-		return
-	}
-
-	r := NewResolver(ResolverConfig{Logf: t.Logf, Forward: true})
-	r.SetMap(dnsMap)
-	r.SetUpstreams([]net.Addr{
-		v4server.PacketConn.LocalAddr(),
-		v6server.PacketConn.LocalAddr(),
-	})
-
-	if err := r.Start(); err != nil {
-		t.Fatalf("start: %v", err)
-	}
+	r := New(t.Logf, nil)
 	defer r.Close()
 
+	cfg := dnsCfg
+	cfg.Routes = map[string][]netaddr.IPPort{
+		".": {
+			netaddr.MustParseIPPort(v4server.PacketConn.LocalAddr().String()),
+			netaddr.MustParseIPPort(v6server.PacketConn.LocalAddr().String()),
+		},
+	}
+	r.SetConfig(cfg)
+
 	tests := []struct {
 		title    string
 		query    []byte
@@ -382,30 +345,85 @@ func TestDelegate(t *testing.T) {
 	}
 }
 
-func TestDelegateCollision(t *testing.T) {
-	dnsHandleFunc("test.site.", resolveToIP(testipv4, testipv6, "dns.test.site."))
+func TestDelegateSplitRoute(t *testing.T) {
+	test4 := netaddr.MustParseIP("2.3.4.5")
+	test6 := netaddr.MustParseIP("ff::1")
 
-	server, errch := serveDNS(t, "127.0.0.1:0")
-	defer func() {
-		if err := <-errch; err != nil {
-			t.Errorf("server error: %v", err)
-		}
-	}()
+	server1 := serveDNS(t, "127.0.0.1:0",
+		"test.site.", resolveToIP(testipv4, testipv6, "dns.test.site."))
+	defer server1.Shutdown()
+	server2 := serveDNS(t, "127.0.0.1:0",
+		"test.other.", resolveToIP(test4, test6, "dns.other."))
+	defer server2.Shutdown()
 
-	if server == nil {
-		return
+	r := New(t.Logf, nil)
+	defer r.Close()
+
+	cfg := dnsCfg
+	cfg.Routes = map[string][]netaddr.IPPort{
+		".":      {netaddr.MustParseIPPort(server1.PacketConn.LocalAddr().String())},
+		"other.": {netaddr.MustParseIPPort(server2.PacketConn.LocalAddr().String())},
 	}
+	r.SetConfig(cfg)
+
+	tests := []struct {
+		title    string
+		query    []byte
+		response dnsResponse
+	}{
+		{
+			"general",
+			dnspacket("test.site.", dns.TypeA),
+			dnsResponse{ip: testipv4, rcode: dns.RCodeSuccess},
+		},
+		{
+			"override",
+			dnspacket("test.other.", dns.TypeA),
+			dnsResponse{ip: test4, rcode: dns.RCodeSuccess},
+		},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.title, func(t *testing.T) {
+			payload, err := syncRespond(r, tt.query)
+			if err != nil {
+				t.Errorf("err = %v; want nil", err)
+				return
+			}
+			response, err := unpackResponse(payload)
+			if err != nil {
+				t.Errorf("extract: err = %v; want nil (in %x)", err, payload)
+				return
+			}
+			if response.rcode != tt.response.rcode {
+				t.Errorf("rcode = %v; want %v", response.rcode, tt.response.rcode)
+			}
+			if response.ip != tt.response.ip {
+				t.Errorf("ip = %v; want %v", response.ip, tt.response.ip)
+			}
+			if response.name != tt.response.name {
+				t.Errorf("name = %v; want %v", response.name, tt.response.name)
+			}
+		})
+	}
+}
+
+func TestDelegateCollision(t *testing.T) {
+	server := serveDNS(t, "127.0.0.1:0",
+		"test.site.", resolveToIP(testipv4, testipv6, "dns.test.site."))
 	defer server.Shutdown()
 
-	r := NewResolver(ResolverConfig{Logf: t.Logf, Forward: true})
-	r.SetMap(dnsMap)
-	r.SetUpstreams([]net.Addr{server.PacketConn.LocalAddr()})
-
-	if err := r.Start(); err != nil {
-		t.Fatalf("start: %v", err)
-	}
+	r := New(t.Logf, nil)
 	defer r.Close()
 
+	cfg := dnsCfg
+	cfg.Routes = map[string][]netaddr.IPPort{
+		".": {
+			netaddr.MustParseIPPort(server.PacketConn.LocalAddr().String()),
+		},
+	}
+	r.SetConfig(cfg)
+
 	packets := []struct {
 		qname string
 		qtype dns.Type
@@ -418,21 +436,20 @@ func TestDelegateCollision(t *testing.T) {
 	// packets will have the same dns txid.
 	for _, p := range packets {
 		payload := dnspacket(p.qname, p.qtype)
-		req := Packet{Payload: payload, Addr: p.addr}
-		err := r.EnqueueRequest(req)
+		err := r.EnqueueRequest(payload, p.addr)
 		if err != nil {
 			t.Error(err)
 		}
 	}
 
 	// Despite the txid collision, the answer(s) should still match the query.
-	resp, err := r.NextResponse()
+	resp, addr, err := r.NextResponse()
 	if err != nil {
 		t.Error(err)
 	}
 
 	var p dns.Parser
-	_, err = p.Start(resp.Payload)
+	_, err = p.Start(resp)
 	if err != nil {
 		t.Error(err)
 	}
@@ -456,72 +473,12 @@ func TestDelegateCollision(t *testing.T) {
 	}
 
 	for _, p := range packets {
-		if p.qtype == wantType && p.addr != resp.Addr {
-			t.Errorf("addr = %v; want %v", resp.Addr, p.addr)
+		if p.qtype == wantType && p.addr != addr {
+			t.Errorf("addr = %v; want %v", addr, p.addr)
 		}
 	}
 }
 
-func TestConcurrentSetMap(t *testing.T) {
-	r := NewResolver(ResolverConfig{Logf: t.Logf, Forward: false})
-
-	if err := r.Start(); err != nil {
-		t.Fatalf("start: %v", err)
-	}
-	defer r.Close()
-
-	// This is purely to ensure that Resolve does not race with SetMap.
-	var wg sync.WaitGroup
-	wg.Add(2)
-	go func() {
-		defer wg.Done()
-		r.SetMap(dnsMap)
-	}()
-	go func() {
-		defer wg.Done()
-		r.Resolve("test1.ipn.dev", dns.TypeA)
-	}()
-	wg.Wait()
-}
-
-func TestConcurrentSetUpstreams(t *testing.T) {
-	dnsHandleFunc("test.site.", resolveToIP(testipv4, testipv6, "dns.test.site."))
-
-	server, errch := serveDNS(t, "127.0.0.1:0")
-	defer func() {
-		if err := <-errch; err != nil {
-			t.Errorf("server error: %v", err)
-		}
-	}()
-
-	if server == nil {
-		return
-	}
-	defer server.Shutdown()
-
-	r := NewResolver(ResolverConfig{Logf: t.Logf, Forward: true})
-	r.SetMap(dnsMap)
-
-	if err := r.Start(); err != nil {
-		t.Fatalf("start: %v", err)
-	}
-	defer r.Close()
-
-	packet := dnspacket("test.site.", dns.TypeA)
-	// This is purely to ensure that delegation does not race with SetUpstreams.
-	var wg sync.WaitGroup
-	wg.Add(2)
-	go func() {
-		defer wg.Done()
-		r.SetUpstreams([]net.Addr{server.PacketConn.LocalAddr()})
-	}()
-	go func() {
-		defer wg.Done()
-		syncRespond(r, packet)
-	}()
-	wg.Wait()
-}
-
 var allResponse = []byte{
 	0x00, 0x00, // transaction id: 0
 	0x84, 0x00, // flags: response, authoritative, no error
@@ -670,14 +627,11 @@ var emptyResponse = []byte{
 }
 
 func TestFull(t *testing.T) {
-	r := NewResolver(ResolverConfig{Logf: t.Logf, Forward: false})
-	r.SetMap(dnsMap)
-
-	if err := r.Start(); err != nil {
-		t.Fatalf("start: %v", err)
-	}
+	r := New(t.Logf, nil)
 	defer r.Close()
 
+	r.SetConfig(dnsCfg)
+
 	// One full packet and one error packet
 	tests := []struct {
 		name     string
@@ -689,8 +643,8 @@ func TestFull(t *testing.T) {
 		{"ipv6", dnspacket("test2.ipn.dev.", dns.TypeAAAA), ipv6Response},
 		{"no-ipv6", dnspacket("test1.ipn.dev.", dns.TypeAAAA), emptyResponse},
 		{"upper", dnspacket("TEST1.IPN.DEV.", dns.TypeA), ipv4UppercaseResponse},
-		{"ptr", dnspacket("4.3.2.1.in-addr.arpa.", dns.TypePTR), ptrResponse},
-		{"ptr", dnspacket("f.0.e.0.d.0.c.0.b.0.a.0.9.0.8.0.7.0.6.0.5.0.4.0.3.0.2.0.1.0.0.0.ip6.arpa.",
+		{"ptr4", dnspacket("4.3.2.1.in-addr.arpa.", dns.TypePTR), ptrResponse},
+		{"ptr6", dnspacket("f.0.e.0.d.0.c.0.b.0.a.0.9.0.8.0.7.0.6.0.5.0.4.0.3.0.2.0.1.0.0.0.ip6.arpa.",
 			dns.TypePTR), ptrResponse6},
 		{"nxdomain", dnspacket("test3.ipn.dev.", dns.TypeA), nxdomainResponse},
 	}
@@ -709,13 +663,9 @@ func TestFull(t *testing.T) {
 }
 
 func TestAllocs(t *testing.T) {
-	r := NewResolver(ResolverConfig{Logf: t.Logf, Forward: false})
-	r.SetMap(dnsMap)
-
-	if err := r.Start(); err != nil {
-		t.Fatalf("start: %v", err)
-	}
+	r := New(t.Logf, nil)
 	defer r.Close()
+	r.SetConfig(dnsCfg)
 
 	// It is seemingly pointless to test allocs in the delegate path,
 	// as dialer.Dial -> Read -> Write alone comprise 12 allocs.
@@ -764,29 +714,20 @@ func TestTrimRDNSBonjourPrefix(t *testing.T) {
 }
 
 func BenchmarkFull(b *testing.B) {
-	dnsHandleFunc("test.site.", resolveToIP(testipv4, testipv6, "dns.test.site."))
-
-	server, errch := serveDNS(b, "127.0.0.1:0")
-	defer func() {
-		if err := <-errch; err != nil {
-			b.Errorf("server error: %v", err)
-		}
-	}()
-
-	if server == nil {
-		return
-	}
+	server := serveDNS(b, "127.0.0.1:0",
+		"test.site.", resolveToIP(testipv4, testipv6, "dns.test.site."))
 	defer server.Shutdown()
 
-	r := NewResolver(ResolverConfig{Logf: b.Logf, Forward: true})
-	r.SetMap(dnsMap)
-	r.SetUpstreams([]net.Addr{server.PacketConn.LocalAddr()})
-
-	if err := r.Start(); err != nil {
-		b.Fatalf("start: %v", err)
-	}
+	r := New(b.Logf, nil)
 	defer r.Close()
 
+	cfg := dnsCfg
+	cfg.Routes = map[string][]netaddr.IPPort{
+		".": {
+			netaddr.MustParseIPPort(server.PacketConn.LocalAddr().String()),
+		},
+	}
+
 	tests := []struct {
 		name    string
 		request []byte

net/interfaces/interfaces.go

@@ -500,7 +500,8 @@ func isPrivateIP(ip netaddr.IP) bool {
 }
 
 func isGlobalV6(ip netaddr.IP) bool {
-	return v6Global1.Contains(ip)
+	return v6Global1.Contains(ip) ||
+		(tsaddr.IsULA(ip) && !tsaddr.TailscaleULARange().Contains(ip))
 }
 
 func mustCIDR(s string) netaddr.IPPrefix {

net/interfaces/interfaces_default_route_test.go

@@ -2,16 +2,85 @@
 // Use of this source code is governed by a BSD-style
 // license that can be found in the LICENSE file.
 
-// +build linux darwin,!redo
+// +build linux,!redo
 
 package interfaces
 
-import "testing"
+import (
+	"fmt"
+	"io/ioutil"
+	"os"
+	"path/filepath"
+	"testing"
+)
 
 func TestDefaultRouteInterface(t *testing.T) {
+	// tests /proc/net/route on the local system, cannot make an assertion about
+	// the correct interface name, but good as a sanity check.
 	v, err := DefaultRouteInterface()
 	if err != nil {
 		t.Fatal(err)
 	}
 	t.Logf("got %q", v)
 }
+
+// test the specific /proc/net/route path as found on Google Cloud Run instances
+func TestGoogleCloudRunDefaultRouteInterface(t *testing.T) {
+	dir := t.TempDir()
+	savedProcNetRoutePath := procNetRoutePath
+	defer func() { procNetRoutePath = savedProcNetRoutePath }()
+	procNetRoutePath = filepath.Join(dir, "CloudRun")
+	buf := []byte("Iface\tDestination\tGateway\tFlags\tRefCnt\tUse\tMetric\tMask\tMTU\tWindow\tIRTT\n" +
+		"eth0\t8008FEA9\t00000000\t0001\t0\t0\t0\t01FFFFFF\t0\t0\t0\n" +
+		"eth1\t00000000\t00000000\t0001\t0\t0\t0\t00000000\t0\t0\t0\n")
+	err := ioutil.WriteFile(procNetRoutePath, buf, 0644)
+	if err != nil {
+		t.Fatal(err)
+	}
+	got, err := DefaultRouteInterface()
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	if got != "eth1" {
+		t.Fatalf("got %s, want eth1", got)
+	}
+}
+
+// we read chunks of /proc/net/route at a time, test that files longer than the chunk
+// size can be handled.
+func TestExtremelyLongProcNetRoute(t *testing.T) {
+	dir := t.TempDir()
+	savedProcNetRoutePath := procNetRoutePath
+	defer func() { procNetRoutePath = savedProcNetRoutePath }()
+	procNetRoutePath = filepath.Join(dir, "VeryLong")
+	f, err := os.Create(procNetRoutePath)
+	if err != nil {
+		t.Fatal(err)
+	}
+	_, err = f.Write([]byte("Iface\tDestination\tGateway\tFlags\tRefCnt\tUse\tMetric\tMask\tMTU\tWindow\tIRTT\n"))
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	for n := 0; n <= 1000; n++ {
+		line := fmt.Sprintf("eth%d\t8008FEA9\t00000000\t0001\t0\t0\t0\t01FFFFFF\t0\t0\t0\n", n)
+		_, err := f.Write([]byte(line))
+		if err != nil {
+			t.Fatal(err)
+		}
+	}
+	_, err = f.Write([]byte("tokenring1\t00000000\t00000000\t0001\t0\t0\t0\t00000000\t0\t0\t0\n"))
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	got, err := DefaultRouteInterface()
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	if got != "tokenring1" {
+		t.Fatalf("got %q, want tokenring1", got)
+	}
+}

net/interfaces/interfaces_linux.go

@@ -8,6 +8,7 @@ import (
 	"bufio"
 	"bytes"
 	"errors"
+	"fmt"
 	"io"
 	"log"
 	"os"
@@ -135,18 +136,20 @@ func DefaultRouteInterface() (string, error) {
 }
 
 var zeroRouteBytes = []byte("00000000")
+var procNetRoutePath = "/proc/net/route"
 
-func defaultRouteInterfaceProcNet() (string, error) {
-	f, err := os.Open("/proc/net/route")
+func defaultRouteInterfaceProcNetInternal(bufsize int) (string, error) {
+	f, err := os.Open(procNetRoutePath)
 	if err != nil {
 		return "", err
 	}
 	defer f.Close()
-	br := bufio.NewReaderSize(f, 128)
+
+	br := bufio.NewReaderSize(f, bufsize)
 	for {
 		line, err := br.ReadSlice('\n')
 		if err == io.EOF {
-			break
+			return "", fmt.Errorf("no default routes found: %w", err)
 		}
 		if err != nil {
 			return "", err
@@ -168,9 +171,28 @@ func defaultRouteInterfaceProcNet() (string, error) {
 			return ifc, nil // interface name
 		}
 	}
+}
 
-	return "", errors.New("no default routes found")
-
+// returns string interface name and an error.
+// io.EOF: full route table processed, no default route found.
+// other io error: something went wrong reading the route file.
+func defaultRouteInterfaceProcNet() (string, error) {
+	rc, err := defaultRouteInterfaceProcNetInternal(128)
+	if rc == "" && (errors.Is(err, io.EOF) || err == nil) {
+		// https://github.com/google/gvisor/issues/5732
+		// On a regular Linux kernel you can read the first 128 bytes of /proc/net/route,
+		// then come back later to read the next 128 bytes and so on.
+		//
+		// In Google Cloud Run, where /proc/net/route comes from gVisor, you have to
+		// read it all at once. If you read only the first few bytes then the second
+		// read returns 0 bytes no matter how much originally appeared to be in the file.
+		//
+		// At the time of this writing (Mar 2021) Google Cloud Run has eth0 and eth1
+		// with a 384 byte /proc/net/route. We allocate a large buffer to ensure we'll
+		// read it all in one call.
+		return defaultRouteInterfaceProcNetInternal(4096)
+	}
+	return rc, err
 }
 
 // defaultRouteInterfaceAndroidIPRoute tries to find the machine's default route interface name

net/interfaces/interfaces_test.go

@@ -7,6 +7,8 @@ package interfaces
 import (
 	"encoding/json"
 	"testing"
+
+	"inet.af/netaddr"
 )
 
 func TestGetState(t *testing.T) {
@@ -43,3 +45,24 @@ func TestLikelyHomeRouterIP(t *testing.T) {
 	}
 	t.Logf("myIP = %v; gw = %v", my, gw)
 }
+
+func TestIsGlobalV6(t *testing.T) {
+	tests := []struct {
+		name string
+		ip   string
+		want bool
+	}{
+		{"first ULA", "fc00::1", true},
+		{"Tailscale", "fd7a:115c:a1e0::1", false},
+		{"Cloud Run", "fddf:3978:feb1:d745::1", true},
+		{"zeros", "0000:0000:0000:0000:0000:0000:0000:0000", false},
+		{"Link Local", "fe80::1", false},
+		{"Global", "2602::1", true},
+	}
+
+	for _, test := range tests {
+		if got := isGlobalV6(netaddr.MustParseIP(test.ip)); got != test.want {
+			t.Errorf("isGlobalV6(%s) = %v, want %v", test.name, got, test.want)
+		}
+	}
+}

net/tsaddr/tsaddr.go

@@ -33,6 +33,7 @@ func CGNATRange() netaddr.IPPrefix {
 var (
 	cgnatRange   oncePrefix
 	ulaRange     oncePrefix
+	tsUlaRange   oncePrefix
 	ula4To6Range oncePrefix
 )
 
@@ -57,8 +58,8 @@ func IsTailscaleIP(ip netaddr.IP) bool {
 // TailscaleULARange returns the IPv6 Unique Local Address range that
 // is the superset range that Tailscale assigns out of.
 func TailscaleULARange() netaddr.IPPrefix {
-	ulaRange.Do(func() { mustPrefix(&ulaRange.v, "fd7a:115c:a1e0::/48") })
-	return ulaRange.v
+	tsUlaRange.Do(func() { mustPrefix(&tsUlaRange.v, "fd7a:115c:a1e0::/48") })
+	return tsUlaRange.v
 }
 
 // Tailscale4To6Range returns the subset of TailscaleULARange used for
@@ -95,6 +96,11 @@ func Tailscale4To6(ipv4 netaddr.IP) netaddr.IP {
 	return netaddr.IPFrom16(ret)
 }
 
+func IsULA(ip netaddr.IP) bool {
+	ulaRange.Do(func() { mustPrefix(&ulaRange.v, "fc00::/7") })
+	return ulaRange.v.Contains(ip)
+}
+
 func mustPrefix(v *netaddr.IPPrefix, prefix string) {
 	var err error
 	*v, err = netaddr.ParseIPPrefix(prefix)

net/tsaddr/tsaddr_test.go

@@ -42,3 +42,25 @@ func TestCGNATRange(t *testing.T) {
 		t.Errorf("got %q; want %q", got, want)
 	}
 }
+
+func TestIsUla(t *testing.T) {
+	tests := []struct {
+		name string
+		ip   string
+		want bool
+	}{
+		{"first ULA", "fc00::1", true},
+		{"not ULA", "fb00::1", false},
+		{"Tailscale", "fd7a:115c:a1e0::1", true},
+		{"Cloud Run", "fddf:3978:feb1:d745::1", true},
+		{"zeros", "0000:0000:0000:0000:0000:0000:0000:0000", false},
+		{"Link Local", "fe80::1", false},
+		{"Global", "2602::1", false},
+	}
+
+	for _, test := range tests {
+		if got := IsULA(netaddr.MustParseIP(test.ip)); got != test.want {
+			t.Errorf("IsULA(%s) = %v, want %v", test.name, got, test.want)
+		}
+	}
+}

paths/paths.go

@@ -10,8 +10,13 @@ import (
 	"os"
 	"path/filepath"
 	"runtime"
+	"sync/atomic"
 )
 
+// IOSSharedDir is a string set by the iOS app on start
+// containing a directory we can read/write in.
+var IOSSharedDir atomic.Value
+
 // LegacyConfigPath returns the path used by the pre-tailscaled
 // "relaynode" daemon's config file. It returns the empty string for
 // platforms where relaynode never ran.

safesocket/safesocket.go

@@ -10,8 +10,6 @@ import (
 	"errors"
 	"net"
 	"runtime"
-
-	"tailscale.com/paths"
 )
 
 type closeable interface {
@@ -31,11 +29,6 @@ func ConnCloseWrite(c net.Conn) error {
 	return c.(closeable).CloseWrite()
 }
 
-// ConnectDefault connects to the local Tailscale daemon.
-func ConnectDefault() (net.Conn, error) {
-	return Connect(paths.DefaultTailscaledSocket(), 41112)
-}
-
 // Connect connects to either path (on Unix) or the provided localhost port (on Windows).
 func Connect(path string, port uint16) (net.Conn, error) {
 	return connect(path, port)

tailcfg/tailcfg.go

@@ -4,7 +4,7 @@
 
 package tailcfg
 
-//go:generate go run tailscale.com/cmd/cloner --type=User,Node,Hostinfo,NetInfo,Group,Role,Capability,Login,DNSConfig,RegisterResponse --clonefunc=true --output=tailcfg_clone.go
+//go:generate go run tailscale.com/cmd/cloner --type=User,Node,Hostinfo,NetInfo,Login,DNSConfig,RegisterResponse --clonefunc=true --output=tailcfg_clone.go
 
 import (
 	"bytes"
@@ -66,20 +66,6 @@ func (u StableNodeID) IsZero() bool {
 	return u == ""
 }
 
-type GroupID ID
-
-func (u GroupID) IsZero() bool {
-	return u == 0
-}
-
-type RoleID ID
-
-func (u RoleID) IsZero() bool {
-	return u == 0
-}
-
-type CapabilityID ID
-
 // MachineKey is the curve25519 public key for a machine.
 type MachineKey [32]byte
 
@@ -90,31 +76,6 @@ type NodeKey [32]byte
 // It's never written to disk or reused between network start-ups.
 type DiscoKey [32]byte
 
-type Group struct {
-	ID      GroupID
-	Name    string
-	Members []ID
-}
-
-type Role struct {
-	ID           RoleID
-	Name         string
-	Capabilities []CapabilityID
-}
-
-type CapType string
-
-const (
-	CapRead  = CapType("read")
-	CapWrite = CapType("write")
-)
-
-type Capability struct {
-	ID   CapabilityID
-	Type CapType
-	Val  ID
-}
-
 // User is an IPN user.
 //
 // A user can have multiple logins associated with it (e.g. gmail and github oauth).
@@ -133,7 +94,6 @@ type User struct {
 	ProfilePicURL string // if non-empty overrides Login field
 	Domain        string
 	Logins        []LoginID
-	Roles         []RoleID
 	Created       time.Time
 }
 
@@ -155,9 +115,22 @@ type UserProfile struct {
 	LoginName     string // "alice@smith.com"; for display purposes only (provider is not listed)
 	DisplayName   string // "Alice Smith"
 	ProfilePicURL string
-	Roles         []RoleID // deprecated; clients should not rely on Roles
+
+	// Roles exists for legacy reasons, to keep old macOS clients
+	// happy. It JSON marshals as [].
+	Roles emptyStructJSONSlice
 }
 
+type emptyStructJSONSlice struct{}
+
+var emptyJSONSliceBytes = []byte("[]")
+
+func (emptyStructJSONSlice) MarshalJSON() ([]byte, error) {
+	return emptyJSONSliceBytes, nil
+}
+
+func (emptyStructJSONSlice) UnmarshalJSON([]byte) error { return nil }
+
 type Node struct {
 	ID       NodeID
 	StableID StableNodeID
@@ -187,6 +160,13 @@ type Node struct {
 
 	MachineAuthorized bool `json:",omitempty"` // TODO(crawshaw): replace with MachineStatus
 
+	// Capabilities are capabilities that the node has.
+	// They're free-form strings, but should be in the form of URLs/URIs
+	// such as:
+	//    "https://tailscale.com/cap/is-admin"
+	//    "https://tailscale.com/cap/recv-file"
+	Capabilities []string `json:",omitempty"`
+
 	// The following three computed fields hold the various names that can
 	// be used for this node in UIs. They are populated from controlclient
 	// (not from control) by calling node.InitDisplayNames. These can be
@@ -789,13 +769,12 @@ type DNSConfig struct {
 	Nameservers []netaddr.IP `json:",omitempty"`
 	// Domains are the search domains to use.
 	Domains []string `json:",omitempty"`
-	// PerDomain indicates whether it is preferred to use Nameservers
-	// only for DNS queries for subdomains of Domains.
-	// Some OSes and OS configurations don't support per-domain DNS configuration,
-	// in which case Nameservers applies to all DNS requests regardless of PerDomain's value.
+	// PerDomain is not set by the control server, and does nothing.
+	// TODO(danderson): revise DNS configuration to make this useful
+	// again.
 	PerDomain bool
 	// Proxied indicates whether DNS requests are proxied through a dns.Resolver.
-	// This enables MagicDNS. It is togglable independently of PerDomain.
+	// This enables MagicDNS.
 	Proxied bool
 }
 
@@ -891,10 +870,6 @@ type MapResponse struct {
 	PacketFilter []FilterRule
 
 	UserProfiles []UserProfile // as of 1.1.541 (mapver 5): may be new or updated user profiles only
-	Roles        []Role        // deprecated; clients should not rely on Roles
-
-	// TODO: Groups       []Group
-	// TODO: Capabilities []Capability
 
 	// Debug is normally nil, except for when the control server
 	// is setting debug settings on a node.
@@ -981,13 +956,10 @@ func (k DiscoKey) ShortString() string              { return fmt.Sprintf("d:%x",
 // IsZero reports whether k is the zero value.
 func (k DiscoKey) IsZero() bool { return k == DiscoKey{} }
 
-func (id ID) String() string           { return fmt.Sprintf("id:%x", int64(id)) }
-func (id UserID) String() string       { return fmt.Sprintf("userid:%x", int64(id)) }
-func (id LoginID) String() string      { return fmt.Sprintf("loginid:%x", int64(id)) }
-func (id NodeID) String() string       { return fmt.Sprintf("nodeid:%x", int64(id)) }
-func (id GroupID) String() string      { return fmt.Sprintf("groupid:%x", int64(id)) }
-func (id RoleID) String() string       { return fmt.Sprintf("roleid:%x", int64(id)) }
-func (id CapabilityID) String() string { return fmt.Sprintf("capid:%x", int64(id)) }
+func (id ID) String() string      { return fmt.Sprintf("id:%x", int64(id)) }
+func (id UserID) String() string  { return fmt.Sprintf("userid:%x", int64(id)) }
+func (id LoginID) String() string { return fmt.Sprintf("loginid:%x", int64(id)) }
+func (id NodeID) String() string  { return fmt.Sprintf("nodeid:%x", int64(id)) }
 
 // Equal reports whether n and n2 are equal.
 func (n *Node) Equal(n2 *Node) bool {
@@ -1012,6 +984,7 @@ func (n *Node) Equal(n2 *Node) bool {
 		n.Created.Equal(n2.Created) &&
 		eqTimePtr(n.LastSeen, n2.LastSeen) &&
 		n.MachineAuthorized == n2.MachineAuthorized &&
+		eqStrings(n.Capabilities, n2.Capabilities) &&
 		n.ComputedName == n2.ComputedName &&
 		n.computedHostIfDifferent == n2.computedHostIfDifferent &&
 		n.ComputedNameWithHost == n2.ComputedNameWithHost

tailcfg/tailcfg_clone.go

@@ -2,7 +2,7 @@
 // Use of this source code is governed by a BSD-style
 // license that can be found in the LICENSE file.
 
-// Code generated by tailscale.com/cmd/cloner -type User,Node,Hostinfo,NetInfo,Group,Role,Capability,Login,DNSConfig,RegisterResponse; DO NOT EDIT.
+// Code generated by tailscale.com/cmd/cloner -type User,Node,Hostinfo,NetInfo,Login,DNSConfig,RegisterResponse; DO NOT EDIT.
 
 package tailcfg
 
@@ -22,12 +22,11 @@ func (src *User) Clone() *User {
 	dst := new(User)
 	*dst = *src
 	dst.Logins = append(src.Logins[:0:0], src.Logins...)
-	dst.Roles = append(src.Roles[:0:0], src.Roles...)
 	return dst
 }
 
 // A compilation failure here means this code must be regenerated, with command:
-//   tailscale.com/cmd/cloner -type User,Node,Hostinfo,NetInfo,Group,Role,Capability,Login,DNSConfig,RegisterResponse
+//   tailscale.com/cmd/cloner -type User,Node,Hostinfo,NetInfo,Login,DNSConfig,RegisterResponse
 var _UserNeedsRegeneration = User(struct {
 	ID            UserID
 	LoginName     string
@@ -35,7 +34,6 @@ var _UserNeedsRegeneration = User(struct {
 	ProfilePicURL string
 	Domain        string
 	Logins        []LoginID
-	Roles         []RoleID
 	Created       time.Time
 }{})
 
@@ -55,11 +53,12 @@ func (src *Node) Clone() *Node {
 		dst.LastSeen = new(time.Time)
 		*dst.LastSeen = *src.LastSeen
 	}
+	dst.Capabilities = append(src.Capabilities[:0:0], src.Capabilities...)
 	return dst
 }
 
 // A compilation failure here means this code must be regenerated, with command:
-//   tailscale.com/cmd/cloner -type User,Node,Hostinfo,NetInfo,Group,Role,Capability,Login,DNSConfig,RegisterResponse
+//   tailscale.com/cmd/cloner -type User,Node,Hostinfo,NetInfo,Login,DNSConfig,RegisterResponse
 var _NodeNeedsRegeneration = Node(struct {
 	ID                      NodeID
 	StableID                StableNodeID
@@ -79,6 +78,7 @@ var _NodeNeedsRegeneration = Node(struct {
 	LastSeen                *time.Time
 	KeepAlive               bool
 	MachineAuthorized       bool
+	Capabilities            []string
 	ComputedName            string
 	computedHostIfDifferent string
 	ComputedNameWithHost    string
@@ -100,7 +100,7 @@ func (src *Hostinfo) Clone() *Hostinfo {
 }
 
 // A compilation failure here means this code must be regenerated, with command:
-//   tailscale.com/cmd/cloner -type User,Node,Hostinfo,NetInfo,Group,Role,Capability,Login,DNSConfig,RegisterResponse
+//   tailscale.com/cmd/cloner -type User,Node,Hostinfo,NetInfo,Login,DNSConfig,RegisterResponse
 var _HostinfoNeedsRegeneration = Hostinfo(struct {
 	IPNVersion    string
 	FrontendLogID string
@@ -137,7 +137,7 @@ func (src *NetInfo) Clone() *NetInfo {
 }
 
 // A compilation failure here means this code must be regenerated, with command:
-//   tailscale.com/cmd/cloner -type User,Node,Hostinfo,NetInfo,Group,Role,Capability,Login,DNSConfig,RegisterResponse
+//   tailscale.com/cmd/cloner -type User,Node,Hostinfo,NetInfo,Login,DNSConfig,RegisterResponse
 var _NetInfoNeedsRegeneration = NetInfo(struct {
 	MappingVariesByDestIP opt.Bool
 	HairPinning           opt.Bool
@@ -152,65 +152,6 @@ var _NetInfoNeedsRegeneration = NetInfo(struct {
 	DERPLatency           map[string]float64
 }{})
 
-// Clone makes a deep copy of Group.
-// The result aliases no memory with the original.
-func (src *Group) Clone() *Group {
-	if src == nil {
-		return nil
-	}
-	dst := new(Group)
-	*dst = *src
-	dst.Members = append(src.Members[:0:0], src.Members...)
-	return dst
-}
-
-// A compilation failure here means this code must be regenerated, with command:
-//   tailscale.com/cmd/cloner -type User,Node,Hostinfo,NetInfo,Group,Role,Capability,Login,DNSConfig,RegisterResponse
-var _GroupNeedsRegeneration = Group(struct {
-	ID      GroupID
-	Name    string
-	Members []ID
-}{})
-
-// Clone makes a deep copy of Role.
-// The result aliases no memory with the original.
-func (src *Role) Clone() *Role {
-	if src == nil {
-		return nil
-	}
-	dst := new(Role)
-	*dst = *src
-	dst.Capabilities = append(src.Capabilities[:0:0], src.Capabilities...)
-	return dst
-}
-
-// A compilation failure here means this code must be regenerated, with command:
-//   tailscale.com/cmd/cloner -type User,Node,Hostinfo,NetInfo,Group,Role,Capability,Login,DNSConfig,RegisterResponse
-var _RoleNeedsRegeneration = Role(struct {
-	ID           RoleID
-	Name         string
-	Capabilities []CapabilityID
-}{})
-
-// Clone makes a deep copy of Capability.
-// The result aliases no memory with the original.
-func (src *Capability) Clone() *Capability {
-	if src == nil {
-		return nil
-	}
-	dst := new(Capability)
-	*dst = *src
-	return dst
-}
-
-// A compilation failure here means this code must be regenerated, with command:
-//   tailscale.com/cmd/cloner -type User,Node,Hostinfo,NetInfo,Group,Role,Capability,Login,DNSConfig,RegisterResponse
-var _CapabilityNeedsRegeneration = Capability(struct {
-	ID   CapabilityID
-	Type CapType
-	Val  ID
-}{})
-
 // Clone makes a deep copy of Login.
 // The result aliases no memory with the original.
 func (src *Login) Clone() *Login {
@@ -223,7 +164,7 @@ func (src *Login) Clone() *Login {
 }
 
 // A compilation failure here means this code must be regenerated, with command:
-//   tailscale.com/cmd/cloner -type User,Node,Hostinfo,NetInfo,Group,Role,Capability,Login,DNSConfig,RegisterResponse
+//   tailscale.com/cmd/cloner -type User,Node,Hostinfo,NetInfo,Login,DNSConfig,RegisterResponse
 var _LoginNeedsRegeneration = Login(struct {
 	_             structs.Incomparable
 	ID            LoginID
@@ -248,7 +189,7 @@ func (src *DNSConfig) Clone() *DNSConfig {
 }
 
 // A compilation failure here means this code must be regenerated, with command:
-//   tailscale.com/cmd/cloner -type User,Node,Hostinfo,NetInfo,Group,Role,Capability,Login,DNSConfig,RegisterResponse
+//   tailscale.com/cmd/cloner -type User,Node,Hostinfo,NetInfo,Login,DNSConfig,RegisterResponse
 var _DNSConfigNeedsRegeneration = DNSConfig(struct {
 	Nameservers []netaddr.IP
 	Domains     []string
@@ -269,7 +210,7 @@ func (src *RegisterResponse) Clone() *RegisterResponse {
 }
 
 // A compilation failure here means this code must be regenerated, with command:
-//   tailscale.com/cmd/cloner -type User,Node,Hostinfo,NetInfo,Group,Role,Capability,Login,DNSConfig,RegisterResponse
+//   tailscale.com/cmd/cloner -type User,Node,Hostinfo,NetInfo,Login,DNSConfig,RegisterResponse
 var _RegisterResponseNeedsRegeneration = RegisterResponse(struct {
 	User              User
 	Login             Login
@@ -280,7 +221,7 @@ var _RegisterResponseNeedsRegeneration = RegisterResponse(struct {
 
 // Clone duplicates src into dst and reports whether it succeeded.
 // To succeed, <src, dst> must be of types <*T, *T> or <*T, **T>,
-// where T is one of User,Node,Hostinfo,NetInfo,Group,Role,Capability,Login,DNSConfig,RegisterResponse.
+// where T is one of User,Node,Hostinfo,NetInfo,Login,DNSConfig,RegisterResponse.
 func Clone(dst, src interface{}) bool {
 	switch src := src.(type) {
 	case *User:
@@ -319,33 +260,6 @@ func Clone(dst, src interface{}) bool {
 			*dst = src.Clone()
 			return true
 		}
-	case *Group:
-		switch dst := dst.(type) {
-		case *Group:
-			*dst = *src.Clone()
-			return true
-		case **Group:
-			*dst = src.Clone()
-			return true
-		}
-	case *Role:
-		switch dst := dst.(type) {
-		case *Role:
-			*dst = *src.Clone()
-			return true
-		case **Role:
-			*dst = src.Clone()
-			return true
-		}
-	case *Capability:
-		switch dst := dst.(type) {
-		case *Capability:
-			*dst = *src.Clone()
-			return true
-		case **Capability:
-			*dst = src.Clone()
-			return true
-		}
 	case *Login:
 		switch dst := dst.(type) {
 		case *Login:

tailcfg/tailcfg_test.go

@@ -6,6 +6,7 @@ package tailcfg
 
 import (
 	"encoding"
+	"encoding/json"
 	"reflect"
 	"strings"
 	"testing"
@@ -164,13 +165,13 @@ func TestHostinfoEqual(t *testing.T) {
 		},
 
 		{
-			&Hostinfo{Services: []Service{Service{Proto: TCP, Port: 1234, Description: "foo"}}},
-			&Hostinfo{Services: []Service{Service{Proto: UDP, Port: 2345, Description: "bar"}}},
+			&Hostinfo{Services: []Service{{Proto: TCP, Port: 1234, Description: "foo"}}},
+			&Hostinfo{Services: []Service{{Proto: UDP, Port: 2345, Description: "bar"}}},
 			false,
 		},
 		{
-			&Hostinfo{Services: []Service{Service{Proto: TCP, Port: 1234, Description: "foo"}}},
-			&Hostinfo{Services: []Service{Service{Proto: TCP, Port: 1234, Description: "foo"}}},
+			&Hostinfo{Services: []Service{{Proto: TCP, Port: 1234, Description: "foo"}}},
+			&Hostinfo{Services: []Service{{Proto: TCP, Port: 1234, Description: "foo"}}},
 			true,
 		},
 		{
@@ -193,6 +194,7 @@ func TestNodeEqual(t *testing.T) {
 		"Key", "KeyExpiry", "Machine", "DiscoKey",
 		"Addresses", "AllowedIPs", "Endpoints", "DERP", "Hostinfo",
 		"Created", "LastSeen", "KeepAlive", "MachineAuthorized",
+		"Capabilities",
 		"ComputedName", "computedHostIfDifferent", "ComputedNameWithHost",
 	}
 	if have := fieldsOf(reflect.TypeOf(Node{})); !reflect.DeepEqual(have, nodeHandles) {
@@ -476,3 +478,25 @@ func TestCloneNode(t *testing.T) {
 		})
 	}
 }
+
+func TestUserProfileJSONMarshalForMac(t *testing.T) {
+	// Old macOS clients had a bug where they required
+	// UserProfile.Roles to be non-null. Lock that in
+	// 1.0.x/1.2.x clients are gone in the wild.
+	// See mac commit 0242c08a2ca496958027db1208f44251bff8488b (Sep 30).
+	// It was fixed in at least 1.4.x, and perhaps 1.2.x.
+	j, err := json.Marshal(UserProfile{})
+	if err != nil {
+		t.Fatal(err)
+	}
+	const wantSub = `"Roles":[]`
+	if !strings.Contains(string(j), wantSub) {
+		t.Fatalf("didn't contain %#q; got: %s", wantSub, j)
+	}
+
+	// And back:
+	var up UserProfile
+	if err := json.Unmarshal(j, &up); err != nil {
+		t.Fatalf("Unmarshal: %v", err)
+	}
+}

types/preftype/netfiltermode.go

@@ -10,10 +10,12 @@ package preftype
 // programming the Linux network stack.
 type NetfilterMode int
 
+// These numbers are persisted to disk in JSON files and thus can't be
+// renumbered or repurposed.
 const (
-	NetfilterOff      NetfilterMode = iota // remove all tailscale netfilter state
-	NetfilterNoDivert                      // manage tailscale chains, but don't call them
-	NetfilterOn                            // manage tailscale chains and call them from main chains
+	NetfilterOff      NetfilterMode = 0 // remove all tailscale netfilter state
+	NetfilterNoDivert NetfilterMode = 1 // manage tailscale chains, but don't call them
+	NetfilterOn       NetfilterMode = 2 // manage tailscale chains and call them from main chains
 )
 
 func (m NetfilterMode) String() string {

util/dnsname/dnsname.go

@@ -124,3 +124,12 @@ func SanitizeHostname(hostname string) string {
 	hostname = TrimCommonSuffixes(hostname)
 	return SanitizeLabel(hostname)
 }
+
+// NumLabels returns the number of DNS labels in hostname.
+// If hostname is empty or the top-level name ".", returns 0.
+func NumLabels(hostname string) int {
+	if hostname == "" || hostname == "." {
+		return 0
+	}
+	return strings.Count(hostname, ".")
+}

version/distro/distro.go

@@ -40,7 +40,7 @@ func haveDir(file string) bool {
 
 func linuxDistro() Distro {
 	switch {
-	case haveDir("usr/syno"):
+	case haveDir("/usr/syno"):
 		return Synology
 	case have("/etc/debian_version"):
 		return Debian

wgengine/magicsock/magicsock.go

@@ -12,7 +12,6 @@ import (
 	crand "crypto/rand"
 	"encoding/binary"
 	"errors"
-	"expvar"
 	"fmt"
 	"hash/fnv"
 	"math"
@@ -25,7 +24,6 @@ import (
 	"strings"
 	"sync"
 	"sync/atomic"
-	"syscall"
 	"time"
 
 	"github.com/tailscale/wireguard-go/conn"
@@ -53,7 +51,6 @@ import (
 	"tailscale.com/types/logger"
 	"tailscale.com/types/netmap"
 	"tailscale.com/types/nettype"
-	"tailscale.com/types/pad32"
 	"tailscale.com/types/wgkey"
 	"tailscale.com/version"
 	"tailscale.com/wgengine/monitor"
@@ -161,17 +158,14 @@ type Conn struct {
 	// Its Loaded value is always non-nil.
 	stunReceiveFunc atomic.Value // of func(p []byte, fromAddr *net.UDPAddr)
 
-	// derpRecvCh is used by ReceiveIPv4 to read DERP messages.
+	// derpRecvCh is used by receiveDERP to read DERP messages.
 	derpRecvCh chan derpReadResult
 
-	_ pad32.Four
-	// derpRecvCountAtomic is how many derpRecvCh sends are pending.
-	// It's incremented by runDerpReader whenever a DERP message
-	// arrives and decremented when they're read.
-	derpRecvCountAtomic int64
+	// bind is the wireguard-go conn.Bind for Conn.
+	bind *connBind
 
-	// ippEndpoint4 and ippEndpoint6 are owned by ReceiveIPv4 and
-	// ReceiveIPv6, respectively, to cache an IPPort->endpoint for
+	// ippEndpoint4 and ippEndpoint6 are owned by receiveIPv4 and
+	// receiveIPv6, respectively, to cache an IPPort->endpoint for
 	// hot flows.
 	ippEndpoint4, ippEndpoint6 ippEndpointCache
 
@@ -467,6 +461,7 @@ func newConn() *Conn {
 		sharedDiscoKey:  make(map[tailcfg.DiscoKey]*[32]byte),
 		discoOfAddr:     make(map[netaddr.IPPort]tailcfg.DiscoKey),
 	}
+	c.bind = &connBind{Conn: c, closed: true}
 	c.muCond = sync.NewCond(&c.mu)
 	c.networkUp.Set(true) // assume up until told otherwise
 	return c
@@ -1499,9 +1494,12 @@ func (c *Conn) runDerpReader(ctx context.Context, derpFakeAddr netaddr.IPPort, d
 			continue
 		}
 
-		if !c.sendDerpReadResult(ctx, res) {
+		select {
+		case <-ctx.Done():
 			return
+		case c.derpRecvCh <- res:
 		}
+
 		select {
 		case <-ctx.Done():
 			return
@@ -1511,49 +1509,6 @@ func (c *Conn) runDerpReader(ctx context.Context, derpFakeAddr netaddr.IPPort, d
 	}
 }
 
-var (
-	testCounterZeroDerpReadResultSend expvar.Int
-	testCounterZeroDerpReadResultRecv expvar.Int
-)
-
-// sendDerpReadResult sends res to c.derpRecvCh and reports whether it
-// was sent. (It reports false if ctx was done first.)
-//
-// This includes doing the whole wake-up dance to interrupt
-// ReceiveIPv4's blocking UDP read.
-func (c *Conn) sendDerpReadResult(ctx context.Context, res derpReadResult) (sent bool) {
-	// Before we wake up ReceiveIPv4 with SetReadDeadline,
-	// note that a DERP packet has arrived. ReceiveIPv4
-	// will read this field to note that its UDP read
-	// error is due to us.
-	atomic.AddInt64(&c.derpRecvCountAtomic, 1)
-	// Cancel the pconn read goroutine.
-	c.pconn4.SetReadDeadline(aLongTimeAgo)
-	select {
-	case <-ctx.Done():
-		select {
-		case <-c.donec:
-			// The whole Conn shut down. The reader of
-			// c.derpRecvCh also selects on c.donec, so it's
-			// safe to abort now.
-		case c.derpRecvCh <- (derpReadResult{}):
-			// Just this DERP reader is closing (perhaps
-			// the user is logging out, or the DERP
-			// connection is too idle for sends). Since we
-			// already incremented c.derpRecvCountAtomic,
-			// we need to send on the channel (unless the
-			// conn is going down).
-			// The receiver treats a derpReadResult zero value
-			// message as a skip.
-			testCounterZeroDerpReadResultSend.Add(1)
-
-		}
-		return false
-	case c.derpRecvCh <- res:
-		return true
-	}
-}
-
 type derpWriteRequest struct {
 	addr   netaddr.IPPort
 	pubKey key.Public
@@ -1607,10 +1562,6 @@ func (c *Conn) findEndpoint(ipp netaddr.IPPort, packet []byte) conn.Endpoint {
 	return c.findLegacyEndpointLocked(ipp, packet)
 }
 
-// aLongTimeAgo is a non-zero time, far in the past, used for
-// immediate cancellation of network operations.
-var aLongTimeAgo = time.Unix(233431200, 0)
-
 // noteRecvActivityFromEndpoint calls the c.noteRecvActivity hook if
 // e is a discovery-capable peer and this is the first receive activity
 // it's got in awhile (in last 10 seconds).
@@ -1623,10 +1574,8 @@ func (c *Conn) noteRecvActivityFromEndpoint(e conn.Endpoint) {
 	}
 }
 
-func (c *Conn) ReceiveIPv6(b []byte) (int, conn.Endpoint, error) {
-	if c.pconn6 == nil {
-		return 0, nil, syscall.EAFNOSUPPORT
-	}
+// receiveIPv6 receives a UDP IPv6 packet. It is called by wireguard-go.
+func (c *Conn) receiveIPv6(b []byte) (int, conn.Endpoint, error) {
 	for {
 		n, ipp, err := c.pconn6.ReadFromNetaddr(b)
 		if err != nil {
@@ -1638,43 +1587,16 @@ func (c *Conn) ReceiveIPv6(b []byte) (int, conn.Endpoint, error) {
 	}
 }
 
-func (c *Conn) derpPacketArrived() bool {
-	return atomic.LoadInt64(&c.derpRecvCountAtomic) > 0
-}
-
-// ReceiveIPv4 is called by wireguard-go to receive an IPv4 packet.
-// In Tailscale's case, that packet might also arrive via DERP. A DERP packet arrival
-// aborts the pconn4 read deadline to make it fail.
-func (c *Conn) ReceiveIPv4(b []byte) (n int, ep conn.Endpoint, err error) {
-	var ipp netaddr.IPPort
+// receiveIPv4 receives a UDP IPv4 packet. It is called by wireguard-go.
+func (c *Conn) receiveIPv4(b []byte) (n int, ep conn.Endpoint, err error) {
 	for {
-		// Drain DERP queues before reading new UDP packets.
-		if c.derpPacketArrived() {
-			goto ReadDERP
-		}
-		n, ipp, err = c.pconn4.ReadFromNetaddr(b)
+		n, ipp, err := c.pconn4.ReadFromNetaddr(b)
 		if err != nil {
-			// If the pconn4 read failed, the likely reason is a DERP reader received
-			// a packet and interrupted us.
-			// It's possible for ReadFrom to return a non deadline exceeded error
-			// and for there to have also had a DERP packet arrive, but that's fine:
-			// we'll get the same error from ReadFrom later.
-			if c.derpPacketArrived() {
-				goto ReadDERP
-			}
 			return 0, nil, err
 		}
 		if ep, ok := c.receiveIP(b[:n], ipp, &c.ippEndpoint4); ok {
 			return n, ep, nil
-		} else {
-			continue
 		}
-	ReadDERP:
-		n, ep, err = c.receiveIPv4DERP(b)
-		if err == errLoopAgain {
-			continue
-		}
-		return n, ep, err
 	}
 }
 
@@ -1692,9 +1614,8 @@ func (c *Conn) receiveIP(b []byte, ipp netaddr.IPPort, cache *ippEndpointCache)
 	}
 	if !c.havePrivateKey.Get() {
 		// If we have no private key, we're logged out or
-		// stopped.  Don't try to pass these wireguard packets
-		// up to wireguard-go; it'll just complain (Issue
-		// 1167).
+		// stopped. Don't try to pass these wireguard packets
+		// up to wireguard-go; it'll just complain (issue 1167).
 		return nil, false
 	}
 	if cache.ipp == ipp && cache.de != nil && cache.gen == cache.de.numStopAndReset() {
@@ -1714,50 +1635,42 @@ func (c *Conn) receiveIP(b []byte, ipp netaddr.IPPort, cache *ippEndpointCache)
 	return ep, true
 }
 
-var errLoopAgain = errors.New("received packet was not a wireguard-go packet or no endpoint found")
-
-// receiveIPv4DERP reads a packet from c.derpRecvCh into b and returns the associated endpoint.
+// receiveDERP reads a packet from c.derpRecvCh into b and returns the associated endpoint.
+// It is called by wireguard-go.
 //
 // If the packet was a disco message or the peer endpoint wasn't
 // found, the returned error is errLoopAgain.
-func (c *Conn) receiveIPv4DERP(b []byte) (n int, ep conn.Endpoint, err error) {
-	var dm derpReadResult
-	select {
-	case <-c.donec:
-		// Socket has been shut down. All the producers of packets
-		// respond to the context cancellation and go away, so we have
-		// to also unblock and return an error, to inform wireguard-go
-		// that this socket has gone away.
-		//
-		// Specifically, wireguard-go depends on its bind.Conn having
-		// the standard socket behavior, which is that a Close()
-		// unblocks any concurrent Read()s. wireguard-go itself calls
-		// Close() on magicsock, and expects ReceiveIPv4 to unblock
-		// with an error so it can clean up.
-		return 0, nil, errors.New("socket closed")
-	case dm = <-c.derpRecvCh:
-		// Below.
-	}
-	if atomic.AddInt64(&c.derpRecvCountAtomic, -1) == 0 {
-		c.pconn4.SetReadDeadline(time.Time{})
-	}
-	if dm.copyBuf == nil {
-		testCounterZeroDerpReadResultRecv.Add(1)
-		return 0, nil, errLoopAgain
+func (c *connBind) receiveDERP(b []byte) (n int, ep conn.Endpoint, err error) {
+	for dm := range c.derpRecvCh {
+		if c.Closed() {
+			break
+		}
+		n, ep := c.processDERPReadResult(dm, b)
+		if n == 0 {
+			// No data read occurred. Wait for another packet.
+			continue
+		}
+		return n, ep, nil
 	}
+	return 0, nil, net.ErrClosed
+}
 
+func (c *Conn) processDERPReadResult(dm derpReadResult, b []byte) (n int, ep conn.Endpoint) {
+	if dm.copyBuf == nil {
+		return 0, nil
+	}
 	var regionID int
 	n, regionID = dm.n, dm.regionID
 	ncopy := dm.copyBuf(b)
 	if ncopy != n {
-		err = fmt.Errorf("received DERP packet of length %d that's too big for WireGuard ReceiveIPv4 buf size %d", n, ncopy)
+		err := fmt.Errorf("received DERP packet of length %d that's too big for WireGuard buf size %d", n, ncopy)
 		c.logf("magicsock: %v", err)
-		return 0, nil, err
+		return 0, nil
 	}
 
 	ipp := netaddr.IPPort{IP: derpMagicIPAddr, Port: uint16(regionID)}
 	if c.handleDiscoMessage(b[:n], ipp) {
-		return 0, nil, errLoopAgain
+		return 0, nil
 	}
 
 	var (
@@ -1799,14 +1712,14 @@ func (c *Conn) receiveIPv4DERP(b []byte) (n int, ep conn.Endpoint, err error) {
 		c.logf("magicsock: DERP packet from unknown key: %s", key.ShortString())
 		ep = c.findEndpoint(ipp, b[:n])
 		if ep == nil {
-			return 0, nil, errLoopAgain
+			return 0, nil
 		}
 	}
 
 	if !didNoteRecvActivity {
 		c.noteRecvActivityFromEndpoint(ep)
 	}
-	return n, ep, nil
+	return n, ep
 }
 
 // discoLogLevel controls the verbosity of discovery log messages.
@@ -2468,8 +2381,72 @@ func (c *Conn) DERPs() int {
 	return len(c.activeDerp)
 }
 
-func (c *Conn) SetMark(value uint32) error { return nil }
-func (c *Conn) LastMark() uint32           { return 0 }
+// Bind returns the wireguard-go conn.Bind for c.
+func (c *Conn) Bind() conn.Bind {
+	return c.bind
+}
+
+// connBind is a wireguard-go conn.Bind for a Conn.
+// It bridges the behavior of wireguard-go and a Conn.
+// wireguard-go calls Close then Open on device.Up.
+// That won't work well for a Conn, which is only closed on shutdown.
+// The subsequent Close is a real close.
+type connBind struct {
+	*Conn
+	mu     sync.Mutex
+	closed bool
+}
+
+// Open is called by WireGuard to create a UDP binding.
+// The ignoredPort comes from wireguard-go, via the wgcfg config.
+// We ignore that port value here, since we have the local port available easily.
+func (c *connBind) Open(ignoredPort uint16) ([]conn.ReceiveFunc, uint16, error) {
+	c.mu.Lock()
+	defer c.mu.Unlock()
+	if !c.closed {
+		return nil, 0, errors.New("magicsock: connBind already open")
+	}
+	c.closed = false
+	fns := []conn.ReceiveFunc{c.receiveIPv4, c.receiveDERP}
+	if c.pconn6 != nil {
+		fns = append(fns, c.receiveIPv6)
+	}
+	// TODO: Combine receiveIPv4 and receiveIPv6 and receiveIP into a single
+	// closure that closes over a *RebindingUDPConn?
+	return fns, c.LocalPort(), nil
+}
+
+// SetMark is used by wireguard-go to set a mark bit for packets to avoid routing loops.
+// We handle that ourselves elsewhere.
+func (c *connBind) SetMark(value uint32) error {
+	return nil
+}
+
+// Close closes the connBind, unless it is already closed.
+func (c *connBind) Close() error {
+	c.mu.Lock()
+	defer c.mu.Unlock()
+	if c.closed {
+		return nil
+	}
+	c.closed = true
+	// Unblock all outstanding receives.
+	c.pconn4.Close()
+	if c.pconn6 != nil {
+		c.pconn6.Close()
+	}
+	// Send an empty read result to unblock receiveDERP,
+	// which will then check connBind.Closed.
+	c.derpRecvCh <- derpReadResult{}
+	return nil
+}
+
+// Closed reports whether c is closed.
+func (c *connBind) Closed() bool {
+	c.mu.Lock()
+	defer c.mu.Unlock()
+	return c.closed
+}
 
 // Close closes the connection.
 //
@@ -2493,10 +2470,12 @@ func (c *Conn) Close() error {
 	c.closed = true
 	c.connCtxCancel()
 	c.closeAllDerpLocked("conn-close")
+	// Ignore errors from c.pconnN.Close.
+	// They will frequently have been closed already by a call to connBind.Close.
 	if c.pconn6 != nil {
 		c.pconn6.Close()
 	}
-	err := c.pconn4.Close()
+	c.pconn4.Close()
 
 	// Wait on goroutines updating right at the end, once everything is
 	// already closed. We want everything else in the Conn to be
@@ -2505,7 +2484,7 @@ func (c *Conn) Close() error {
 	for c.goroutinesRunningLocked() {
 		c.muCond.Wait()
 	}
-	return err
+	return nil
 }
 
 func (c *Conn) goroutinesRunningLocked() bool {
@@ -2731,27 +2710,27 @@ func packIPPort(ua netaddr.IPPort) []byte {
 	return b
 }
 
-// CreateBind is called by WireGuard to create a UDP binding.
-func (c *Conn) CreateBind(uint16) (conn.Bind, uint16, error) {
-	return c, c.LocalPort(), nil
-}
-
-// CreateEndpoint is called by WireGuard to connect to an endpoint.
+// ParseEndpoint is called by WireGuard to connect to an endpoint.
 //
-// The key is the public key of the peer and addrs is either:
+// keyAddrs is the 32 byte public key of the peer followed by addrs.
+// Addrs is either:
 //
 //  1) a comma-separated list of UDP ip:ports (the peer doesn't have a discovery key)
 //  2) "<hex-discovery-key>.disco.tailscale:12345", a magic value that means the peer
 //     is running code that supports active discovery, so CreateEndpoint returns
 //     a discoEndpoint.
-//
-
-func (c *Conn) CreateEndpoint(pubKey [32]byte, addrs string) (conn.Endpoint, error) {
+func (c *Conn) ParseEndpoint(keyAddrs string) (conn.Endpoint, error) {
+	if len(keyAddrs) < 32 {
+		c.logf("[unexpected] ParseEndpoint keyAddrs too short: %q", keyAddrs)
+		return nil, errors.New("endpoint string too short")
+	}
+	var pk key.Public
+	copy(pk[:], keyAddrs)
+	addrs := keyAddrs[len(pk):]
 	c.mu.Lock()
 	defer c.mu.Unlock()
 
-	pk := key.Public(pubKey)
-	c.logf("magicsock: CreateEndpoint: key=%s: %s", pk.ShortString(), derpStr(addrs))
+	c.logf("magicsock: ParseEndpoint: key=%s: %s", pk.ShortString(), derpStr(addrs))
 
 	if !strings.HasSuffix(addrs, wgcfg.EndpointDiscoSuffix) {
 		return c.createLegacyEndpointLocked(pk, addrs)
@@ -2784,6 +2763,13 @@ type RebindingUDPConn struct {
 	pconn net.PacketConn
 }
 
+// currentConn returns c's current pconn and whether it is (fake) closed.
+func (c *RebindingUDPConn) currentConn() (pconn net.PacketConn) {
+	c.mu.Lock()
+	defer c.mu.Unlock()
+	return c.pconn
+}
+
 func (c *RebindingUDPConn) Reset(pconn net.PacketConn) {
 	c.mu.Lock()
 	old := c.pconn
@@ -2799,19 +2785,10 @@ func (c *RebindingUDPConn) Reset(pconn net.PacketConn) {
 // It returns the number of bytes copied and the source address.
 func (c *RebindingUDPConn) ReadFrom(b []byte) (int, net.Addr, error) {
 	for {
-		c.mu.Lock()
-		pconn := c.pconn
-		c.mu.Unlock()
-
+		pconn := c.currentConn()
 		n, addr, err := pconn.ReadFrom(b)
-		if err != nil {
-			c.mu.Lock()
-			pconn2 := c.pconn
-			c.mu.Unlock()
-
-			if pconn != pconn2 {
-				continue
-			}
+		if err != nil && pconn != c.currentConn() {
+			continue
 		}
 		return n, addr, err
 	}
@@ -2826,9 +2803,7 @@ func (c *RebindingUDPConn) ReadFrom(b []byte) (int, net.Addr, error) {
 // when c's underlying connection is a net.UDPConn.
 func (c *RebindingUDPConn) ReadFromNetaddr(b []byte) (n int, ipp netaddr.IPPort, err error) {
 	for {
-		c.mu.Lock()
-		pconn := c.pconn
-		c.mu.Unlock()
+		pconn := c.currentConn()
 
 		// Optimization: Treat *net.UDPConn specially.
 		// ReadFromUDP gets partially inlined, avoiding allocating a *net.UDPAddr,
@@ -2849,11 +2824,7 @@ func (c *RebindingUDPConn) ReadFromNetaddr(b []byte) (n int, ipp netaddr.IPPort,
 		}
 
 		if err != nil {
-			c.mu.Lock()
-			pconn2 := c.pconn
-			c.mu.Unlock()
-
-			if pconn != pconn2 {
+			if pconn != c.currentConn() {
 				continue
 			}
 		} else {
@@ -2885,12 +2856,6 @@ func (c *RebindingUDPConn) Close() error {
 	return c.pconn.Close()
 }
 
-func (c *RebindingUDPConn) SetReadDeadline(t time.Time) {
-	c.mu.Lock()
-	defer c.mu.Unlock()
-	c.pconn.SetReadDeadline(t)
-}
-
 func (c *RebindingUDPConn) WriteToUDP(b []byte, addr *net.UDPAddr) (int, error) {
 	for {
 		c.mu.Lock()

wgengine/magicsock/magicsock_test.go

@@ -22,7 +22,6 @@ import (
 	"strconv"
 	"strings"
 	"sync"
-	"sync/atomic"
 	"testing"
 	"time"
 	"unsafe"
@@ -93,7 +92,7 @@ func runDERPAndStun(t *testing.T, logf logger.Logf, l nettype.PacketListener, st
 
 	m := &tailcfg.DERPMap{
 		Regions: map[int]*tailcfg.DERPRegion{
-			1: &tailcfg.DERPRegion{
+			1: {
 				RegionID:   1,
 				RegionCode: "test",
 				Nodes: []*tailcfg.DERPNode{
@@ -170,12 +169,7 @@ func newMagicStack(t testing.TB, logf logger.Logf, l nettype.PacketListener, der
 	tsTun.SetFilter(filter.NewAllowAllForTest(logf))
 
 	wgLogger := wglog.NewLogger(logf)
-	opts := &device.DeviceOptions{
-		CreateEndpoint: conn.CreateEndpoint,
-		CreateBind:     conn.CreateBind,
-		SkipBindUpdate: true,
-	}
-	dev := device.NewDevice(tsTun, wgLogger.DeviceLogger, opts)
+	dev := device.NewDevice(tsTun, conn.Bind(), wgLogger.DeviceLogger, new(device.DeviceOptions))
 	dev.Up()
 
 	// Wait for magicsock to connect up to DERP.
@@ -228,15 +222,14 @@ func (s *magicStack) Status() *ipnstate.Status {
 // Something external needs to provide a NetworkMap and WireGuard
 // configs to the magicStack in order for it to acquire an IP
 // address. See meshStacks for one possible source of netmaps and IPs.
-func (s *magicStack) IP(t *testing.T) netaddr.IP {
+func (s *magicStack) IP() netaddr.IP {
 	for deadline := time.Now().Add(5 * time.Second); time.Now().Before(deadline); time.Sleep(10 * time.Millisecond) {
 		st := s.Status()
 		if len(st.TailscaleIPs) > 0 {
 			return st.TailscaleIPs[0]
 		}
 	}
-	t.Fatal("timed out waiting for magicstack to get an IP assigned")
-	panic("unreachable") // compiler doesn't know t.Fatal panics
+	panic("timed out waiting for magicstack to get an IP assigned")
 }
 
 // meshStacks monitors epCh on all given ms, and plumbs network maps
@@ -365,7 +358,7 @@ func TestNewConn(t *testing.T) {
 	go func() {
 		var pkt [64 << 10]byte
 		for {
-			_, _, err := conn.ReceiveIPv4(pkt[:])
+			_, _, err := conn.receiveIPv4(pkt[:])
 			if err != nil {
 				return
 			}
@@ -441,7 +434,7 @@ func TestPickDERPFallback(t *testing.T) {
 	// But move if peers are elsewhere.
 	const otherNode = 789
 	c.addrsByKey = map[key.Public]*addrSet{
-		key.Public{1}: &addrSet{ipPorts: []netaddr.IPPort{{IP: derpMagicIPAddr, Port: otherNode}}},
+		{1}: {ipPorts: []netaddr.IPPort{{IP: derpMagicIPAddr, Port: otherNode}}},
 	}
 	if got := c.pickDERPFallback(); got != otherNode {
 		t.Errorf("didn't join peers: got %v; want %v", got, someNode)
@@ -467,12 +460,11 @@ func makeConfigs(t *testing.T, addrs []netaddr.IPPort) []wgcfg.Config {
 	}
 
 	var cfgs []wgcfg.Config
-	for i, addr := range addrs {
+	for i := range addrs {
 		cfg := wgcfg.Config{
 			Name:       fmt.Sprintf("peer%d", i+1),
 			PrivateKey: privKeys[i],
 			Addresses:  addresses[i],
-			ListenPort: addr.Port,
 		}
 		for peerNum, addr := range addrs {
 			if peerNum == i {
@@ -523,12 +515,7 @@ func TestDeviceStartStop(t *testing.T) {
 
 	tun := tuntest.NewChannelTUN()
 	wgLogger := wglog.NewLogger(t.Logf)
-	opts := &device.DeviceOptions{
-		CreateEndpoint: conn.CreateEndpoint,
-		CreateBind:     conn.CreateBind,
-		SkipBindUpdate: true,
-	}
-	dev := device.NewDevice(tun.TUN(), wgLogger.DeviceLogger, opts)
+	dev := device.NewDevice(tun.TUN(), conn.Bind(), wgLogger.DeviceLogger, new(device.DeviceOptions))
 	dev.Up()
 	dev.Close()
 }
@@ -566,7 +553,7 @@ func TestConnClosed(t *testing.T) {
 	cleanup = meshStacks(t.Logf, []*magicStack{ms1, ms2})
 	defer cleanup()
 
-	pkt := tuntest.Ping(ms2.IP(t).IPAddr().IP, ms1.IP(t).IPAddr().IP)
+	pkt := tuntest.Ping(ms2.IP().IPAddr().IP, ms1.IP().IPAddr().IP)
 
 	if len(ms1.conn.activeDerp) == 0 {
 		t.Errorf("unexpected DERP empty got: %v want: >0", len(ms1.conn.activeDerp))
@@ -767,7 +754,7 @@ func newPinger(t *testing.T, logf logger.Logf, src, dst *magicStack) (cleanup fu
 		// failure). Figure out what kind of thing would be
 		// acceptable to test instead of "every ping must
 		// transit".
-		pkt := tuntest.Ping(dst.IP(t).IPAddr().IP, src.IP(t).IPAddr().IP)
+		pkt := tuntest.Ping(dst.IP().IPAddr().IP, src.IP().IPAddr().IP)
 		select {
 		case src.tun.Outbound <- pkt:
 		case <-ctx.Done():
@@ -812,7 +799,7 @@ func newPinger(t *testing.T, logf logger.Logf, src, dst *magicStack) (cleanup fu
 	}
 
 	go func() {
-		logf("sending ping stream from %s (%s) to %s (%s)", src, src.IP(t), dst, dst.IP(t))
+		logf("sending ping stream from %s (%s) to %s (%s)", src, src.IP(), dst, dst.IP())
 		defer close(done)
 		for one() {
 		}
@@ -852,8 +839,8 @@ func testActiveDiscovery(t *testing.T, d *devices) {
 	cleanup = meshStacks(logf, []*magicStack{m1, m2})
 	defer cleanup()
 
-	m1IP := m1.IP(t)
-	m2IP := m2.IP(t)
+	m1IP := m1.IP()
+	m2IP := m2.IP()
 	logf("IPs: %s %s", m1IP, m2IP)
 
 	cleanup = newPinger(t, logf, m1, m2)
@@ -1331,12 +1318,12 @@ func TestDiscoMessage(t *testing.T) {
 	peer1Pub := c.DiscoPublicKey()
 	peer1Priv := c.discoPrivate
 	c.endpointOfDisco = map[tailcfg.DiscoKey]*discoEndpoint{
-		tailcfg.DiscoKey(peer1Pub): &discoEndpoint{
+		tailcfg.DiscoKey(peer1Pub): {
 			// ... (enough for this test)
 		},
 	}
 	c.nodeOfDisco = map[tailcfg.DiscoKey]*tailcfg.Node{
-		tailcfg.DiscoKey(peer1Pub): &tailcfg.Node{
+		tailcfg.DiscoKey(peer1Pub): {
 			// ... (enough for this test)
 		},
 	}
@@ -1385,14 +1372,10 @@ func stringifyConfig(cfg wgcfg.Config) string {
 
 func Test32bitAlignment(t *testing.T) {
 	var de discoEndpoint
-	var c Conn
 
 	if off := unsafe.Offsetof(de.lastRecvUnixAtomic); off%8 != 0 {
 		t.Fatalf("discoEndpoint.lastRecvUnixAtomic is not 8-byte aligned")
 	}
-	if off := unsafe.Offsetof(c.derpRecvCountAtomic); off%8 != 0 {
-		t.Fatalf("Conn.derpRecvCountAtomic is not 8-byte aligned")
-	}
 
 	if !de.isFirstRecvActivityInAwhile() { // verify this doesn't panic on 32-bit
 		t.Error("expected true")
@@ -1400,7 +1383,6 @@ func Test32bitAlignment(t *testing.T) {
 	if de.isFirstRecvActivityInAwhile() {
 		t.Error("expected false on second call")
 	}
-	atomic.AddInt64(&c.derpRecvCountAtomic, 1)
 }
 
 // newNonLegacyTestConn returns a new Conn with DisableLegacyNetworking set true.
@@ -1421,92 +1403,6 @@ func newNonLegacyTestConn(t testing.TB) *Conn {
 	return conn
 }
 
-// Tests concurrent DERP readers pushing DERP data into ReceiveIPv4
-// (which should blend all DERP reads into UDP reads).
-func TestDerpReceiveFromIPv4(t *testing.T) {
-	conn := newNonLegacyTestConn(t)
-	defer conn.Close()
-
-	sendConn, err := net.ListenPacket("udp4", "127.0.0.1:0")
-	if err != nil {
-		t.Fatal(err)
-	}
-	defer sendConn.Close()
-	nodeKey, _ := addTestEndpoint(t, conn, sendConn)
-
-	var sends int = 250e3 // takes about a second
-	if testing.Short() {
-		sends /= 10
-	}
-	senders := runtime.NumCPU()
-	sends -= (sends % senders)
-	var wg sync.WaitGroup
-	defer wg.Wait()
-	t.Logf("doing %v sends over %d senders", sends, senders)
-
-	ctx, cancel := context.WithCancel(context.Background())
-	defer conn.Close()
-	defer cancel()
-
-	doneCtx, cancelDoneCtx := context.WithCancel(context.Background())
-	cancelDoneCtx()
-
-	for i := 0; i < senders; i++ {
-		wg.Add(1)
-		regionID := i + 1
-		go func() {
-			defer wg.Done()
-			for i := 0; i < sends/senders; i++ {
-				res := derpReadResult{
-					regionID: regionID,
-					n:        123,
-					src:      key.Public(nodeKey),
-					copyBuf:  func(dst []byte) int { return 123 },
-				}
-				// First send with the closed context. ~50% of
-				// these should end up going through the
-				// send-a-zero-derpReadResult path, returning
-				// true, in which case we don't want to send again.
-				// We test later that we hit the other path.
-				if conn.sendDerpReadResult(doneCtx, res) {
-					continue
-				}
-
-				if !conn.sendDerpReadResult(ctx, res) {
-					t.Error("unexpected false")
-					return
-				}
-			}
-		}()
-	}
-
-	zeroSendsStart := testCounterZeroDerpReadResultSend.Value()
-
-	buf := make([]byte, 1500)
-	for i := 0; i < sends; i++ {
-		n, ep, err := conn.ReceiveIPv4(buf)
-		if err != nil {
-			t.Fatal(err)
-		}
-		_ = n
-		_ = ep
-	}
-
-	t.Logf("did %d ReceiveIPv4 calls", sends)
-
-	zeroSends, zeroRecv := testCounterZeroDerpReadResultSend.Value(), testCounterZeroDerpReadResultRecv.Value()
-	if zeroSends != zeroRecv {
-		t.Errorf("did %d zero sends != %d corresponding receives", zeroSends, zeroRecv)
-	}
-	zeroSendDelta := zeroSends - zeroSendsStart
-	if zeroSendDelta == 0 {
-		t.Errorf("didn't see any sends of derpReadResult zero value")
-	}
-	if zeroSendDelta == int64(sends) {
-		t.Errorf("saw %v sends of the derpReadResult zero value which was unexpectedly high (100%% of our %v sends)", zeroSendDelta, sends)
-	}
-}
-
 // addTestEndpoint sets conn's network map to a single peer expected
 // to receive packets from sendConn (or DERP), and returns that peer's
 // nodekey and discokey.
@@ -1526,7 +1422,7 @@ func addTestEndpoint(tb testing.TB, conn *Conn, sendConn net.PacketConn) (tailcf
 		},
 	})
 	conn.SetPrivateKey(wgkey.Private{0: 1})
-	_, err := conn.CreateEndpoint([32]byte(nodeKey), "0000000000000000000000000000000000000000000000000000000000000001.disco.tailscale:12345")
+	_, err := conn.ParseEndpoint(string(nodeKey[:]) + "0000000000000000000000000000000000000000000000000000000000000001.disco.tailscale:12345")
 	if err != nil {
 		tb.Fatal(err)
 	}
@@ -1557,7 +1453,7 @@ func setUpReceiveFrom(tb testing.TB) (roundTrip func()) {
 		if _, err := sendConn.WriteTo(sendBuf, dstAddr); err != nil {
 			tb.Fatalf("WriteTo: %v", err)
 		}
-		n, ep, err := conn.ReceiveIPv4(buf)
+		n, ep, err := conn.receiveIPv4(buf)
 		if err != nil {
 			tb.Fatal(err)
 		}
@@ -1700,7 +1596,7 @@ func TestSetNetworkMapChangingNodeKey(t *testing.T) {
 			},
 		},
 	})
-	_, err := conn.CreateEndpoint([32]byte(nodeKey1), "0000000000000000000000000000000000000000000000000000000000000001.disco.tailscale:12345")
+	_, err := conn.ParseEndpoint(string(nodeKey1[:]) + "0000000000000000000000000000000000000000000000000000000000000001.disco.tailscale:12345")
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -1758,7 +1654,7 @@ func TestRebindStress(t *testing.T) {
 	go func() {
 		buf := make([]byte, 1500)
 		for {
-			_, _, err := conn.ReceiveIPv4(buf)
+			_, _, err := conn.receiveIPv4(buf)
 			if ctx.Err() != nil {
 				errc <- nil
 				return

wgengine/monitor/monitor_linux.go

@@ -7,7 +7,6 @@
 package monitor
 
 import (
-	"fmt"
 	"net"
 	"time"
 
@@ -37,7 +36,7 @@ type nlConn struct {
 	buffered []netlink.Message
 }
 
-func newOSMon(logf logger.Logf, _ *Mon) (osMon, error) {
+func newOSMon(logf logger.Logf, m *Mon) (osMon, error) {
 	conn, err := netlink.Dial(unix.NETLINK_ROUTE, &netlink.Config{
 		// Routes get us most of the events of interest, but we need
 		// address as well to cover things like DHCP deciding to give
@@ -46,7 +45,9 @@ func newOSMon(logf logger.Logf, _ *Mon) (osMon, error) {
 		Groups: unix.RTMGRP_IPV4_IFADDR | unix.RTMGRP_IPV6_IFADDR | unix.RTMGRP_IPV4_ROUTE | unix.RTMGRP_IPV6_ROUTE,
 	})
 	if err != nil {
-		return nil, fmt.Errorf("dialing netlink socket: %v", err)
+		// Google Cloud Run does not implement NETLINK_ROUTE RTMGRP support
+		logf("monitor_linux: AF_NETLINK RTMGRP failed, falling back to polling")
+		return newPollingMon(logf, m)
 	}
 	return &nlConn{logf: logf, conn: conn}, nil
 }

wgengine/monitor/monitor_polling.go

@@ -7,62 +7,11 @@
 package monitor
 
 import (
-	"errors"
-	"runtime"
-	"sync"
-	"time"
-
-	"tailscale.com/net/interfaces"
 	"tailscale.com/types/logger"
 )
 
 func newOSMon(logf logger.Logf, m *Mon) (osMon, error) {
-	return &pollingMon{
-		logf: logf,
-		m:    m,
-		stop: make(chan struct{}),
-	}, nil
-}
-
-// pollingMon is a bad but portable implementation of the link monitor
-// that works by polling the interface state every 10 seconds, in lieu
-// of anything to subscribe to. A good implementation
-type pollingMon struct {
-	logf logger.Logf
-	m    *Mon
-
-	closeOnce sync.Once
-	stop      chan struct{}
-}
-
-func (pm *pollingMon) Close() error {
-	pm.closeOnce.Do(func() {
-		close(pm.stop)
-	})
-	return nil
-}
-
-func (pm *pollingMon) Receive() (message, error) {
-	d := 10 * time.Second
-	if runtime.GOOS == "android" {
-		// We'll have Android notify the link monitor to wake up earlier,
-		// so this can go very slowly there, to save battery.
-		// https://github.com/tailscale/tailscale/issues/1427
-		d = 10 * time.Minute
-	}
-	ticker := time.NewTicker(d)
-	defer ticker.Stop()
-	base := pm.m.InterfaceState()
-	for {
-		if cur, err := pm.m.interfaceStateUncached(); err == nil && !cur.EqualFiltered(base, interfaces.FilterInteresting) {
-			return unspecifiedMessage{}, nil
-		}
-		select {
-		case <-ticker.C:
-		case <-pm.stop:
-			return nil, errors.New("stopped")
-		}
-	}
+	return newPollingMon(logf, m)
 }
 
 // unspecifiedMessage is a minimal message implementation that should not

wgengine/monitor/polling.go

@@ -0,0 +1,68 @@
+// Copyright (c) 2020 Tailscale Inc & AUTHORS All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+//  +build !freebsd,!windows,!darwin
+
+package monitor
+
+import (
+	"errors"
+	"runtime"
+	"sync"
+	"time"
+
+	"tailscale.com/net/interfaces"
+	"tailscale.com/types/logger"
+)
+
+func newPollingMon(logf logger.Logf, m *Mon) (osMon, error) {
+	return &pollingMon{
+		logf: logf,
+		m:    m,
+		stop: make(chan struct{}),
+	}, nil
+}
+
+// pollingMon is a bad but portable implementation of the link monitor
+// that works by polling the interface state every 10 seconds, in lieu
+// of anything to subscribe to.
+type pollingMon struct {
+	logf logger.Logf
+	m    *Mon
+
+	closeOnce sync.Once
+	stop      chan struct{}
+}
+
+func (pm *pollingMon) Close() error {
+	pm.closeOnce.Do(func() {
+		close(pm.stop)
+	})
+	return nil
+}
+
+func (pm *pollingMon) Receive() (message, error) {
+	d := 10 * time.Second
+	if runtime.GOOS == "android" {
+		// We'll have Android notify the link monitor to wake up earlier,
+		// so this can go very slowly there, to save battery.
+		// https://github.com/tailscale/tailscale/issues/1427
+		d = 10 * time.Minute
+	}
+	// TODO: detect if we're running in Cloud Run, and reduce frequency of
+	// polling as its routes never change.
+	ticker := time.NewTicker(d)
+	defer ticker.Stop()
+	base := pm.m.InterfaceState()
+	for {
+		if cur, err := pm.m.interfaceStateUncached(); err == nil && !cur.EqualFiltered(base, interfaces.FilterInteresting) {
+			return unspecifiedMessage{}, nil
+		}
+		select {
+		case <-ticker.C:
+		case <-pm.stop:
+			return nil, errors.New("stopped")
+		}
+	}
+}

wgengine/netstack/netstack.go

@@ -242,7 +242,6 @@ func (ns *Impl) updateIPs(nm *netmap.NetworkMap) {
 	ns.mu.Lock()
 	for ip := range ns.connsOpenBySubnetIP {
 		ipp := tcpip.Address(ip.IPAddr().IP).WithPrefix()
-		ipsToBeAdded[ipp] = true
 		delete(ipsToBeRemoved, ipp)
 	}
 	ns.mu.Unlock()

wgengine/router/callback.go

@@ -0,0 +1,54 @@
+// Copyright (c) 2021 Tailscale Inc & AUTHORS All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package router
+
+import (
+	"sync"
+
+	"tailscale.com/net/dns"
+)
+
+// CallbackRouter is an implementation of both Router and dns.OSConfigurator.
+// When either network or DNS settings are changed, SetBoth is called with both configs.
+// Mainly used as a shim for OSes that want to set both network and
+// DNS configuration simultaneously (iOS, android).
+type CallbackRouter struct {
+	SetBoth func(rcfg *Config, dcfg *dns.OSConfig) error
+	DNSMode dns.RoutingMode
+
+	mu   sync.Mutex    // protects all the following
+	rcfg *Config       // last applied router config
+	dcfg *dns.OSConfig // last applied DNS config
+}
+
+// Up implements Router.
+func (r *CallbackRouter) Up() error {
+	return nil // TODO: check that all callers have no need for initialization
+}
+
+// Set implements Router.
+func (r *CallbackRouter) Set(rcfg *Config) error {
+	r.mu.Lock()
+	defer r.mu.Unlock()
+	r.rcfg = rcfg
+	return r.SetBoth(r.rcfg, r.dcfg)
+}
+
+// SetDNS implements dns.OSConfigurator.
+func (r *CallbackRouter) SetDNS(dcfg dns.OSConfig) error {
+	r.mu.Lock()
+	defer r.mu.Unlock()
+	r.dcfg = &dcfg
+	return r.SetBoth(r.rcfg, r.dcfg)
+}
+
+// RoutingMode implements dns.OSConfigurator.
+func (r *CallbackRouter) RoutingMode() dns.RoutingMode {
+	return r.DNSMode
+}
+
+func (r *CallbackRouter) Close() error {
+	return r.SetBoth(nil, nil) // TODO: check if makes sense
+}

wgengine/router/router.go

@@ -9,7 +9,6 @@ package router
 import (
 	"github.com/tailscale/wireguard-go/tun"
 	"inet.af/netaddr"
-	"tailscale.com/net/dns"
 	"tailscale.com/types/logger"
 	"tailscale.com/types/preftype"
 )
@@ -41,28 +40,24 @@ func New(logf logger.Logf, tundev tun.Device) (Router, error) {
 // in case the Tailscale daemon terminated without closing the router.
 // No other state needs to be instantiated before this runs.
 func Cleanup(logf logger.Logf, interfaceName string) {
-	mconfig := dns.ManagerConfig{
-		Logf:          logf,
-		InterfaceName: interfaceName,
-		Cleanup:       true,
-	}
-	dns := dns.NewManager(mconfig)
-	if err := dns.Down(); err != nil {
-		logf("dns down: %v", err)
-	}
 	cleanup(logf, interfaceName)
 }
 
 // Config is the subset of Tailscale configuration that is relevant to
 // the OS's network stack.
 type Config struct {
+	// LocalAddrs are the address(es) for this node. This is
+	// typically one IPv4/32 (the 100.x.y.z CGNAT) and one
+	// IPv6/128 (Tailscale ULA).
 	LocalAddrs []netaddr.IPPrefix
-	Routes     []netaddr.IPPrefix // routes to point into the Tailscale interface
 
-	DNS dns.Config
+	// Routes are the routes that point in to the Tailscale
+	// interface.  These are the /32 and /128 routes to peers, as
+	// well as any other subnets that peers are advertising and
+	// this node has chosen to use.
+	Routes []netaddr.IPPrefix
 
 	// Linux-only things below, ignored on other platforms.
-
 	SubnetRoutes     []netaddr.IPPrefix     // subnets being advertised to other Tailscale nodes
 	SNATSubnetRoutes bool                   // SNAT traffic to local subnets
 	NetfilterMode    preftype.NetfilterMode // how much to manage netfilter rules

wgengine/router/router_linux.go

@@ -18,7 +18,6 @@ import (
 	"github.com/go-multierror/multierror"
 	"github.com/tailscale/wireguard-go/tun"
 	"inet.af/netaddr"
-	"tailscale.com/net/dns"
 	"tailscale.com/net/tsaddr"
 	"tailscale.com/types/logger"
 	"tailscale.com/types/preftype"
@@ -102,8 +101,6 @@ type linuxRouter struct {
 	v6Available     bool
 	v6NATAvailable  bool
 
-	dns *dns.Manager
-
 	ipt4 netfilterRunner
 	ipt6 netfilterRunner
 	cmd  commandRunner
@@ -146,11 +143,6 @@ func newUserspaceRouter(logf logger.Logf, tunDev tun.Device) (Router, error) {
 func newUserspaceRouterAdvanced(logf logger.Logf, tunname string, netfilter4, netfilter6 netfilterRunner, cmd commandRunner, supportsV6, supportsV6NAT bool) (Router, error) {
 	ipRuleAvailable := (cmd.run("ip", "rule") == nil)
 
-	mconfig := dns.ManagerConfig{
-		Logf:          logf,
-		InterfaceName: tunname,
-	}
-
 	return &linuxRouter{
 		logf:          logf,
 		tunname:       tunname,
@@ -163,7 +155,6 @@ func newUserspaceRouterAdvanced(logf logger.Logf, tunname string, netfilter4, ne
 		ipt4: netfilter4,
 		ipt6: netfilter6,
 		cmd:  cmd,
-		dns:  dns.NewManager(mconfig),
 	}, nil
 }
 
@@ -185,9 +176,6 @@ func (r *linuxRouter) Up() error {
 }
 
 func (r *linuxRouter) Close() error {
-	if err := r.dns.Down(); err != nil {
-		return fmt.Errorf("dns down: %w", err)
-	}
 	if err := r.downInterface(); err != nil {
 		return err
 	}
@@ -211,10 +199,6 @@ func (r *linuxRouter) Set(cfg *Config) error {
 		cfg = &shutdownConfig
 	}
 
-	if err := r.dns.Set(cfg.DNS); err != nil {
-		errs = append(errs, fmt.Errorf("dns set: %w", err))
-	}
-
 	if err := r.setNetfilterMode(cfg.NetfilterMode); err != nil {
 		errs = append(errs, err)
 	}

wgengine/router/router_openbsd.go

@@ -12,7 +12,6 @@ import (
 
 	"github.com/tailscale/wireguard-go/tun"
 	"inet.af/netaddr"
-	"tailscale.com/net/dns"
 	"tailscale.com/types/logger"
 )
 
@@ -26,8 +25,6 @@ type openbsdRouter struct {
 	local4  netaddr.IPPrefix
 	local6  netaddr.IPPrefix
 	routes  map[netaddr.IPPrefix]struct{}
-
-	dns *dns.Manager
 }
 
 func newUserspaceRouter(logf logger.Logf, tundev tun.Device) (Router, error) {
@@ -36,15 +33,9 @@ func newUserspaceRouter(logf logger.Logf, tundev tun.Device) (Router, error) {
 		return nil, err
 	}
 
-	mconfig := dns.ManagerConfig{
-		Logf:          logf,
-		InterfaceName: tunname,
-	}
-
 	return &openbsdRouter{
 		logf:    logf,
 		tunname: tunname,
-		dns:     dns.NewManager(mconfig),
 	}, nil
 }
 
@@ -215,17 +206,10 @@ func (r *openbsdRouter) Set(cfg *Config) error {
 	r.local6 = localAddr6
 	r.routes = newRoutes
 
-	if err := r.dns.Set(cfg.DNS); err != nil {
-		errq = fmt.Errorf("dns set: %v", err)
-	}
-
 	return errq
 }
 
 func (r *openbsdRouter) Close() error {
-	if err := r.dns.Down(); err != nil {
-		return fmt.Errorf("dns down: %v", err)
-	}
 	cleanup(r.logf, r.tunname)
 	return nil
 }

wgengine/router/router_userspace_bsd.go

@@ -14,7 +14,6 @@ import (
 
 	"github.com/tailscale/wireguard-go/tun"
 	"inet.af/netaddr"
-	"tailscale.com/net/dns"
 	"tailscale.com/types/logger"
 	"tailscale.com/version"
 )
@@ -24,8 +23,6 @@ type userspaceBSDRouter struct {
 	tunname string
 	local   []netaddr.IPPrefix
 	routes  map[netaddr.IPPrefix]struct{}
-
-	dns *dns.Manager
 }
 
 func newUserspaceBSDRouter(logf logger.Logf, tundev tun.Device) (Router, error) {
@@ -34,15 +31,9 @@ func newUserspaceBSDRouter(logf logger.Logf, tundev tun.Device) (Router, error)
 		return nil, err
 	}
 
-	mconfig := dns.ManagerConfig{
-		Logf:          logf,
-		InterfaceName: tunname,
-	}
-
 	return &userspaceBSDRouter{
 		logf:    logf,
 		tunname: tunname,
-		dns:     dns.NewManager(mconfig),
 	}, nil
 }
 
@@ -188,18 +179,9 @@ func (r *userspaceBSDRouter) Set(cfg *Config) (reterr error) {
 	}
 	r.routes = newRoutes
 
-	if err := r.dns.Set(cfg.DNS); err != nil {
-		r.logf("DNS set: %v", err)
-		setErr(err)
-	}
-
 	return errq
 }
 
 func (r *userspaceBSDRouter) Close() error {
-	if err := r.dns.Down(); err != nil {
-		r.logf("dns down: %v", err)
-	}
-	// No interface cleanup is necessary during normal shutdown.
 	return nil
 }

wgengine/router/router_windows.go

@@ -27,10 +27,8 @@ import (
 
 type winRouter struct {
 	logf                func(fmt string, args ...interface{})
-	tunname             string
 	nativeTun           *tun.NativeTun
 	routeChangeCallback *winipcfg.RouteChangeCallback
-	dns                 *dns.Manager
 	firewall            *firewallTweaker
 
 	// firewallSubproc is a subprocess that runs a tweaked version of
@@ -44,11 +42,6 @@ type winRouter struct {
 }
 
 func newUserspaceRouter(logf logger.Logf, tundev tun.Device) (Router, error) {
-	tunname, err := tundev.Name()
-	if err != nil {
-		return nil, err
-	}
-
 	nativeTun := tundev.(*tun.NativeTun)
 	luid := winipcfg.LUID(nativeTun.LUID())
 	guid, err := luid.GUID()
@@ -56,16 +49,9 @@ func newUserspaceRouter(logf logger.Logf, tundev tun.Device) (Router, error) {
 		return nil, err
 	}
 
-	mconfig := dns.ManagerConfig{
-		Logf:          logf,
-		InterfaceName: guid.String(),
-	}
-
 	return &winRouter{
 		logf:      logf,
-		tunname:   tunname,
 		nativeTun: nativeTun,
-		dns:       dns.NewManager(mconfig),
 		firewall: &firewallTweaker{
 			logf:    logger.WithPrefix(logf, "firewall: "),
 			tunGUID: *guid,
@@ -104,10 +90,6 @@ func (r *winRouter) Set(cfg *Config) error {
 		return err
 	}
 
-	if err := r.dns.Set(cfg.DNS); err != nil {
-		return fmt.Errorf("dns set: %w", err)
-	}
-
 	// Flush DNS on router config change to clear cached DNS entries (solves #1430)
 	if err := dns.Flush(); err != nil {
 		r.logf("flushdns error: %v", err)
@@ -128,9 +110,6 @@ func hasDefaultRoute(routes []netaddr.IPPrefix) bool {
 func (r *winRouter) Close() error {
 	r.firewall.clear()
 
-	if err := r.dns.Down(); err != nil {
-		return fmt.Errorf("dns down: %w", err)
-	}
 	if r.routeChangeCallback != nil {
 		r.routeChangeCallback.Unregister()
 	}

wgengine/userspace.go

@@ -30,6 +30,7 @@ import (
 	"tailscale.com/internal/deepprint"
 	"tailscale.com/ipn/ipnstate"
 	"tailscale.com/net/dns"
+	"tailscale.com/net/dns/resolver"
 	"tailscale.com/net/flowtrack"
 	"tailscale.com/net/interfaces"
 	"tailscale.com/net/packet"
@@ -83,7 +84,7 @@ type userspaceEngine struct {
 	tundev            *tstun.Wrapper
 	wgdev             *device.Device
 	router            router.Router
-	resolver          *dns.Resolver
+	dns               *dns.Manager
 	magicConn         *magicsock.Conn
 	linkMon           *monitor.Mon
 	linkMonOwned      bool   // whether we created linkMon (and thus need to close it)
@@ -142,6 +143,10 @@ type Config struct {
 	// If nil, a fake Router that does nothing is used.
 	Router router.Router
 
+	// DNS interfaces the Engine to the OS DNS resolver configuration.
+	// If nil, a fake OSConfigurator that does nothing is used.
+	DNS dns.OSConfigurator
+
 	// LinkMonitor optionally provides an existing link monitor to re-use.
 	// If nil, a new link monitor is created.
 	LinkMonitor *monitor.Mon
@@ -164,6 +169,20 @@ func NewFakeUserspaceEngine(logf logger.Logf, listenPort uint16) (Engine, error)
 	})
 }
 
+// IsNetstack reports whether e is a netstack-based TUN-free engine.
+func IsNetstack(e Engine) bool {
+	ig, ok := e.(InternalsGetter)
+	if !ok {
+		return false
+	}
+	tw, _, ok := ig.GetInternals()
+	if !ok {
+		return false
+	}
+	name, err := tw.Name()
+	return err == nil && name == "FakeTUN"
+}
+
 // NewUserspaceEngine creates the named tun device and returns a
 // Tailscale Engine running on it.
 func NewUserspaceEngine(logf logger.Logf, conf Config) (_ Engine, reterr error) {
@@ -178,6 +197,10 @@ func NewUserspaceEngine(logf logger.Logf, conf Config) (_ Engine, reterr error)
 		logf("[v1] using fake (no-op) OS network configurator")
 		conf.Router = router.NewFake(logf)
 	}
+	if conf.DNS == nil {
+		logf("[v1] using fake (no-op) DNS configurator")
+		conf.DNS = dns.NewNoopManager()
+	}
 
 	tsTUNDev := tstun.Wrap(logf, conf.Tun)
 	closePool.add(tsTUNDev)
@@ -205,11 +228,7 @@ func NewUserspaceEngine(logf logger.Logf, conf Config) (_ Engine, reterr error)
 		e.linkMonOwned = true
 	}
 
-	e.resolver = dns.NewResolver(dns.ResolverConfig{
-		Logf:        logf,
-		Forward:     true,
-		LinkMonitor: e.linkMon,
-	})
+	e.dns = dns.NewManager(logf, conf.DNS, e.linkMon)
 
 	logf("link state: %+v", e.linkMon.InterfaceState())
 
@@ -236,6 +255,7 @@ func NewUserspaceEngine(logf logger.Logf, conf Config) (_ Engine, reterr error)
 		NoteRecvActivity: e.noteReceiveActivity,
 		LinkMonitor:      e.linkMon,
 	}
+
 	var err error
 	e.magicConn, err = magicsock.NewConn(magicsockOpts)
 	if err != nil {
@@ -305,9 +325,6 @@ func NewUserspaceEngine(logf logger.Logf, conf Config) (_ Engine, reterr error)
 				logf("[unexpected] peer %s has no single-IP routes: %v", peerWGKey.ShortString(), allowedIPs)
 			}
 		},
-		CreateBind:     e.magicConn.CreateBind,
-		CreateEndpoint: e.magicConn.CreateEndpoint,
-		SkipBindUpdate: true,
 	}
 
 	e.tundev.OnTSMPPongReceived = func(pong packet.TSMPPongReply) {
@@ -322,8 +339,13 @@ func NewUserspaceEngine(logf logger.Logf, conf Config) (_ Engine, reterr error)
 
 	// wgdev takes ownership of tundev, will close it when closed.
 	e.logf("Creating wireguard device...")
-	e.wgdev = device.NewDevice(e.tundev, e.wgLogger.DeviceLogger, opts)
+	e.wgdev = device.NewDevice(e.tundev, e.magicConn.Bind(), e.wgLogger.DeviceLogger, opts)
 	closePool.addFunc(e.wgdev.Close)
+	closePool.addFunc(func() {
+		if err := e.magicConn.Close(); err != nil {
+			e.logf("error closing magicconn: %v", err)
+		}
+	})
 
 	go func() {
 		up := false
@@ -364,8 +386,6 @@ func NewUserspaceEngine(logf logger.Logf, conf Config) (_ Engine, reterr error)
 	e.logf("Starting magicsock...")
 	e.magicConn.Start()
 
-	e.logf("Starting resolver...")
-	e.resolver.Start()
 	go e.pollResolver()
 
 	e.logf("Engine created.")
@@ -421,11 +441,7 @@ func (e *userspaceEngine) handleLocalPackets(p *packet.Parsed, t *tstun.Wrapper)
 // handleDNS is an outbound pre-filter resolving Tailscale domains.
 func (e *userspaceEngine) handleDNS(p *packet.Parsed, t *tstun.Wrapper) filter.Response {
 	if p.Dst.IP == magicDNSIP && p.Dst.Port == magicDNSPort && p.IPProto == ipproto.UDP {
-		request := dns.Packet{
-			Payload: append([]byte(nil), p.Payload()...),
-			Addr:    netaddr.IPPort{IP: p.Src.IP, Port: p.Src.Port},
-		}
-		err := e.resolver.EnqueueRequest(request)
+		err := e.dns.EnqueueRequest(append([]byte(nil), p.Payload()...), p.Src)
 		if err != nil {
 			e.logf("dns: enqueue: %v", err)
 		}
@@ -437,8 +453,8 @@ func (e *userspaceEngine) handleDNS(p *packet.Parsed, t *tstun.Wrapper) filter.R
 // pollResolver reads responses from the DNS resolver and injects them inbound.
 func (e *userspaceEngine) pollResolver() {
 	for {
-		resp, err := e.resolver.NextResponse()
-		if err == dns.ErrClosed {
+		bs, to, err := e.dns.NextResponse()
+		if err == resolver.ErrClosed {
 			return
 		}
 		if err != nil {
@@ -449,17 +465,17 @@ func (e *userspaceEngine) pollResolver() {
 		h := packet.UDP4Header{
 			IP4Header: packet.IP4Header{
 				Src: magicDNSIP,
-				Dst: resp.Addr.IP,
+				Dst: to.IP,
 			},
 			SrcPort: magicDNSPort,
-			DstPort: resp.Addr.Port,
+			DstPort: to.Port,
 		}
 		hlen := h.Len()
 
 		// TODO(dmytro): avoid this allocation without importing tstun quirks into dns.
 		const offset = tstun.PacketStartOffset
-		buf := make([]byte, offset+hlen+len(resp.Payload))
-		copy(buf[offset+hlen:], resp.Payload)
+		buf := make([]byte, offset+hlen+len(bs))
+		copy(buf[offset+hlen:], bs)
 		h.Marshal(buf[offset:])
 
 		e.tundev.InjectInboundDirect(buf, offset)
@@ -905,10 +921,13 @@ func genLocalAddrFunc(addrs []netaddr.IPPrefix) func(netaddr.IP) bool {
 	return func(t netaddr.IP) bool { return m[t] }
 }
 
-func (e *userspaceEngine) Reconfig(cfg *wgcfg.Config, routerCfg *router.Config) error {
+func (e *userspaceEngine) Reconfig(cfg *wgcfg.Config, routerCfg *router.Config, dnsCfg *dns.Config) error {
 	if routerCfg == nil {
 		panic("routerCfg must not be nil")
 	}
+	if dnsCfg == nil {
+		panic("dnsCfg must not be nil")
+	}
 
 	e.isLocalAddr.Store(genLocalAddrFunc(routerCfg.LocalAddrs))
 
@@ -925,7 +944,7 @@ func (e *userspaceEngine) Reconfig(cfg *wgcfg.Config, routerCfg *router.Config)
 	e.mu.Unlock()
 
 	engineChanged := deepprint.UpdateHash(&e.lastEngineSigFull, cfg)
-	routerChanged := deepprint.UpdateHash(&e.lastRouterSig, routerCfg)
+	routerChanged := deepprint.UpdateHash(&e.lastRouterSig, routerCfg, dnsCfg)
 	if !engineChanged && !routerChanged {
 		return ErrNoChanges
 	}
@@ -971,22 +990,14 @@ func (e *userspaceEngine) Reconfig(cfg *wgcfg.Config, routerCfg *router.Config)
 	}
 
 	if routerChanged {
-		if routerCfg.DNS.Proxied {
-			ips := routerCfg.DNS.Nameservers
-			upstreams := make([]net.Addr, len(ips))
-			for i, ip := range ips {
-				stdIP := ip.IPAddr()
-				upstreams[i] = &net.UDPAddr{
-					IP:   stdIP.IP,
-					Port: 53,
-					Zone: stdIP.Zone,
-				}
-			}
-			e.resolver.SetUpstreams(upstreams)
-			routerCfg.DNS.Nameservers = []netaddr.IP{tsaddr.TailscaleServiceIP()}
+		e.logf("wgengine: Reconfig: configuring DNS")
+		err := e.dns.Set(*dnsCfg)
+		health.SetDNSHealth(err)
+		if err != nil {
+			return err
 		}
 		e.logf("wgengine: Reconfig: configuring router")
-		err := e.router.Set(routerCfg)
+		err = e.router.Set(routerCfg)
 		health.SetRouterHealth(err)
 		if err != nil {
 			return err
@@ -1010,10 +1021,6 @@ func (e *userspaceEngine) SetFilter(filt *filter.Filter) {
 	e.tundev.SetFilter(filt)
 }
 
-func (e *userspaceEngine) SetDNSMap(dm *dns.Map) {
-	e.resolver.SetMap(dm)
-}
-
 func (e *userspaceEngine) SetStatusCallback(cb StatusCallback) {
 	e.mu.Lock()
 	defer e.mu.Unlock()
@@ -1203,12 +1210,12 @@ func (e *userspaceEngine) Close() {
 
 	r := bufio.NewReader(strings.NewReader(""))
 	e.wgdev.IpcSetOperation(r)
-	e.resolver.Close()
 	e.magicConn.Close()
 	e.linkMonUnregister()
 	if e.linkMonOwned {
 		e.linkMon.Close()
 	}
+	e.dns.Down()
 	e.router.Close()
 	e.wgdev.Close()
 	e.tundev.Close()
@@ -1438,11 +1445,30 @@ func (e *userspaceEngine) UnregisterIPPortIdentity(ipport netaddr.IPPort) {
 	delete(e.tsIPByIPPort, ipport)
 }
 
+var whoIsSleeps = [...]time.Duration{
+	0,
+	10 * time.Millisecond,
+	20 * time.Millisecond,
+	50 * time.Millisecond,
+	100 * time.Millisecond,
+}
+
 func (e *userspaceEngine) WhoIsIPPort(ipport netaddr.IPPort) (tsIP netaddr.IP, ok bool) {
-	e.mu.Lock()
-	defer e.mu.Unlock()
-	tsIP, ok = e.tsIPByIPPort[ipport]
-	return tsIP, ok
+	// We currently have a registration race,
+	// https://github.com/tailscale/tailscale/issues/1616,
+	// so loop a few times for now waiting for the registration
+	// to appear.
+	// TODO(bradfitz,namansood): remove this once #1616 is fixed.
+	for _, d := range whoIsSleeps {
+		time.Sleep(d)
+		e.mu.Lock()
+		tsIP, ok = e.tsIPByIPPort[ipport]
+		e.mu.Unlock()
+		if ok {
+			return tsIP, true
+		}
+	}
+	return tsIP, false
 }
 
 // peerForIP returns the Node in the wireguard config

wgengine/userspace_test.go

@@ -13,6 +13,7 @@ import (
 
 	"go4.org/mem"
 	"inet.af/netaddr"
+	"tailscale.com/net/dns"
 	"tailscale.com/net/tstun"
 	"tailscale.com/tailcfg"
 	"tailscale.com/types/key"
@@ -108,7 +109,7 @@ func TestUserspaceEngineReconfig(t *testing.T) {
 			},
 		}
 
-		err = e.Reconfig(cfg, routerCfg)
+		err = e.Reconfig(cfg, routerCfg, &dns.Config{})
 		if err != nil {
 			t.Fatal(err)
 		}
@@ -186,3 +187,14 @@ func BenchmarkGenLocalAddrFunc(b *testing.B) {
 	})
 	b.Logf("x = %v", x)
 }
+
+func TestIsNetstack(t *testing.T) {
+	e, err := NewUserspaceEngine(t.Logf, Config{})
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer e.Close()
+	if !IsNetstack(e) {
+		t.Errorf("IsNetstack = false; want true")
+	}
+}

wgengine/watchdog.go

@@ -74,8 +74,8 @@ func (e *watchdogEngine) watchdog(name string, fn func()) {
 	})
 }
 
-func (e *watchdogEngine) Reconfig(cfg *wgcfg.Config, routerCfg *router.Config) error {
-	return e.watchdogErr("Reconfig", func() error { return e.wrap.Reconfig(cfg, routerCfg) })
+func (e *watchdogEngine) Reconfig(cfg *wgcfg.Config, routerCfg *router.Config, dnsCfg *dns.Config) error {
+	return e.watchdogErr("Reconfig", func() error { return e.wrap.Reconfig(cfg, routerCfg, dnsCfg) })
 }
 func (e *watchdogEngine) GetLinkMonitor() *monitor.Mon {
 	return e.wrap.GetLinkMonitor()
@@ -86,9 +86,6 @@ func (e *watchdogEngine) GetFilter() *filter.Filter {
 func (e *watchdogEngine) SetFilter(filt *filter.Filter) {
 	e.watchdog("SetFilter", func() { e.wrap.SetFilter(filt) })
 }
-func (e *watchdogEngine) SetDNSMap(dm *dns.Map) {
-	e.watchdog("SetDNSMap", func() { e.wrap.SetDNSMap(dm) })
-}
 func (e *watchdogEngine) SetStatusCallback(cb StatusCallback) {
 	e.watchdog("SetStatusCallback", func() { e.wrap.SetStatusCallback(cb) })
 }

wgengine/wgcfg/config.go

@@ -20,7 +20,6 @@ type Config struct {
 	Name       string
 	PrivateKey PrivateKey
 	Addresses  []netaddr.IPPrefix
-	ListenPort uint16
 	MTU        uint16
 	DNS        []netaddr.IP
 	Peers      []Peer

wgengine/wgcfg/device_test.go

@@ -14,6 +14,7 @@ import (
 	"sync"
 	"testing"
 
+	"github.com/tailscale/wireguard-go/conn"
 	"github.com/tailscale/wireguard-go/device"
 	"github.com/tailscale/wireguard-go/tun"
 	"inet.af/netaddr"
@@ -55,8 +56,8 @@ func TestDeviceConfig(t *testing.T) {
 		}},
 	}
 
-	device1 := device.NewDevice(newNilTun(), device.NewLogger(device.LogLevelError, "device1"))
-	device2 := device.NewDevice(newNilTun(), device.NewLogger(device.LogLevelError, "device2"))
+	device1 := device.NewDevice(newNilTun(), conn.NewDefaultBind(), device.NewLogger(device.LogLevelError, "device1"))
+	device2 := device.NewDevice(newNilTun(), conn.NewDefaultBind(), device.NewLogger(device.LogLevelError, "device2"))
 	defer device1.Close()
 	defer device2.Close()
 

wgengine/wgcfg/nmcfg/nmcfg.go

@@ -57,7 +57,6 @@ func WGCfg(nm *netmap.NetworkMap, logf logger.Logf, flags netmap.WGConfigFlags,
 		Name:       "tailscale",
 		PrivateKey: wgcfg.PrivateKey(nm.PrivateKey),
 		Addresses:  nm.Addresses,
-		ListenPort: nm.LocalPort,
 		Peers:      make([]wgcfg.Peer, 0, len(nm.Peers)),
 	}
 
@@ -74,9 +73,6 @@ func WGCfg(nm *netmap.NetworkMap, logf logger.Logf, flags netmap.WGConfigFlags,
 		}
 
 		if !peer.DiscoKey.IsZero() {
-			if err := appendEndpoint(cpeer, fmt.Sprintf("%x%s", peer.DiscoKey[:], wgcfg.EndpointDiscoSuffix)); err != nil {
-				return nil, err
-			}
 			cpeer.Endpoints = fmt.Sprintf("%x.disco.tailscale:12345", peer.DiscoKey[:])
 		} else {
 			if err := appendEndpoint(cpeer, peer.DERP); err != nil {
@@ -88,8 +84,14 @@ func WGCfg(nm *netmap.NetworkMap, logf logger.Logf, flags netmap.WGConfigFlags,
 				}
 			}
 		}
+		didExitNodeWarn := false
 		for _, allowedIP := range peer.AllowedIPs {
 			if allowedIP.Bits == 0 && peer.StableID != exitNode {
+				if didExitNodeWarn {
+					// Don't log about both the IPv4 /0 and IPv6 /0.
+					continue
+				}
+				didExitNodeWarn = true
 				logf("[v1] wgcfg: skipping unselected default route from %q (%v)", nodeDebugName(peer), peer.Key.ShortString())
 				continue
 			} else if allowedIP.IsSingleIP() && tsaddr.IsTailscaleIP(allowedIP.IP) && (flags&netmap.AllowSingleHosts) == 0 {

wgengine/wgcfg/parser.go

@@ -144,11 +144,7 @@ func (cfg *Config) handleDeviceLine(key, value string) error {
 		// wireguard-go guarantees not to send zero value; private keys are already clamped.
 		cfg.PrivateKey = PrivateKey(*k)
 	case "listen_port":
-		port, err := strconv.ParseUint(value, 10, 16)
-		if err != nil {
-			return fmt.Errorf("failed to parse listen_port: %w", err)
-		}
-		cfg.ListenPort = uint16(port)
+		// ignore
 	case "fwmark":
 		// ignore
 	default:

wgengine/wgcfg/writer.go

@@ -40,9 +40,6 @@ func (cfg *Config) ToUAPI(w io.Writer, prev *Config) error {
 	if prev.PrivateKey != cfg.PrivateKey {
 		set("private_key", cfg.PrivateKey.HexString())
 	}
-	if prev.ListenPort != cfg.ListenPort {
-		setUint16("listen_port", cfg.ListenPort)
-	}
 
 	old := make(map[Key]Peer)
 	for _, p := range prev.Peers {

wgengine/wgengine.go

@@ -57,7 +57,7 @@ type Engine interface {
 	// sends an updated network map.
 	//
 	// The returned error is ErrNoChanges if no changes were made.
-	Reconfig(*wgcfg.Config, *router.Config) error
+	Reconfig(*wgcfg.Config, *router.Config, *dns.Config) error
 
 	// GetFilter returns the current packet filter, if any.
 	GetFilter() *filter.Filter
@@ -65,9 +65,6 @@ type Engine interface {
 	// SetFilter updates the packet filter.
 	SetFilter(*filter.Filter)
 
-	// SetDNSMap updates the DNS map.
-	SetDNSMap(*dns.Map)
-
 	// SetStatusCallback sets the function to call when the
 	// WireGuard status changes.
 	SetStatusCallback(StatusCallback)