cmd/cloner, tailcfg: fix nil vs len 0 issues, add tests, use for Hostinfo

Also use go:generate and https://golang.org/s/generatedcode header style.

.github/workflows/linux32.yml

@@ -0,0 +1,48 @@
+name: Linux 32-bit
+
+on:
+  push:
+    branches:
+      - main
+  pull_request:
+    branches:
+      - '*'
+
+jobs:
+  build:
+    runs-on: ubuntu-latest
+
+    if: "!contains(github.event.head_commit.message, '[ci skip]')"
+
+    steps:
+
+    - name: Set up Go
+      uses: actions/setup-go@v1
+      with:
+        go-version: 1.14
+      id: go
+
+    - name: Check out code into the Go module directory
+      uses: actions/checkout@v1
+
+    - name: Basic build
+      run: GOARCH=386 go build ./cmd/...
+
+    - name: Run tests on linux
+      run: GOARCH=386 go test ./...
+
+    - uses: k0kubun/action-slack@v2.0.0
+      with:
+        payload: |
+          {
+            "attachments": [{
+              "text": "${{ job.status }}: ${{ github.workflow }} <https://github.com/${{ github.repository }}/commit/${{ github.sha }}/checks|${{ env.COMMIT_DATE }} #${{ env.COMMIT_NUMBER_OF_DAY }}> " +
+                      "(<https://github.com/${{ github.repository }}/commit/${{ github.sha }}|" + "${{ github.sha }}".substring(0, 10) + ">) " +
+                      "of ${{ github.repository }}@" + "${{ github.ref }}".split('/').reverse()[0] + " by ${{ github.event.head_commit.committer.name }}",
+              "color": "danger"
+            }]
+          }
+      env:
+        SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK_URL }}
+      if: failure() && github.event_name == 'push'
+

.github/workflows/staticcheck.yml

@@ -21,6 +21,9 @@ jobs:
     - name: Check out code
       uses: actions/checkout@v1
 
+    - name: Run go vet
+      run: go vet ./...
+
     - name: Print staticcheck version
       run: go run honnef.co/go/tools/cmd/staticcheck -version
 

atomicfile/atomicfile.go

@@ -9,20 +9,39 @@
 package atomicfile // import "tailscale.com/atomicfile"
 
 import (
-	"fmt"
 	"io/ioutil"
 	"os"
+	"path/filepath"
+	"runtime"
 )
 
 // WriteFile writes data to filename+some suffix, then renames it
 // into filename.
-func WriteFile(filename string, data []byte, perm os.FileMode) error {
-	tmpname := filename + ".new.tmp"
-	if err := ioutil.WriteFile(tmpname, data, perm); err != nil {
-		return fmt.Errorf("%#v: %v", tmpname, err)
+func WriteFile(filename string, data []byte, perm os.FileMode) (err error) {
+	f, err := ioutil.TempFile(filepath.Dir(filename), filepath.Base(filename)+".tmp")
+	if err != nil {
+		return err
 	}
-	if err := os.Rename(tmpname, filename); err != nil {
-		return fmt.Errorf("%#v->%#v: %v", tmpname, filename, err)
+	tmpName := f.Name()
+	defer func() {
+		if err != nil {
+			f.Close()
+			os.Remove(tmpName)
+		}
+	}()
+	if _, err := f.Write(data); err != nil {
+		return err
 	}
-	return nil
+	if runtime.GOOS != "windows" {
+		if err := f.Chmod(perm); err != nil {
+			return err
+		}
+	}
+	if err := f.Sync(); err != nil {
+		return err
+	}
+	if err := f.Close(); err != nil {
+		return err
+	}
+	return os.Rename(tmpName, filename)
 }

cmd/cloner/cloner.go

@@ -0,0 +1,264 @@
+// Copyright (c) 2020 Tailscale Inc & AUTHORS All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+// Cloner is a tool to automate the creation of a Clone method.
+//
+// The result of the Clone method aliases no memory that can be edited
+// with the original.
+//
+// This tool makes lots of implicit assumptions about the types you feed it.
+// In particular, it can only write relatively "shallow" Clone methods.
+// That is, if a type contains another named struct type, cloner assumes that
+// named type will also have a Clone method.
+package main
+
+import (
+	"bytes"
+	"flag"
+	"fmt"
+	"go/ast"
+	"go/format"
+	"go/token"
+	"go/types"
+	"io/ioutil"
+	"log"
+	"os"
+	"strings"
+
+	"golang.org/x/tools/go/packages"
+)
+
+var (
+	flagTypes     = flag.String("type", "", "comma-separated list of types; required")
+	flagOutput    = flag.String("output", "", "output file; required")
+	flagBuildTags = flag.String("tags", "", "compiler build tags to apply")
+)
+
+func main() {
+	log.SetFlags(0)
+	log.SetPrefix("cloner: ")
+	flag.Parse()
+	if len(*flagTypes) == 0 {
+		flag.Usage()
+		os.Exit(2)
+	}
+	typeNames := strings.Split(*flagTypes, ",")
+
+	cfg := &packages.Config{
+		Mode:  packages.NeedTypes | packages.NeedTypesInfo | packages.NeedSyntax | packages.NeedName,
+		Tests: false,
+	}
+	if *flagBuildTags != "" {
+		cfg.BuildFlags = []string{"-tags=" + *flagBuildTags}
+	}
+	pkgs, err := packages.Load(cfg, ".")
+	if err != nil {
+		log.Fatal(err)
+	}
+	if len(pkgs) != 1 {
+		log.Fatalf("wrong number of packages: %d", len(pkgs))
+	}
+	pkg := pkgs[0]
+	buf := new(bytes.Buffer)
+	imports := make(map[string]struct{})
+	for _, typeName := range typeNames {
+		found := false
+		for _, file := range pkg.Syntax {
+			//var fbuf bytes.Buffer
+			//ast.Fprint(&fbuf, pkg.Fset, file, nil)
+			//fmt.Println(fbuf.String())
+
+			for _, d := range file.Decls {
+				decl, ok := d.(*ast.GenDecl)
+				if !ok || decl.Tok != token.TYPE {
+					continue
+				}
+				for _, s := range decl.Specs {
+					spec, ok := s.(*ast.TypeSpec)
+					if !ok || spec.Name.Name != typeName {
+						continue
+					}
+					typeNameObj := pkg.TypesInfo.Defs[spec.Name]
+					typ, ok := typeNameObj.Type().(*types.Named)
+					if !ok {
+						continue
+					}
+					pkg := typeNameObj.Pkg()
+					gen(buf, imports, typeName, typ, pkg)
+				}
+				found = true
+			}
+		}
+		if !found {
+			log.Fatalf("could not find type %s", typeName)
+		}
+	}
+
+	contents := new(bytes.Buffer)
+	fmt.Fprintf(contents, header, *flagTypes, pkg.Name)
+	fmt.Fprintf(contents, "import (\n")
+	for s := range imports {
+		fmt.Fprintf(contents, "\t%q\n", s)
+	}
+	fmt.Fprintf(contents, ")\n\n")
+	contents.Write(buf.Bytes())
+
+	out, err := format.Source(contents.Bytes())
+	if err != nil {
+		log.Fatalf("%s, in source:\n%s", err, contents.Bytes())
+	}
+
+	output := *flagOutput
+	if output == "" {
+		flag.Usage()
+		os.Exit(2)
+	}
+	if err := ioutil.WriteFile(output, out, 0666); err != nil {
+		log.Fatal(err)
+	}
+}
+
+const header = `// Copyright (c) 2020 Tailscale Inc & AUTHORS All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+// Code generated by tailscale.com/cmd/cloner -type %s; DO NOT EDIT.
+
+package %s
+
+`
+
+func gen(buf *bytes.Buffer, imports map[string]struct{}, name string, typ *types.Named, thisPkg *types.Package) {
+	pkgQual := func(pkg *types.Package) string {
+		if thisPkg == pkg {
+			return ""
+		}
+		imports[pkg.Path()] = struct{}{}
+		return pkg.Name()
+	}
+	importedName := func(t types.Type) string {
+		return types.TypeString(t, pkgQual)
+	}
+
+	switch t := typ.Underlying().(type) {
+	case *types.Struct:
+		_ = t
+		name := typ.Obj().Name()
+		fmt.Fprintf(buf, "// Clone makes a deep copy of %s.\n", name)
+		fmt.Fprintf(buf, "// The result aliases no memory with the original.\n")
+		fmt.Fprintf(buf, "func (src *%s) Clone() *%s {\n", name, name)
+		writef := func(format string, args ...interface{}) {
+			fmt.Fprintf(buf, "\t"+format+"\n", args...)
+		}
+		writef("if src == nil {")
+		writef("\treturn nil")
+		writef("}")
+		writef("dst := new(%s)", name)
+		writef("*dst = *src")
+		for i := 0; i < t.NumFields(); i++ {
+			fname := t.Field(i).Name()
+			ft := t.Field(i).Type()
+			if !containsPointers(ft) {
+				continue
+			}
+			if named, _ := ft.(*types.Named); named != nil && !hasBasicUnderlying(ft) {
+				writef("dst.%s = *src.%s.Clone()", fname, fname)
+				continue
+			}
+			switch ft := ft.Underlying().(type) {
+			case *types.Slice:
+				if containsPointers(ft.Elem()) {
+					n := importedName(ft.Elem())
+					writef("dst.%s = make([]%s, len(src.%s))", fname, n, fname)
+					writef("for i := range dst.%s {", fname)
+					if _, isPtr := ft.Elem().(*types.Pointer); isPtr {
+						writef("\tdst.%s[i] = src.%s[i].Clone()", fname, fname)
+					} else {
+						writef("\tdst.%s[i] = *src.%s[i].Clone()", fname, fname)
+					}
+					writef("}")
+				} else {
+					writef("dst.%s = append(src.%s[:0:0], src.%s...)", fname, fname, fname)
+				}
+			case *types.Pointer:
+				if named, _ := ft.Elem().(*types.Named); named != nil && containsPointers(ft.Elem()) {
+					writef("dst.%s = src.%s.Clone()", fname, fname)
+					continue
+				}
+				n := importedName(ft.Elem())
+				writef("if dst.%s != nil {", fname)
+				writef("\tdst.%s = new(%s)", fname, n)
+				writef("\t*dst.%s = *src.%s", fname, fname)
+				if containsPointers(ft.Elem()) {
+					writef("\t" + `panic("TODO pointers in pointers")`)
+				}
+				writef("}")
+			case *types.Map:
+				writef("if dst.%s != nil {", fname)
+				writef("\tdst.%s = map[%s]%s{}", fname, importedName(ft.Key()), importedName(ft.Elem()))
+				if sliceType, isSlice := ft.Elem().(*types.Slice); isSlice {
+					n := importedName(sliceType.Elem())
+					writef("\tfor k := range src.%s {", fname)
+					// use zero-length slice instead of nil to ensure
+					// the key is always copied.
+					writef("\t\tdst.%s[k] = append([]%s{}, src.%s[k]...)", fname, n, fname)
+					writef("\t}")
+				} else if containsPointers(ft.Elem()) {
+					writef("\t\t" + `panic("TODO map value pointers")`)
+				} else {
+					writef("\tfor k, v := range src.%s {", fname)
+					writef("\t\tdst.%s[k] = v", fname)
+					writef("\t}")
+				}
+				writef("}")
+			case *types.Struct:
+				writef(`panic("TODO struct %s")`, fname)
+			default:
+				writef(`panic(fmt.Sprintf("TODO: %T", ft))`)
+			}
+		}
+		writef("return dst")
+		fmt.Fprintf(buf, "}\n\n")
+	}
+}
+
+func hasBasicUnderlying(typ types.Type) bool {
+	switch typ.Underlying().(type) {
+	case *types.Slice, *types.Map:
+		return true
+	default:
+		return false
+	}
+}
+
+func containsPointers(typ types.Type) bool {
+	switch typ.String() {
+	case "time.Time":
+		// time.Time contains a pointer that does not need copying
+		return false
+	case "inet.af/netaddr.IP":
+		return false
+	}
+	switch ft := typ.Underlying().(type) {
+	case *types.Array:
+		return containsPointers(ft.Elem())
+	case *types.Chan:
+		return true
+	case *types.Interface:
+		return true // a little too broad
+	case *types.Map:
+		return true
+	case *types.Pointer:
+		return true
+	case *types.Slice:
+		return true
+	case *types.Struct:
+		for i := 0; i < ft.NumFields(); i++ {
+			if containsPointers(ft.Field(i).Type()) {
+				return true
+			}
+		}
+	}
+	return false
+}

cmd/derper/derper.go

@@ -185,7 +185,7 @@ func main() {
 		}
 		httpsrv.TLSConfig = certManager.TLSConfig()
 		go func() {
-			err := http.ListenAndServe(":80", certManager.HTTPHandler(tsweb.Port80Handler{mux}))
+			err := http.ListenAndServe(":80", certManager.HTTPHandler(tsweb.Port80Handler{Main: mux}))
 			if err != nil {
 				if err != http.ErrServerClosed {
 					log.Fatal(err)

cmd/tailscale/cli/cli.go

@@ -0,0 +1,123 @@
+// Copyright (c) 2020 Tailscale Inc & AUTHORS All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+// Package cli contains the cmd/tailscale CLI code in a package that can be included
+// in other wrapper binaries such as the Mac and Windows clients.
+package cli
+
+import (
+	"context"
+	"flag"
+	"log"
+	"net"
+	"os"
+	"os/signal"
+	"runtime"
+	"strings"
+	"syscall"
+
+	"github.com/peterbourgon/ff/v2/ffcli"
+	"tailscale.com/ipn"
+	"tailscale.com/paths"
+	"tailscale.com/safesocket"
+)
+
+// ActLikeCLI reports whether a GUI application should act like the
+// CLI based on os.Args, GOOS, the context the process is running in
+// (pty, parent PID), etc.
+func ActLikeCLI() bool {
+	if len(os.Args) < 2 {
+		return false
+	}
+	switch os.Args[1] {
+	case "up", "status", "netcheck", "version",
+		"-V", "--version", "-h", "--help":
+		return true
+	}
+	return false
+}
+
+// Run runs the CLI. The args do not include the binary name.
+func Run(args []string) error {
+	if len(args) == 1 && (args[0] == "-V" || args[0] == "--version") {
+		args = []string{"version"}
+	}
+
+	rootfs := flag.NewFlagSet("tailscale", flag.ExitOnError)
+	rootfs.StringVar(&rootArgs.socket, "socket", paths.DefaultTailscaledSocket(), "path to tailscaled's unix socket")
+
+	rootCmd := &ffcli.Command{
+		Name:       "tailscale",
+		ShortUsage: "tailscale subcommand [flags]",
+		ShortHelp:  "The easiest, most secure way to use WireGuard.",
+		LongHelp: strings.TrimSpace(`
+This CLI is still under active development. Commands and flags will
+change in the future.
+`),
+		Subcommands: []*ffcli.Command{
+			upCmd,
+			netcheckCmd,
+			statusCmd,
+			versionCmd,
+		},
+		FlagSet: rootfs,
+		Exec:    func(context.Context, []string) error { return flag.ErrHelp },
+	}
+
+	if err := rootCmd.Parse(args); err != nil {
+		return err
+	}
+
+	err := rootCmd.Run(context.Background())
+	if err == flag.ErrHelp {
+		return nil
+	}
+	return err
+}
+
+var rootArgs struct {
+	socket string
+}
+
+func connect(ctx context.Context) (net.Conn, *ipn.BackendClient, context.Context, context.CancelFunc) {
+	c, err := safesocket.Connect(rootArgs.socket, 41112)
+	if err != nil {
+		if runtime.GOOS != "windows" && rootArgs.socket == "" {
+			log.Fatalf("--socket cannot be empty")
+		}
+		log.Fatalf("Failed to connect to connect to tailscaled. (safesocket.Connect: %v)\n", err)
+	}
+	clientToServer := func(b []byte) {
+		ipn.WriteMsg(c, b)
+	}
+
+	ctx, cancel := context.WithCancel(ctx)
+
+	go func() {
+		interrupt := make(chan os.Signal, 1)
+		signal.Notify(interrupt, syscall.SIGINT, syscall.SIGTERM)
+		<-interrupt
+		c.Close()
+		cancel()
+	}()
+
+	bc := ipn.NewBackendClient(log.Printf, clientToServer)
+	return c, bc, ctx, cancel
+}
+
+// pump receives backend messages on conn and pushes them into bc.
+func pump(ctx context.Context, bc *ipn.BackendClient, conn net.Conn) {
+	defer conn.Close()
+	for ctx.Err() == nil {
+		msg, err := ipn.ReadMsg(conn)
+		if err != nil {
+			if ctx.Err() != nil {
+				return
+			}
+			log.Printf("ReadMsg: %v\n", err)
+			break
+		}
+		bc.GotNotifyMsg(msg)
+	}
+}

cmd/tailscale/cli/netcheck.go

@@ -2,7 +2,7 @@
 // Use of this source code is governed by a BSD-style
 // license that can be found in the LICENSE file.
 
-package main
+package cli
 
 import (
 	"context"
@@ -117,6 +117,7 @@ func printReport(dm *tailcfg.DERPMap, report *netcheck.Report) error {
 	}
 	fmt.Printf("\t* MappingVariesByDestIP: %v\n", report.MappingVariesByDestIP)
 	fmt.Printf("\t* HairPinning: %v\n", report.HairPinning)
+	fmt.Printf("\t* PortMapping: %v\n", portMapping(report))
 
 	// When DERP latency checking failed,
 	// magicsock will try to pick the DERP server that
@@ -142,3 +143,20 @@ func printReport(dm *tailcfg.DERPMap, report *netcheck.Report) error {
 	}
 	return nil
 }
+
+func portMapping(r *netcheck.Report) string {
+	if !r.AnyPortMappingChecked() {
+		return "not checked"
+	}
+	var got []string
+	if r.UPnP.EqualBool(true) {
+		got = append(got, "UPnP")
+	}
+	if r.PMP.EqualBool(true) {
+		got = append(got, "NAT-PMP")
+	}
+	if r.PCP.EqualBool(true) {
+		got = append(got, "PCP")
+	}
+	return strings.Join(got, ", ")
+}

cmd/tailscale/cli/status.go

@@ -2,7 +2,7 @@
 // Use of this source code is governed by a BSD-style
 // license that can be found in the LICENSE file.
 
-package main
+package cli
 
 import (
 	"bytes"
@@ -25,13 +25,14 @@ import (
 
 var statusCmd = &ffcli.Command{
 	Name:       "status",
-	ShortUsage: "status [-web] [-json]",
+	ShortUsage: "status [-active] [-web] [-json]",
 	ShortHelp:  "Show state of tailscaled and its connections",
 	Exec:       runStatus,
 	FlagSet: (func() *flag.FlagSet {
 		fs := flag.NewFlagSet("status", flag.ExitOnError)
 		fs.BoolVar(&statusArgs.json, "json", false, "output in JSON format (WARNING: format subject to change)")
 		fs.BoolVar(&statusArgs.web, "web", false, "run webserver with HTML showing status")
+		fs.BoolVar(&statusArgs.active, "active", false, "filter output to only peers with active sessions (not applicable to web mode)")
 		fs.StringVar(&statusArgs.listen, "listen", "127.0.0.1:8384", "listen address; use port 0 for automatic")
 		fs.BoolVar(&statusArgs.browser, "browser", true, "Open a browser in web mode")
 		return fs
@@ -43,6 +44,7 @@ var statusArgs struct {
 	web     bool   // run webserver
 	listen  string // in web mode, webserver address to listen on, empty means auto
 	browser bool   // in web mode, whether to open browser
+	active  bool   // in CLI mode, filter output to only peers with active sessions
 }
 
 func runStatus(ctx context.Context, args []string) error {
@@ -76,6 +78,13 @@ func runStatus(ctx context.Context, args []string) error {
 		return err
 	}
 	if statusArgs.json {
+		if statusArgs.active {
+			for peer, ps := range st.Peer {
+				if !peerActive(ps) {
+					delete(st.Peer, peer)
+				}
+			}
+		}
 		j, err := json.MarshalIndent(st, "", "  ")
 		if err != nil {
 			return err
@@ -120,6 +129,10 @@ func runStatus(ctx context.Context, args []string) error {
 	f := func(format string, a ...interface{}) { fmt.Fprintf(&buf, format, a...) }
 	for _, peer := range st.Peers() {
 		ps := st.Peer[peer]
+		active := peerActive(ps)
+		if statusArgs.active && !active {
+			continue
+		}
 		f("%s %-7s %-15s %-18s tx=%8d rx=%8d ",
 			peer.ShortString(),
 			ps.OS,
@@ -128,8 +141,6 @@ func runStatus(ctx context.Context, args []string) error {
 			ps.TxBytes,
 			ps.RxBytes,
 		)
-		// TODO: let server report this active bool instead
-		active := !ps.LastWrite.IsZero() && time.Since(ps.LastWrite) < 2*time.Minute
 		relay := ps.Relay
 		if active && relay != "" && ps.CurAddr == "" {
 			relay = "*" + relay + "*"
@@ -152,3 +163,10 @@ func runStatus(ctx context.Context, args []string) error {
 	os.Stdout.Write(buf.Bytes())
 	return nil
 }
+
+// peerActive reports whether ps has recent activity.
+//
+// TODO: have the server report this bool instead.
+func peerActive(ps *ipnstate.PeerStatus) bool {
+	return !ps.LastWrite.IsZero() && time.Since(ps.LastWrite) < 2*time.Minute
+}

cmd/tailscale/cli/up.go

@@ -0,0 +1,252 @@
+// Copyright (c) 2020 Tailscale Inc & AUTHORS All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package cli
+
+import (
+	"bytes"
+	"context"
+	"flag"
+	"fmt"
+	"log"
+	"os"
+	"os/exec"
+	"runtime"
+	"strconv"
+	"strings"
+
+	"github.com/peterbourgon/ff/v2/ffcli"
+	"github.com/tailscale/wireguard-go/wgcfg"
+	"tailscale.com/ipn"
+	"tailscale.com/tailcfg"
+	"tailscale.com/wgengine/router"
+)
+
+// globalStateKey is the ipn.StateKey that tailscaled loads on
+// startup.
+//
+// We have to support multiple state keys for other OSes (Windows in
+// particular), but right now Unix daemons run with a single
+// node-global state. To keep open the option of having per-user state
+// later, the global state key doesn't look like a username.
+const globalStateKey = "_daemon"
+
+var upCmd = &ffcli.Command{
+	Name:       "up",
+	ShortUsage: "up [flags]",
+	ShortHelp:  "Connect to your Tailscale network",
+
+	LongHelp: strings.TrimSpace(`
+"tailscale up" connects this machine to your Tailscale network,
+triggering authentication if necessary.
+
+The flags passed to this command are specific to this machine. If you don't
+specify any flags, options are reset to their default.
+`),
+	FlagSet: (func() *flag.FlagSet {
+		upf := flag.NewFlagSet("up", flag.ExitOnError)
+		upf.StringVar(&upArgs.server, "login-server", "https://login.tailscale.com", "base URL of control server")
+		upf.BoolVar(&upArgs.acceptRoutes, "accept-routes", false, "accept routes advertised by other Tailscale nodes")
+		upf.BoolVar(&upArgs.acceptDNS, "accept-dns", true, "accept DNS configuration from the admin panel")
+		upf.BoolVar(&upArgs.singleRoutes, "host-routes", true, "install host routes to other Tailscale nodes")
+		upf.BoolVar(&upArgs.shieldsUp, "shields-up", false, "don't allow incoming connections")
+		upf.StringVar(&upArgs.advertiseTags, "advertise-tags", "", "ACL tags to request (comma-separated, e.g. eng,montreal,ssh)")
+		upf.StringVar(&upArgs.authKey, "authkey", "", "node authorization key")
+		upf.StringVar(&upArgs.hostname, "hostname", "", "hostname to use instead of the one provided by the OS")
+		upf.BoolVar(&upArgs.enableDERP, "enable-derp", true, "enable the use of DERP servers")
+		if runtime.GOOS == "linux" || isBSD(runtime.GOOS) {
+			upf.StringVar(&upArgs.advertiseRoutes, "advertise-routes", "", "routes to advertise to other nodes (comma-separated, e.g. 10.0.0.0/8,192.168.0.0/24)")
+		}
+		if runtime.GOOS == "linux" {
+			upf.BoolVar(&upArgs.snat, "snat-subnet-routes", true, "source NAT traffic to local routes advertised with -advertise-routes")
+			upf.StringVar(&upArgs.netfilterMode, "netfilter-mode", "on", "netfilter mode (one of on, nodivert, off)")
+		}
+		return upf
+	})(),
+	Exec: runUp,
+}
+
+var upArgs struct {
+	server          string
+	acceptRoutes    bool
+	acceptDNS       bool
+	singleRoutes    bool
+	shieldsUp       bool
+	advertiseRoutes string
+	advertiseTags   string
+	enableDERP      bool
+	snat            bool
+	netfilterMode   string
+	authKey         string
+	hostname        string
+}
+
+// parseIPOrCIDR parses an IP address or a CIDR prefix. If the input
+// is an IP address, it is returned in CIDR form with a /32 mask for
+// IPv4 or a /128 mask for IPv6.
+func parseIPOrCIDR(s string) (wgcfg.CIDR, bool) {
+	if strings.Contains(s, "/") {
+		ret, err := wgcfg.ParseCIDR(s)
+		if err != nil {
+			return wgcfg.CIDR{}, false
+		}
+		return ret, true
+	}
+
+	ip, ok := wgcfg.ParseIP(s)
+	if !ok {
+		return wgcfg.CIDR{}, false
+	}
+	if ip.Is4() {
+		return wgcfg.CIDR{IP: ip, Mask: 32}, true
+	} else {
+		return wgcfg.CIDR{IP: ip, Mask: 128}, true
+	}
+}
+
+func isBSD(s string) bool {
+	return s == "dragonfly" || s == "freebsd" || s == "netbsd" || s == "openbsd"
+}
+
+func warning(format string, args ...interface{}) {
+	fmt.Printf("Warning: "+format+"\n", args...)
+}
+
+// checkIPForwarding prints warnings on linux if IP forwarding is not
+// enabled, or if we were unable to verify the state of IP forwarding.
+func checkIPForwarding() {
+	var key string
+
+	if runtime.GOOS == "linux" {
+		key = "net.ipv4.ip_forward"
+	} else if isBSD(runtime.GOOS) {
+		key = "net.inet.ip.forwarding"
+	} else {
+		return
+	}
+
+	bs, err := exec.Command("sysctl", "-n", key).Output()
+	if err != nil {
+		warning("couldn't check %s (%v).\nSubnet routes won't work without IP forwarding.", key, err)
+		return
+	}
+	on, err := strconv.ParseBool(string(bytes.TrimSpace(bs)))
+	if err != nil {
+		warning("couldn't parse %s (%v).\nSubnet routes won't work without IP forwarding.", key, err)
+		return
+	}
+	if !on {
+		warning("%s is disabled. Subnet routes won't work.", key)
+	}
+}
+
+func runUp(ctx context.Context, args []string) error {
+	if len(args) > 0 {
+		log.Fatalf("too many non-flag arguments: %q", args)
+	}
+
+	var routes []wgcfg.CIDR
+	if upArgs.advertiseRoutes != "" {
+		checkIPForwarding()
+		advroutes := strings.Split(upArgs.advertiseRoutes, ",")
+		for _, s := range advroutes {
+			cidr, ok := parseIPOrCIDR(s)
+			if !ok {
+				log.Fatalf("%q is not a valid IP address or CIDR prefix", s)
+			}
+			routes = append(routes, cidr)
+		}
+	}
+
+	var tags []string
+	if upArgs.advertiseTags != "" {
+		tags = strings.Split(upArgs.advertiseTags, ",")
+		for _, tag := range tags {
+			err := tailcfg.CheckTag(tag)
+			if err != nil {
+				log.Fatalf("tag: %q: %s", tag, err)
+			}
+		}
+	}
+
+	if len(upArgs.hostname) > 256 {
+		log.Fatalf("hostname too long: %d bytes (max 256)", len(upArgs.hostname))
+	}
+
+	// TODO(apenwarr): fix different semantics between prefs and uflags
+	// TODO(apenwarr): allow setting/using CorpDNS
+	prefs := ipn.NewPrefs()
+	prefs.ControlURL = upArgs.server
+	prefs.WantRunning = true
+	prefs.RouteAll = upArgs.acceptRoutes
+	prefs.CorpDNS = upArgs.acceptDNS
+	prefs.AllowSingleHosts = upArgs.singleRoutes
+	prefs.ShieldsUp = upArgs.shieldsUp
+	prefs.AdvertiseRoutes = routes
+	prefs.AdvertiseTags = tags
+	prefs.NoSNAT = !upArgs.snat
+	prefs.DisableDERP = !upArgs.enableDERP
+	prefs.Hostname = upArgs.hostname
+	if runtime.GOOS == "linux" {
+		switch upArgs.netfilterMode {
+		case "on":
+			prefs.NetfilterMode = router.NetfilterOn
+		case "nodivert":
+			prefs.NetfilterMode = router.NetfilterNoDivert
+			warning("netfilter=nodivert; add iptables calls to ts-* chains manually.")
+		case "off":
+			prefs.NetfilterMode = router.NetfilterOff
+			warning("netfilter=off; configure iptables yourself.")
+		default:
+			log.Fatalf("invalid value --netfilter-mode: %q", upArgs.netfilterMode)
+		}
+	}
+
+	c, bc, ctx, cancel := connect(ctx)
+	defer cancel()
+
+	var printed bool
+
+	bc.SetPrefs(prefs)
+	opts := ipn.Options{
+		StateKey: globalStateKey,
+		AuthKey:  upArgs.authKey,
+		Notify: func(n ipn.Notify) {
+			if n.ErrMessage != nil {
+				log.Fatalf("backend error: %v\n", *n.ErrMessage)
+			}
+			if s := n.State; s != nil {
+				switch *s {
+				case ipn.NeedsLogin:
+					printed = true
+					bc.StartLoginInteractive()
+				case ipn.NeedsMachineAuth:
+					printed = true
+					fmt.Fprintf(os.Stderr, "\nTo authorize your machine, visit (as admin):\n\n\t%s/admin/machines\n\n", upArgs.server)
+				case ipn.Starting, ipn.Running:
+					// Done full authentication process
+					if printed {
+						// Only need to print an update if we printed the "please click" message earlier.
+						fmt.Fprintf(os.Stderr, "Success.\n")
+					}
+					cancel()
+				}
+			}
+			if url := n.BrowseToURL; url != nil {
+				fmt.Fprintf(os.Stderr, "\nTo authenticate, visit:\n\n\t%s\n\n", *url)
+			}
+		},
+	}
+	// We still have to Start right now because it's the only way to
+	// set up notifications and whatnot. This causes a bunch of churn
+	// every time the CLI touches anything.
+	//
+	// TODO(danderson): redo the frontend/backend API to assume
+	// ephemeral frontends that read/modify/write state, once
+	// Windows/Mac state is moved into backend.
+	bc.Start(opts)
+	pump(ctx, bc, c)
+
+	return nil
+}

cmd/tailscale/cli/version.go

@@ -0,0 +1,69 @@
+// Copyright (c) 2020 Tailscale Inc & AUTHORS All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package cli
+
+import (
+	"context"
+	"flag"
+	"fmt"
+	"log"
+
+	"github.com/peterbourgon/ff/v2/ffcli"
+	"tailscale.com/ipn"
+	"tailscale.com/version"
+)
+
+var versionCmd = &ffcli.Command{
+	Name:       "version",
+	ShortUsage: "version [flags]",
+	ShortHelp:  "Print Tailscale version",
+	FlagSet: (func() *flag.FlagSet {
+		fs := flag.NewFlagSet("version", flag.ExitOnError)
+		fs.BoolVar(&versionArgs.daemon, "daemon", false, "also print local node's daemon version")
+		return fs
+	})(),
+	Exec: runVersion,
+}
+
+var versionArgs struct {
+	daemon bool // also check local node's daemon version
+}
+
+func runVersion(ctx context.Context, args []string) error {
+	if len(args) > 0 {
+		log.Fatalf("too many non-flag arguments: %q", args)
+	}
+	if !versionArgs.daemon {
+		fmt.Println(version.LONG)
+		return nil
+	}
+	fmt.Printf("Client: %s\n", version.LONG)
+
+	c, bc, ctx, cancel := connect(ctx)
+	defer cancel()
+
+	bc.AllowVersionSkew = true
+
+	done := make(chan struct{})
+
+	bc.SetNotifyCallback(func(n ipn.Notify) {
+		if n.ErrMessage != nil {
+			log.Fatal(*n.ErrMessage)
+		}
+		if n.Status != nil {
+			fmt.Printf("Daemon: %s\n", n.Version)
+			close(done)
+		}
+	})
+	go pump(ctx, bc, c)
+
+	bc.RequestStatus()
+	select {
+	case <-done:
+		return nil
+	case <-ctx.Done():
+		return ctx.Err()
+	}
+}

cmd/tailscale/tailscale.go

@@ -7,319 +7,22 @@
 package main // import "tailscale.com/cmd/tailscale"
 
 import (
-	"bytes"
-	"context"
-	"flag"
 	"fmt"
 	"log"
-	"net"
 	"os"
-	"os/exec"
-	"os/signal"
-	"runtime"
-	"strconv"
-	"strings"
-	"syscall"
 
 	"github.com/apenwarr/fixconsole"
-	"github.com/peterbourgon/ff/v2/ffcli"
-	"github.com/tailscale/wireguard-go/wgcfg"
-	"tailscale.com/ipn"
-	"tailscale.com/paths"
-	"tailscale.com/safesocket"
-	"tailscale.com/tailcfg"
-	"tailscale.com/wgengine/router"
+	"tailscale.com/cmd/tailscale/cli"
 )
 
-// globalStateKey is the ipn.StateKey that tailscaled loads on
-// startup.
-//
-// We have to support multiple state keys for other OSes (Windows in
-// particular), but right now Unix daemons run with a single
-// node-global state. To keep open the option of having per-user state
-// later, the global state key doesn't look like a username.
-const globalStateKey = "_daemon"
-
-var rootArgs struct {
-	socket string
-}
-
 func main() {
 	err := fixconsole.FixConsoleIfNeeded()
 	if err != nil {
 		log.Printf("fixConsoleOutput: %v\n", err)
 	}
 
-	upf := flag.NewFlagSet("up", flag.ExitOnError)
-	upf.StringVar(&upArgs.server, "login-server", "https://login.tailscale.com", "base URL of control server")
-	upf.BoolVar(&upArgs.acceptRoutes, "accept-routes", false, "accept routes advertised by other Tailscale nodes")
-	upf.BoolVar(&upArgs.singleRoutes, "host-routes", true, "install host routes to other Tailscale nodes")
-	upf.BoolVar(&upArgs.shieldsUp, "shields-up", false, "don't allow incoming connections")
-	upf.StringVar(&upArgs.advertiseTags, "advertise-tags", "", "ACL tags to request (comma-separated, e.g. eng,montreal,ssh)")
-	upf.StringVar(&upArgs.authKey, "authkey", "", "node authorization key")
-	upf.BoolVar(&upArgs.enableDERP, "enable-derp", true, "enable the use of DERP servers")
-	if runtime.GOOS == "linux" || isBSD(runtime.GOOS) {
-		upf.StringVar(&upArgs.advertiseRoutes, "advertise-routes", "", "routes to advertise to other nodes (comma-separated, e.g. 10.0.0.0/8,192.168.0.0/24)")
-	}
-	if runtime.GOOS == "linux" {
-		upf.BoolVar(&upArgs.snat, "snat-subnet-routes", true, "source NAT traffic to local routes advertised with -advertise-routes")
-		upf.StringVar(&upArgs.netfilterMode, "netfilter-mode", "on", "netfilter mode (one of on, nodivert, off)")
-	}
-	upCmd := &ffcli.Command{
-		Name:       "up",
-		ShortUsage: "up [flags]",
-		ShortHelp:  "Connect to your Tailscale network",
-
-		LongHelp: strings.TrimSpace(`
-"tailscale up" connects this machine to your Tailscale network,
-triggering authentication if necessary.
-
-The flags passed to this command are specific to this machine. If you don't
-specify any flags, options are reset to their default.
-`),
-		FlagSet: upf,
-		Exec:    runUp,
-	}
-
-	rootfs := flag.NewFlagSet("tailscale", flag.ExitOnError)
-	rootfs.StringVar(&rootArgs.socket, "socket", paths.DefaultTailscaledSocket(), "path to tailscaled's unix socket")
-
-	rootCmd := &ffcli.Command{
-		Name:       "tailscale",
-		ShortUsage: "tailscale subcommand [flags]",
-		ShortHelp:  "The easiest, most secure way to use WireGuard.",
-		LongHelp: strings.TrimSpace(`
-This CLI is still under active development. Commands and flags will
-change in the future.
-`),
-		Subcommands: []*ffcli.Command{
-			upCmd,
-			netcheckCmd,
-			statusCmd,
-		},
-		FlagSet: rootfs,
-		Exec:    func(context.Context, []string) error { return flag.ErrHelp },
-	}
-
-	if err := rootCmd.ParseAndRun(context.Background(), os.Args[1:]); err != nil && err != flag.ErrHelp {
-		log.Fatal(err)
-	}
-}
-
-var upArgs struct {
-	server          string
-	acceptRoutes    bool
-	singleRoutes    bool
-	shieldsUp       bool
-	advertiseRoutes string
-	advertiseTags   string
-	enableDERP      bool
-	snat            bool
-	netfilterMode   string
-	authKey         string
-}
-
-// parseIPOrCIDR parses an IP address or a CIDR prefix. If the input
-// is an IP address, it is returned in CIDR form with a /32 mask for
-// IPv4 or a /128 mask for IPv6.
-func parseIPOrCIDR(s string) (wgcfg.CIDR, bool) {
-	if strings.Contains(s, "/") {
-		ret, err := wgcfg.ParseCIDR(s)
-		if err != nil {
-			return wgcfg.CIDR{}, false
-		}
-		return ret, true
-	}
-
-	ip, ok := wgcfg.ParseIP(s)
-	if !ok {
-		return wgcfg.CIDR{}, false
-	}
-	if ip.Is4() {
-		return wgcfg.CIDR{ip, 32}, true
-	} else {
-		return wgcfg.CIDR{ip, 128}, true
-	}
-}
-
-func isBSD(s string) bool {
-	return s == "dragonfly" || s == "freebsd" || s == "netbsd" || s == "openbsd"
-}
-
-func warning(format string, args ...interface{}) {
-	fmt.Printf("Warning: "+format+"\n", args...)
-}
-
-// checkIPForwarding prints warnings on linux if IP forwarding is not
-// enabled, or if we were unable to verify the state of IP forwarding.
-func checkIPForwarding() {
-	var key string
-
-	if runtime.GOOS == "linux" {
-		key = "net.ipv4.ip_forward"
-	} else if isBSD(runtime.GOOS) {
-		key = "net.inet.ip.forwarding"
-	} else {
-		return
-	}
-
-	bs, err := exec.Command("sysctl", "-n", key).Output()
-	if err != nil {
-		warning("couldn't check %s (%v).\nSubnet routes won't work without IP forwarding.", key, err)
-		return
-	}
-	on, err := strconv.ParseBool(string(bytes.TrimSpace(bs)))
-	if err != nil {
-		warning("couldn't parse %s (%v).\nSubnet routes won't work without IP forwarding.", key, err)
-		return
-	}
-	if !on {
-		warning("%s is disabled. Subnet routes won't work.", key)
-	}
-}
-
-func runUp(ctx context.Context, args []string) error {
-	if len(args) > 0 {
-		log.Fatalf("too many non-flag arguments: %q", args)
-	}
-
-	var routes []wgcfg.CIDR
-	if upArgs.advertiseRoutes != "" {
-		checkIPForwarding()
-		advroutes := strings.Split(upArgs.advertiseRoutes, ",")
-		for _, s := range advroutes {
-			cidr, ok := parseIPOrCIDR(s)
-			if !ok {
-				log.Fatalf("%q is not a valid IP address or CIDR prefix", s)
-			}
-			routes = append(routes, cidr)
-		}
-	}
-
-	var tags []string
-	if upArgs.advertiseTags != "" {
-		tags = strings.Split(upArgs.advertiseTags, ",")
-		for _, tag := range tags {
-			err := tailcfg.CheckTag(tag)
-			if err != nil {
-				log.Fatalf("tag: %q: %s", tag, err)
-			}
-		}
-	}
-
-	// TODO(apenwarr): fix different semantics between prefs and uflags
-	// TODO(apenwarr): allow setting/using CorpDNS
-	prefs := ipn.NewPrefs()
-	prefs.ControlURL = upArgs.server
-	prefs.WantRunning = true
-	prefs.RouteAll = upArgs.acceptRoutes
-	prefs.AllowSingleHosts = upArgs.singleRoutes
-	prefs.ShieldsUp = upArgs.shieldsUp
-	prefs.AdvertiseRoutes = routes
-	prefs.AdvertiseTags = tags
-	prefs.NoSNAT = !upArgs.snat
-	prefs.DisableDERP = !upArgs.enableDERP
-	if runtime.GOOS == "linux" {
-		switch upArgs.netfilterMode {
-		case "on":
-			prefs.NetfilterMode = router.NetfilterOn
-		case "nodivert":
-			prefs.NetfilterMode = router.NetfilterNoDivert
-			warning("netfilter=nodivert; add iptables calls to ts-* chains manually.")
-		case "off":
-			prefs.NetfilterMode = router.NetfilterOff
-			warning("netfilter=off; configure iptables yourself.")
-		default:
-			log.Fatalf("invalid value --netfilter-mode: %q", upArgs.netfilterMode)
-		}
-	}
-
-	c, bc, ctx, cancel := connect(ctx)
-	defer cancel()
-
-	var printed bool
-
-	bc.SetPrefs(prefs)
-	opts := ipn.Options{
-		StateKey: globalStateKey,
-		AuthKey:  upArgs.authKey,
-		Notify: func(n ipn.Notify) {
-			if n.ErrMessage != nil {
-				log.Fatalf("backend error: %v\n", *n.ErrMessage)
-			}
-			if s := n.State; s != nil {
-				switch *s {
-				case ipn.NeedsLogin:
-					printed = true
-					bc.StartLoginInteractive()
-				case ipn.NeedsMachineAuth:
-					printed = true
-					fmt.Fprintf(os.Stderr, "\nTo authorize your machine, visit (as admin):\n\n\t%s/admin/machines\n\n", upArgs.server)
-				case ipn.Starting, ipn.Running:
-					// Done full authentication process
-					if printed {
-						// Only need to print an update if we printed the "please click" message earlier.
-						fmt.Fprintf(os.Stderr, "Success.\n")
-					}
-					cancel()
-				}
-			}
-			if url := n.BrowseToURL; url != nil {
-				fmt.Fprintf(os.Stderr, "\nTo authenticate, visit:\n\n\t%s\n\n", *url)
-			}
-		},
-	}
-	// We still have to Start right now because it's the only way to
-	// set up notifications and whatnot. This causes a bunch of churn
-	// every time the CLI touches anything.
-	//
-	// TODO(danderson): redo the frontend/backend API to assume
-	// ephemeral frontends that read/modify/write state, once
-	// Windows/Mac state is moved into backend.
-	bc.Start(opts)
-	pump(ctx, bc, c)
-
-	return nil
-}
-
-func connect(ctx context.Context) (net.Conn, *ipn.BackendClient, context.Context, context.CancelFunc) {
-	c, err := safesocket.Connect(rootArgs.socket, 41112)
-	if err != nil {
-		if runtime.GOOS != "windows" && rootArgs.socket == "" {
-			log.Fatalf("--socket cannot be empty")
-		}
-		log.Fatalf("Failed to connect to connect to tailscaled. (safesocket.Connect: %v)\n", err)
-	}
-	clientToServer := func(b []byte) {
-		ipn.WriteMsg(c, b)
-	}
-
-	ctx, cancel := context.WithCancel(ctx)
-
-	go func() {
-		interrupt := make(chan os.Signal, 1)
-		signal.Notify(interrupt, syscall.SIGINT, syscall.SIGTERM)
-		<-interrupt
-		c.Close()
-		cancel()
-	}()
-
-	bc := ipn.NewBackendClient(log.Printf, clientToServer)
-	return c, bc, ctx, cancel
-}
-
-// pump receives backend messages on conn and pushes them into bc.
-func pump(ctx context.Context, bc *ipn.BackendClient, conn net.Conn) {
-	defer conn.Close()
-	for ctx.Err() == nil {
-		msg, err := ipn.ReadMsg(conn)
-		if err != nil {
-			if ctx.Err() != nil {
-				return
-			}
-			log.Printf("ReadMsg: %v\n", err)
-			break
-		}
-		bc.GotNotifyMsg(msg)
+	if err := cli.Run(os.Args[1:]); err != nil {
+		fmt.Fprintln(os.Stderr, err)
+		os.Exit(1)
 	}
 }

cmd/tailscaled/tailscaled.go

@@ -15,8 +15,10 @@ import (
 	"net/http"
 	"net/http/pprof"
 	"os"
+	"os/signal"
 	"runtime"
 	"runtime/debug"
+	"syscall"
 	"time"
 
 	"github.com/apenwarr/fixconsole"
@@ -27,6 +29,7 @@ import (
 	"tailscale.com/types/logger"
 	"tailscale.com/wgengine"
 	"tailscale.com/wgengine/magicsock"
+	"tailscale.com/wgengine/router"
 )
 
 // globalStateKey is the ipn.StateKey that tailscaled loads on
@@ -38,6 +41,27 @@ import (
 // later, the global state key doesn't look like a username.
 const globalStateKey = "_daemon"
 
+// defaultTunName returns the default tun device name for the platform.
+func defaultTunName() string {
+	switch runtime.GOOS {
+	case "openbsd":
+		return "tun"
+	case "windows":
+		return "Tailscale"
+	}
+	return "tailscale0"
+}
+
+var args struct {
+	cleanup    bool
+	fake       bool
+	debug      string
+	tunname    string
+	port       uint16
+	statepath  string
+	socketpath string
+}
+
 func main() {
 	// We aren't very performance sensitive, and the parts that are
 	// performance sensitive (wireguard) try hard not to do any memory
@@ -47,77 +71,112 @@ func main() {
 		debug.SetGCPercent(10)
 	}
 
-	defaultTunName := "tailscale0"
-	if runtime.GOOS == "openbsd" {
-		defaultTunName = "tun"
-	}
+	// Set default values for getopt.
+	args.tunname = defaultTunName()
+	args.port = magicsock.DefaultPort
+	args.statepath = paths.DefaultTailscaledStateFile()
+	args.socketpath = paths.DefaultTailscaledSocket()
 
-	fake := getopt.BoolLong("fake", 0, "fake tunnel+routing instead of tuntap")
-	debug := getopt.StringLong("debug", 0, "", "Address of debug server")
-	tunname := getopt.StringLong("tun", 0, defaultTunName, "tunnel interface name")
-	listenport := getopt.Uint16Long("port", 'p', magicsock.DefaultPort, "WireGuard port (0=autoselect)")
-	statepath := getopt.StringLong("state", 0, paths.DefaultTailscaledStateFile(), "Path of state file")
-	socketpath := getopt.StringLong("socket", 's', paths.DefaultTailscaledSocket(), "Path of the service unix socket")
-
-	logf := wgengine.RusagePrefixLog(log.Printf)
-	logf = logger.RateLimitedFn(logf, 5*time.Second, 5, 100)
+	getopt.FlagLong(&args.cleanup, "cleanup", 0, "clean up system state and exit")
+	getopt.FlagLong(&args.fake, "fake", 0, "fake tunnel+routing instead of tuntap")
+	getopt.FlagLong(&args.debug, "debug", 0, "address of debug server")
+	getopt.FlagLong(&args.tunname, "tun", 0, "tunnel interface name")
+	getopt.FlagLong(&args.port, "port", 'p', "WireGuard port (0=autoselect)")
+	getopt.FlagLong(&args.statepath, "state", 0, "path of state file")
+	getopt.FlagLong(&args.socketpath, "socket", 's', "path of the service unix socket")
 
 	err := fixconsole.FixConsoleIfNeeded()
 	if err != nil {
-		logf("fixConsoleOutput: %v", err)
+		log.Fatalf("fixConsoleOutput: %v", err)
 	}
-	pol := logpolicy.New("tailnode.log.tailscale.io")
 
 	getopt.Parse()
 	if len(getopt.Args()) > 0 {
 		log.Fatalf("too many non-flag arguments: %#v", getopt.Args()[0])
 	}
 
-	if *statepath == "" {
+	if args.statepath == "" {
 		log.Fatalf("--state is required")
 	}
 
-	if *socketpath == "" {
+	if args.socketpath == "" && runtime.GOOS != "windows" {
 		log.Fatalf("--socket is required")
 	}
 
+	if err := run(); err != nil {
+		// No need to log; the func already did
+		os.Exit(1)
+	}
+}
+
+func run() error {
+	var err error
+
+	pol := logpolicy.New("tailnode.log.tailscale.io")
+	defer func() {
+		// Finish uploading logs after closing everything else.
+		ctx, cancel := context.WithTimeout(context.Background(), time.Second)
+		defer cancel()
+		pol.Shutdown(ctx)
+	}()
+
+	logf := wgengine.RusagePrefixLog(log.Printf)
+	logf = logger.RateLimitedFn(logf, 5*time.Second, 5, 100)
+
+	if args.cleanup {
+		router.Cleanup(logf, args.tunname)
+		return nil
+	}
+
 	var debugMux *http.ServeMux
-	if *debug != "" {
+	if args.debug != "" {
 		debugMux = newDebugMux()
-		go runDebugServer(debugMux, *debug)
+		go runDebugServer(debugMux, args.debug)
 	}
 
 	var e wgengine.Engine
-	if *fake {
+	if args.fake {
 		e, err = wgengine.NewFakeUserspaceEngine(logf, 0)
 	} else {
-		e, err = wgengine.NewUserspaceEngine(logf, *tunname, *listenport)
+		e, err = wgengine.NewUserspaceEngine(logf, args.tunname, args.port)
 	}
 	if err != nil {
-		log.Fatalf("wgengine.New: %v", err)
+		logf("wgengine.New: %v", err)
+		return err
 	}
 	e = wgengine.NewWatchdog(e)
 
+	ctx, cancel := context.WithCancel(context.Background())
+	// Exit gracefully by cancelling the ipnserver context in most common cases:
+	// interrupted from the TTY or killed by a service manager.
+	go func() {
+		interrupt := make(chan os.Signal, 1)
+		signal.Notify(interrupt, syscall.SIGINT, syscall.SIGTERM)
+		select {
+		case <-interrupt:
+			cancel()
+		case <-ctx.Done():
+			// continue
+		}
+	}()
+
 	opts := ipnserver.Options{
-		SocketPath:         *socketpath,
+		SocketPath:         args.socketpath,
 		Port:               41112,
-		StatePath:          *statepath,
+		StatePath:          args.statepath,
 		AutostartStateKey:  globalStateKey,
-		LegacyConfigPath:   paths.LegacyConfigPath,
+		LegacyConfigPath:   paths.LegacyConfigPath(),
 		SurviveDisconnects: true,
 		DebugMux:           debugMux,
 	}
-	err = ipnserver.Run(context.Background(), logf, pol.PublicID.String(), opts, e)
-	if err != nil {
-		log.Fatalf("tailscaled: %v", err)
+	err = ipnserver.Run(ctx, logf, pol.PublicID.String(), opts, e)
+	// Cancelation is not an error: it is the only way to stop ipnserver.
+	if err != nil && err != context.Canceled {
+		logf("ipnserver.Run: %v", err)
+		return err
 	}
 
-	// TODO(crawshaw): It would be nice to start a timeout context the moment a signal
-	// is received and use that timeout to give us a moment to finish uploading logs
-	// here. But the signal is handled inside ipnserver.Run, so some plumbing is needed.
-	ctx, cancel := context.WithCancel(context.Background())
-	cancel()
-	pol.Shutdown(ctx)
+	return nil
 }
 
 func newDebugMux() *http.ServeMux {

cmd/tailscaled/tailscaled.service

@@ -9,6 +9,7 @@ StartLimitBurst=0
 [Service]
 EnvironmentFile=/etc/default/tailscaled
 ExecStart=/usr/sbin/tailscaled --state=/var/lib/tailscale/tailscaled.state --socket=/run/tailscale/tailscaled.sock --port $PORT $FLAGS
+ExecStopPost=/usr/sbin/tailscaled --cleanup
 
 Restart=on-failure
 

control/controlclient/auto.go

@@ -355,12 +355,13 @@ func (c *Client) authRoutine() {
 					err = fmt.Errorf("weird: server required a new url?")
 					report(err, "WaitLoginURL")
 				}
-				goal.url = url
-				goal.token = nil
-				goal.flags = LoginDefault
 
 				c.mu.Lock()
-				c.loginGoal = goal
+				c.loginGoal = &LoginGoal{
+					wantLoggedIn: true,
+					flags:        LoginDefault,
+					url:          url,
+				}
 				c.state = StateURLVisitRequired
 				c.synced = false
 				c.mu.Unlock()

control/controlclient/direct.go

@@ -266,7 +266,7 @@ func (c *Direct) doLogin(ctx context.Context, t *oauth2.Token, flags LoginFlags,
 	tryingNewKey := c.tryingNewKey
 	serverKey := c.serverKey
 	authKey := c.authKey
-	hostinfo := c.hostinfo
+	hostinfo := c.hostinfo.Clone()
 	backendLogID := hostinfo.BackendLogID
 	expired := c.expiry != nil && !c.expiry.IsZero() && c.expiry.Before(c.timeNow())
 	c.mu.Unlock()
@@ -456,7 +456,7 @@ func (c *Direct) PollNetMap(ctx context.Context, maxPolls int, cb func(*NetworkM
 	persist := c.persist
 	serverURL := c.serverURL
 	serverKey := c.serverKey
-	hostinfo := c.hostinfo
+	hostinfo := c.hostinfo.Clone()
 	backendLogID := hostinfo.BackendLogID
 	localPort := c.localPort
 	ep := append([]string(nil), c.endpoints...)
@@ -678,8 +678,10 @@ func decode(res *http.Response, v interface{}, serverKey *wgcfg.Key, mkey *wgcfg
 }
 
 func (c *Direct) decodeMsg(msg []byte, v interface{}) error {
+	c.mu.Lock()
 	mkey := c.persist.PrivateMachineKey
 	serverKey := c.serverKey
+	c.mu.Unlock()
 
 	decrypted, err := decryptMsg(msg, &serverKey, &mkey)
 	if err != nil {

control/controlclient/netmap.go

@@ -5,12 +5,12 @@
 package controlclient
 
 import (
-	"bytes"
-	"encoding/base64"
 	"encoding/json"
 	"fmt"
 	"log"
+	"net"
 	"reflect"
+	"strconv"
 	"strings"
 	"time"
 
@@ -29,7 +29,7 @@ type NetworkMap struct {
 	Addresses     []wgcfg.CIDR
 	LocalPort     uint16 // used for debugging
 	MachineStatus tailcfg.MachineStatus
-	Peers         []*tailcfg.Node
+	Peers         []*tailcfg.Node // sorted by Node.ID
 	DNS           []wgcfg.IP
 	DNSDomains    []string
 	Hostinfo      tailcfg.Hostinfo
@@ -54,25 +54,25 @@ type NetworkMap struct {
 	// TODO(crawshaw): Capabilities []tailcfg.Capability
 }
 
-func (n *NetworkMap) Equal(n2 *NetworkMap) bool {
-	// TODO(crawshaw): this is crude, but is an easy way to avoid bugs.
-	b, err := json.Marshal(n)
-	if err != nil {
-		panic(err)
-	}
-	b2, err := json.Marshal(n2)
-	if err != nil {
-		panic(err)
-	}
-	return bytes.Equal(b, b2)
-}
-
 func (nm NetworkMap) String() string {
 	return nm.Concise()
 }
 
 func (nm *NetworkMap) Concise() string {
 	buf := new(strings.Builder)
+
+	nm.printConciseHeader(buf)
+	for _, p := range nm.Peers {
+		printPeerConcise(buf, p)
+	}
+	return buf.String()
+}
+
+// printConciseHeader prints a concise header line representing nm to buf.
+//
+// If this function is changed to access different fields of nm, keep
+// in equalConciseHeader in sync.
+func (nm *NetworkMap) printConciseHeader(buf *strings.Builder) {
 	fmt.Fprintf(buf, "netmap: self: %v auth=%v",
 		nm.NodeKey.ShortString(), nm.MachineStatus)
 	if nm.LocalPort != 0 {
@@ -84,72 +84,116 @@ func (nm *NetworkMap) Concise() string {
 	}
 	fmt.Fprintf(buf, " %v", nm.Addresses)
 	buf.WriteByte('\n')
-	for _, p := range nm.Peers {
-		aip := make([]string, len(p.AllowedIPs))
-		for i, a := range p.AllowedIPs {
-			s := strings.TrimSuffix(fmt.Sprint(a), "/32")
-			aip[i] = s
-		}
+}
 
-		ep := make([]string, len(p.Endpoints))
-		for i, e := range p.Endpoints {
-			// Align vertically on the ':' between IP and port
-			colon := strings.IndexByte(e, ':')
-			spaces := 0
-			for colon > 0 && len(e)+spaces-colon < 6 {
-				spaces++
-				colon--
-			}
-			ep[i] = fmt.Sprintf("%21v", e+strings.Repeat(" ", spaces))
-		}
-
-		derp := p.DERP
-		const derpPrefix = "127.3.3.40:"
-		if strings.HasPrefix(derp, derpPrefix) {
-			derp = "D" + derp[len(derpPrefix):]
-		}
-
-		// Most of the time, aip is just one element, so format the
-		// table to look good in that case. This will also make multi-
-		// subnet nodes stand out visually.
-		fmt.Fprintf(buf, " %v %-2v %-15v : %v\n",
-			p.Key.ShortString(), derp,
-			strings.Join(aip, " "),
-			strings.Join(ep, " "))
+// equalConciseHeader reports whether a and b are equal for the fields
+// used by printConciseHeader.
+func (a *NetworkMap) equalConciseHeader(b *NetworkMap) bool {
+	if a.NodeKey != b.NodeKey ||
+		a.MachineStatus != b.MachineStatus ||
+		a.LocalPort != b.LocalPort ||
+		len(a.Addresses) != len(b.Addresses) {
+		return false
 	}
-	return buf.String()
+	for i, a := range a.Addresses {
+		if b.Addresses[i] != a {
+			return false
+		}
+	}
+	return (a.Debug == nil && b.Debug == nil) || reflect.DeepEqual(a.Debug, b.Debug)
+}
+
+// printPeerConcise appends to buf a line repsenting the peer p.
+//
+// If this function is changed to access different fields of p, keep
+// in nodeConciseEqual in sync.
+func printPeerConcise(buf *strings.Builder, p *tailcfg.Node) {
+	aip := make([]string, len(p.AllowedIPs))
+	for i, a := range p.AllowedIPs {
+		s := strings.TrimSuffix(fmt.Sprint(a), "/32")
+		aip[i] = s
+	}
+
+	ep := make([]string, len(p.Endpoints))
+	for i, e := range p.Endpoints {
+		// Align vertically on the ':' between IP and port
+		colon := strings.IndexByte(e, ':')
+		spaces := 0
+		for colon > 0 && len(e)+spaces-colon < 6 {
+			spaces++
+			colon--
+		}
+		ep[i] = fmt.Sprintf("%21v", e+strings.Repeat(" ", spaces))
+	}
+
+	derp := p.DERP
+	const derpPrefix = "127.3.3.40:"
+	if strings.HasPrefix(derp, derpPrefix) {
+		derp = "D" + derp[len(derpPrefix):]
+	}
+
+	// Most of the time, aip is just one element, so format the
+	// table to look good in that case. This will also make multi-
+	// subnet nodes stand out visually.
+	fmt.Fprintf(buf, " %v %-2v %-15v : %v\n",
+		p.Key.ShortString(), derp,
+		strings.Join(aip, " "),
+		strings.Join(ep, " "))
+}
+
+// nodeConciseEqual reports whether a and b are equal for the fields accessed by printPeerConcise.
+func nodeConciseEqual(a, b *tailcfg.Node) bool {
+	return a.Key == b.Key &&
+		a.DERP == b.DERP &&
+		eqCIDRsIgnoreNil(a.AllowedIPs, b.AllowedIPs) &&
+		eqStringsIgnoreNil(a.Endpoints, b.Endpoints)
 }
 
 func (b *NetworkMap) ConciseDiffFrom(a *NetworkMap) string {
-	if reflect.DeepEqual(a, b) {
-		// Fast path that only does one allocation.
-		return ""
-	}
-	out := []string{}
-	ra := strings.Split(a.Concise(), "\n")
-	rb := strings.Split(b.Concise(), "\n")
+	var diff strings.Builder
 
-	ma := map[string]struct{}{}
-	for _, s := range ra {
-		ma[s] = struct{}{}
+	// See if header (non-peers, "bare") part of the network map changed.
+	// If so, print its diff lines first.
+	if !a.equalConciseHeader(b) {
+		diff.WriteByte('-')
+		a.printConciseHeader(&diff)
+		diff.WriteByte('+')
+		b.printConciseHeader(&diff)
 	}
 
-	mb := map[string]struct{}{}
-	for _, s := range rb {
-		mb[s] = struct{}{}
-	}
-
-	for _, s := range ra {
-		if _, ok := mb[s]; !ok {
-			out = append(out, "-"+s)
+	aps, bps := a.Peers, b.Peers
+	for len(aps) > 0 && len(bps) > 0 {
+		pa, pb := aps[0], bps[0]
+		switch {
+		case pa.ID == pb.ID:
+			if !nodeConciseEqual(pa, pb) {
+				diff.WriteByte('-')
+				printPeerConcise(&diff, pa)
+				diff.WriteByte('+')
+				printPeerConcise(&diff, pb)
+			}
+			aps, bps = aps[1:], bps[1:]
+		case pa.ID > pb.ID:
+			// New peer in b.
+			diff.WriteByte('+')
+			printPeerConcise(&diff, pb)
+			bps = bps[1:]
+		case pb.ID > pa.ID:
+			// Deleted peer in b.
+			diff.WriteByte('-')
+			printPeerConcise(&diff, pa)
+			aps = aps[1:]
 		}
 	}
-	for _, s := range rb {
-		if _, ok := ma[s]; !ok {
-			out = append(out, "+"+s)
-		}
+	for _, pa := range aps {
+		diff.WriteByte('-')
+		printPeerConcise(&diff, pa)
 	}
-	return strings.Join(out, "\n")
+	for _, pb := range bps {
+		diff.WriteByte('+')
+		printPeerConcise(&diff, pb)
+	}
+	return diff.String()
 }
 
 func (nm *NetworkMap) JSON() string {
@@ -160,141 +204,141 @@ func (nm *NetworkMap) JSON() string {
 	return string(b)
 }
 
+// WGConfigFlags is a bitmask of flags to control the behavior of the
+// wireguard configuration generation done by NetMap.WGCfg.
+type WGConfigFlags int
+
 const (
-	UAllowSingleHosts = 1 << iota
-	UAllowSubnetRoutes
-	UAllowDefaultRoute
-	UHackDefaultRoute
-
-	UDefault = 0
+	AllowSingleHosts WGConfigFlags = 1 << iota
+	AllowSubnetRoutes
+	AllowDefaultRoute
+	HackDefaultRoute
 )
 
-// Several programs need to parse these arguments into uflags, so let's
-// centralize it here.
-func UFlagsHelper(uroutes, rroutes, droutes bool) int {
-	uflags := 0
-	if uroutes {
-		uflags |= UAllowSingleHosts
-	}
-	if rroutes {
-		uflags |= UAllowSubnetRoutes
-	}
-	if droutes {
-		uflags |= UAllowDefaultRoute
-	}
-	return uflags
-}
-
 // TODO(bradfitz): UAPI seems to only be used by the old confnode and
 // pingnode; delete this when those are deleted/rewritten?
-func (nm *NetworkMap) UAPI(uflags int, dnsOverride []wgcfg.IP) string {
-	wgcfg, err := nm.WGCfg(log.Printf, uflags, dnsOverride)
+func (nm *NetworkMap) UAPI(flags WGConfigFlags, dnsOverride []wgcfg.IP) string {
+	wgcfg, err := nm.WGCfg(log.Printf, flags, dnsOverride)
 	if err != nil {
-		log.Fatalf("WGCfg() failed unexpectedly: %v\n", err)
+		log.Fatalf("WGCfg() failed unexpectedly: %v", err)
 	}
 	s, err := wgcfg.ToUAPI()
 	if err != nil {
-		log.Fatalf("ToUAPI() failed unexpectedly: %v\n", err)
+		log.Fatalf("ToUAPI() failed unexpectedly: %v", err)
 	}
 	return s
 }
 
-func (nm *NetworkMap) WGCfg(logf logger.Logf, uflags int, dnsOverride []wgcfg.IP) (*wgcfg.Config, error) {
-	s := nm._WireGuardConfig(logf, uflags, dnsOverride, true)
-	return wgcfg.FromWgQuick(s, "tailscale")
-}
-
 // EndpointDiscoSuffix is appended to the hex representation of a peer's discovery key
 // and is then the sole wireguard endpoint for peers with a non-zero discovery key.
 // This form is then recognize by magicsock's CreateEndpoint.
 const EndpointDiscoSuffix = ".disco.tailscale:12345"
 
-func (nm *NetworkMap) _WireGuardConfig(logf logger.Logf, uflags int, dnsOverride []wgcfg.IP, allEndpoints bool) string {
-	buf := new(strings.Builder)
-	fmt.Fprintf(buf, "[Interface]\n")
-	fmt.Fprintf(buf, "PrivateKey = %s\n", base64.StdEncoding.EncodeToString(nm.PrivateKey[:]))
-	if len(nm.Addresses) > 0 {
-		fmt.Fprintf(buf, "Address = ")
-		for i, cidr := range nm.Addresses {
-			if i > 0 {
-				fmt.Fprintf(buf, ", ")
-			}
-			fmt.Fprintf(buf, "%s", cidr)
-		}
-		fmt.Fprintf(buf, "\n")
+// WGCfg returns the NetworkMaps's Wireguard configuration.
+func (nm *NetworkMap) WGCfg(logf logger.Logf, flags WGConfigFlags, dnsOverride []wgcfg.IP) (*wgcfg.Config, error) {
+	cfg := &wgcfg.Config{
+		Name:       "tailscale",
+		PrivateKey: nm.PrivateKey,
+		Addresses:  nm.Addresses,
+		ListenPort: nm.LocalPort,
+		DNS:        append([]wgcfg.IP(nil), dnsOverride...),
+		Peers:      make([]wgcfg.Peer, 0, len(nm.Peers)),
 	}
-	fmt.Fprintf(buf, "ListenPort = %d\n", nm.LocalPort)
-	if len(dnsOverride) > 0 {
-		dnss := []string{}
-		for _, ip := range dnsOverride {
-			dnss = append(dnss, ip.String())
-		}
-		fmt.Fprintf(buf, "DNS = %s\n", strings.Join(dnss, ","))
-	}
-	fmt.Fprintf(buf, "\n")
 
-	for i, peer := range nm.Peers {
+	for _, peer := range nm.Peers {
 		if Debug.OnlyDisco && peer.DiscoKey.IsZero() {
 			continue
 		}
-		if (uflags&UAllowSingleHosts) == 0 && len(peer.AllowedIPs) < 2 {
-			logf("wgcfg: %v skipping a single-host peer.\n", peer.Key.ShortString())
+		if (flags&AllowSingleHosts) == 0 && len(peer.AllowedIPs) < 2 {
+			logf("wgcfg: %v skipping a single-host peer.", peer.Key.ShortString())
 			continue
 		}
-		if i > 0 {
-			fmt.Fprintf(buf, "\n")
+		cfg.Peers = append(cfg.Peers, wgcfg.Peer{
+			PublicKey: wgcfg.Key(peer.Key),
+		})
+		cpeer := &cfg.Peers[len(cfg.Peers)-1]
+		if peer.KeepAlive {
+			cpeer.PersistentKeepalive = 25 // seconds
 		}
-		fmt.Fprintf(buf, "[Peer]\n")
-		fmt.Fprintf(buf, "PublicKey = %s\n", base64.StdEncoding.EncodeToString(peer.Key[:]))
-		var endpoints []string
+
 		if !peer.DiscoKey.IsZero() {
-			fmt.Fprintf(buf, "Endpoint = %x%s\n", peer.DiscoKey[:], EndpointDiscoSuffix)
-		} else {
-			if peer.DERP != "" {
-				endpoints = append(endpoints, peer.DERP)
+			if err := appendEndpoint(cpeer, fmt.Sprintf("%x%s", peer.DiscoKey[:], EndpointDiscoSuffix)); err != nil {
+				return nil, err
 			}
-			endpoints = append(endpoints, peer.Endpoints...)
-			if len(endpoints) > 0 {
-				if len(endpoints) == 1 {
-					fmt.Fprintf(buf, "Endpoint = %s", endpoints[0])
-				} else if allEndpoints {
-					// TODO(apenwarr): This mode is incompatible.
-					// Normal wireguard clients don't know how to
-					// parse it (yet?)
-					fmt.Fprintf(buf, "Endpoint = %s", strings.Join(endpoints, ","))
-				} else {
-					fmt.Fprintf(buf, "Endpoint = %s # other endpoints: %s",
-						endpoints[0],
-						strings.Join(endpoints[1:], ", "))
+			cpeer.Endpoints = []wgcfg.Endpoint{{Host: fmt.Sprintf("%x.disco.tailscale", peer.DiscoKey[:]), Port: 12345}}
+		} else {
+			if err := appendEndpoint(cpeer, peer.DERP); err != nil {
+				return nil, err
+			}
+			for _, ep := range peer.Endpoints {
+				if err := appendEndpoint(cpeer, ep); err != nil {
+					return nil, err
 				}
-				buf.WriteByte('\n')
 			}
 		}
-		var aips []string
 		for _, allowedIP := range peer.AllowedIPs {
-			aip := allowedIP.String()
 			if allowedIP.Mask == 0 {
-				if (uflags & UAllowDefaultRoute) == 0 {
-					logf("wgcfg: %v skipping default route\n", peer.Key.ShortString())
+				if (flags & AllowDefaultRoute) == 0 {
+					logf("wgcfg: %v skipping default route", peer.Key.ShortString())
 					continue
 				}
-				if (uflags & UHackDefaultRoute) != 0 {
-					aip = "10.0.0.0/8"
-					logf("wgcfg: %v converting default route => %v\n", peer.Key.ShortString(), aip)
+				if (flags & HackDefaultRoute) != 0 {
+					allowedIP = wgcfg.CIDR{IP: wgcfg.IPv4(10, 0, 0, 0), Mask: 8}
+					logf("wgcfg: %v converting default route => %v", peer.Key.ShortString(), allowedIP.String())
 				}
 			} else if allowedIP.Mask < 32 {
-				if (uflags & UAllowSubnetRoutes) == 0 {
-					logf("wgcfg: %v skipping subnet route\n", peer.Key.ShortString())
+				if (flags & AllowSubnetRoutes) == 0 {
+					logf("wgcfg: %v skipping subnet route", peer.Key.ShortString())
 					continue
 				}
 			}
-			aips = append(aips, aip)
-		}
-		fmt.Fprintf(buf, "AllowedIPs = %s\n", strings.Join(aips, ", "))
-		if peer.KeepAlive {
-			fmt.Fprintf(buf, "PersistentKeepalive = 25\n")
+			cpeer.AllowedIPs = append(cpeer.AllowedIPs, allowedIP)
 		}
 	}
 
-	return buf.String()
+	return cfg, nil
+}
+
+func appendEndpoint(peer *wgcfg.Peer, epStr string) error {
+	if epStr == "" {
+		return nil
+	}
+	host, port, err := net.SplitHostPort(epStr)
+	if err != nil {
+		return fmt.Errorf("malformed endpoint %q for peer %v", epStr, peer.PublicKey.ShortString())
+	}
+	port16, err := strconv.ParseUint(port, 10, 16)
+	if err != nil {
+		return fmt.Errorf("invalid port in endpoint %q for peer %v", epStr, peer.PublicKey.ShortString())
+	}
+	peer.Endpoints = append(peer.Endpoints, wgcfg.Endpoint{Host: host, Port: uint16(port16)})
+	return nil
+}
+
+// eqStringsIgnoreNil reports whether a and b have the same length and
+// contents, but ignore whether a or b are nil.
+func eqStringsIgnoreNil(a, b []string) bool {
+	if len(a) != len(b) {
+		return false
+	}
+	for i, v := range a {
+		if v != b[i] {
+			return false
+		}
+	}
+	return true
+}
+
+// eqCIDRsIgnoreNil reports whether a and b have the same length and
+// contents, but ignore whether a or b are nil.
+func eqCIDRsIgnoreNil(a, b []wgcfg.CIDR) bool {
+	if len(a) != len(b) {
+		return false
+	}
+	for i, v := range a {
+		if v != b[i] {
+			return false
+		}
+	}
+	return true
 }

control/controlclient/netmap_test.go

@@ -10,13 +10,14 @@ import (
 	"tailscale.com/tailcfg"
 )
 
-func TestNetworkMapConcise(t *testing.T) {
-	nodekey := func(b byte) (ret tailcfg.NodeKey) {
-		for i := range ret {
-			ret[i] = b
-		}
-		return
+func testNodeKey(b byte) (ret tailcfg.NodeKey) {
+	for i := range ret {
+		ret[i] = b
 	}
+	return
+}
+
+func TestNetworkMapConcise(t *testing.T) {
 	for _, tt := range []struct {
 		name string
 		nm   *NetworkMap
@@ -25,15 +26,15 @@ func TestNetworkMapConcise(t *testing.T) {
 		{
 			name: "basic",
 			nm: &NetworkMap{
-				NodeKey: nodekey(1),
+				NodeKey: testNodeKey(1),
 				Peers: []*tailcfg.Node{
 					{
-						Key:       nodekey(2),
+						Key:       testNodeKey(2),
 						DERP:      "127.3.3.40:2",
 						Endpoints: []string{"192.168.0.100:12", "192.168.0.100:12354"},
 					},
 					{
-						Key:       nodekey(3),
+						Key:       testNodeKey(3),
 						DERP:      "127.3.3.40:4",
 						Endpoints: []string{"10.2.0.100:12", "10.1.0.100:12345"},
 					},
@@ -44,7 +45,7 @@ func TestNetworkMapConcise(t *testing.T) {
 		{
 			name: "debug_non_nil",
 			nm: &NetworkMap{
-				NodeKey: nodekey(1),
+				NodeKey: testNodeKey(1),
 				Debug:   &tailcfg.Debug{},
 			},
 			want: "netmap: self: [AQEBA] auth=machine-unknown debug={} []\n",
@@ -52,7 +53,7 @@ func TestNetworkMapConcise(t *testing.T) {
 		{
 			name: "debug_values",
 			nm: &NetworkMap{
-				NodeKey: nodekey(1),
+				NodeKey: testNodeKey(1),
 				Debug:   &tailcfg.Debug{LogHeapPprof: true},
 			},
 			want: "netmap: self: [AQEBA] auth=machine-unknown debug={\"LogHeapPprof\":true} []\n",
@@ -70,3 +71,147 @@ func TestNetworkMapConcise(t *testing.T) {
 		})
 	}
 }
+
+func TestConciseDiffFrom(t *testing.T) {
+	for _, tt := range []struct {
+		name string
+		a, b *NetworkMap
+		want string
+	}{
+		{
+			name: "no_change",
+			a: &NetworkMap{
+				NodeKey: testNodeKey(1),
+				Peers: []*tailcfg.Node{
+					{
+						Key:       testNodeKey(2),
+						DERP:      "127.3.3.40:2",
+						Endpoints: []string{"192.168.0.100:12", "192.168.0.100:12354"},
+					},
+				},
+			},
+			b: &NetworkMap{
+				NodeKey: testNodeKey(1),
+				Peers: []*tailcfg.Node{
+					{
+						Key:       testNodeKey(2),
+						DERP:      "127.3.3.40:2",
+						Endpoints: []string{"192.168.0.100:12", "192.168.0.100:12354"},
+					},
+				},
+			},
+			want: "",
+		},
+		{
+			name: "header_change",
+			a: &NetworkMap{
+				NodeKey: testNodeKey(1),
+				Peers: []*tailcfg.Node{
+					{
+						Key:       testNodeKey(2),
+						DERP:      "127.3.3.40:2",
+						Endpoints: []string{"192.168.0.100:12", "192.168.0.100:12354"},
+					},
+				},
+			},
+			b: &NetworkMap{
+				NodeKey: testNodeKey(2),
+				Peers: []*tailcfg.Node{
+					{
+						Key:       testNodeKey(2),
+						DERP:      "127.3.3.40:2",
+						Endpoints: []string{"192.168.0.100:12", "192.168.0.100:12354"},
+					},
+				},
+			},
+			want: "-netmap: self: [AQEBA] auth=machine-unknown []\n+netmap: self: [AgICA] auth=machine-unknown []\n",
+		},
+		{
+			name: "peer_add",
+			a: &NetworkMap{
+				NodeKey: testNodeKey(1),
+				Peers: []*tailcfg.Node{
+					{
+						ID:        2,
+						Key:       testNodeKey(2),
+						DERP:      "127.3.3.40:2",
+						Endpoints: []string{"192.168.0.100:12", "192.168.0.100:12354"},
+					},
+				},
+			},
+			b: &NetworkMap{
+				NodeKey: testNodeKey(1),
+				Peers: []*tailcfg.Node{
+					{
+						ID:        1,
+						Key:       testNodeKey(1),
+						DERP:      "127.3.3.40:1",
+						Endpoints: []string{"192.168.0.100:12", "192.168.0.100:12354"},
+					},
+					{
+						ID:        2,
+						Key:       testNodeKey(2),
+						DERP:      "127.3.3.40:2",
+						Endpoints: []string{"192.168.0.100:12", "192.168.0.100:12354"},
+					},
+					{
+						ID:        3,
+						Key:       testNodeKey(3),
+						DERP:      "127.3.3.40:3",
+						Endpoints: []string{"192.168.0.100:12", "192.168.0.100:12354"},
+					},
+				},
+			},
+			want: "+ [AQEBA] D1                 :    192.168.0.100:12     192.168.0.100:12354\n+ [AwMDA] D3                 :    192.168.0.100:12     192.168.0.100:12354\n",
+		},
+		{
+			name: "peer_remove",
+			a: &NetworkMap{
+				NodeKey: testNodeKey(1),
+				Peers: []*tailcfg.Node{
+					{
+						ID:        1,
+						Key:       testNodeKey(1),
+						DERP:      "127.3.3.40:1",
+						Endpoints: []string{"192.168.0.100:12", "192.168.0.100:12354"},
+					},
+					{
+						ID:        2,
+						Key:       testNodeKey(2),
+						DERP:      "127.3.3.40:2",
+						Endpoints: []string{"192.168.0.100:12", "192.168.0.100:12354"},
+					},
+					{
+						ID:        3,
+						Key:       testNodeKey(3),
+						DERP:      "127.3.3.40:3",
+						Endpoints: []string{"192.168.0.100:12", "192.168.0.100:12354"},
+					},
+				},
+			},
+			b: &NetworkMap{
+				NodeKey: testNodeKey(1),
+				Peers: []*tailcfg.Node{
+					{
+						ID:        2,
+						Key:       testNodeKey(2),
+						DERP:      "127.3.3.40:2",
+						Endpoints: []string{"192.168.0.100:12", "192.168.0.100:12354"},
+					},
+				},
+			},
+			want: "- [AQEBA] D1                 :    192.168.0.100:12     192.168.0.100:12354\n- [AwMDA] D3                 :    192.168.0.100:12     192.168.0.100:12354\n",
+		},
+	} {
+		t.Run(tt.name, func(t *testing.T) {
+			var got string
+			n := int(testing.AllocsPerRun(50, func() {
+				got = tt.b.ConciseDiffFrom(tt.a)
+			}))
+			t.Logf("Allocs = %d", n)
+			if got != tt.want {
+				t.Errorf("Wrong output\n Got: %q\nWant: %q\n## Got (unescaped):\n%s\n## Want (unescaped):\n%s\n", got, tt.want, got, tt.want)
+			}
+		})
+	}
+}

derp/derp_server.go

@@ -26,6 +26,7 @@ import (
 
 	"golang.org/x/crypto/nacl/box"
 	"golang.org/x/sync/errgroup"
+	"tailscale.com/disco"
 	"tailscale.com/metrics"
 	"tailscale.com/types/key"
 	"tailscale.com/types/logger"
@@ -38,12 +39,26 @@ const (
 	writeTimeout            = 2 * time.Second
 )
 
+const host64bit = (^uint(0) >> 32) & 1 // 1 on 64-bit, 0 on 32-bit
+
+// pad32bit is 4 on 32-bit machines and 0 on 64-bit.
+// It exists so the Server struct's atomic fields can be aligned to 8
+// byte boundaries. (As tested by GOARCH=386 go test, etc)
+const pad32bit = 4 - host64bit*4 // 0 on 64-bit, 4 on 32-bit
+
 // Server is a DERP server.
 type Server struct {
 	// WriteTimeout, if non-zero, specifies how long to wait
 	// before failing when writing to a client.
 	WriteTimeout time.Duration
 
+	// OnlyDisco controls whether, for tests, non-discovery packets
+	// are dropped. This is used by magicsock tests to verify that
+	// NAT traversal works (using DERP for out-of-band messaging)
+	// but the packets themselves aren't going via DERP.
+	OnlyDisco bool
+	_         [pad32bit]byte
+
 	privateKey key.Private
 	publicKey  key.Public
 	logf       logger.Logf
@@ -51,6 +66,7 @@ type Server struct {
 	meshKey    string
 
 	// Counters:
+	_                        [pad32bit]byte
 	packetsSent, bytesSent   expvar.Int
 	packetsRecv, bytesRecv   expvar.Int
 	packetsDropped           expvar.Int
@@ -61,6 +77,7 @@ type Server struct {
 	packetsDroppedQueueHead  *expvar.Int // queue full, drop head packet
 	packetsDroppedQueueTail  *expvar.Int // queue full, drop tail packet
 	packetsDroppedWrite      *expvar.Int // error writing to dst conn
+	_                        [pad32bit]byte
 	packetsForwardedOut      expvar.Int
 	packetsForwardedIn       expvar.Int
 	peerGoneFrames           expvar.Int // number of peer gone frames sent
@@ -542,6 +559,11 @@ func (c *sclient) handleFrameSendPacket(ft frameType, fl uint32) error {
 		return fmt.Errorf("client %x: recvPacket: %v", c.key, err)
 	}
 
+	if s.OnlyDisco && !disco.LooksLikeDiscoWrapper(contents) {
+		s.packetsDropped.Add(1)
+		return nil
+	}
+
 	var fwd PacketForwarder
 	s.mu.Lock()
 	dst := s.clients[dstKey]

derp/derphttp/derphttp_client.go

@@ -338,6 +338,9 @@ func (c *Client) dialRegion(ctx context.Context, reg *tailcfg.DERPRegion) (net.C
 	var firstErr error
 	for _, n := range reg.Nodes {
 		if n.STUNOnly {
+			if firstErr == nil {
+				firstErr = fmt.Errorf("no non-STUNOnly nodes for %s", c.targetString(reg))
+			}
 			continue
 		}
 		c, err := c.dialNode(ctx, n)

disco/disco.go

@@ -31,6 +31,8 @@ import (
 // Magic is the 6 byte header of all discovery messages.
 const Magic = "TS💬" // 6 bytes: 0x54 53 f0 9f 92 ac
 
+const keyLen = 32
+
 // NonceLen is the length of the nonces used by nacl secretboxes.
 const NonceLen = 24
 
@@ -46,6 +48,15 @@ const v0 = byte(0)
 
 var errShort = errors.New("short message")
 
+// LooksLikeDiscoWrapper reports whether p looks like it's a packet
+// containing an encrypted disco message.
+func LooksLikeDiscoWrapper(p []byte) bool {
+	if len(p) < len(Magic)+keyLen+NonceLen {
+		return false
+	}
+	return string(p[:len(Magic)]) == Magic
+}
+
 // Parse parses the encrypted part of the message from inside the
 // nacl secretbox.
 func Parse(p []byte) (Message, error) {

go.mod

@@ -9,26 +9,30 @@ require (
 	github.com/flynn/go-shlex v0.0.0-20150515145356-3f9db97f8568 // indirect
 	github.com/gliderlabs/ssh v0.2.2
 	github.com/go-ole/go-ole v1.2.4
+	github.com/godbus/dbus/v5 v5.0.3
 	github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e
 	github.com/google/go-cmp v0.4.0
 	github.com/goreleaser/nfpm v1.1.10
+	github.com/jsimonetti/rtnetlink v0.0.0-20200117123717-f846d4f6c1f4
 	github.com/klauspost/compress v1.10.10
 	github.com/kr/pty v1.1.1
 	github.com/mdlayher/netlink v1.1.0
+	github.com/miekg/dns v1.1.30
 	github.com/pborman/getopt v0.0.0-20190409184431-ee0cd42419d3
 	github.com/peterbourgon/ff/v2 v2.0.0
 	github.com/tailscale/winipcfg-go v0.0.0-20200413171540-609dcf2df55f
-	github.com/tailscale/wireguard-go v0.0.0-20200624060658-de1f1af1f35f
+	github.com/tailscale/wireguard-go v0.0.0-20200724155040-d554a2a5e7e1
 	github.com/tcnksm/go-httpstat v0.2.0
 	github.com/toqueteos/webbrowser v1.2.0
-	go4.org/mem v0.0.0-20200601023850-d8ee1dfa5518
+	go4.org/mem v0.0.0-20200706164138-185c595c3ecc
 	golang.org/x/crypto v0.0.0-20200317142112-1b76d66859c6
 	golang.org/x/net v0.0.0-20200301022130-244492dfa37a
 	golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d
 	golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e
 	golang.org/x/sys v0.0.0-20200501052902-10377860bb8e
 	golang.org/x/time v0.0.0-20191024005414-555d28b269f0
+	golang.org/x/tools v0.0.0-20191216052735-49a3e744a425
 	honnef.co/go/tools v0.0.1-2020.1.4
-	inet.af/netaddr v0.0.0-20200702150737-4591d218f82c
+	inet.af/netaddr v0.0.0-20200718043157-99321d6ad24c
 	rsc.io/goversion v1.2.0
 )

go.sum

@@ -30,6 +30,8 @@ github.com/gliderlabs/ssh v0.2.2 h1:6zsha5zo/TWhRhwqCD3+EarCAgZ2yN28ipRnGPnwkI0=
 github.com/gliderlabs/ssh v0.2.2/go.mod h1:U7qILu1NlMHj9FlMhZLlkCdDnU1DBEAqr0aevW3Awn0=
 github.com/go-ole/go-ole v1.2.4 h1:nNBDSCOigTSiarFpYE9J/KtEA1IOW4CNeqT9TQDqCxI=
 github.com/go-ole/go-ole v1.2.4/go.mod h1:XCwSNxSkXRo4vlyPy93sltvi/qJq0jqQhjqQNIwKuxM=
+github.com/godbus/dbus/v5 v5.0.3 h1:ZqHaoEF7TBzh4jzPmqVhE/5A1z9of6orkAe5uHoAeME=
+github.com/godbus/dbus/v5 v5.0.3/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA=
 github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e h1:1r7pUrabqp18hOBcwBwiTsbnFeTZHV9eER/QT5JVZxY=
 github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
 github.com/golang/protobuf v1.2.0 h1:P3YflyNX/ehuJFLhxviNdFxQPkGK5cDcApsge1SqnvM=
@@ -63,6 +65,8 @@ github.com/mdlayher/netlink v0.0.0-20190409211403-11939a169225/go.mod h1:eQB3mZE
 github.com/mdlayher/netlink v1.0.0/go.mod h1:KxeJAFOFLG6AjpyDkQ/iIhxygIUKD+vcwqcnu43w/+M=
 github.com/mdlayher/netlink v1.1.0 h1:mpdLgm+brq10nI9zM1BpX1kpDbh3NLl3RSnVq6ZSkfg=
 github.com/mdlayher/netlink v1.1.0/go.mod h1:H4WCitaheIsdF9yOYu8CFmCgQthAPIWZmcKp9uZHgmY=
+github.com/miekg/dns v1.1.30 h1:Qww6FseFn8PRfw07jueqIXqodm0JKiiKuK0DeXSqfyo=
+github.com/miekg/dns v1.1.30/go.mod h1:KNUDUusw/aVsxyTYZM1oqvCicbwhgbNgztCETuNZ7xM=
 github.com/op/go-logging v0.0.0-20160315200505-970db520ece7 h1:lDH9UUVJtmYCjyT0CI4q8xvlXPxeZ0gYCVvWbmPlp88=
 github.com/op/go-logging v0.0.0-20160315200505-970db520ece7/go.mod h1:HzydrMdWErDVzsI23lYNej1Htcns9BCg93Dk0bBINWk=
 github.com/pborman/getopt v0.0.0-20190409184431-ee0cd42419d3 h1:YtFkrqsMEj7YqpIhRteVxJxCeC3jJBieuLr0d4C4rSA=
@@ -82,8 +86,6 @@ github.com/stretchr/testify v1.4.0 h1:2E4SXV/wtOkTonXsotYi4li6zVWxYlZuYNCXe9XRJy
 github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4=
 github.com/tailscale/winipcfg-go v0.0.0-20200413171540-609dcf2df55f h1:uFj5bslHsMzxIM8UTjAhq4VXeo6GfNW91rpoh/WMJaY=
 github.com/tailscale/winipcfg-go v0.0.0-20200413171540-609dcf2df55f/go.mod h1:x880GWw5fvrl2DVTQ04ttXQD4DuppTt1Yz6wLibbjNE=
-github.com/tailscale/wireguard-go v0.0.0-20200615180905-687c10194779 h1:zg0rgvhBZGA4nvh17nDKcqkEXw6Nbc/Ma2VBvLaW7LU=
-github.com/tailscale/wireguard-go v0.0.0-20200615180905-687c10194779/go.mod h1:JPm5cTfu1K+qDFRbiHy0sOlHUylYQbpl356sdYFD8V4=
 github.com/tcnksm/go-httpstat v0.2.0 h1:rP7T5e5U2HfmOBmZzGgGZjBQ5/GluWUylujl0tJ04I0=
 github.com/tcnksm/go-httpstat v0.2.0/go.mod h1:s3JVJFtQxtBEBC9dwcdTTXS9xFnM3SXAZwPG41aurT8=
 github.com/toqueteos/webbrowser v1.2.0 h1:tVP/gpK69Fx+qMJKsLE7TD8LuGWPnEV71wBN9rrstGQ=
@@ -92,20 +94,23 @@ github.com/ulikunitz/xz v0.5.6 h1:jGHAfXawEGZQ3blwU5wnWKQJvAraT7Ftq9EXjnXYgt8=
 github.com/ulikunitz/xz v0.5.6/go.mod h1:2bypXElzHzzJZwzH67Y6wb67pO62Rzfn7BSiF4ABRW8=
 github.com/xi2/xz v0.0.0-20171230120015-48954b6210f8 h1:nIPpBwaJSVYIxUFsDv3M8ofmx9yWTog9BfvIu0q41lo=
 github.com/xi2/xz v0.0.0-20171230120015-48954b6210f8/go.mod h1:HUYIGzjTL3rfEspMxjDjgmT5uz5wzYJKVo23qUhYTos=
-go4.org/mem v0.0.0-20200601023850-d8ee1dfa5518 h1:AA3bSGklCgkrqIGnvL4894oa/2K9ltE0RejXh8CgyvA=
-go4.org/mem v0.0.0-20200601023850-d8ee1dfa5518/go.mod h1:NEYvpHWemiG/E5UWfaN5QAIGZeT1sa0Z2UNk6oeMb/k=
+go4.org/mem v0.0.0-20200706164138-185c595c3ecc h1:paujszgN6SpsO/UsXC7xax3gQAKz/XQKCYZLQdU34Tw=
+go4.org/mem v0.0.0-20200706164138-185c595c3ecc/go.mod h1:NEYvpHWemiG/E5UWfaN5QAIGZeT1sa0Z2UNk6oeMb/k=
 golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
 golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
 golang.org/x/crypto v0.0.0-20191002192127-34f69633bfdc/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
+golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
 golang.org/x/crypto v0.0.0-20200317142112-1b76d66859c6 h1:TjszyFsQsyZNHwdVdZ5m7bjmreu0znc2kRYsEml9/Ww=
 golang.org/x/crypto v0.0.0-20200317142112-1b76d66859c6/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
 golang.org/x/mod v0.0.0-20190513183733-4bf6d317e70e/go.mod h1:mXi4GBBbnImb6dmsKGUJ2LatrhH/nqhxcFungHvyanc=
+golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg=
 golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/net v0.0.0-20190108225652-1e06a53dbb7e/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
 golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
 golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
 golang.org/x/net v0.0.0-20190827160401-ba9fcec4b297/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
+golang.org/x/net v0.0.0-20190923162816-aa69164e4478/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
 golang.org/x/net v0.0.0-20191003171128-d98b1b443823/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
 golang.org/x/net v0.0.0-20191007182048-72f939374954/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
 golang.org/x/net v0.0.0-20200202094626-16171245cfb2 h1:CCH4IOTTfewWjGOlSp+zGcjutRKlBEZQ6wTn8ozI/nI=
@@ -126,6 +131,7 @@ golang.org/x/sys v0.0.0-20190405154228-4b34438f7a67/go.mod h1:h1NjWce9XRLGQEsW7w
 golang.org/x/sys v0.0.0-20190411185658-b44545bcd369/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20190826190057-c7b8b68b1456/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20190924154521-2837fb4f24fe/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20191003212358-c178f38b412c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20191008105621-543471e840be/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20200202164722-d101bd2416d5 h1:LfCXLvNmTYH9kEmVgqbnsWfruoXZIrh4YBgqVHtDvw0=
@@ -140,7 +146,10 @@ golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxb
 golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
 golang.org/x/tools v0.0.0-20191130070609-6e064ea0cf2d h1:/iIZNFGxc/a7C3yWjGcnboV+Tkc7mxr+p6fDztwoxuM=
 golang.org/x/tools v0.0.0-20191130070609-6e064ea0cf2d/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
+golang.org/x/tools v0.0.0-20191216052735-49a3e744a425 h1:VvQyQJN0tSuecqgcIxMWnnfG5kSmgy9KZR9sW3W5QeA=
+golang.org/x/tools v0.0.0-20191216052735-49a3e744a425/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=
 golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
+golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
 golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543 h1:E7g+9GITq07hpfrRu66IVDexMakfv52eLZ2CXBWiKr4=
 golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
 google.golang.org/appengine v1.4.0 h1:/wp5JvzpHIxhs/dumFmF7BXTf3Z+dd4uXta4kVyO508=
@@ -156,7 +165,7 @@ gopkg.in/yaml.v2 v2.2.7 h1:VUgggvou5XRW9mHwD/yXxIYSMtY0zoKQf/v226p2nyo=
 gopkg.in/yaml.v2 v2.2.7/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
 honnef.co/go/tools v0.0.1-2020.1.4 h1:UoveltGrhghAA7ePc+e+QYDHXrBps2PqFZiHkGR/xK8=
 honnef.co/go/tools v0.0.1-2020.1.4/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k=
-inet.af/netaddr v0.0.0-20200702150737-4591d218f82c h1:j3Z4HL4KcLBDU1kmRpXTD5fikKBqIkE+7vFKS5mCz3Y=
-inet.af/netaddr v0.0.0-20200702150737-4591d218f82c/go.mod h1:qqYzz/2whtrbWJvt+DNWQyvekNN4ePQZcg2xc2/Yjww=
+inet.af/netaddr v0.0.0-20200718043157-99321d6ad24c h1:si3Owrfem175Ry6gKqnh59eOXxDojyBTIHxUKuvK/Eo=
+inet.af/netaddr v0.0.0-20200718043157-99321d6ad24c/go.mod h1:qqYzz/2whtrbWJvt+DNWQyvekNN4ePQZcg2xc2/Yjww=
 rsc.io/goversion v1.2.0 h1:SPn+NLTiAG7w30IRK/DKp1BjvpWabYgxlLp/+kx5J8w=
 rsc.io/goversion v1.2.0/go.mod h1:Eih9y/uIBS3ulggl7KNJ09xGSLcuNaLgmvvqa07sgfo=

internal/deepprint/deepprint_test.go

@@ -50,7 +50,10 @@ func getVal() []interface{} {
 			},
 		},
 		&router.Config{
-			DNS: []netaddr.IP{netaddr.IPv4(8, 8, 8, 8)},
+			DNSConfig: router.DNSConfig{
+				Nameservers: []netaddr.IP{netaddr.IPv4(8, 8, 8, 8)},
+				Domains:     []string{"tailscale.net"},
+			},
 		},
 		map[string]string{
 			"key1": "val1",

ipn/backend.go

@@ -8,6 +8,7 @@ import (
 	"net/http"
 	"time"
 
+	"golang.org/x/oauth2"
 	"tailscale.com/control/controlclient"
 	"tailscale.com/ipn/ipnstate"
 	"tailscale.com/tailcfg"
@@ -27,6 +28,10 @@ const (
 	Running
 )
 
+// GoogleIDToken Type is the oauth2.Token.TokenType for the Google
+// ID tokens used by the Android client.
+const GoogleIDTokenType = "ts_android_google_login"
+
 func (s State) String() string {
 	return [...]string{"NoState", "NeedsLogin", "NeedsMachineAuth",
 		"Stopped", "Starting", "Running"}[s]
@@ -58,6 +63,12 @@ type Notify struct {
 	BrowseToURL   *string                   // UI should open a browser right now
 	BackendLogID  *string                   // public logtail id used by backend
 
+	// LocalTCPPort, if non-nil, informs the UI frontend which
+	// (non-zero) localhost TCP port it's listening on.
+	// This is currently only used by Tailscale when run in the
+	// macOS Network Extension.
+	LocalTCPPort *uint16 `json:",omitempty"`
+
 	// type is mirrored in xcode/Shared/IPN.swift
 }
 
@@ -123,6 +134,8 @@ type Backend interface {
 	// flow. This should trigger a new BrowseToURL notification
 	// eventually.
 	StartLoginInteractive()
+	// Login logs in with an OAuth2 token.
+	Login(token *oauth2.Token)
 	// Logout terminates the current login session and stops the
 	// wireguard engine.
 	Logout()

ipn/fake_test.go

@@ -8,6 +8,7 @@ import (
 	"log"
 	"time"
 
+	"golang.org/x/oauth2"
 	"tailscale.com/control/controlclient"
 	"tailscale.com/ipn/ipnstate"
 )
@@ -42,6 +43,14 @@ func (b *FakeBackend) newState(s State) {
 func (b *FakeBackend) StartLoginInteractive() {
 	u := b.serverURL + "/this/is/fake"
 	b.notify(Notify{BrowseToURL: &u})
+	b.login()
+}
+
+func (b *FakeBackend) Login(token *oauth2.Token) {
+	b.login()
+}
+
+func (b *FakeBackend) login() {
 	b.newState(NeedsMachineAuth)
 	b.newState(Stopped)
 	// TODO(apenwarr): Fill in a more interesting netmap here.

ipn/handle.go

@@ -9,6 +9,7 @@ import (
 	"time"
 
 	"github.com/tailscale/wireguard-go/wgcfg"
+	"golang.org/x/oauth2"
 	"tailscale.com/control/controlclient"
 	"tailscale.com/types/logger"
 )
@@ -154,6 +155,10 @@ func (h *Handle) StartLoginInteractive() {
 	h.b.StartLoginInteractive()
 }
 
+func (h *Handle) Login(token *oauth2.Token) {
+	h.b.Login(token)
+}
+
 func (h *Handle) Logout() {
 	h.b.Logout()
 }

ipn/ipnserver/server.go

@@ -33,15 +33,19 @@ type Options struct {
 	// SocketPath, on unix systems, is the unix socket path to listen
 	// on for frontend connections.
 	SocketPath string
+
 	// Port, on windows, is the localhost TCP port to listen on for
 	// frontend connections.
 	Port int
+
 	// StatePath is the path to the stored agent state.
 	StatePath string
+
 	// AutostartStateKey, if non-empty, immediately starts the agent
 	// using the given StateKey. If empty, the agent stays idle and
 	// waits for a frontend to start it.
 	AutostartStateKey ipn.StateKey
+
 	// LegacyConfigPath optionally specifies the old-style relaynode
 	// relay.conf location. If both LegacyConfigPath and
 	// AutostartStateKey are specified and the requested state doesn't
@@ -51,53 +55,151 @@ type Options struct {
 	// TODO(danderson): remove some time after the transition to
 	// tailscaled is done.
 	LegacyConfigPath string
+
 	// SurviveDisconnects specifies how the server reacts to its
 	// frontend disconnecting. If true, the server keeps running on
 	// its existing state, and accepts new frontend connections. If
 	// false, the server dumps its state and becomes idle.
+	//
+	// To support CLI connections (notably, "tailscale status"),
+	// the actual definition of "disconnect" is when the
+	// connection count transitions from 1 to 0.
 	SurviveDisconnects bool
 
 	// DebugMux, if non-nil, specifies an HTTP ServeMux in which
 	// to register a debug handler.
 	DebugMux *http.ServeMux
+
+	// ErrorMessage, if not empty, signals that the server will exist
+	// only to relay the provided critical error message to the user.
+	ErrorMessage string
 }
 
-func pump(logf logger.Logf, ctx context.Context, bs *ipn.BackendServer, s net.Conn) {
-	defer logf("Control connection done.")
+// server is an IPN backend and its set of 0 or more active connections
+// talking to an IPN backend.
+type server struct {
+	resetOnZero bool // call bs.Reset on transition from 1->0 connections
 
-	for ctx.Err() == nil && !bs.GotQuit {
-		msg, err := ipn.ReadMsg(s)
+	bsMu sync.Mutex // lock order: bsMu, then mu
+	bs   *ipn.BackendServer
+
+	mu      sync.Mutex
+	clients map[net.Conn]bool
+}
+
+func (s *server) serveConn(ctx context.Context, c net.Conn, logf logger.Logf) {
+	s.addConn(c)
+	logf("incoming control connection")
+	defer s.removeAndCloseConn(c)
+	for ctx.Err() == nil {
+		msg, err := ipn.ReadMsg(c)
 		if err != nil {
-			logf("ReadMsg: %v", err)
-			break
+			if ctx.Err() == nil {
+				logf("ReadMsg: %v", err)
+			}
+			return
 		}
-		err = bs.GotCommandMsg(msg)
-		if err != nil {
+		s.bsMu.Lock()
+		if err := s.bs.GotCommandMsg(msg); err != nil {
 			logf("GotCommandMsg: %v", err)
-			break
+		}
+		gotQuit := s.bs.GotQuit
+		s.bsMu.Unlock()
+		if gotQuit {
+			return
 		}
 	}
 }
 
-func Run(rctx context.Context, logf logger.Logf, logid string, opts Options, e wgengine.Engine) (err error) {
-	runDone := make(chan error, 1)
-	defer func() { runDone <- err }()
+func (s *server) addConn(c net.Conn) {
+	s.mu.Lock()
+	defer s.mu.Unlock()
+	if s.clients == nil {
+		s.clients = map[net.Conn]bool{}
+	}
+	s.clients[c] = true
+}
+
+func (s *server) removeAndCloseConn(c net.Conn) {
+	s.mu.Lock()
+	delete(s.clients, c)
+	remain := len(s.clients)
+	s.mu.Unlock()
+
+	if remain == 0 && s.resetOnZero {
+		s.bsMu.Lock()
+		s.bs.Reset()
+		s.bsMu.Unlock()
+	}
+	c.Close()
+}
+
+func (s *server) stopAll() {
+	s.mu.Lock()
+	defer s.mu.Unlock()
+	for c := range s.clients {
+		safesocket.ConnCloseRead(c)
+		safesocket.ConnCloseWrite(c)
+	}
+	s.clients = nil
+}
+
+func (s *server) writeToClients(b []byte) {
+	s.mu.Lock()
+	defer s.mu.Unlock()
+	for c := range s.clients {
+		ipn.WriteMsg(c, b)
+	}
+}
+
+func Run(ctx context.Context, logf logger.Logf, logid string, opts Options, e wgengine.Engine) error {
+	runDone := make(chan struct{})
+	defer close(runDone)
 
 	listen, _, err := safesocket.Listen(opts.SocketPath, uint16(opts.Port))
 	if err != nil {
 		return fmt.Errorf("safesocket.Listen: %v", err)
 	}
 
-	// Go listeners can't take a context, close it instead.
+	server := &server{
+		resetOnZero: !opts.SurviveDisconnects,
+	}
+
+	// When the context is closed or when we return, whichever is first, close our listner
+	// and all open connections.
 	go func() {
 		select {
-		case <-rctx.Done():
+		case <-ctx.Done():
 		case <-runDone:
 		}
+		server.stopAll()
 		listen.Close()
 	}()
 	logf("Listening on %v", listen.Addr())
 
+	bo := backoff.NewBackoff("ipnserver", logf)
+
+	if opts.ErrorMessage != "" {
+		for i := 1; ctx.Err() == nil; i++ {
+			s, err := listen.Accept()
+			if err != nil {
+				logf("%d: Accept: %v", i, err)
+				bo.BackOff(ctx, err)
+				continue
+			}
+			serverToClient := func(b []byte) {
+				ipn.WriteMsg(s, b)
+			}
+			go func() {
+				defer s.Close()
+				bs := ipn.NewBackendServer(logf, nil, serverToClient)
+				bs.SendErrorMessage(opts.ErrorMessage)
+				s.Read(make([]byte, 1))
+			}()
+		}
+		return ctx.Err()
+	}
+
 	var store ipn.StateStore
 	if opts.StatePath != "" {
 		store, err = ipn.NewFileStore(opts.StatePath)
@@ -112,6 +214,7 @@ func Run(rctx context.Context, logf logger.Logf, logid string, opts Options, e w
 	if err != nil {
 		return fmt.Errorf("NewLocalBackend: %v", err)
 	}
+	defer b.Shutdown()
 	b.SetDecompressor(func() (controlclient.Decompressor, error) {
 		return smallzstd.NewDecoder(nil)
 	})
@@ -125,17 +228,10 @@ func Run(rctx context.Context, logf logger.Logf, logid string, opts Options, e w
 		})
 	}
 
-	var s net.Conn
-	serverToClient := func(b []byte) {
-		if s != nil { // TODO: racy access to s?
-			ipn.WriteMsg(s, b)
-		}
-	}
-
-	bs := ipn.NewBackendServer(logf, b, serverToClient)
+	server.bs = ipn.NewBackendServer(logf, b, server.writeToClients)
 
 	if opts.AutostartStateKey != "" {
-		bs.GotCommand(&ipn.Command{
+		server.bs.GotCommand(&ipn.Command{
 			Version: version.LONG,
 			Start: &ipn.StartArgs{
 				Opts: ipn.Options{
@@ -146,55 +242,18 @@ func Run(rctx context.Context, logf logger.Logf, logid string, opts Options, e w
 		})
 	}
 
-	var (
-		oldS   net.Conn
-		ctx    context.Context
-		cancel context.CancelFunc
-	)
-	stopAll := func() {
-		// Currently we only support one client connection at a time.
-		// Theoretically we could allow multiple clients, by passing
-		// notifications to all of them and accepting commands from
-		// any of them, but there doesn't seem to be much need for
-		// that right now.
-		if oldS != nil {
-			cancel()
-			safesocket.ConnCloseRead(oldS)
-			safesocket.ConnCloseWrite(oldS)
-		}
-	}
-
-	bo := backoff.NewBackoff("ipnserver", logf)
-
-	for i := 1; rctx.Err() == nil; i++ {
-		s, err = listen.Accept()
+	for i := 1; ctx.Err() == nil; i++ {
+		c, err := listen.Accept()
 		if err != nil {
-			logf("%d: Accept: %v", i, err)
-			bo.BackOff(rctx, err)
+			if ctx.Err() == nil {
+				logf("ipnserver: Accept: %v", err)
+				bo.BackOff(ctx, err)
+			}
 			continue
 		}
-		logf("%d: Incoming control connection.", i)
-		stopAll()
-
-		ctx, cancel = context.WithCancel(rctx)
-		oldS = s
-
-		go func(ctx context.Context, s net.Conn, i int) {
-			logf := logger.WithPrefix(logf, fmt.Sprintf("%d: ", i))
-			pump(logf, ctx, bs, s)
-			if !opts.SurviveDisconnects || bs.GotQuit {
-				bs.Reset()
-				s.Close()
-			}
-			// Quitting not allowed, just keep going.
-			bs.GotQuit = false
-		}(ctx, s, i)
-
-		bo.BackOff(ctx, nil)
+		go server.serveConn(ctx, c, logger.WithPrefix(logf, fmt.Sprintf("ipnserver: conn%d: ", i)))
 	}
-	stopAll()
-
-	return rctx.Err()
+	return ctx.Err()
 }
 
 func BabysitProc(ctx context.Context, args []string, logf logger.Logf) {

ipn/local.go

@@ -13,6 +13,7 @@ import (
 	"time"
 
 	"github.com/tailscale/wireguard-go/wgcfg"
+	"golang.org/x/oauth2"
 	"inet.af/netaddr"
 	"tailscale.com/control/controlclient"
 	"tailscale.com/ipn/ipnstate"
@@ -49,6 +50,7 @@ type LocalBackend struct {
 	store           StateStore
 	backendLogID    string
 	portpoll        *portlist.Poller // may be nil
+	portpollOnce    sync.Once
 	newDecompressor func() (controlclient.Decompressor, error)
 
 	// TODO: these fields are accessed unsafely by concurrent
@@ -351,6 +353,7 @@ func (b *LocalBackend) Start(opts Options) error {
 	b.serverURL = b.prefs.ControlURL
 	hostinfo.RoutableIPs = append(hostinfo.RoutableIPs, b.prefs.AdvertiseRoutes...)
 	hostinfo.RequestTags = append(hostinfo.RequestTags, b.prefs.AdvertiseTags...)
+	applyPrefsToHostinfo(hostinfo, b.prefs)
 
 	b.notify = opts.Notify
 	b.netMap = nil
@@ -361,9 +364,7 @@ func (b *LocalBackend) Start(opts Options) error {
 
 	var discoPublic tailcfg.DiscoKey
 	if controlclient.Debug.Disco {
-		discoPrivate := key.NewPrivate()
-		b.e.SetDiscoPrivateKey(discoPrivate)
-		discoPublic = tailcfg.DiscoKey(discoPrivate.Public())
+		discoPublic = b.e.DiscoPublicKey()
 	}
 
 	var err error
@@ -389,8 +390,10 @@ func (b *LocalBackend) Start(opts Options) error {
 	// At this point, we have finished using hostinfo without synchronization,
 	// so it is safe to start readPoller which concurrently writes to it.
 	if b.portpoll != nil {
-		go b.portpoll.Run(b.ctx)
-		go b.readPoller()
+		b.portpollOnce.Do(func() {
+			go b.portpoll.Run(b.ctx)
+			go b.readPoller()
+		})
 	}
 
 	b.mu.Lock()
@@ -460,18 +463,25 @@ func (b *LocalBackend) updateDNSMap(netMap *controlclient.NetworkMap) {
 	if netMap == nil {
 		return
 	}
+
 	domainToIP := make(map[string]netaddr.IP)
-	for _, peer := range netMap.Peers {
-		if len(peer.Addresses) == 0 {
-			continue
+	set := func(hostname string, addrs []wgcfg.CIDR) {
+		if len(addrs) == 0 {
+			return
 		}
-		domain := peer.Hostinfo.Hostname
+		domain := hostname
 		// Like PeerStatus.SimpleHostName()
 		domain = strings.TrimSuffix(domain, ".local")
 		domain = strings.TrimSuffix(domain, ".localdomain")
-		domain = domain + ".ipn.dev"
-		domainToIP[domain] = netaddr.IPFrom16(peer.Addresses[0].IP.Addr)
+		domain = domain + ".b.tailscale.net"
+		domainToIP[domain] = netaddr.IPFrom16(addrs[0].IP.Addr)
 	}
+
+	for _, peer := range netMap.Peers {
+		set(peer.Hostinfo.Hostname, peer.Addresses)
+	}
+	set(netMap.Hostinfo.Hostname, netMap.Addresses)
+
 	b.e.SetDNSMap(tsdns.NewMap(domainToIP))
 }
 
@@ -517,6 +527,8 @@ func (b *LocalBackend) send(n Notify) {
 	if notify != nil {
 		n.Version = version.LONG
 		notify(n)
+	} else {
+		b.logf("nil notify callback; dropping %+v", n)
 	}
 }
 
@@ -569,7 +581,7 @@ func (b *LocalBackend) loadStateLocked(key StateKey, prefs *Prefs, legacyPath st
 	if err != nil {
 		if errors.Is(err, ErrStateNotExist) {
 			if legacyPath != "" {
-				b.prefs, err = LoadPrefs(legacyPath, true)
+				b.prefs, err = LoadPrefs(legacyPath)
 				if err != nil {
 					b.logf("Failed to load legacy prefs: %v", err)
 					b.prefs = NewPrefs()
@@ -611,6 +623,16 @@ func (b *LocalBackend) getEngineStatus() EngineStatus {
 	return b.engineStatus
 }
 
+// Login implements Backend.
+func (b *LocalBackend) Login(token *oauth2.Token) {
+	b.mu.Lock()
+	b.assertClientLocked()
+	c := b.c
+	b.mu.Unlock()
+
+	c.Login(token, controlclient.LoginInteractive)
+}
+
 // StartLoginInteractive implements Backend. It requests a new
 // interactive login from controlclient, unless such a flow is already
 // in progress, in which case StartLoginInteractive attempts to pick
@@ -710,15 +732,14 @@ func (b *LocalBackend) SetPrefs(new *Prefs) {
 	oldHi := b.hostinfo
 	newHi := oldHi.Clone()
 	newHi.RoutableIPs = append([]wgcfg.CIDR(nil), b.prefs.AdvertiseRoutes...)
-	if h := new.Hostname; h != "" {
-		newHi.Hostname = h
-	}
+	applyPrefsToHostinfo(newHi, new)
 	b.hostinfo = newHi
+	hostInfoChanged := !oldHi.Equal(newHi)
 	b.mu.Unlock()
 
 	b.logf("SetPrefs: %v", new.Pretty())
 
-	if old.ShieldsUp != new.ShieldsUp || !oldHi.Equal(newHi) {
+	if old.ShieldsUp != new.ShieldsUp || hostInfoChanged {
 		b.doSetHostinfoFilterServices(newHi)
 	}
 
@@ -807,20 +828,20 @@ func (b *LocalBackend) authReconfig() {
 		return
 	}
 
-	uflags := controlclient.UDefault
+	var flags controlclient.WGConfigFlags
 	if uc.RouteAll {
-		uflags |= controlclient.UAllowDefaultRoute
+		flags |= controlclient.AllowDefaultRoute
 		// TODO(apenwarr): Make subnet routes a different pref?
-		uflags |= controlclient.UAllowSubnetRoutes
+		flags |= controlclient.AllowSubnetRoutes
 		// TODO(apenwarr): Remove this once we sort out subnet routes.
 		//  Right now default routes are broken in Windows, but
 		//  controlclient doesn't properly send subnet routes. So
 		//  let's convert a default route into a subnet route in order
 		//  to allow experimentation.
-		uflags |= controlclient.UHackDefaultRoute
+		flags |= controlclient.HackDefaultRoute
 	}
 	if uc.AllowSingleHosts {
-		uflags |= controlclient.UAllowSingleHosts
+		flags |= controlclient.AllowSingleHosts
 	}
 
 	dns := nm.DNS
@@ -829,7 +850,7 @@ func (b *LocalBackend) authReconfig() {
 		dns = []wgcfg.IP{}
 		dom = []string{}
 	}
-	cfg, err := nm.WGCfg(b.logf, uflags, dns)
+	cfg, err := nm.WGCfg(b.logf, flags, dns)
 	if err != nil {
 		b.logf("wgcfg: %v", err)
 		return
@@ -839,7 +860,7 @@ func (b *LocalBackend) authReconfig() {
 	if err == wgengine.ErrNoChanges {
 		return
 	}
-	b.logf("authReconfig: ra=%v dns=%v 0x%02x: %v", uc.RouteAll, uc.CorpDNS, uflags, err)
+	b.logf("authReconfig: ra=%v dns=%v 0x%02x: %v", uc.RouteAll, uc.CorpDNS, flags, err)
 }
 
 // routerConfig produces a router.Config from a wireguard config,
@@ -855,11 +876,13 @@ func routerConfig(cfg *wgcfg.Config, prefs *Prefs, dnsDomains []string) *router.
 
 	rs := &router.Config{
 		LocalAddrs:       wgCIDRToNetaddr(addrs),
-		DNS:              wgIPToNetaddr(cfg.DNS),
-		DNSDomains:       dnsDomains,
 		SubnetRoutes:     wgCIDRToNetaddr(prefs.AdvertiseRoutes),
 		SNATSubnetRoutes: !prefs.NoSNAT,
 		NetfilterMode:    prefs.NetfilterMode,
+		DNSConfig: router.DNSConfig{
+			Nameservers: wgIPToNetaddr(cfg.DNS),
+			Domains:     dnsDomains,
+		},
 	}
 
 	for _, peer := range cfg.Peers {
@@ -916,6 +939,18 @@ func wgCIDRToNetaddr(cidrs []wgcfg.CIDR) (ret []netaddr.IPPrefix) {
 	return ret
 }
 
+func applyPrefsToHostinfo(hi *tailcfg.Hostinfo, prefs *Prefs) {
+	if h := prefs.Hostname; h != "" {
+		hi.Hostname = h
+	}
+	if v := prefs.OSVersion; v != "" {
+		hi.OSVersion = v
+	}
+	if m := prefs.DeviceModel; m != "" {
+		hi.DeviceModel = m
+	}
+}
+
 // enterState transitions the backend into newState, updating internal
 // state and propagating events out as needed.
 //
@@ -1015,7 +1050,7 @@ func (b *LocalBackend) RequestEngineStatus() {
 // RequestStatus implements Backend.
 func (b *LocalBackend) RequestStatus() {
 	st := b.Status()
-	b.notify(Notify{Status: st})
+	b.send(Notify{Status: st})
 }
 
 // stateMachine updates the state machine state based on other things

ipn/message.go

@@ -13,6 +13,7 @@ import (
 	"log"
 	"time"
 
+	"golang.org/x/oauth2"
 	"tailscale.com/types/logger"
 	"tailscale.com/types/structs"
 	"tailscale.com/version"
@@ -49,6 +50,7 @@ type Command struct {
 	Quit                  *NoArgs
 	Start                 *StartArgs
 	StartLoginInteractive *NoArgs
+	Login                 *oauth2.Token
 	Logout                *NoArgs
 	SetPrefs              *SetPrefsArgs
 	RequestEngineStatus   *NoArgs
@@ -80,6 +82,10 @@ func (bs *BackendServer) send(n Notify) {
 	bs.sendNotifyMsg(b)
 }
 
+func (bs *BackendServer) SendErrorMessage(msg string) {
+	bs.send(Notify{ErrMessage: &msg})
+}
+
 // GotCommandMsg parses the incoming message b as a JSON Command and
 // calls GotCommand with it.
 func (bs *BackendServer) GotCommandMsg(b []byte) error {
@@ -124,6 +130,9 @@ func (bs *BackendServer) GotCommand(cmd *Command) error {
 	} else if c := cmd.StartLoginInteractive; c != nil {
 		bs.b.StartLoginInteractive()
 		return nil
+	} else if c := cmd.Login; c != nil {
+		bs.b.Login(c)
+		return nil
 	} else if c := cmd.Logout; c != nil {
 		bs.b.Logout()
 		return nil
@@ -221,6 +230,10 @@ func (bc *BackendClient) StartLoginInteractive() {
 	bc.send(Command{StartLoginInteractive: &NoArgs{}})
 }
 
+func (bc *BackendClient) Login(token *oauth2.Token) {
+	bc.send(Command{Login: token})
+}
+
 func (bc *BackendClient) Logout() {
 	bc.send(Command{Logout: &NoArgs{}})
 }

ipn/message_test.go

@@ -9,6 +9,7 @@ import (
 	"testing"
 	"time"
 
+	"golang.org/x/oauth2"
 	"tailscale.com/tstest"
 )
 
@@ -177,4 +178,10 @@ func TestClientServer(t *testing.T) {
 
 	h.Logout()
 	flushUntil(NeedsLogin)
+
+	h.Login(&oauth2.Token{
+		AccessToken: "google_id_token",
+		TokenType:   GoogleIDTokenType,
+	})
+	flushUntil(Running)
 }

ipn/prefs.go

@@ -54,6 +54,10 @@ type Prefs struct {
 	// Hostname is the hostname to use for identifying the node. If
 	// not set, os.Hostname is used.
 	Hostname string
+	// OSVersion overrides tailcfg.Hostinfo's OSVersion.
+	OSVersion string
+	// DeviceModel overrides tailcfg.Hostinfo's DeviceModel.
+	DeviceModel string
 
 	// NotepadURLs is a debugging setting that opens OAuth URLs in
 	// notepad.exe on Windows, rather than loading them in a browser.
@@ -138,6 +142,8 @@ func (p *Prefs) Equals(p2 *Prefs) bool {
 		p.NoSNAT == p2.NoSNAT &&
 		p.NetfilterMode == p2.NetfilterMode &&
 		p.Hostname == p2.Hostname &&
+		p.OSVersion == p2.OSVersion &&
+		p.DeviceModel == p2.DeviceModel &&
 		compareIPNets(p.AdvertiseRoutes, p2.AdvertiseRoutes) &&
 		compareStrings(p.AdvertiseTags, p2.AdvertiseTags) &&
 		p.Persist.Equals(p2.Persist)
@@ -217,10 +223,9 @@ func (p *Prefs) Clone() *Prefs {
 	return p2
 }
 
-// LoadLegacyPrefs loads a legacy relaynode config file into Prefs
-// with sensible migration defaults set. If enforceDefaults is true,
-// Prefs.RouteAll and Prefs.AllowSingleHosts are forced on.
-func LoadPrefs(filename string, enforceDefaults bool) (*Prefs, error) {
+// LoadPrefs loads a legacy relaynode config file into Prefs
+// with sensible migration defaults set.
+func LoadPrefs(filename string) (*Prefs, error) {
 	data, err := ioutil.ReadFile(filename)
 	if err != nil {
 		return nil, fmt.Errorf("loading prefs from %q: %v", filename, err)

ipn/prefs_test.go

@@ -24,7 +24,7 @@ func fieldsOf(t reflect.Type) (fields []string) {
 func TestPrefsEqual(t *testing.T) {
 	tstest.PanicOnLog()
 
-	prefsHandles := []string{"ControlURL", "RouteAll", "AllowSingleHosts", "CorpDNS", "WantRunning", "ShieldsUp", "AdvertiseTags", "Hostname", "NotepadURLs", "DisableDERP", "AdvertiseRoutes", "NoSNAT", "NetfilterMode", "Persist"}
+	prefsHandles := []string{"ControlURL", "RouteAll", "AllowSingleHosts", "CorpDNS", "WantRunning", "ShieldsUp", "AdvertiseTags", "Hostname", "OSVersion", "DeviceModel", "NotepadURLs", "DisableDERP", "AdvertiseRoutes", "NoSNAT", "NetfilterMode", "Persist"}
 	if have := fieldsOf(reflect.TypeOf(Prefs{})); !reflect.DeepEqual(have, prefsHandles) {
 		t.Errorf("Prefs.Equal check might be out of sync\nfields: %q\nhandled: %q\n",
 			have, prefsHandles)

metrics/metrics.go

@@ -40,3 +40,10 @@ func (m *LabelMap) Get(key string) *expvar.Int {
 	m.Add(key, 0)
 	return m.Map.Get(key).(*expvar.Int)
 }
+
+// GetFloat returns a direct pointer to the expvar.Float for key, creating it
+// if necessary.
+func (m *LabelMap) GetFloat(key string) *expvar.Float {
+	m.AddFloat(key, 0.0)
+	return m.Map.Get(key).(*expvar.Float)
+}

net/interfaces/interfaces.go

@@ -12,6 +12,7 @@ import (
 	"strings"
 
 	"inet.af/netaddr"
+	"tailscale.com/net/tsaddr"
 )
 
 // Tailscale returns the current machine's Tailscale interface, if any.
@@ -52,7 +53,7 @@ func maybeTailscaleInterfaceName(s string) bool {
 // Tailscale virtual network interfaces.
 func IsTailscaleIP(ip net.IP) bool {
 	nip, _ := netaddr.FromStdIP(ip) // TODO: push this up to caller, change func signature
-	return cgNAT.Contains(nip)
+	return tsaddr.IsTailscaleIP(nip)
 }
 
 func isUp(nif *net.Interface) bool       { return nif.Flags&net.FlagUp != 0 }
@@ -95,7 +96,7 @@ func LocalAddresses() (regular, loopback []string, err error) {
 				// very well be something we can route to
 				// directly, because both nodes are
 				// behind the same CGNAT router.
-				if cgNAT.Contains(ip) {
+				if tsaddr.IsTailscaleIP(ip) {
 					continue
 				}
 				if linkLocalIPv4.Contains(ip) {
@@ -230,6 +231,38 @@ func HTTPOfListener(ln net.Listener) string {
 
 }
 
+var likelyHomeRouterIP func() (netaddr.IP, bool)
+
+// LikelyHomeRouterIP returns the likely IP of the residential router,
+// which will always be an IPv4 private address, if found.
+// In addition, it returns the IP address of the current machine on
+// the LAN using that gateway.
+// This is used as the destination for UPnP, NAT-PMP, PCP, etc queries.
+func LikelyHomeRouterIP() (gateway, myIP netaddr.IP, ok bool) {
+	if likelyHomeRouterIP != nil {
+		gateway, ok = likelyHomeRouterIP()
+		if !ok {
+			return
+		}
+	}
+	if !ok {
+		return
+	}
+	ForeachInterfaceAddress(func(i Interface, ip netaddr.IP) {
+		if !i.IsUp() || ip.IsZero() || !myIP.IsZero() {
+			return
+		}
+		for _, prefix := range privatev4s {
+			if prefix.Contains(gateway) && prefix.Contains(ip) {
+				myIP = ip
+				ok = true
+				return
+			}
+		}
+	})
+	return gateway, myIP, !myIP.IsZero()
+}
+
 func isPrivateIP(ip netaddr.IP) bool {
 	return private1.Contains(ip) || private2.Contains(ip) || private3.Contains(ip)
 }
@@ -250,7 +283,7 @@ var (
 	private1      = mustCIDR("10.0.0.0/8")
 	private2      = mustCIDR("172.16.0.0/12")
 	private3      = mustCIDR("192.168.0.0/16")
-	cgNAT         = mustCIDR("100.64.0.0/10")
+	privatev4s    = []netaddr.IPPrefix{private1, private2, private3}
 	linkLocalIPv4 = mustCIDR("169.254.0.0/16")
 	v6Global1     = mustCIDR("2000::/3")
 )

net/interfaces/interfaces_darwin.go

@@ -0,0 +1,66 @@
+// Copyright (c) 2020 Tailscale Inc & AUTHORS All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package interfaces
+
+import (
+	"os/exec"
+
+	"go4.org/mem"
+	"inet.af/netaddr"
+	"tailscale.com/util/lineread"
+)
+
+func init() {
+	likelyHomeRouterIP = likelyHomeRouterIPDarwin
+}
+
+/*
+Parse out 10.0.0.1 from:
+
+$ netstat -r -n -f inet
+Routing tables
+
+Internet:
+Destination        Gateway            Flags        Netif Expire
+default            10.0.0.1           UGSc           en0
+default            link#14            UCSI         utun2
+10/16              link#4             UCS            en0      !
+10.0.0.1/32        link#4             UCS            en0      !
+...
+
+*/
+func likelyHomeRouterIPDarwin() (ret netaddr.IP, ok bool) {
+	cmd := exec.Command("/usr/sbin/netstat", "-r", "-n", "-f", "inet")
+	stdout, err := cmd.StdoutPipe()
+	if err != nil {
+		return
+	}
+	if err := cmd.Start(); err != nil {
+		return
+	}
+	defer cmd.Wait()
+
+	var f []mem.RO
+	lineread.Reader(stdout, func(lineb []byte) error {
+		line := mem.B(lineb)
+		if !mem.Contains(line, mem.S("default")) {
+			return nil
+		}
+		f = mem.AppendFields(f[:0], line)
+		if len(f) < 3 || !f[0].EqualString("default") {
+			return nil
+		}
+		ipm, flagsm := f[1], f[2]
+		if !mem.Contains(flagsm, mem.S("G")) {
+			return nil
+		}
+		ip, err := netaddr.ParseIP(string(mem.Append(nil, ipm)))
+		if err == nil && isPrivateIP(ip) {
+			ret = ip
+		}
+		return nil
+	})
+	return ret, !ret.IsZero()
+}

net/interfaces/interfaces_linux.go

@@ -0,0 +1,59 @@
+// Copyright (c) 2020 Tailscale Inc & AUTHORS All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package interfaces
+
+import (
+	"go4.org/mem"
+	"inet.af/netaddr"
+	"tailscale.com/util/lineread"
+)
+
+func init() {
+	likelyHomeRouterIP = likelyHomeRouterIPLinux
+}
+
+/*
+Parse 10.0.0.1 out of:
+
+$ cat /proc/net/route
+Iface   Destination     Gateway         Flags   RefCnt  Use     Metric  Mask            MTU     Window  IRTT
+ens18   00000000        0100000A        0003    0       0       0       00000000        0       0       0
+ens18   0000000A        00000000        0001    0       0       0       0000FFFF        0       0       0
+*/
+func likelyHomeRouterIPLinux() (ret netaddr.IP, ok bool) {
+	lineNum := 0
+	var f []mem.RO
+	lineread.File("/proc/net/route", func(line []byte) error {
+		lineNum++
+		if lineNum == 1 {
+			// Skip header line.
+			return nil
+		}
+		f = mem.AppendFields(f[:0], mem.B(line))
+		if len(f) < 4 {
+			return nil
+		}
+		gwHex, flagsHex := f[2], f[3]
+		flags, err := mem.ParseUint(flagsHex, 16, 16)
+		if err != nil {
+			return nil // ignore error, skip line and keep going
+		}
+		const RTF_UP = 0x0001
+		const RTF_GATEWAY = 0x0002
+		if flags&(RTF_UP|RTF_GATEWAY) != RTF_UP|RTF_GATEWAY {
+			return nil
+		}
+		ipu32, err := mem.ParseUint(gwHex, 16, 32)
+		if err != nil {
+			return nil // ignore error, skip line and keep going
+		}
+		ip := netaddr.IPv4(byte(ipu32), byte(ipu32>>8), byte(ipu32>>16), byte(ipu32>>24))
+		if isPrivateIP(ip) {
+			ret = ip
+		}
+		return nil
+	})
+	return ret, !ret.IsZero()
+}

net/interfaces/interfaces_test.go

@@ -47,3 +47,12 @@ func TestGetState(t *testing.T) {
 		t.Fatal("two States back-to-back were not equal")
 	}
 }
+
+func TestLikelyHomeRouterIP(t *testing.T) {
+	gw, my, ok := LikelyHomeRouterIP()
+	if !ok {
+		t.Logf("no result")
+		return
+	}
+	t.Logf("myIP = %v; gw = %v", my, gw)
+}

net/interfaces/interfaces_windows.go

@@ -0,0 +1,73 @@
+// Copyright (c) 2020 Tailscale Inc & AUTHORS All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package interfaces
+
+import (
+	"os/exec"
+	"syscall"
+
+	"go4.org/mem"
+	"inet.af/netaddr"
+	"tailscale.com/util/lineread"
+)
+
+func init() {
+	likelyHomeRouterIP = likelyHomeRouterIPWindows
+}
+
+/*
+Parse out 10.0.0.1 from:
+
+Z:\>route print -4
+===========================================================================
+Interface List
+ 15...aa 15 48 ff 1c 72 ......Red Hat VirtIO Ethernet Adapter
+  5...........................Tailscale Tunnel
+  1...........................Software Loopback Interface 1
+===========================================================================
+
+IPv4 Route Table
+===========================================================================
+Active Routes:
+Network Destination        Netmask          Gateway       Interface  Metric
+          0.0.0.0          0.0.0.0         10.0.0.1       10.0.28.63      5
+         10.0.0.0      255.255.0.0         On-link        10.0.28.63    261
+       10.0.28.63  255.255.255.255         On-link        10.0.28.63    261
+        10.0.42.0    255.255.255.0   100.103.42.106   100.103.42.106      5
+     10.0.255.255  255.255.255.255         On-link        10.0.28.63    261
+   34.193.248.174  255.255.255.255   100.103.42.106   100.103.42.106      5
+
+*/
+func likelyHomeRouterIPWindows() (ret netaddr.IP, ok bool) {
+	cmd := exec.Command("route", "print", "-4")
+	cmd.SysProcAttr = &syscall.SysProcAttr{HideWindow: true}
+	stdout, err := cmd.StdoutPipe()
+	if err != nil {
+		return
+	}
+	if err := cmd.Start(); err != nil {
+		return
+	}
+	defer cmd.Wait()
+
+	var f []mem.RO
+	lineread.Reader(stdout, func(lineb []byte) error {
+		line := mem.B(lineb)
+		if !mem.Contains(line, mem.S("0.0.0.0")) {
+			return nil
+		}
+		f = mem.AppendFields(f[:0], line)
+		if len(f) < 3 || !f[0].EqualString("0.0.0.0") || !f[1].EqualString("0.0.0.0") {
+			return nil
+		}
+		ipm := f[2]
+		ip, err := netaddr.ParseIP(string(mem.Append(nil, ipm)))
+		if err == nil && isPrivateIP(ip) {
+			ret = ip
+		}
+		return nil
+	})
+	return ret, !ret.IsZero()
+}

net/netcheck/netcheck.go

@@ -8,7 +8,9 @@ package netcheck
 import (
 	"bufio"
 	"context"
+	"crypto/rand"
 	"crypto/tls"
+	"encoding/binary"
 	"errors"
 	"fmt"
 	"io"
@@ -21,6 +23,7 @@ import (
 	"time"
 
 	"github.com/tcnksm/go-httpstat"
+	"go4.org/mem"
 	"inet.af/netaddr"
 	"tailscale.com/derp/derphttp"
 	"tailscale.com/net/dnscache"
@@ -34,15 +37,26 @@ import (
 )
 
 type Report struct {
-	UDP                   bool                  // UDP works
-	IPv6                  bool                  // IPv6 works
-	IPv4                  bool                  // IPv4 works
-	MappingVariesByDestIP opt.Bool              // for IPv4
-	HairPinning           opt.Bool              // for IPv4
-	PreferredDERP         int                   // or 0 for unknown
-	RegionLatency         map[int]time.Duration // keyed by DERP Region ID
-	RegionV4Latency       map[int]time.Duration // keyed by DERP Region ID
-	RegionV6Latency       map[int]time.Duration // keyed by DERP Region ID
+	UDP                   bool     // UDP works
+	IPv6                  bool     // IPv6 works
+	IPv4                  bool     // IPv4 works
+	MappingVariesByDestIP opt.Bool // for IPv4
+	HairPinning           opt.Bool // for IPv4
+
+	// UPnP is whether UPnP appears present on the LAN.
+	// Empty means not checked.
+	UPnP opt.Bool
+	// PMP is whether NAT-PMP appears present on the LAN.
+	// Empty means not checked.
+	PMP opt.Bool
+	// PCP is whether PCP appears present on the LAN.
+	// Empty means not checked.
+	PCP opt.Bool
+
+	PreferredDERP   int                   // or 0 for unknown
+	RegionLatency   map[int]time.Duration // keyed by DERP Region ID
+	RegionV4Latency map[int]time.Duration // keyed by DERP Region ID
+	RegionV6Latency map[int]time.Duration // keyed by DERP Region ID
 
 	GlobalV4 string // ip:port of global IPv4
 	GlobalV6 string // [ip]:port of global IPv6
@@ -50,6 +64,11 @@ type Report struct {
 	// TODO: update Clone when adding new fields
 }
 
+// AnyPortMappingChecked reports whether any of UPnP, PMP, or PCP are non-empty.
+func (r *Report) AnyPortMappingChecked() bool {
+	return r.UPnP != "" || r.PMP != "" || r.PCP != ""
+}
+
 func (r *Report) Clone() *Report {
 	if r == nil {
 		return nil
@@ -434,6 +453,7 @@ type reportState struct {
 	pc4Hair     net.PacketConn
 	incremental bool // doing a lite, follow-up netcheck
 	stopProbeCh chan struct{}
+	waitPortMap sync.WaitGroup
 
 	mu            sync.Mutex
 	sentHairCheck bool
@@ -599,6 +619,102 @@ func (rs *reportState) stopProbes() {
 	}
 }
 
+func (rs *reportState) setOptBool(b *opt.Bool, v bool) {
+	rs.mu.Lock()
+	defer rs.mu.Unlock()
+	b.Set(v)
+}
+
+func (rs *reportState) probePortMapServices() {
+	defer rs.waitPortMap.Done()
+	gw, myIP, ok := interfaces.LikelyHomeRouterIP()
+	if !ok {
+		return
+	}
+
+	rs.setOptBool(&rs.report.UPnP, false)
+	rs.setOptBool(&rs.report.PMP, false)
+	rs.setOptBool(&rs.report.PCP, false)
+
+	port1900 := netaddr.IPPort{IP: gw, Port: 1900}.UDPAddr()
+	port5351 := netaddr.IPPort{IP: gw, Port: 5351}.UDPAddr()
+
+	rs.c.logf("probePortMapServices: me %v -> gw %v", myIP, gw)
+
+	// Create a UDP4 socket used just for querying for UPnP, NAT-PMP, and PCP.
+	uc, err := netns.Listener().ListenPacket(context.Background(), "udp4", ":0")
+	if err != nil {
+		rs.c.logf("probePortMapServices: %v", err)
+		return
+	}
+	defer uc.Close()
+	tempPort := uc.LocalAddr().(*net.UDPAddr).Port
+	uc.SetReadDeadline(time.Now().Add(100 * time.Millisecond))
+
+	// Send request packets for all three protocols.
+	uc.WriteTo(uPnPPacket, port1900)
+	uc.WriteTo(pmpPacket, port5351)
+	uc.WriteTo(pcpPacket(myIP, tempPort, false), port5351)
+
+	res := make([]byte, 1500)
+	for {
+		n, addr, err := uc.ReadFrom(res)
+		if err != nil {
+			return
+		}
+		switch addr.(*net.UDPAddr).Port {
+		case 1900:
+			if mem.Contains(mem.B(res[:n]), mem.S(":InternetGatewayDevice:")) {
+				rs.setOptBool(&rs.report.UPnP, true)
+			}
+		case 5351:
+			if n == 12 && res[0] == 0x00 { // right length and version 0
+				rs.setOptBool(&rs.report.PMP, true)
+			}
+			if n == 60 && res[0] == 0x02 { // right length and version 2
+				rs.setOptBool(&rs.report.PCP, true)
+
+				// And now delete the mapping.
+				// (PCP is the only protocol of the three that requires
+				// we cause a side effect to detect whether it's present,
+				// so we need to redo that side effect now.)
+				uc.WriteTo(pcpPacket(myIP, tempPort, true), port5351)
+			}
+		}
+	}
+}
+
+var pmpPacket = []byte{0, 0} // version 0, opcode 0 = "Public address request"
+
+var uPnPPacket = []byte("M-SEARCH * HTTP/1.1\r\n" +
+	"HOST: 239.255.255.250:1900\r\n" +
+	"ST: ssdp:all\r\n" +
+	"MAN: \"ssdp:discover\"\r\n" +
+	"MX: 2\r\n\r\n")
+
+var v4unspec, _ = netaddr.ParseIP("0.0.0.0")
+
+func pcpPacket(myIP netaddr.IP, mapToLocalPort int, delete bool) []byte {
+	const udpProtoNumber = 17
+	lifetimeSeconds := uint32(1)
+	if delete {
+		lifetimeSeconds = 0
+	}
+	const opMap = 1
+	pkt := make([]byte, (32+32+128)/8+(96+8+24+16+16+128)/8)
+	pkt[0] = 2 // version
+	pkt[1] = opMap
+	binary.BigEndian.PutUint32(pkt[4:8], lifetimeSeconds)
+	myIP16 := myIP.As16()
+	copy(pkt[8:], myIP16[:])
+	rand.Read(pkt[24 : 24+12])
+	pkt[36] = udpProtoNumber
+	binary.BigEndian.PutUint16(pkt[40:], uint16(mapToLocalPort))
+	v4unspec16 := v4unspec.As16()
+	copy(pkt[40:], v4unspec16[:])
+	return pkt
+}
+
 func newReport() *Report {
 	return &Report{
 		RegionLatency:   make(map[int]time.Duration),
@@ -671,6 +787,22 @@ func (c *Client) GetReport(ctx context.Context, dm *tailcfg.DERPMap) (*Report, e
 	}
 	defer rs.pc4Hair.Close()
 
+	rs.waitPortMap.Add(1)
+	go rs.probePortMapServices()
+
+	// At least the Apple Airport Extreme doesn't allow hairpin
+	// sends from a private socket until it's seen traffic from
+	// that src IP:port to something else out on the internet.
+	//
+	// See https://github.com/tailscale/tailscale/issues/188#issuecomment-600728643
+	//
+	// And it seems that even sending to a likely-filtered RFC 5737
+	// documentation-only IPv4 range is enough to set up the mapping.
+	// So do that for now. In the future we might want to classify networks
+	// that do and don't require this separately. But for now help it.
+	const documentationIP = "203.0.113.1"
+	rs.pc4Hair.WriteTo([]byte("tailscale netcheck; see https://github.com/tailscale/tailscale/issues/188"), &net.UDPAddr{IP: net.ParseIP(documentationIP), Port: 12345})
+
 	if f := c.GetSTUNConn4; f != nil {
 		rs.pc4 = f()
 	} else {
@@ -725,6 +857,7 @@ func (c *Client) GetReport(ctx context.Context, dm *tailcfg.DERPMap) (*Report, e
 	}
 
 	rs.waitHairCheck(ctx)
+	rs.waitPortMap.Wait()
 	rs.stopTimers()
 
 	// Try HTTPS latency check if all STUN probes failed due to UDP presumably being blocked.
@@ -848,6 +981,11 @@ func (c *Client) logConciseReport(r *Report, dm *tailcfg.DERPMap) {
 		fmt.Fprintf(w, " v6=%v", r.IPv6)
 		fmt.Fprintf(w, " mapvarydest=%v", r.MappingVariesByDestIP)
 		fmt.Fprintf(w, " hair=%v", r.HairPinning)
+		if r.AnyPortMappingChecked() {
+			fmt.Fprintf(w, " portmap=%v%v%v", conciseOptBool(r.UPnP, "U"), conciseOptBool(r.PMP, "M"), conciseOptBool(r.PCP, "C"))
+		} else {
+			fmt.Fprintf(w, " portmap=?")
+		}
 		if r.GlobalV4 != "" {
 			fmt.Fprintf(w, " v4a=%v", r.GlobalV4)
 		}
@@ -1008,6 +1146,20 @@ func (c *Client) nodeAddr(ctx context.Context, n *tailcfg.DERPNode, proto probeP
 	if port < 0 || port > 1<<16-1 {
 		return nil
 	}
+	if n.STUNTestIP != "" {
+		ip, err := netaddr.ParseIP(n.STUNTestIP)
+		if err != nil {
+			return nil
+		}
+		if proto == probeIPv4 && ip.Is6() {
+			return nil
+		}
+		if proto == probeIPv6 && ip.Is4() {
+			return nil
+		}
+		return netaddr.IPPort{IP: ip, Port: uint16(port)}.UDPAddr()
+	}
+
 	switch proto {
 	case probeIPv4:
 		if n.IPv4 != "" {
@@ -1015,7 +1167,7 @@ func (c *Client) nodeAddr(ctx context.Context, n *tailcfg.DERPNode, proto probeP
 			if !ip.Is4() {
 				return nil
 			}
-			return netaddr.IPPort{ip, uint16(port)}.UDPAddr()
+			return netaddr.IPPort{IP: ip, Port: uint16(port)}.UDPAddr()
 		}
 	case probeIPv6:
 		if n.IPv6 != "" {
@@ -1023,7 +1175,7 @@ func (c *Client) nodeAddr(ctx context.Context, n *tailcfg.DERPNode, proto probeP
 			if !ip.Is6() {
 				return nil
 			}
-			return netaddr.IPPort{ip, uint16(port)}.UDPAddr()
+			return netaddr.IPPort{IP: ip, Port: uint16(port)}.UDPAddr()
 		}
 	default:
 		return nil
@@ -1056,3 +1208,17 @@ func maxDurationValue(m map[int]time.Duration) (max time.Duration) {
 	}
 	return max
 }
+
+func conciseOptBool(b opt.Bool, trueVal string) string {
+	if b == "" {
+		return "_"
+	}
+	v, ok := b.Get()
+	if !ok {
+		return "x"
+	}
+	if v {
+		return trueVal
+	}
+	return ""
+}

net/netcheck/netcheck_test.go

@@ -100,6 +100,9 @@ func TestWorksWhenUDPBlocked(t *testing.T) {
 		t.Fatal(err)
 	}
 	want := newReport()
+	r.UPnP = ""
+	r.PMP = ""
+	r.PCP = ""
 
 	if !reflect.DeepEqual(r, want) {
 		t.Errorf("mismatch\n got: %+v\nwant: %+v\n", r, want)
@@ -463,7 +466,7 @@ func TestLogConciseReport(t *testing.T) {
 		{
 			name: "no_udp",
 			r:    &Report{},
-			want: "udp=false v4=false v6=false mapvarydest= hair= derp=0",
+			want: "udp=false v4=false v6=false mapvarydest= hair= portmap=? derp=0",
 		},
 		{
 			name: "ipv4_one_region",
@@ -478,7 +481,7 @@ func TestLogConciseReport(t *testing.T) {
 					1: 10 * ms,
 				},
 			},
-			want: "udp=true v6=false mapvarydest= hair= derp=1 derpdist=1v4:10ms",
+			want: "udp=true v6=false mapvarydest= hair= portmap=? derp=1 derpdist=1v4:10ms",
 		},
 		{
 			name: "ipv4_all_region",
@@ -497,7 +500,7 @@ func TestLogConciseReport(t *testing.T) {
 					3: 30 * ms,
 				},
 			},
-			want: "udp=true v6=false mapvarydest= hair= derp=1 derpdist=1v4:10ms,2v4:20ms,3v4:30ms",
+			want: "udp=true v6=false mapvarydest= hair= portmap=? derp=1 derpdist=1v4:10ms,2v4:20ms,3v4:30ms",
 		},
 		{
 			name: "ipboth_all_region",
@@ -522,7 +525,27 @@ func TestLogConciseReport(t *testing.T) {
 					3: 30 * ms,
 				},
 			},
-			want: "udp=true v6=true mapvarydest= hair= derp=1 derpdist=1v4:10ms,1v6:10ms,2v4:20ms,2v6:20ms,3v4:30ms,3v6:30ms",
+			want: "udp=true v6=true mapvarydest= hair= portmap=? derp=1 derpdist=1v4:10ms,1v6:10ms,2v4:20ms,2v6:20ms,3v4:30ms,3v6:30ms",
+		},
+		{
+			name: "portmap_all",
+			r: &Report{
+				UDP:  true,
+				UPnP: "true",
+				PMP:  "true",
+				PCP:  "true",
+			},
+			want: "udp=true v4=false v6=false mapvarydest= hair= portmap=UMC derp=0",
+		},
+		{
+			name: "portmap_some",
+			r: &Report{
+				UDP:  true,
+				UPnP: "true",
+				PMP:  "false",
+				PCP:  "true",
+			},
+			want: "udp=true v4=false v6=false mapvarydest= hair= portmap=UC derp=0",
 		},
 	}
 	for _, tt := range tests {

net/netns/netns_linux.go

@@ -26,7 +26,7 @@ import (
 //
 // Keep this in sync with tailscaleBypassMark in
 // wgengine/router/router_linux.go.
-const tailscaleBypassMark = 0x20000
+const tailscaleBypassMark = 0x80000
 
 // ipRuleOnce is the sync.Once & cached value for ipRuleAvailable.
 var ipRuleOnce struct {

net/stun/stuntest/stuntest.go

@@ -6,6 +6,7 @@
 package stuntest
 
 import (
+	"context"
 	"fmt"
 	"net"
 	"strconv"
@@ -16,6 +17,7 @@ import (
 	"inet.af/netaddr"
 	"tailscale.com/net/stun"
 	"tailscale.com/tailcfg"
+	"tailscale.com/types/nettype"
 )
 
 type stunStats struct {
@@ -25,18 +27,22 @@ type stunStats struct {
 }
 
 func Serve(t *testing.T) (addr *net.UDPAddr, cleanupFn func()) {
+	return ServeWithPacketListener(t, nettype.Std{})
+}
+
+func ServeWithPacketListener(t *testing.T, ln nettype.PacketListener) (addr *net.UDPAddr, cleanupFn func()) {
 	t.Helper()
 
 	// TODO(crawshaw): use stats to test re-STUN logic
 	var stats stunStats
 
-	pc, err := net.ListenPacket("udp4", ":0")
+	pc, err := ln.ListenPacket(context.Background(), "udp4", ":0")
 	if err != nil {
 		t.Fatalf("failed to open STUN listener: %v", err)
 	}
-	addr = &net.UDPAddr{
-		IP:   net.ParseIP("127.0.0.1"),
-		Port: pc.LocalAddr().(*net.UDPAddr).Port,
+	addr = pc.LocalAddr().(*net.UDPAddr)
+	if len(addr.IP) == 0 || addr.IP.IsUnspecified() {
+		addr.IP = net.ParseIP("127.0.0.1")
 	}
 	doneCh := make(chan struct{})
 	go runSTUN(t, pc, &stats, doneCh)

net/tsaddr/tsaddr.go

@@ -0,0 +1,52 @@
+// Copyright (c) 2020 Tailscale Inc & AUTHORS All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+// Package tsaddr handles Tailscale-specific IPs and ranges.
+package tsaddr
+
+import (
+	"sync"
+
+	"inet.af/netaddr"
+)
+
+// ChromeOSVMRange returns the subset of the CGNAT IPv4 range used by
+// ChromeOS to interconnect the host OS to containers and VMs. We
+// avoid allocating Tailscale IPs from it, to avoid conflicts.
+func ChromeOSVMRange() netaddr.IPPrefix {
+	chromeOSRange.Do(func() { mustPrefix(&chromeOSRange.v, "100.115.92.0/23") })
+	return chromeOSRange.v
+}
+
+var chromeOSRange oncePrefix
+
+// CGNATRange returns the Carrier Grade NAT address range that
+// is the superset range that Tailscale assigns out of.
+// See https://tailscale.com/kb/1015/100.x-addresses.
+// Note that Tailscale does not assign out of the ChromeOSVMRange.
+func CGNATRange() netaddr.IPPrefix {
+	cgnatRange.Do(func() { mustPrefix(&cgnatRange.v, "100.64.0.0/10") })
+	return cgnatRange.v
+}
+
+var cgnatRange oncePrefix
+
+// IsTailscaleIP reports whether ip is an IP address in a range that
+// Tailscale assigns from.
+func IsTailscaleIP(ip netaddr.IP) bool {
+	return CGNATRange().Contains(ip) && !ChromeOSVMRange().Contains(ip)
+}
+
+func mustPrefix(v *netaddr.IPPrefix, prefix string) {
+	var err error
+	*v, err = netaddr.ParseIPPrefix(prefix)
+	if err != nil {
+		panic(err)
+	}
+}
+
+type oncePrefix struct {
+	sync.Once
+	v netaddr.IPPrefix
+}

net/tsaddr/tsaddr_test.go

@@ -0,0 +1,19 @@
+// Copyright (c) 2020 Tailscale Inc & AUTHORS All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package tsaddr
+
+import "testing"
+
+func TestChromeOSVMRange(t *testing.T) {
+	if got, want := ChromeOSVMRange().String(), "100.115.92.0/23"; got != want {
+		t.Errorf("got %q; want %q", got, want)
+	}
+}
+
+func TestCGNATRange(t *testing.T) {
+	if got, want := CGNATRange().String(), "100.64.0.0/10"; got != want {
+		t.Errorf("got %q; want %q", got, want)
+	}
+}

paths/paths.go

@@ -11,9 +11,15 @@ import (
 	"runtime"
 )
 
-// LegacyConfigPath is the path used by the pre-tailscaled "relaynode"
-// daemon's config file.
-const LegacyConfigPath = "/var/lib/tailscale/relay.conf"
+// LegacyConfigPath returns the path used by the pre-tailscaled
+// "relaynode" daemon's config file. It returns the empty string for
+// platforms where relaynode never ran.
+func LegacyConfigPath() string {
+	if runtime.GOOS == "windows" {
+		return ""
+	}
+	return "/var/lib/tailscale/relay.conf"
+}
 
 // DefaultTailscaledSocket returns the path to the tailscaled Unix socket
 // or the empty string if there's no reasonable default.

safesocket/unixsocket.go

@@ -11,6 +11,7 @@ import (
 	"bytes"
 	"fmt"
 	"io"
+	"io/ioutil"
 	"log"
 	"net"
 	"os"
@@ -77,6 +78,16 @@ func listen(path string, port uint16) (ln net.Listener, _ uint16, err error) {
 //   * it also picks a random hex string that acts as an auth token
 //   * it then creates a file named "sameuserproof-$PORT-$TOKEN" and leaves
 //     that file descriptor open forever.
+//
+// Then, we do different things depending on whether the user is
+// running cmd/tailscale that they built themselves (running as
+// themselves, outside the App Sandbox), or whether the user is
+// running the CLI via the GUI binary
+// (e.g. /Applications/Tailscale.app/Contents/MacOS/Tailscale <args>),
+// in which case we're running within the App Sandbox.
+//
+// If we're outside the App Sandbox:
+//
 //   * then we come along here, running as the same UID, but outside
 //     of the sandbox, and look for it. We can run lsof on our own processes,
 //     but other users on the system can't.
@@ -86,7 +97,38 @@ func listen(path string, port uint16) (ln net.Listener, _ uint16, err error) {
 //   * server verifies $TOKEN, sends "#IPN\n" if okay.
 //   * server is now protocol switched
 //   * we return the net.Conn and the caller speaks the normal protocol
+//
+// If we're inside the App Sandbox, then TS_MACOS_CLI_SHARED_DIR has
+// been set to our shared directory. We now have to find the most
+// recent "sameuserproof" file (there should only be 1, but previous
+// versions of the macOS app didn't clean them up).
 func connectMacOSAppSandbox() (net.Conn, error) {
+	// Are we running the Tailscale.app GUI binary as a CLI, running within the App Sandbox?
+	if d := os.Getenv("TS_MACOS_CLI_SHARED_DIR"); d != "" {
+		fis, err := ioutil.ReadDir(d)
+		if err != nil {
+			return nil, fmt.Errorf("reading TS_MACOS_CLI_SHARED_DIR: %w", err)
+		}
+		var best os.FileInfo
+		for _, fi := range fis {
+			if !strings.HasPrefix(fi.Name(), "sameuserproof-") || strings.Count(fi.Name(), "-") != 2 {
+				continue
+			}
+			if best == nil || fi.ModTime().After(best.ModTime()) {
+				best = fi
+			}
+		}
+		if best == nil {
+			return nil, fmt.Errorf("no sameuserproof token found in TS_MACOS_CLI_SHARED_DIR %q", d)
+		}
+		f := strings.SplitN(best.Name(), "-", 3)
+		portStr, token := f[1], f[2]
+		return connectMacTCP(portStr, token)
+	}
+
+	// Otherwise, assume we're running the cmd/tailscale binary from outside the
+	// App Sandbox.
+
 	out, err := exec.Command("lsof",
 		"-n",                             // numeric sockets; don't do DNS lookups, etc
 		"-a",                             // logical AND remaining options
@@ -110,22 +152,26 @@ func connectMacOSAppSandbox() (net.Conn, error) {
 			continue
 		}
 		portStr, token := f[0], f[1]
-		c, err := net.Dial("tcp", "localhost:"+portStr)
-		if err != nil {
-			return nil, fmt.Errorf("error dialing IPNExtension: %w", err)
-		}
-		if _, err := io.WriteString(c, token+"\n"); err != nil {
-			return nil, fmt.Errorf("error writing auth token: %w", err)
-		}
-		buf := make([]byte, 5)
-		const authOK = "#IPN\n"
-		if _, err := io.ReadFull(c, buf); err != nil {
-			return nil, fmt.Errorf("error reading from IPNExtension post-auth: %w", err)
-		}
-		if string(buf) != authOK {
-			return nil, fmt.Errorf("invalid response reading from IPNExtension post-auth")
-		}
-		return c, nil
+		return connectMacTCP(portStr, token)
 	}
 	return nil, fmt.Errorf("failed to find Tailscale's IPNExtension process")
 }
+
+func connectMacTCP(portStr, token string) (net.Conn, error) {
+	c, err := net.Dial("tcp", "localhost:"+portStr)
+	if err != nil {
+		return nil, fmt.Errorf("error dialing IPNExtension: %w", err)
+	}
+	if _, err := io.WriteString(c, token+"\n"); err != nil {
+		return nil, fmt.Errorf("error writing auth token: %w", err)
+	}
+	buf := make([]byte, 5)
+	const authOK = "#IPN\n"
+	if _, err := io.ReadFull(c, buf); err != nil {
+		return nil, fmt.Errorf("error reading from IPNExtension post-auth: %w", err)
+	}
+	if string(buf) != authOK {
+		return nil, fmt.Errorf("invalid response reading from IPNExtension post-auth")
+	}
+	return c, nil
+}

tailcfg/derpmap.go

@@ -117,4 +117,8 @@ type DERPNode struct {
 	// of using the default port of 443. If non-zero, TLS
 	// verification is skipped.
 	DERPTestPort int `json:",omitempty"`
+
+	// STUNTestIP is used in tests to override the STUN server's IP.
+	// If empty, it's assumed to be the same as the DERP server.
+	STUNTestIP string `json:",omitempty"`
 }

tailcfg/tailcfg.go

@@ -4,6 +4,8 @@
 
 package tailcfg
 
+//go:generate go run tailscale.com/cmd/cloner -type=User,Node,Hostinfo,NetInfo -output=tailcfg_clone.go
+
 import (
 	"bytes"
 	"errors"
@@ -93,18 +95,6 @@ type User struct {
 	// Note: be sure to update Clone when adding new fields
 }
 
-// Clone returns a copy of u that aliases no memory with the original.
-func (u *User) Clone() *User {
-	if u == nil {
-		return nil
-	}
-	u2 := new(User)
-	*u2 = *u
-	u2.Logins = append([]LoginID(nil), u.Logins...)
-	u2.Roles = append([]RoleID(nil), u.Roles...)
-	return u2
-}
-
 type Login struct {
 	_             structs.Incomparable
 	ID            LoginID
@@ -150,23 +140,6 @@ type Node struct {
 	//       require changes to Node.Clone.
 }
 
-// Clone makes a deep copy of Node.
-// The result aliases no memory with the original.
-func (n *Node) Clone() (res *Node) {
-	res = new(Node)
-	*res = *n
-
-	res.Addresses = append([]wgcfg.CIDR{}, res.Addresses...)
-	res.AllowedIPs = append([]wgcfg.CIDR{}, res.AllowedIPs...)
-	res.Endpoints = append([]string{}, res.Endpoints...)
-	if res.LastSeen != nil {
-		lastSeen := *res.LastSeen
-		res.LastSeen = &lastSeen
-	}
-	res.Hostinfo = *res.Hostinfo.Clone()
-	return res
-}
-
 type MachineStatus int
 
 const (
@@ -284,6 +257,8 @@ type Hostinfo struct {
 	FrontendLogID string       // logtail ID of frontend instance
 	BackendLogID  string       // logtail ID of backend instance
 	OS            string       // operating system the client runs on (a version.OS value)
+	OSVersion     string       // operating system version, with optional distro prefix ("Debian 10.4", "Windows 10 Pro 10.0.19041")
+	DeviceModel   string       // mobile phone model ("Pixel 3a", "iPhone 11 Pro")
 	Hostname      string       // name of the host the client runs on
 	RoutableIPs   []wgcfg.CIDR `json:",omitempty"` // set of IP ranges this client can route
 	RequestTags   []string     `json:",omitempty"` // set of ACL tags this node wants to claim
@@ -310,6 +285,18 @@ type NetInfo struct {
 	// WorkingUDP is whether UDP works.
 	WorkingUDP opt.Bool
 
+	// UPnP is whether UPnP appears present on the LAN.
+	// Empty means not checked.
+	UPnP opt.Bool
+
+	// PMP is whether NAT-PMP appears present on the LAN.
+	// Empty means not checked.
+	PMP opt.Bool
+
+	// PCP is whether PCP appears present on the LAN.
+	// Empty means not checked.
+	PCP opt.Bool
+
 	// PreferredDERP is this node's preferred DERP server
 	// for incoming traffic. The node might be be temporarily
 	// connected to multiple DERP servers (to send to other nodes)
@@ -338,9 +325,32 @@ func (ni *NetInfo) String() string {
 	if ni == nil {
 		return "NetInfo(nil)"
 	}
-	return fmt.Sprintf("NetInfo{varies=%v hairpin=%v ipv6=%v udp=%v derp=#%v link=%q}",
+	return fmt.Sprintf("NetInfo{varies=%v hairpin=%v ipv6=%v udp=%v derp=#%v portmap=%v link=%q}",
 		ni.MappingVariesByDestIP, ni.HairPinning, ni.WorkingIPv6,
-		ni.WorkingUDP, ni.PreferredDERP, ni.LinkType)
+		ni.WorkingUDP, ni.PreferredDERP,
+		ni.portMapSummary(),
+		ni.LinkType)
+}
+
+func (ni *NetInfo) portMapSummary() string {
+	if ni.UPnP == "" && ni.PMP == "" && ni.PCP == "" {
+		return "?"
+	}
+	return conciseOptBool(ni.UPnP, "U") + conciseOptBool(ni.PMP, "M") + conciseOptBool(ni.PCP, "C")
+}
+
+func conciseOptBool(b opt.Bool, trueVal string) string {
+	if b == "" {
+		return "_"
+	}
+	v, ok := b.Get()
+	if !ok {
+		return "x"
+	}
+	if v {
+		return trueVal
+	}
+	return ""
 }
 
 // BasicallyEqual reports whether ni and ni2 are basically equal, ignoring
@@ -356,39 +366,21 @@ func (ni *NetInfo) BasicallyEqual(ni2 *NetInfo) bool {
 		ni.HairPinning == ni2.HairPinning &&
 		ni.WorkingIPv6 == ni2.WorkingIPv6 &&
 		ni.WorkingUDP == ni2.WorkingUDP &&
+		ni.UPnP == ni2.UPnP &&
+		ni.PMP == ni2.PMP &&
+		ni.PCP == ni2.PCP &&
 		ni.PreferredDERP == ni2.PreferredDERP &&
 		ni.LinkType == ni2.LinkType
 }
 
-func (ni *NetInfo) Clone() (res *NetInfo) {
-	if ni == nil {
-		return nil
-	}
-	res = new(NetInfo)
-	*res = *ni
-	if ni.DERPLatency != nil {
-		res.DERPLatency = map[string]float64{}
-		for k, v := range ni.DERPLatency {
-			res.DERPLatency[k] = v
-		}
-	}
-	return res
-}
-
-// Clone makes a deep copy of Hostinfo.
-// The result aliases no memory with the original.
-func (h *Hostinfo) Clone() (res *Hostinfo) {
-	res = new(Hostinfo)
-	*res = *h
-
-	res.RoutableIPs = append([]wgcfg.CIDR{}, h.RoutableIPs...)
-	res.Services = append([]Service{}, h.Services...)
-	res.NetInfo = h.NetInfo.Clone()
-	return res
-}
-
 // Equal reports whether h and h2 are equal.
 func (h *Hostinfo) Equal(h2 *Hostinfo) bool {
+	if h == nil && h2 == nil {
+		return true
+	}
+	if (h == nil) != (h2 == nil) {
+		return false
+	}
 	return reflect.DeepEqual(h, h2)
 }
 
@@ -415,6 +407,8 @@ type RegisterRequest struct {
 
 // Clone makes a deep copy of RegisterRequest.
 // The result aliases no memory with the original.
+//
+// TODO: extend cmd/cloner to generate this method.
 func (req *RegisterRequest) Clone() *RegisterRequest {
 	res := new(RegisterRequest)
 	*res = *req
@@ -598,11 +592,39 @@ func (n *Node) Equal(n2 *Node) bool {
 		n.KeyExpiry.Equal(n2.KeyExpiry) &&
 		n.Machine == n2.Machine &&
 		n.DiscoKey == n2.DiscoKey &&
-		reflect.DeepEqual(n.Addresses, n2.Addresses) &&
-		reflect.DeepEqual(n.AllowedIPs, n2.AllowedIPs) &&
-		reflect.DeepEqual(n.Endpoints, n2.Endpoints) &&
-		reflect.DeepEqual(n.Hostinfo, n2.Hostinfo) &&
+		eqCIDRs(n.Addresses, n2.Addresses) &&
+		eqCIDRs(n.AllowedIPs, n2.AllowedIPs) &&
+		eqStrings(n.Endpoints, n2.Endpoints) &&
+		n.Hostinfo.Equal(&n2.Hostinfo) &&
 		n.Created.Equal(n2.Created) &&
-		reflect.DeepEqual(n.LastSeen, n2.LastSeen) &&
+		eqTimePtr(n.LastSeen, n2.LastSeen) &&
 		n.MachineAuthorized == n2.MachineAuthorized
 }
+
+func eqStrings(a, b []string) bool {
+	if len(a) != len(b) || ((a == nil) != (b == nil)) {
+		return false
+	}
+	for i, v := range a {
+		if v != b[i] {
+			return false
+		}
+	}
+	return true
+}
+
+func eqCIDRs(a, b []wgcfg.CIDR) bool {
+	if len(a) != len(b) || ((a == nil) != (b == nil)) {
+		return false
+	}
+	for i, v := range a {
+		if v != b[i] {
+			return false
+		}
+	}
+	return true
+}
+
+func eqTimePtr(a, b *time.Time) bool {
+	return ((a == nil) == (b == nil)) && (a == nil || a.Equal(*b))
+}

tailcfg/tailcfg_clone.go

@@ -0,0 +1,75 @@
+// Copyright (c) 2020 Tailscale Inc & AUTHORS All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+// Code generated by tailscale.com/cmd/cloner -type User,Node,Hostinfo,NetInfo; DO NOT EDIT.
+
+package tailcfg
+
+import (
+	"time"
+)
+
+// Clone makes a deep copy of User.
+// The result aliases no memory with the original.
+func (src *User) Clone() *User {
+	if src == nil {
+		return nil
+	}
+	dst := new(User)
+	*dst = *src
+	dst.Logins = append(src.Logins[:0:0], src.Logins...)
+	dst.Roles = append(src.Roles[:0:0], src.Roles...)
+	return dst
+}
+
+// Clone makes a deep copy of Node.
+// The result aliases no memory with the original.
+func (src *Node) Clone() *Node {
+	if src == nil {
+		return nil
+	}
+	dst := new(Node)
+	*dst = *src
+	dst.Addresses = append(src.Addresses[:0:0], src.Addresses...)
+	dst.AllowedIPs = append(src.AllowedIPs[:0:0], src.AllowedIPs...)
+	dst.Endpoints = append(src.Endpoints[:0:0], src.Endpoints...)
+	dst.Hostinfo = *src.Hostinfo.Clone()
+	if dst.LastSeen != nil {
+		dst.LastSeen = new(time.Time)
+		*dst.LastSeen = *src.LastSeen
+	}
+	return dst
+}
+
+// Clone makes a deep copy of Hostinfo.
+// The result aliases no memory with the original.
+func (src *Hostinfo) Clone() *Hostinfo {
+	if src == nil {
+		return nil
+	}
+	dst := new(Hostinfo)
+	*dst = *src
+	dst.RoutableIPs = append(src.RoutableIPs[:0:0], src.RoutableIPs...)
+	dst.RequestTags = append(src.RequestTags[:0:0], src.RequestTags...)
+	dst.Services = append(src.Services[:0:0], src.Services...)
+	dst.NetInfo = src.NetInfo.Clone()
+	return dst
+}
+
+// Clone makes a deep copy of NetInfo.
+// The result aliases no memory with the original.
+func (src *NetInfo) Clone() *NetInfo {
+	if src == nil {
+		return nil
+	}
+	dst := new(NetInfo)
+	*dst = *src
+	if dst.DERPLatency != nil {
+		dst.DERPLatency = map[string]float64{}
+		for k, v := range src.DERPLatency {
+			dst.DERPLatency[k] = v
+		}
+	}
+	return dst
+}

tailcfg/tailcfg_test.go

@@ -23,7 +23,8 @@ func fieldsOf(t reflect.Type) (fields []string) {
 
 func TestHostinfoEqual(t *testing.T) {
 	hiHandles := []string{
-		"IPNVersion", "FrontendLogID", "BackendLogID", "OS", "Hostname", "RoutableIPs", "RequestTags", "Services",
+		"IPNVersion", "FrontendLogID", "BackendLogID", "OS", "OSVersion",
+		"DeviceModel", "Hostname", "RoutableIPs", "RequestTags", "Services",
 		"NetInfo",
 	}
 	if have := fieldsOf(reflect.TypeOf(Hostinfo{})); !reflect.DeepEqual(have, hiHandles) {
@@ -329,6 +330,9 @@ func TestNetInfoFields(t *testing.T) {
 		"HairPinning",
 		"WorkingIPv6",
 		"WorkingUDP",
+		"UPnP",
+		"PMP",
+		"PCP",
 		"PreferredDERP",
 		"LinkType",
 		"DERPLatency",
@@ -386,3 +390,43 @@ func testKey(t *testing.T, prefix string, in keyIn, out encoding.TextUnmarshaler
 		t.Errorf("mismatch after unmarshal")
 	}
 }
+
+func TestCloneUser(t *testing.T) {
+	tests := []struct {
+		name string
+		u    *User
+	}{
+		{"nil_logins", &User{}},
+		{"zero_logins", &User{Logins: make([]LoginID, 0)}},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			u2 := tt.u.Clone()
+			if !reflect.DeepEqual(tt.u, u2) {
+				t.Errorf("not equal")
+			}
+		})
+	}
+}
+
+func TestCloneNode(t *testing.T) {
+	tests := []struct {
+		name string
+		v    *Node
+	}{
+		{"nil_fields", &Node{}},
+		{"zero_fields", &Node{
+			Addresses:  make([]wgcfg.CIDR, 0),
+			AllowedIPs: make([]wgcfg.CIDR, 0),
+			Endpoints:  make([]string, 0),
+		}},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			v2 := tt.v.Clone()
+			if !reflect.DeepEqual(tt.v, v2) {
+				t.Errorf("not equal")
+			}
+		})
+	}
+}

tempfork/pprof/README.md

@@ -0,0 +1,4 @@
+This is a fork of net/http/pprof that doesn't use init side effects
+and doesn't use html/template (which ends up calling
+reflect.Value.MethodByName, which disables some linker deadcode
+optimizations).

tempfork/pprof/pprof.go

@@ -0,0 +1,301 @@
+// Copyright 2010 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+// Package pprof serves via its HTTP server runtime profiling data
+// in the format expected by the pprof visualization tool.
+//
+// See Go's net/http/pprof for docs.
+//
+// This is a fork of net/http/pprof that doesn't use init side effects
+// and doesn't use html/template (which ends up calling
+// reflect.Value.MethodByName, which disables some linker deadcode
+// optimizations).
+package pprof
+
+import (
+	"bufio"
+	"bytes"
+	"fmt"
+	"html"
+	"io"
+	"net/http"
+	"os"
+	"runtime"
+	"runtime/pprof"
+	"runtime/trace"
+	"sort"
+	"strconv"
+	"strings"
+	"time"
+)
+
+func AddHandlers(mux *http.ServeMux) {
+	mux.HandleFunc("/debug/pprof/", Index)
+	mux.HandleFunc("/debug/pprof/cmdline", Cmdline)
+	mux.HandleFunc("/debug/pprof/profile", Profile)
+	mux.HandleFunc("/debug/pprof/symbol", Symbol)
+	mux.HandleFunc("/debug/pprof/trace", Trace)
+}
+
+// Cmdline responds with the running program's
+// command line, with arguments separated by NUL bytes.
+// The package initialization registers it as /debug/pprof/cmdline.
+func Cmdline(w http.ResponseWriter, r *http.Request) {
+	w.Header().Set("X-Content-Type-Options", "nosniff")
+	w.Header().Set("Content-Type", "text/plain; charset=utf-8")
+	fmt.Fprintf(w, strings.Join(os.Args, "\x00"))
+}
+
+func sleep(w http.ResponseWriter, d time.Duration) {
+	var clientGone <-chan bool
+	if cn, ok := w.(http.CloseNotifier); ok {
+		clientGone = cn.CloseNotify()
+	}
+	select {
+	case <-time.After(d):
+	case <-clientGone:
+	}
+}
+
+func durationExceedsWriteTimeout(r *http.Request, seconds float64) bool {
+	srv, ok := r.Context().Value(http.ServerContextKey).(*http.Server)
+	return ok && srv.WriteTimeout != 0 && seconds >= srv.WriteTimeout.Seconds()
+}
+
+func serveError(w http.ResponseWriter, status int, txt string) {
+	w.Header().Set("Content-Type", "text/plain; charset=utf-8")
+	w.Header().Set("X-Go-Pprof", "1")
+	w.Header().Del("Content-Disposition")
+	w.WriteHeader(status)
+	fmt.Fprintln(w, txt)
+}
+
+// Profile responds with the pprof-formatted cpu profile.
+// Profiling lasts for duration specified in seconds GET parameter, or for 30 seconds if not specified.
+// The package initialization registers it as /debug/pprof/profile.
+func Profile(w http.ResponseWriter, r *http.Request) {
+	w.Header().Set("X-Content-Type-Options", "nosniff")
+	sec, err := strconv.ParseInt(r.FormValue("seconds"), 10, 64)
+	if sec <= 0 || err != nil {
+		sec = 30
+	}
+
+	if durationExceedsWriteTimeout(r, float64(sec)) {
+		serveError(w, http.StatusBadRequest, "profile duration exceeds server's WriteTimeout")
+		return
+	}
+
+	// Set Content Type assuming StartCPUProfile will work,
+	// because if it does it starts writing.
+	w.Header().Set("Content-Type", "application/octet-stream")
+	w.Header().Set("Content-Disposition", `attachment; filename="profile"`)
+	if err := pprof.StartCPUProfile(w); err != nil {
+		// StartCPUProfile failed, so no writes yet.
+		serveError(w, http.StatusInternalServerError,
+			fmt.Sprintf("Could not enable CPU profiling: %s", err))
+		return
+	}
+	sleep(w, time.Duration(sec)*time.Second)
+	pprof.StopCPUProfile()
+}
+
+// Trace responds with the execution trace in binary form.
+// Tracing lasts for duration specified in seconds GET parameter, or for 1 second if not specified.
+// The package initialization registers it as /debug/pprof/trace.
+func Trace(w http.ResponseWriter, r *http.Request) {
+	w.Header().Set("X-Content-Type-Options", "nosniff")
+	sec, err := strconv.ParseFloat(r.FormValue("seconds"), 64)
+	if sec <= 0 || err != nil {
+		sec = 1
+	}
+
+	if durationExceedsWriteTimeout(r, sec) {
+		serveError(w, http.StatusBadRequest, "profile duration exceeds server's WriteTimeout")
+		return
+	}
+
+	// Set Content Type assuming trace.Start will work,
+	// because if it does it starts writing.
+	w.Header().Set("Content-Type", "application/octet-stream")
+	w.Header().Set("Content-Disposition", `attachment; filename="trace"`)
+	if err := trace.Start(w); err != nil {
+		// trace.Start failed, so no writes yet.
+		serveError(w, http.StatusInternalServerError,
+			fmt.Sprintf("Could not enable tracing: %s", err))
+		return
+	}
+	sleep(w, time.Duration(sec*float64(time.Second)))
+	trace.Stop()
+}
+
+// Symbol looks up the program counters listed in the request,
+// responding with a table mapping program counters to function names.
+// The package initialization registers it as /debug/pprof/symbol.
+func Symbol(w http.ResponseWriter, r *http.Request) {
+	w.Header().Set("X-Content-Type-Options", "nosniff")
+	w.Header().Set("Content-Type", "text/plain; charset=utf-8")
+
+	// We have to read the whole POST body before
+	// writing any output. Buffer the output here.
+	var buf bytes.Buffer
+
+	// We don't know how many symbols we have, but we
+	// do have symbol information. Pprof only cares whether
+	// this number is 0 (no symbols available) or > 0.
+	fmt.Fprintf(&buf, "num_symbols: 1\n")
+
+	var b *bufio.Reader
+	if r.Method == "POST" {
+		b = bufio.NewReader(r.Body)
+	} else {
+		b = bufio.NewReader(strings.NewReader(r.URL.RawQuery))
+	}
+
+	for {
+		word, err := b.ReadSlice('+')
+		if err == nil {
+			word = word[0 : len(word)-1] // trim +
+		}
+		pc, _ := strconv.ParseUint(string(word), 0, 64)
+		if pc != 0 {
+			f := runtime.FuncForPC(uintptr(pc))
+			if f != nil {
+				fmt.Fprintf(&buf, "%#x %s\n", pc, f.Name())
+			}
+		}
+
+		// Wait until here to check for err; the last
+		// symbol will have an err because it doesn't end in +.
+		if err != nil {
+			if err != io.EOF {
+				fmt.Fprintf(&buf, "reading request: %v\n", err)
+			}
+			break
+		}
+	}
+
+	w.Write(buf.Bytes())
+}
+
+// Handler returns an HTTP handler that serves the named profile.
+func Handler(name string) http.Handler {
+	return handler(name)
+}
+
+type handler string
+
+func (name handler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
+	w.Header().Set("X-Content-Type-Options", "nosniff")
+	p := pprof.Lookup(string(name))
+	if p == nil {
+		serveError(w, http.StatusNotFound, "Unknown profile")
+		return
+	}
+	gc, _ := strconv.Atoi(r.FormValue("gc"))
+	if name == "heap" && gc > 0 {
+		runtime.GC()
+	}
+	debug, _ := strconv.Atoi(r.FormValue("debug"))
+	if debug != 0 {
+		w.Header().Set("Content-Type", "text/plain; charset=utf-8")
+	} else {
+		w.Header().Set("Content-Type", "application/octet-stream")
+		w.Header().Set("Content-Disposition", fmt.Sprintf(`attachment; filename="%s"`, name))
+	}
+	p.WriteTo(w, debug)
+}
+
+var profileDescriptions = map[string]string{
+	"allocs":       "A sampling of all past memory allocations",
+	"block":        "Stack traces that led to blocking on synchronization primitives",
+	"cmdline":      "The command line invocation of the current program",
+	"goroutine":    "Stack traces of all current goroutines",
+	"heap":         "A sampling of memory allocations of live objects. You can specify the gc GET parameter to run GC before taking the heap sample.",
+	"mutex":        "Stack traces of holders of contended mutexes",
+	"profile":      "CPU profile. You can specify the duration in the seconds GET parameter. After you get the profile file, use the go tool pprof command to investigate the profile.",
+	"threadcreate": "Stack traces that led to the creation of new OS threads",
+	"trace":        "A trace of execution of the current program. You can specify the duration in the seconds GET parameter. After you get the trace file, use the go tool trace command to investigate the trace.",
+}
+
+// Index responds with the pprof-formatted profile named by the request.
+// For example, "/debug/pprof/heap" serves the "heap" profile.
+// Index responds to a request for "/debug/pprof/" with an HTML page
+// listing the available profiles.
+func Index(w http.ResponseWriter, r *http.Request) {
+	if strings.HasPrefix(r.URL.Path, "/debug/pprof/") {
+		name := strings.TrimPrefix(r.URL.Path, "/debug/pprof/")
+		if name != "" {
+			handler(name).ServeHTTP(w, r)
+			return
+		}
+	}
+
+	type profile struct {
+		Name  string
+		Href  string
+		Desc  string
+		Count int
+	}
+	var profiles []profile
+	for _, p := range pprof.Profiles() {
+		profiles = append(profiles, profile{
+			Name:  p.Name(),
+			Href:  p.Name() + "?debug=1",
+			Desc:  profileDescriptions[p.Name()],
+			Count: p.Count(),
+		})
+	}
+
+	// Adding other profiles exposed from within this package
+	for _, p := range []string{"cmdline", "profile", "trace"} {
+		profiles = append(profiles, profile{
+			Name: p,
+			Href: p,
+			Desc: profileDescriptions[p],
+		})
+	}
+
+	sort.Slice(profiles, func(i, j int) bool {
+		return profiles[i].Name < profiles[j].Name
+	})
+
+	io.WriteString(w, `<html>
+<head>
+<title>/debug/pprof/</title>
+<style>
+.profile-name{
+	display:inline-block;
+	width:6rem;
+}
+</style>
+</head>
+<body>
+/debug/pprof/<br>
+<br>
+Types of profiles available:
+<table>
+<thead><td>Count</td><td>Profile</td></thead>
+`)
+	for _, p := range profiles {
+		fmt.Fprintf(w, "<tr><td>%d</td><td><a href=%v>%v</td></tr>\n",
+			p.Count, html.EscapeString(p.Href), html.EscapeString(p.Name))
+	}
+	io.WriteString(w, `</table>
+<a href="goroutine?debug=2">full goroutine stack dump</a>
+<br/>
+<p>
+Profile Descriptions:
+<ul>
+`)
+	for _, p := range profiles {
+		fmt.Fprintf(w, "<li><div class=profile-name>%s:</div> %s</li>\n",
+			html.EscapeString(p.Name), html.EscapeString(p.Desc))
+	}
+	io.WriteString(w, `
+</ul>
+</p>
+</body>
+</html>
+`)
+}

tempfork/pprof/pprof_test.go

@@ -0,0 +1,81 @@
+// Copyright 2018 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package pprof
+
+import (
+	"bytes"
+	"io/ioutil"
+	"net/http"
+	"net/http/httptest"
+	"runtime/pprof"
+	"testing"
+)
+
+// TestDescriptions checks that the profile names under runtime/pprof package
+// have a key in the description map.
+func TestDescriptions(t *testing.T) {
+	for _, p := range pprof.Profiles() {
+		_, ok := profileDescriptions[p.Name()]
+		if ok != true {
+			t.Errorf("%s does not exist in profileDescriptions map\n", p.Name())
+		}
+	}
+}
+
+func TestHandlers(t *testing.T) {
+	testCases := []struct {
+		path               string
+		handler            http.HandlerFunc
+		statusCode         int
+		contentType        string
+		contentDisposition string
+		resp               []byte
+	}{
+		{"/debug/pprof/<script>scripty<script>", Index, http.StatusNotFound, "text/plain; charset=utf-8", "", []byte("Unknown profile\n")},
+		{"/debug/pprof/heap", Index, http.StatusOK, "application/octet-stream", `attachment; filename="heap"`, nil},
+		{"/debug/pprof/heap?debug=1", Index, http.StatusOK, "text/plain; charset=utf-8", "", nil},
+		{"/debug/pprof/cmdline", Cmdline, http.StatusOK, "text/plain; charset=utf-8", "", nil},
+		{"/debug/pprof/profile?seconds=1", Profile, http.StatusOK, "application/octet-stream", `attachment; filename="profile"`, nil},
+		{"/debug/pprof/symbol", Symbol, http.StatusOK, "text/plain; charset=utf-8", "", nil},
+		{"/debug/pprof/trace", Trace, http.StatusOK, "application/octet-stream", `attachment; filename="trace"`, nil},
+	}
+	for _, tc := range testCases {
+		t.Run(tc.path, func(t *testing.T) {
+			req := httptest.NewRequest("GET", "http://example.com"+tc.path, nil)
+			w := httptest.NewRecorder()
+			tc.handler(w, req)
+
+			resp := w.Result()
+			if got, want := resp.StatusCode, tc.statusCode; got != want {
+				t.Errorf("status code: got %d; want %d", got, want)
+			}
+
+			body, err := ioutil.ReadAll(resp.Body)
+			if err != nil {
+				t.Errorf("when reading response body, expected non-nil err; got %v", err)
+			}
+			if got, want := resp.Header.Get("X-Content-Type-Options"), "nosniff"; got != want {
+				t.Errorf("X-Content-Type-Options: got %q; want %q", got, want)
+			}
+			if got, want := resp.Header.Get("Content-Type"), tc.contentType; got != want {
+				t.Errorf("Content-Type: got %q; want %q", got, want)
+			}
+			if got, want := resp.Header.Get("Content-Disposition"), tc.contentDisposition; got != want {
+				t.Errorf("Content-Disposition: got %q; want %q", got, want)
+			}
+
+			if resp.StatusCode == http.StatusOK {
+				return
+			}
+			if got, want := resp.Header.Get("X-Go-Pprof"), "1"; got != want {
+				t.Errorf("X-Go-Pprof: got %q; want %q", got, want)
+			}
+			if !bytes.Equal(body, tc.resp) {
+				t.Errorf("response: got %q; want %q", body, tc.resp)
+			}
+		})
+	}
+
+}

tstest/clock.go

@@ -30,16 +30,27 @@ type Clock struct {
 func (c *Clock) Now() time.Time {
 	c.Lock()
 	defer c.Unlock()
+	c.initLocked()
+	step := c.Step
+	ret := c.Present
+	c.Present = c.Present.Add(step)
+	return ret
+}
+
+func (c *Clock) Advance(d time.Duration) {
+	c.Lock()
+	defer c.Unlock()
+	c.initLocked()
+	c.Present = c.Present.Add(d)
+}
+
+func (c *Clock) initLocked() {
 	if c.Start.IsZero() {
 		c.Start = time.Now()
 	}
 	if c.Present.Before(c.Start) {
 		c.Present = c.Start
 	}
-	step := c.Step
-	ret := c.Present
-	c.Present = c.Present.Add(step)
-	return ret
 }
 
 // Reset rewinds the virtual clock to its start time.

tstest/natlab/firewall.go

@@ -0,0 +1,157 @@
+// Copyright (c) 2020 Tailscale Inc & AUTHORS All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package natlab
+
+import (
+	"fmt"
+	"sync"
+	"time"
+
+	"inet.af/netaddr"
+)
+
+// FirewallType is the type of filtering a stateful firewall
+// does. Values express different modes defined by RFC 4787.
+type FirewallType int
+
+const (
+	// AddressAndPortDependentFirewall specifies a destination
+	// address-and-port dependent firewall. Outbound traffic to an
+	// ip:port authorizes traffic from that ip:port exactly, and
+	// nothing else.
+	AddressAndPortDependentFirewall FirewallType = iota
+	// AddressDependentFirewall specifies a destination address
+	// dependent firewall. Once outbound traffic has been seen to an
+	// IP address, that IP address can talk back from any port.
+	AddressDependentFirewall
+	// EndpointIndependentFirewall specifies a destination endpoint
+	// independent firewall. Once outbound traffic has been seen from
+	// a source, anyone can talk back to that source.
+	EndpointIndependentFirewall
+)
+
+// fwKey is the lookup key for a firewall session. While it contains a
+// 4-tuple ({src,dst} {ip,port}), some FirewallTypes will zero out
+// some fields, so in practice the key is either a 2-tuple (src only),
+// 3-tuple (src ip+port and dst ip) or 4-tuple (src+dst ip+port).
+type fwKey struct {
+	src netaddr.IPPort
+	dst netaddr.IPPort
+}
+
+// key returns an fwKey for the given src and dst, trimmed according
+// to the FirewallType. fwKeys are always constructed from the
+// "outbound" point of view (i.e. src is the "trusted" side of the
+// world), it's the caller's responsibility to swap src and dst in the
+// call to key when processing packets inbound from the "untrusted"
+// world.
+func (s FirewallType) key(src, dst netaddr.IPPort) fwKey {
+	k := fwKey{src: src}
+	switch s {
+	case EndpointIndependentFirewall:
+	case AddressDependentFirewall:
+		k.dst.IP = dst.IP
+	case AddressAndPortDependentFirewall:
+		k.dst = dst
+	default:
+		panic(fmt.Sprintf("unknown firewall selectivity %v", s))
+	}
+	return k
+}
+
+// DefaultSessionTimeout is the default timeout for a firewall
+// session.
+const DefaultSessionTimeout = 30 * time.Second
+
+// Firewall is a simple stateful firewall that allows all outbound
+// traffic and filters inbound traffic based on recently seen outbound
+// traffic. Its HandlePacket method should be attached to a Machine to
+// give it a stateful firewall.
+type Firewall struct {
+	// SessionTimeout is the lifetime of idle sessions in the firewall
+	// state. Packets transiting from the TrustedInterface reset the
+	// session lifetime to SessionTimeout. If zero,
+	// DefaultSessionTimeout is used.
+	SessionTimeout time.Duration
+	// Type specifies how precisely return traffic must match
+	// previously seen outbound traffic to be allowed. Defaults to
+	// AddressAndPortDependentFirewall.
+	Type FirewallType
+	// TrustedInterface is an optional interface that is considered
+	// trusted in addition to PacketConns local to the Machine. All
+	// other interfaces can only respond to traffic from
+	// TrustedInterface or the local host.
+	TrustedInterface *Interface
+	// TimeNow is a function returning the current time. If nil,
+	// time.Now is used.
+	TimeNow func() time.Time
+
+	// TODO: refresh directionality: outbound-only, both
+
+	mu   sync.Mutex
+	seen map[fwKey]time.Time // session -> deadline
+}
+
+func (f *Firewall) timeNow() time.Time {
+	if f.TimeNow != nil {
+		return f.TimeNow()
+	}
+	return time.Now()
+}
+
+func (f *Firewall) init() {
+	if f.seen == nil {
+		f.seen = map[fwKey]time.Time{}
+	}
+}
+
+func (f *Firewall) HandleOut(p *Packet, oif *Interface) *Packet {
+	f.mu.Lock()
+	defer f.mu.Unlock()
+	f.init()
+
+	k := f.Type.key(p.Src, p.Dst)
+	f.seen[k] = f.timeNow().Add(f.sessionTimeoutLocked())
+	p.Trace("firewall out ok")
+	return p
+}
+
+func (f *Firewall) HandleIn(p *Packet, iif *Interface) *Packet {
+	f.mu.Lock()
+	defer f.mu.Unlock()
+	f.init()
+
+	// reverse src and dst because the session table is from the POV
+	// of outbound packets.
+	k := f.Type.key(p.Dst, p.Src)
+	now := f.timeNow()
+	if now.After(f.seen[k]) {
+		p.Trace("firewall drop")
+		return nil
+	}
+	p.Trace("firewall in ok")
+	return p
+}
+
+func (f *Firewall) HandleForward(p *Packet, iif *Interface, oif *Interface) *Packet {
+	if iif == f.TrustedInterface {
+		// Treat just like a locally originated packet
+		return f.HandleOut(p, oif)
+	}
+	if oif != f.TrustedInterface {
+		// Not a possible return packet from our trusted interface, drop.
+		p.Trace("firewall drop, unexpected oif")
+		return nil
+	}
+	// Otherwise, a session must exist, same as HandleIn.
+	return f.HandleIn(p, iif)
+}
+
+func (f *Firewall) sessionTimeoutLocked() time.Duration {
+	if f.SessionTimeout == 0 {
+		return DefaultSessionTimeout
+	}
+	return f.SessionTimeout
+}

tstest/natlab/nat.go

@@ -0,0 +1,257 @@
+// Copyright (c) 2020 Tailscale Inc & AUTHORS All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package natlab
+
+import (
+	"context"
+	"fmt"
+	"net"
+	"sync"
+	"time"
+
+	"inet.af/netaddr"
+)
+
+// mapping is the state of an allocated NAT session.
+type mapping struct {
+	lanSrc   netaddr.IPPort
+	lanDst   netaddr.IPPort
+	wanSrc   netaddr.IPPort
+	deadline time.Time
+
+	// pc is a PacketConn that reserves an outbound port on the NAT's
+	// WAN interface. We do this because ListenPacket already has
+	// random port selection logic built in. Additionally this means
+	// that concurrent use of ListenPacket for connections originating
+	// from the NAT box won't conflict with NAT mappings, since both
+	// use PacketConn to reserve ports on the machine.
+	pc net.PacketConn
+}
+
+// NATType is the mapping behavior of a NAT device. Values express
+// different modes defined by RFC 4787.
+type NATType int
+
+const (
+	// EndpointIndependentNAT specifies a destination endpoint
+	// independent NAT. All traffic from a source ip:port gets mapped
+	// to a single WAN ip:port.
+	EndpointIndependentNAT NATType = iota
+	// AddressDependentNAT specifies a destination address dependent
+	// NAT. Every distinct destination IP gets its own WAN ip:port
+	// allocation.
+	AddressDependentNAT
+	// AddressAndPortDependentNAT specifies a destination
+	// address-and-port dependent NAT. Every distinct destination
+	// ip:port gets its own WAN ip:port allocation.
+	AddressAndPortDependentNAT
+)
+
+// natKey is the lookup key for a NAT session. While it contains a
+// 4-tuple ({src,dst} {ip,port}), some NATTypes will zero out some
+// fields, so in practice the key is either a 2-tuple (src only),
+// 3-tuple (src ip+port and dst ip) or 4-tuple (src+dst ip+port).
+type natKey struct {
+	src, dst netaddr.IPPort
+}
+
+func (t NATType) key(src, dst netaddr.IPPort) natKey {
+	k := natKey{src: src}
+	switch t {
+	case EndpointIndependentNAT:
+	case AddressDependentNAT:
+		k.dst.IP = dst.IP
+	case AddressAndPortDependentNAT:
+		k.dst = dst
+	default:
+		panic(fmt.Sprintf("unknown NAT type %v", t))
+	}
+	return k
+}
+
+// DefaultMappingTimeout is the default timeout for a NAT mapping.
+const DefaultMappingTimeout = 30 * time.Second
+
+// SNAT44 implements an IPv4-to-IPv4 source NAT (SNAT) translator, with
+// optional builtin firewall.
+type SNAT44 struct {
+	// Machine is the machine to which this NAT is attached. Altered
+	// packets are injected back into this Machine for processing.
+	Machine *Machine
+	// ExternalInterface is the "WAN" interface of Machine. Packets
+	// from other sources get NATed onto this interface.
+	ExternalInterface *Interface
+	// Type specifies the mapping allocation behavior for this NAT.
+	Type NATType
+	// MappingTimeout is the lifetime of individual NAT sessions. Once
+	// a session expires, the mapped port effectively "closes" to new
+	// traffic. If MappingTimeout is 0, DefaultMappingTimeout is used.
+	MappingTimeout time.Duration
+	// Firewall is an optional packet handler that will be invoked as
+	// a firewall during NAT translation. The firewall always sees
+	// packets in their "LAN form", i.e. before translation in the
+	// outbound direction and after translation in the inbound
+	// direction.
+	Firewall PacketHandler
+	// TimeNow is a function that returns the current time. If
+	// nil, time.Now is used.
+	TimeNow func() time.Time
+
+	mu    sync.Mutex
+	byLAN map[natKey]*mapping         // lookup by outbound packet tuple
+	byWAN map[netaddr.IPPort]*mapping // lookup by wan ip:port only
+}
+
+func (n *SNAT44) timeNow() time.Time {
+	if n.TimeNow != nil {
+		return n.TimeNow()
+	}
+	return time.Now()
+}
+
+func (n *SNAT44) mappingTimeout() time.Duration {
+	if n.MappingTimeout == 0 {
+		return DefaultMappingTimeout
+	}
+	return n.MappingTimeout
+}
+
+func (n *SNAT44) initLocked() {
+	if n.byLAN == nil {
+		n.byLAN = map[natKey]*mapping{}
+		n.byWAN = map[netaddr.IPPort]*mapping{}
+	}
+	if n.ExternalInterface.Machine() != n.Machine {
+		panic(fmt.Sprintf("NAT given interface %s that is not part of given machine %s", n.ExternalInterface, n.Machine.Name))
+	}
+}
+
+func (n *SNAT44) HandleOut(p *Packet, oif *Interface) *Packet {
+	// NATs don't affect locally originated packets.
+	if n.Firewall != nil {
+		return n.Firewall.HandleOut(p, oif)
+	}
+	return p
+}
+
+func (n *SNAT44) HandleIn(p *Packet, iif *Interface) *Packet {
+	if iif != n.ExternalInterface {
+		// NAT can't apply, defer to firewall.
+		if n.Firewall != nil {
+			return n.Firewall.HandleIn(p, iif)
+		}
+		return p
+	}
+
+	n.mu.Lock()
+	defer n.mu.Unlock()
+	n.initLocked()
+
+	now := n.timeNow()
+	mapping := n.byWAN[p.Dst]
+	if mapping == nil || now.After(mapping.deadline) {
+		// NAT didn't hit, defer to firewall or allow in for local
+		// socket handling.
+		if n.Firewall != nil {
+			return n.Firewall.HandleIn(p, iif)
+		}
+		return p
+	}
+
+	p.Dst = mapping.lanSrc
+	p.Trace("dnat to %v", p.Dst)
+	// Don't process firewall here. We mutated the packet such that
+	// it's no longer destined locally, so we'll get reinvoked as
+	// HandleForward and need to process the altered packet there.
+	return p
+}
+
+func (n *SNAT44) HandleForward(p *Packet, iif, oif *Interface) *Packet {
+	switch {
+	case oif == n.ExternalInterface:
+		if p.Src.IP == oif.V4() {
+			// Packet already NATed and is just retraversing Forward,
+			// don't touch it again.
+			return p
+		}
+
+		if n.Firewall != nil {
+			p2 := n.Firewall.HandleForward(p, iif, oif)
+			if p2 == nil {
+				// firewall dropped, done
+				return nil
+			}
+			if !p.Equivalent(p2) {
+				// firewall mutated packet? Weird, but okay.
+				return p2
+			}
+		}
+
+		n.mu.Lock()
+		defer n.mu.Unlock()
+		n.initLocked()
+
+		k := n.Type.key(p.Src, p.Dst)
+		now := n.timeNow()
+		m := n.byLAN[k]
+		if m == nil || now.After(m.deadline) {
+			pc, wanAddr := n.allocateMappedPort()
+			m = &mapping{
+				lanSrc: p.Src,
+				lanDst: p.Dst,
+				wanSrc: wanAddr,
+				pc:     pc,
+			}
+			n.byLAN[k] = m
+			n.byWAN[wanAddr] = m
+		}
+		m.deadline = now.Add(n.mappingTimeout())
+		p.Src = m.wanSrc
+		p.Trace("snat from %v", p.Src)
+		return p
+	case iif == n.ExternalInterface:
+		// Packet was already un-NAT-ed, we just need to either
+		// firewall it or let it through.
+		if n.Firewall != nil {
+			return n.Firewall.HandleForward(p, iif, oif)
+		}
+		return p
+	default:
+		// No NAT applies, invoke firewall or drop.
+		if n.Firewall != nil {
+			return n.Firewall.HandleForward(p, iif, oif)
+		}
+		return nil
+	}
+}
+
+func (n *SNAT44) allocateMappedPort() (net.PacketConn, netaddr.IPPort) {
+	// Clean up old entries before trying to allocate, to free up any
+	// expired ports.
+	n.gc()
+
+	ip := n.ExternalInterface.V4()
+	pc, err := n.Machine.ListenPacket(context.Background(), "udp", net.JoinHostPort(ip.String(), "0"))
+	if err != nil {
+		panic(fmt.Sprintf("ran out of NAT ports: %v", err))
+	}
+	addr := netaddr.IPPort{
+		IP:   ip,
+		Port: uint16(pc.LocalAddr().(*net.UDPAddr).Port),
+	}
+	return pc, addr
+}
+
+func (n *SNAT44) gc() {
+	now := n.timeNow()
+	for _, m := range n.byLAN {
+		if !now.After(m.deadline) {
+			continue
+		}
+		m.pc.Close()
+		delete(n.byLAN, n.Type.key(m.lanSrc, m.lanDst))
+		delete(n.byWAN, m.wanSrc)
+	}
+}

tstest/natlab/natlab.go

@@ -12,10 +12,13 @@
 package natlab
 
 import (
+	"bytes"
 	"context"
 	"crypto/sha256"
 	"encoding/base64"
+	"errors"
 	"fmt"
+	"math/rand"
 	"net"
 	"os"
 	"sort"
@@ -26,23 +29,57 @@ import (
 	"inet.af/netaddr"
 )
 
-var traceOn = os.Getenv("NATLAB_TRACE")
+var traceOn, _ = strconv.ParseBool(os.Getenv("NATLAB_TRACE"))
 
-func trace(p []byte, msg string, args ...interface{}) {
-	if traceOn == "" {
-		return
-	}
-	id := packetShort(p)
-	as := []interface{}{id}
-	as = append(as, args...)
-	fmt.Fprintf(os.Stderr, "[%s] "+msg+"\n", as...)
+// Packet represents a UDP packet flowing through the virtual network.
+type Packet struct {
+	Src, Dst netaddr.IPPort
+	Payload  []byte
+
+	// Prefix set by various internal methods of natlab, to locate
+	// where in the network a trace occured.
+	locator string
 }
 
-// packetShort returns a short identifier for a packet payload,
-// suitable for pritning trace information.
-func packetShort(p []byte) string {
-	s := sha256.Sum256(p)
-	return base64.RawStdEncoding.EncodeToString(s[:])[:4]
+// Equivalent returns true if Src, Dst and Payload are the same in p
+// and p2.
+func (p *Packet) Equivalent(p2 *Packet) bool {
+	return p.Src == p2.Src && p.Dst == p2.Dst && bytes.Equal(p.Payload, p2.Payload)
+}
+
+// Clone returns a copy of p that shares nothing with p.
+func (p *Packet) Clone() *Packet {
+	return &Packet{
+		Src:     p.Src,
+		Dst:     p.Dst,
+		Payload: append([]byte(nil), p.Payload...),
+		locator: p.locator,
+	}
+}
+
+// short returns a short identifier for a packet payload,
+// suitable for printing trace information.
+func (p *Packet) short() string {
+	s := sha256.Sum256(p.Payload)
+	payload := base64.RawStdEncoding.EncodeToString(s[:])[:2]
+
+	s = sha256.Sum256([]byte(p.Src.String() + "_" + p.Dst.String()))
+	tuple := base64.RawStdEncoding.EncodeToString(s[:])[:2]
+
+	return fmt.Sprintf("%s/%s", payload, tuple)
+}
+
+func (p *Packet) Trace(msg string, args ...interface{}) {
+	if !traceOn {
+		return
+	}
+	allArgs := []interface{}{p.short(), p.locator, p.Src, p.Dst}
+	allArgs = append(allArgs, args...)
+	fmt.Fprintf(os.Stderr, "[%s]%s src=%s dst=%s "+msg+"\n", allArgs...)
+}
+
+func (p *Packet) setLocator(msg string, args ...interface{}) {
+	p.locator = fmt.Sprintf(" "+msg, args...)
 }
 
 func mustPrefix(s string) netaddr.IPPrefix {
@@ -68,29 +105,32 @@ type Network struct {
 	Prefix6 netaddr.IPPrefix
 
 	mu        sync.Mutex
-	machine   map[netaddr.IP]*Machine
-	defaultGW *Machine // optional
+	machine   map[netaddr.IP]*Interface
+	defaultGW *Interface // optional
 	lastV4    netaddr.IP
 	lastV6    netaddr.IP
 }
 
-func (n *Network) SetDefaultGateway(gw *Machine) {
+func (n *Network) SetDefaultGateway(gwIf *Interface) {
 	n.mu.Lock()
 	defer n.mu.Unlock()
-	n.defaultGW = gw
+	if gwIf.net != n {
+		panic(fmt.Sprintf("can't set if=%s as net=%s's default gw, if not connected to net", gwIf.name, gwIf.net.Name))
+	}
+	n.defaultGW = gwIf
 }
 
-func (n *Network) addMachineLocked(ip netaddr.IP, m *Machine) {
-	if m == nil {
+func (n *Network) addMachineLocked(ip netaddr.IP, iface *Interface) {
+	if iface == nil {
 		return // for tests
 	}
 	if n.machine == nil {
-		n.machine = map[netaddr.IP]*Machine{}
+		n.machine = map[netaddr.IP]*Interface{}
 	}
-	n.machine[ip] = m
+	n.machine[ip] = iface
 }
 
-func (n *Network) allocIPv4(m *Machine) netaddr.IP {
+func (n *Network) allocIPv4(iface *Interface) netaddr.IP {
 	n.mu.Lock()
 	defer n.mu.Unlock()
 	if n.Prefix4.IsZero() {
@@ -105,11 +145,11 @@ func (n *Network) allocIPv4(m *Machine) netaddr.IP {
 	if !n.Prefix4.Contains(n.lastV4) {
 		panic("pool exhausted")
 	}
-	n.addMachineLocked(n.lastV4, m)
+	n.addMachineLocked(n.lastV4, iface)
 	return n.lastV4
 }
 
-func (n *Network) allocIPv6(m *Machine) netaddr.IP {
+func (n *Network) allocIPv6(iface *Interface) netaddr.IP {
 	n.mu.Lock()
 	defer n.mu.Unlock()
 	if n.Prefix6.IsZero() {
@@ -124,7 +164,7 @@ func (n *Network) allocIPv6(m *Machine) netaddr.IP {
 	if !n.Prefix6.Contains(n.lastV6) {
 		panic("pool exhausted")
 	}
-	n.addMachineLocked(n.lastV6, m)
+	n.addMachineLocked(n.lastV6, iface)
 	return n.lastV6
 }
 
@@ -137,30 +177,40 @@ func addOne(a *[16]byte, index int) {
 	}
 }
 
-func (n *Network) write(p []byte, dst, src netaddr.IPPort) (num int, err error) {
+func (n *Network) write(p *Packet) (num int, err error) {
+	p.setLocator("net=%s", n.Name)
+
 	n.mu.Lock()
 	defer n.mu.Unlock()
-	m, ok := n.machine[dst.IP]
+	iface, ok := n.machine[p.Dst.IP]
 	if !ok {
 		if n.defaultGW == nil {
-			trace(p, "net=%s dropped, no route to %v", n.Name, dst.IP)
-			return len(p), nil
+			p.Trace("no route to %v", p.Dst.IP)
+			return len(p.Payload), nil
 		}
-		m = n.defaultGW
+		iface = n.defaultGW
 	}
 
 	// Pretend it went across the network. Make a copy so nobody
 	// can later mess with caller's memory.
-	trace(p, "net=%s src=%v dst=%v -> mach=%s", n.Name, src, dst, m.Name)
-	pcopy := append([]byte(nil), p...)
-	go m.deliverIncomingPacket(pcopy, dst, src)
-	return len(p), nil
+	p.Trace("-> mach=%s if=%s", iface.machine.Name, iface.name)
+	go iface.machine.deliverIncomingPacket(p, iface)
+	return len(p.Payload), nil
 }
 
 type Interface struct {
-	net  *Network
-	name string       // optional
-	ips  []netaddr.IP // static; not mutated once created
+	machine *Machine
+	net     *Network
+	name    string       // optional
+	ips     []netaddr.IP // static; not mutated once created
+}
+
+func (f *Interface) Machine() *Machine {
+	return f.machine
+}
+
+func (f *Interface) Network() *Network {
+	return f.net
 }
 
 // V4 returns the machine's first IPv4 address, or the zero value if none.
@@ -223,8 +273,41 @@ func (v PacketVerdict) String() string {
 	}
 }
 
-// A PacketHandler is a function that can process packets.
-type PacketHandler func(p []byte, dst, src netaddr.IPPort) PacketVerdict
+// A PacketHandler can look at packets arriving at, departing, and
+// transiting a Machine, and filter or mutate them.
+//
+// Each method is invoked with a Packet that natlab would like to keep
+// processing. Handlers can return that same Packet to allow
+// processing to continue; nil to drop the Packet; or a different
+// Packet that should be processed instead of the original.
+//
+// Packets passed to handlers share no state with anything else, and
+// are therefore safe to mutate. It's safe to return the original
+// packet mutated in-place, or a brand new packet initialized from
+// scratch.
+//
+// Packets mutated by a PacketHandler are processed anew by the
+// associated Machine, as if the packet had always been the mutated
+// one. For example, if HandleForward is invoked with a Packet, and
+// the handler changes the destination IP address to one of the
+// Machine's own IPs, the Machine restarts delivery, but this time
+// going to a local PacketConn (which in turn will invoke HandleIn,
+// since the packet is now destined for local delivery).
+type PacketHandler interface {
+	// HandleIn processes a packet arriving on iif, whose destination
+	// is an IP address owned by the attached Machine. If p is
+	// returned unmodified, the Machine will go on to deliver the
+	// Packet to the appropriate listening PacketConn, if one exists.
+	HandleIn(p *Packet, iif *Interface) *Packet
+	// HandleOut processes a packet about to depart on oif from a
+	// local PacketConn. If p is returned unmodified, the Machine will
+	// transmit the Packet on oif.
+	HandleOut(p *Packet, oif *Interface) *Packet
+	// HandleForward is called when the Machine wants to forward a
+	// packet from iif to oif. If p is returned unmodified, the
+	// Machine will transmit the packet on oif.
+	HandleForward(p *Packet, iif, oif *Interface) *Packet
+}
 
 // A Machine is a representation of an operating system's network
 // stack. It has a network routing table and can have multiple
@@ -235,13 +318,14 @@ type Machine struct {
 	// not be globally unique.
 	Name string
 
-	// HandlePacket, if not nil, is a function that gets invoked for
-	// every packet this Machine receives. Returns a verdict for how
-	// the packet should continue to be handled (or not).
+	// PacketHandler, if not nil, is a PacketHandler implementation
+	// that inspects all packets arriving, departing, or transiting
+	// the Machine. See the definition of the PacketHandler interface
+	// for semantics.
 	//
-	// This can be used to implement things like stateful firewalls
-	// and NAT boxes.
-	HandlePacket PacketHandler
+	// If PacketHandler is nil, the machine allows all inbound
+	// traffic, all outbound traffic, and drops forwarded packets.
+	PacketHandler PacketHandler
 
 	mu         sync.Mutex
 	interfaces []*Interface
@@ -251,22 +335,42 @@ type Machine struct {
 	conns6 map[netaddr.IPPort]*conn // conns that want IPv6 packets
 }
 
-// Inject transmits p from src to dst, without the need for a local socket.
-// It's useful for implementing e.g. NAT boxes that need to mangle IPs.
-func (m *Machine) Inject(p []byte, dst, src netaddr.IPPort) error {
-	trace(p, "mach=%s src=%s dst=%s packet injected", m.Name, src, dst)
-	_, err := m.writePacket(p, dst, src)
-	return err
+func (m *Machine) isLocalIP(ip netaddr.IP) bool {
+	m.mu.Lock()
+	defer m.mu.Unlock()
+	for _, intf := range m.interfaces {
+		for _, iip := range intf.ips {
+			if ip == iip {
+				return true
+			}
+		}
+	}
+	return false
 }
 
-func (m *Machine) deliverIncomingPacket(p []byte, dst, src netaddr.IPPort) {
+func (m *Machine) deliverIncomingPacket(p *Packet, iface *Interface) {
+	p.setLocator("mach=%s if=%s", m.Name, iface.name)
+
+	if m.isLocalIP(p.Dst.IP) {
+		m.deliverLocalPacket(p, iface)
+	} else {
+		m.forwardPacket(p, iface)
+	}
+}
+
+func (m *Machine) deliverLocalPacket(p *Packet, iface *Interface) {
 	// TODO: can't hold lock while handling packet. This is safe as
 	// long as you set HandlePacket before traffic starts flowing.
-	if m.HandlePacket != nil {
-		verdict := m.HandlePacket(p, dst, src)
-		trace(p, "mach=%s src=%v dst=%v packethandler verdict=%s", m.Name, src, dst, verdict)
-		if verdict == Drop {
-			// Custom packet handler ate the packet, we're done.
+	if m.PacketHandler != nil {
+		p2 := m.PacketHandler.HandleIn(p.Clone(), iface)
+		if p2 == nil {
+			// Packet dropped, nothing left to do.
+			return
+		}
+		if !p.Equivalent(p2) {
+			// Restart delivery, this packet might be a forward packet
+			// now.
+			m.deliverIncomingPacket(p2, iface)
 			return
 		}
 	}
@@ -275,13 +379,13 @@ func (m *Machine) deliverIncomingPacket(p []byte, dst, src netaddr.IPPort) {
 	defer m.mu.Unlock()
 
 	conns := m.conns4
-	if dst.IP.Is6() {
+	if p.Dst.IP.Is6() {
 		conns = m.conns6
 	}
 	possibleDsts := []netaddr.IPPort{
-		dst,
-		netaddr.IPPort{IP: v6unspec, Port: dst.Port},
-		netaddr.IPPort{IP: v4unspec, Port: dst.Port},
+		p.Dst,
+		netaddr.IPPort{IP: v6unspec, Port: p.Dst.Port},
+		netaddr.IPPort{IP: v4unspec, Port: p.Dst.Port},
 	}
 	for _, dest := range possibleDsts {
 		c, ok := conns[dest]
@@ -289,15 +393,44 @@ func (m *Machine) deliverIncomingPacket(p []byte, dst, src netaddr.IPPort) {
 			continue
 		}
 		select {
-		case c.in <- incomingPacket{src: src, p: p}:
-			trace(p, "mach=%s src=%v dst=%v queued to conn", m.Name, src, dst)
+		case c.in <- p:
+			p.Trace("queued to conn")
 		default:
-			trace(p, "mach=%s src=%v dst=%v dropped, queue overflow", m.Name, src, dst)
+			p.Trace("dropped, queue overflow")
 			// Queue overflow. Just drop it.
 		}
 		return
 	}
-	trace(p, "mach=%s src=%v dst=%v dropped, no listening conn", m.Name, src, dst)
+	p.Trace("dropped, no listening conn")
+}
+
+func (m *Machine) forwardPacket(p *Packet, iif *Interface) {
+	oif, err := m.interfaceForIP(p.Dst.IP)
+	if err != nil {
+		p.Trace("%v", err)
+		return
+	}
+
+	if m.PacketHandler == nil {
+		// Forwarding not allowed by default
+		p.Trace("drop, forwarding not allowed")
+		return
+	}
+	p2 := m.PacketHandler.HandleForward(p.Clone(), iif, oif)
+	if p2 == nil {
+		p.Trace("drop")
+		// Packet dropped, done.
+		return
+	}
+	if !p.Equivalent(p2) {
+		// Packet changed, restart delivery.
+		p2.Trace("PacketHandler mutated packet")
+		m.deliverIncomingPacket(p2, iif)
+		return
+	}
+
+	p.Trace("-> net=%s oif=%s", oif.net.Name, oif)
+	oif.net.write(p)
 }
 
 func unspecOf(ip netaddr.IP) netaddr.IP {
@@ -316,13 +449,14 @@ func unspecOf(ip netaddr.IP) netaddr.IP {
 // default route.
 func (m *Machine) Attach(interfaceName string, n *Network) *Interface {
 	f := &Interface{
-		net:  n,
-		name: interfaceName,
+		machine: m,
+		net:     n,
+		name:    interfaceName,
 	}
-	if ip := n.allocIPv4(m); !ip.IsZero() {
+	if ip := n.allocIPv4(f); !ip.IsZero() {
 		f.ips = append(f.ips, ip)
 	}
-	if ip := n.allocIPv6(m); !ip.IsZero() {
+	if ip := n.allocIPv6(f); !ip.IsZero() {
 		f.ips = append(f.ips, ip)
 	}
 
@@ -366,38 +500,56 @@ var (
 	v6unspec = netaddr.IPv6Unspecified()
 )
 
-func (m *Machine) writePacket(p []byte, dst, src netaddr.IPPort) (n int, err error) {
-	iface, err := m.interfaceForIP(dst.IP)
+func (m *Machine) writePacket(p *Packet) (n int, err error) {
+	p.setLocator("mach=%s", m.Name)
+
+	iface, err := m.interfaceForIP(p.Dst.IP)
 	if err != nil {
-		trace(p, "%v", err)
+		p.Trace("%v", err)
 		return 0, err
 	}
-	origSrcIP := src.IP
+	origSrcIP := p.Src.IP
 	switch {
-	case src.IP == v4unspec:
-		src.IP = iface.V4()
-	case src.IP == v6unspec:
+	case p.Src.IP == v4unspec:
+		p.Trace("assigning srcIP=%s", iface.V4())
+		p.Src.IP = iface.V4()
+	case p.Src.IP == v6unspec:
 		// v6unspec in Go means "any src, but match address families"
-		if dst.IP.Is6() {
-			src.IP = iface.V6()
-		} else if dst.IP.Is4() {
-			src.IP = iface.V4()
+		if p.Dst.IP.Is6() {
+			p.Trace("assigning srcIP=%s", iface.V6())
+			p.Src.IP = iface.V6()
+		} else if p.Dst.IP.Is4() {
+			p.Trace("assigning srcIP=%s", iface.V4())
+			p.Src.IP = iface.V4()
 		}
 	default:
-		if !iface.Contains(src.IP) {
-			err := fmt.Errorf("can't send to %v with src %v on interface %v", dst.IP, src.IP, iface)
-			trace(p, "%v", err)
+		if !iface.Contains(p.Src.IP) {
+			err := fmt.Errorf("can't send to %v with src %v on interface %v", p.Dst.IP, p.Src.IP, iface)
+			p.Trace("%v", err)
 			return 0, err
 		}
 	}
-	if src.IP.IsZero() {
+	if p.Src.IP.IsZero() {
 		err := fmt.Errorf("no matching address for address family for %v", origSrcIP)
-		trace(p, "%v", err)
+		p.Trace("%v", err)
 		return 0, err
 	}
 
-	trace(p, "mach=%s src=%s dst=%s -> net=%s", m.Name, src, dst, iface.net.Name)
-	return iface.net.write(p, dst, src)
+	if m.PacketHandler != nil {
+		p2 := m.PacketHandler.HandleOut(p.Clone(), iface)
+		if p2 == nil {
+			// Packet dropped, done.
+			return len(p.Payload), nil
+		}
+		if !p.Equivalent(p2) {
+			// Restart transmission, src may have changed weirdly
+			m.writePacket(p2)
+			return
+		}
+	}
+
+	p.Trace("-> net=%s if=%s", iface.net.Name, iface)
+	return iface.net.write(p)
 }
 
 func (m *Machine) interfaceForIP(ip netaddr.IP) (*Interface, error) {
@@ -424,6 +576,32 @@ func (m *Machine) hasv6() bool {
 	return false
 }
 
+func (m *Machine) pickEphemPort() (port uint16, err error) {
+	m.mu.Lock()
+	defer m.mu.Unlock()
+	for tries := 0; tries < 500; tries++ {
+		port := uint16(rand.Intn(32<<10) + 32<<10)
+		if !m.portInUseLocked(port) {
+			return port, nil
+		}
+	}
+	return 0, errors.New("failed to find an ephemeral port")
+}
+
+func (m *Machine) portInUseLocked(port uint16) bool {
+	for ipp := range m.conns4 {
+		if ipp.Port == port {
+			return true
+		}
+	}
+	for ipp := range m.conns6 {
+		if ipp.Port == port {
+			return true
+		}
+	}
+	return false
+}
+
 func (m *Machine) registerConn4(c *conn) error {
 	m.mu.Lock()
 	defer m.mu.Unlock()
@@ -467,7 +645,7 @@ func registerConn(conns *map[netaddr.IPPort]*conn, c *conn) error {
 
 func (m *Machine) AddNetwork(n *Network) {}
 
-func (m *Machine) ListenPacket(network, address string) (net.PacketConn, error) {
+func (m *Machine) ListenPacket(ctx context.Context, network, address string) (net.PacketConn, error) {
 	// if udp4, udp6, etc... look at address IP vs unspec
 	var (
 		fam uint8
@@ -496,18 +674,34 @@ func (m *Machine) ListenPacket(network, address string) (net.PacketConn, error)
 		if err != nil {
 			return nil, err
 		}
+		if fam == 0 && (ip != v4unspec && ip != v6unspec) {
+			// We got an explicit IP address, need to switch the
+			// family to the right one.
+			if ip.Is4() {
+				fam = 4
+			} else {
+				fam = 6
+			}
+		}
 	}
-	port, err := strconv.ParseUint(portStr, 10, 16)
+	porti, err := strconv.ParseUint(portStr, 10, 16)
 	if err != nil {
 		return nil, err
 	}
-	ipp := netaddr.IPPort{IP: ip, Port: uint16(port)}
+	port := uint16(porti)
+	if port == 0 {
+		port, err = m.pickEphemPort()
+		if err != nil {
+			return nil, nil
+		}
+	}
+	ipp := netaddr.IPPort{IP: ip, Port: port}
 
 	c := &conn{
 		m:   m,
 		fam: fam,
 		ipp: ipp,
-		in:  make(chan incomingPacket, 100), // arbitrary
+		in:  make(chan *Packet, 100), // arbitrary
 	}
 	switch c.fam {
 	case 0:
@@ -540,23 +734,24 @@ type conn struct {
 	closed       bool
 	readDeadline time.Time
 	activeReads  map[*activeRead]bool
-	in           chan incomingPacket
-}
-
-type incomingPacket struct {
-	p   []byte
-	src netaddr.IPPort
+	in           chan *Packet
 }
 
 type activeRead struct {
 	cancel context.CancelFunc
 }
 
-// readDeadlineExceeded reports whether the read deadline is set and has already passed.
-func (c *conn) readDeadlineExceeded() bool {
+// canRead reports whether we can do a read.
+func (c *conn) canRead() error {
 	c.mu.Lock()
 	defer c.mu.Unlock()
-	return !c.readDeadline.IsZero() && c.readDeadline.Before(time.Now())
+	if c.closed {
+		return errors.New("closed network connection") // sadface: magic string used by other; don't change
+	}
+	if !c.readDeadline.IsZero() && c.readDeadline.Before(time.Now()) {
+		return errors.New("read deadline exceeded")
+	}
+	return nil
 }
 
 func (c *conn) registerActiveRead(ar *activeRead, active bool) {
@@ -609,8 +804,8 @@ func (c *conn) ReadFrom(p []byte) (n int, addr net.Addr, err error) {
 
 	ar := &activeRead{cancel: cancel}
 
-	if c.readDeadlineExceeded() {
-		return 0, nil, context.DeadlineExceeded
+	if err := c.canRead(); err != nil {
+		return 0, nil, err
 	}
 
 	c.registerActiveRead(ar, true)
@@ -618,9 +813,9 @@ func (c *conn) ReadFrom(p []byte) (n int, addr net.Addr, err error) {
 
 	select {
 	case pkt := <-c.in:
-		n = copy(p, pkt.p)
-		trace(pkt.p, "mach=%s src=%s PacketConn.ReadFrom", c.m.Name, pkt.src)
-		return n, pkt.src.UDPAddr(), nil
+		n = copy(p, pkt.Payload)
+		pkt.Trace("PacketConn.ReadFrom")
+		return n, pkt.Src.UDPAddr(), nil
 	case <-ctx.Done():
 		return 0, nil, context.DeadlineExceeded
 	}
@@ -631,7 +826,14 @@ func (c *conn) WriteTo(p []byte, addr net.Addr) (n int, err error) {
 	if err != nil {
 		return 0, fmt.Errorf("bogus addr %T %q", addr, addr.String())
 	}
-	return c.m.writePacket(p, ipp, c.ipp)
+	pkt := &Packet{
+		Src:     c.ipp,
+		Dst:     ipp,
+		Payload: append([]byte(nil), p...),
+	}
+	pkt.setLocator("mach=%s", c.m.Name)
+	pkt.Trace("PacketConn.WriteTo")
+	return c.m.writePacket(pkt)
 }
 
 func (c *conn) SetDeadline(t time.Time) error {

tstest/natlab/natlab_test.go

@@ -5,17 +5,20 @@
 package natlab
 
 import (
+	"context"
 	"fmt"
 	"testing"
+	"time"
 
 	"inet.af/netaddr"
+	"tailscale.com/tstest"
 )
 
 func TestAllocIPs(t *testing.T) {
 	n := NewInternet()
 	saw := map[netaddr.IP]bool{}
 	for i := 0; i < 255; i++ {
-		for _, f := range []func(*Machine) netaddr.IP{n.allocIPv4, n.allocIPv6} {
+		for _, f := range []func(*Interface) netaddr.IP{n.allocIPv4, n.allocIPv6} {
 			ip := f(nil)
 			if saw[ip] {
 				t.Fatalf("got duplicate %v", ip)
@@ -49,11 +52,12 @@ func TestSendPacket(t *testing.T) {
 	fooAddr := netaddr.IPPort{IP: ifFoo.V4(), Port: 123}
 	barAddr := netaddr.IPPort{IP: ifBar.V4(), Port: 456}
 
-	fooPC, err := foo.ListenPacket("udp4", fooAddr.String())
+	ctx := context.Background()
+	fooPC, err := foo.ListenPacket(ctx, "udp4", fooAddr.String())
 	if err != nil {
 		t.Fatal(err)
 	}
-	barPC, err := bar.ListenPacket("udp4", barAddr.String())
+	barPC, err := bar.ListenPacket(ctx, "udp4", barAddr.String())
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -93,15 +97,16 @@ func TestMultiNetwork(t *testing.T) {
 	ifNATLAN := nat.Attach("ethlan", lan)
 	ifServer := server.Attach("eth0", internet)
 
-	clientPC, err := client.ListenPacket("udp", ":123")
+	ctx := context.Background()
+	clientPC, err := client.ListenPacket(ctx, "udp", ":123")
 	if err != nil {
 		t.Fatal(err)
 	}
-	natPC, err := nat.ListenPacket("udp", ":456")
+	natPC, err := nat.ListenPacket(ctx, "udp", ":456")
 	if err != nil {
 		t.Fatal(err)
 	}
-	serverPC, err := server.ListenPacket("udp", ":789")
+	serverPC, err := server.ListenPacket(ctx, "udp", ":789")
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -143,6 +148,38 @@ func TestMultiNetwork(t *testing.T) {
 	}
 }
 
+type trivialNAT struct {
+	clientIP     netaddr.IP
+	lanIf, wanIf *Interface
+}
+
+func (n *trivialNAT) HandleIn(p *Packet, iface *Interface) *Packet {
+	if iface == n.wanIf && p.Dst.IP == n.wanIf.V4() {
+		p.Dst.IP = n.clientIP
+	}
+	return p
+}
+
+func (n trivialNAT) HandleOut(p *Packet, iface *Interface) *Packet {
+	return p
+}
+
+func (n *trivialNAT) HandleForward(p *Packet, iif, oif *Interface) *Packet {
+	// Outbound from LAN -> apply NAT, continue
+	if iif == n.lanIf && oif == n.wanIf {
+		if p.Src.IP == n.clientIP {
+			p.Src.IP = n.wanIf.V4()
+		}
+		return p
+	}
+	// Return traffic to LAN, allow if right dst.
+	if iif == n.wanIf && oif == n.lanIf && p.Dst.IP == n.clientIP {
+		return p
+	}
+	// Else drop.
+	return nil
+}
+
 func TestPacketHandler(t *testing.T) {
 	lan := &Network{
 		Name:    "lan",
@@ -153,42 +190,27 @@ func TestPacketHandler(t *testing.T) {
 
 	client := &Machine{Name: "client"}
 	nat := &Machine{Name: "nat"}
-	lan.SetDefaultGateway(nat)
 	server := &Machine{Name: "server"}
 
 	ifClient := client.Attach("eth0", lan)
 	ifNATWAN := nat.Attach("wan", internet)
-	_ = nat.Attach("lan", lan)
+	ifNATLAN := nat.Attach("lan", lan)
 	ifServer := server.Attach("server", internet)
 
-	// This HandlePacket implements a basic (some might say "broken")
-	// 1:1 NAT, where client's IP gets replaced with the NAT's WAN IP,
-	// and vice versa.
-	//
-	// This NAT is not suitable for actual use, since it doesn't do
-	// port remappings or any other things that NATs usually to. But
-	// it works as a demonstrator for a single client behind the NAT,
-	// where the NAT box itself doesn't also make PacketConns.
-	nat.HandlePacket = func(p []byte, dst, src netaddr.IPPort) PacketVerdict {
-		switch {
-		case dst.IP.Is6():
-			return Continue // no NAT for ipv6
-		case src.IP == ifClient.V4():
-			nat.Inject(p, dst, netaddr.IPPort{IP: ifNATWAN.V4(), Port: src.Port})
-			return Drop
-		case dst.IP == ifNATWAN.V4():
-			nat.Inject(p, netaddr.IPPort{IP: ifClient.V4(), Port: dst.Port}, src)
-			return Drop
-		default:
-			return Continue
-		}
+	lan.SetDefaultGateway(ifNATLAN)
+
+	nat.PacketHandler = &trivialNAT{
+		clientIP: ifClient.V4(),
+		lanIf:    ifNATLAN,
+		wanIf:    ifNATWAN,
 	}
 
-	clientPC, err := client.ListenPacket("udp4", ":123")
+	ctx := context.Background()
+	clientPC, err := client.ListenPacket(ctx, "udp4", ":123")
 	if err != nil {
 		t.Fatal(err)
 	}
-	serverPC, err := server.ListenPacket("udp4", ":456")
+	serverPC, err := server.ListenPacket(ctx, "udp4", ":456")
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -212,5 +234,276 @@ func TestPacketHandler(t *testing.T) {
 	if addr.String() != mappedAddr.String() {
 		t.Errorf("addr = %q; want %q", addr, mappedAddr)
 	}
-
+}
+
+func TestFirewall(t *testing.T) {
+	wan := NewInternet()
+	lan := &Network{
+		Name:    "lan",
+		Prefix4: mustPrefix("10.0.0.0/8"),
+	}
+	m := &Machine{Name: "test"}
+	trust := m.Attach("trust", lan)
+	untrust := m.Attach("untrust", wan)
+
+	client := ipp("192.168.0.2:1234")
+	serverA := ipp("2.2.2.2:5678")
+	serverB1 := ipp("7.7.7.7:9012")
+	serverB2 := ipp("7.7.7.7:3456")
+
+	t.Run("ip_port_dependent", func(t *testing.T) {
+		f := &Firewall{
+			TrustedInterface: trust,
+			SessionTimeout:   30 * time.Second,
+			Type:             AddressAndPortDependentFirewall,
+		}
+		testFirewall(t, f, []fwTest{
+			// client -> A authorizes A -> client
+			{trust, untrust, client, serverA, true},
+			{untrust, trust, serverA, client, true},
+			{untrust, trust, serverA, client, true},
+
+			// B1 -> client fails until client -> B1
+			{untrust, trust, serverB1, client, false},
+			{trust, untrust, client, serverB1, true},
+			{untrust, trust, serverB1, client, true},
+
+			// B2 -> client still fails
+			{untrust, trust, serverB2, client, false},
+		})
+	})
+	t.Run("ip_dependent", func(t *testing.T) {
+		f := &Firewall{
+			TrustedInterface: trust,
+			SessionTimeout:   30 * time.Second,
+			Type:             AddressDependentFirewall,
+		}
+		testFirewall(t, f, []fwTest{
+			// client -> A authorizes A -> client
+			{trust, untrust, client, serverA, true},
+			{untrust, trust, serverA, client, true},
+			{untrust, trust, serverA, client, true},
+
+			// B1 -> client fails until client -> B1
+			{untrust, trust, serverB1, client, false},
+			{trust, untrust, client, serverB1, true},
+			{untrust, trust, serverB1, client, true},
+
+			// B2 -> client also works now
+			{untrust, trust, serverB2, client, true},
+		})
+	})
+	t.Run("endpoint_independent", func(t *testing.T) {
+		f := &Firewall{
+			TrustedInterface: trust,
+			SessionTimeout:   30 * time.Second,
+			Type:             EndpointIndependentFirewall,
+		}
+		testFirewall(t, f, []fwTest{
+			// client -> A authorizes A -> client
+			{trust, untrust, client, serverA, true},
+			{untrust, trust, serverA, client, true},
+			{untrust, trust, serverA, client, true},
+
+			// B1 -> client also works
+			{untrust, trust, serverB1, client, true},
+
+			// B2 -> client also works
+			{untrust, trust, serverB2, client, true},
+		})
+	})
+}
+
+type fwTest struct {
+	iif, oif *Interface
+	src, dst netaddr.IPPort
+	ok       bool
+}
+
+func testFirewall(t *testing.T, f *Firewall, tests []fwTest) {
+	t.Helper()
+	clock := &tstest.Clock{}
+	f.TimeNow = clock.Now
+	for _, test := range tests {
+		clock.Advance(time.Second)
+		p := &Packet{
+			Src:     test.src,
+			Dst:     test.dst,
+			Payload: []byte{},
+		}
+		got := f.HandleForward(p, test.iif, test.oif)
+		gotOK := got != nil
+		if gotOK != test.ok {
+			t.Errorf("iif=%s oif=%s src=%s dst=%s got ok=%v, want ok=%v", test.iif, test.oif, test.src, test.dst, gotOK, test.ok)
+		}
+	}
+}
+
+func ipp(str string) netaddr.IPPort {
+	ipp, err := netaddr.ParseIPPort(str)
+	if err != nil {
+		panic(err)
+	}
+	return ipp
+}
+
+func TestNAT(t *testing.T) {
+	internet := NewInternet()
+	lan := &Network{
+		Name:    "LAN",
+		Prefix4: mustPrefix("192.168.0.0/24"),
+	}
+	m := &Machine{Name: "NAT"}
+	wanIf := m.Attach("wan", internet)
+	lanIf := m.Attach("lan", lan)
+
+	t.Run("endpoint_independent_mapping", func(t *testing.T) {
+		n := &SNAT44{
+			Machine:           m,
+			ExternalInterface: wanIf,
+			Type:              EndpointIndependentNAT,
+			Firewall: &Firewall{
+				TrustedInterface: lanIf,
+			},
+		}
+		testNAT(t, n, lanIf, wanIf, []natTest{
+			{
+				src:            ipp("192.168.0.20:1234"),
+				dst:            ipp("2.2.2.2:5678"),
+				wantNewMapping: true,
+			},
+			{
+				src:            ipp("192.168.0.20:1234"),
+				dst:            ipp("7.7.7.7:9012"),
+				wantNewMapping: false,
+			},
+			{
+				src:            ipp("192.168.0.20:2345"),
+				dst:            ipp("7.7.7.7:9012"),
+				wantNewMapping: true,
+			},
+		})
+	})
+
+	t.Run("address_dependent_mapping", func(t *testing.T) {
+		n := &SNAT44{
+			Machine:           m,
+			ExternalInterface: wanIf,
+			Type:              AddressDependentNAT,
+			Firewall: &Firewall{
+				TrustedInterface: lanIf,
+			},
+		}
+		testNAT(t, n, lanIf, wanIf, []natTest{
+			{
+				src:            ipp("192.168.0.20:1234"),
+				dst:            ipp("2.2.2.2:5678"),
+				wantNewMapping: true,
+			},
+			{
+				src:            ipp("192.168.0.20:1234"),
+				dst:            ipp("2.2.2.2:9012"),
+				wantNewMapping: false,
+			},
+			{
+				src:            ipp("192.168.0.20:1234"),
+				dst:            ipp("7.7.7.7:9012"),
+				wantNewMapping: true,
+			},
+			{
+				src:            ipp("192.168.0.20:1234"),
+				dst:            ipp("7.7.7.7:1234"),
+				wantNewMapping: false,
+			},
+		})
+	})
+
+	t.Run("address_and_port_dependent_mapping", func(t *testing.T) {
+		n := &SNAT44{
+			Machine:           m,
+			ExternalInterface: wanIf,
+			Type:              AddressAndPortDependentNAT,
+			Firewall: &Firewall{
+				TrustedInterface: lanIf,
+			},
+		}
+		testNAT(t, n, lanIf, wanIf, []natTest{
+			{
+				src:            ipp("192.168.0.20:1234"),
+				dst:            ipp("2.2.2.2:5678"),
+				wantNewMapping: true,
+			},
+			{
+				src:            ipp("192.168.0.20:1234"),
+				dst:            ipp("2.2.2.2:9012"),
+				wantNewMapping: true,
+			},
+			{
+				src:            ipp("192.168.0.20:1234"),
+				dst:            ipp("7.7.7.7:9012"),
+				wantNewMapping: true,
+			},
+			{
+				src:            ipp("192.168.0.20:1234"),
+				dst:            ipp("7.7.7.7:1234"),
+				wantNewMapping: true,
+			},
+		})
+	})
+}
+
+type natTest struct {
+	src, dst       netaddr.IPPort
+	wantNewMapping bool
+}
+
+func testNAT(t *testing.T, n *SNAT44, lanIf, wanIf *Interface, tests []natTest) {
+	clock := &tstest.Clock{}
+	n.TimeNow = clock.Now
+
+	mappings := map[netaddr.IPPort]bool{}
+	for _, test := range tests {
+		clock.Advance(time.Second)
+		p := &Packet{
+			Src:     test.src,
+			Dst:     test.dst,
+			Payload: []byte("foo"),
+		}
+		gotPacket := n.HandleForward(p.Clone(), lanIf, wanIf)
+		if gotPacket == nil {
+			t.Errorf("n.HandleForward(%v) dropped packet", p)
+			continue
+		}
+
+		if gotPacket.Dst != p.Dst {
+			t.Errorf("n.HandleForward(%v) mutated dest ip:port, got %v", p, gotPacket.Dst)
+		}
+		gotNewMapping := !mappings[gotPacket.Src]
+		if gotNewMapping != test.wantNewMapping {
+			t.Errorf("n.HandleForward(%v) mapping was new=%v, want %v", p, gotNewMapping, test.wantNewMapping)
+		}
+		mappings[gotPacket.Src] = true
+
+		// Check that the return path works and translates back
+		// correctly.
+		clock.Advance(time.Second)
+		p2 := &Packet{
+			Src:     test.dst,
+			Dst:     gotPacket.Src,
+			Payload: []byte("bar"),
+		}
+		gotPacket2 := n.HandleIn(p2.Clone(), wanIf)
+
+		if gotPacket2 == nil {
+			t.Errorf("return packet was dropped")
+			continue
+		}
+
+		if gotPacket2.Src != test.dst {
+			t.Errorf("return packet has src=%v, want %v", gotPacket2.Src, test.dst)
+		}
+		if gotPacket2.Dst != test.src {
+			t.Errorf("return packet has dst=%v, want %v", gotPacket2.Dst, test.src)
+		}
+	}
 }

tsweb/jsonhandler.go

@@ -32,8 +32,8 @@ func responseError(e string) *response {
 
 func writeResponse(w http.ResponseWriter, s int, resp *response) {
 	b, _ := json.Marshal(resp)
-	w.WriteHeader(s)
 	w.Header().Set("Content-Type", "application/json")
+	w.WriteHeader(s)
 	w.Write(b)
 }
 

tsweb/jsonhandler_test.go

@@ -44,6 +44,10 @@ func TestNewJSONHandler(t *testing.T) {
 			t.Fatalf("wrong status: %s %s", d.Status, status)
 		}
 
+		if w.Header().Get("Content-Type") != "application/json" {
+			t.Fatalf("wrong content type: %s", w.Header().Get("Content-Type"))
+		}
+
 		return d
 	}
 

types/logger/logger_test.go

@@ -60,7 +60,7 @@ func TestRateLimiter(t *testing.T) {
 		lg("templated format string no. %d", i)
 		if i == 4 {
 			lg("Make sure this string makes it through the rest (that are blocked) %d", i)
-			prefixed = WithPrefix(lg, string('0'+i))
+			prefixed = WithPrefix(lg, string(rune('0'+i)))
 			prefixed(" shouldn't get filtered.")
 		}
 	}

types/nettype/nettype.go

@@ -0,0 +1,25 @@
+// Copyright (c) 2020 Tailscale Inc & AUTHORS All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+// Package nettype defines an interface that doesn't exist in the Go net package.
+package nettype
+
+import (
+	"context"
+	"net"
+)
+
+// PacketListener defines the ListenPacket method as implemented
+// by net.ListenConfig, net.ListenPacket, and tstest/natlab.
+type PacketListener interface {
+	ListenPacket(ctx context.Context, network, address string) (net.PacketConn, error)
+}
+
+// Std implements PacketListener using the Go net package's ListenPacket func.
+type Std struct{}
+
+func (Std) ListenPacket(ctx context.Context, network, address string) (net.PacketConn, error) {
+	var conf net.ListenConfig
+	return conf.ListenPacket(ctx, network, address)
+}

util/lineread/lineread.go

@@ -0,0 +1,33 @@
+// Copyright (c) 2020 Tailscale Inc & AUTHORS All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+// Package lineread reads lines from files. It's not fancy, but it got repetitive.
+package lineread
+
+import (
+	"bufio"
+	"io"
+	"os"
+)
+
+// File opens name and calls fn for each line. It returns an error if the Open failed
+// or once fn returns an error.
+func File(name string, fn func(line []byte) error) error {
+	f, err := os.Open(name)
+	if err != nil {
+		return err
+	}
+	defer f.Close()
+	return Reader(f, fn)
+}
+
+func Reader(r io.Reader, fn func(line []byte) error) error {
+	bs := bufio.NewScanner(r)
+	for bs.Scan() {
+		if err := fn(bs.Bytes()); err != nil {
+			return err
+		}
+	}
+	return bs.Err()
+}

version/describe.txt.do

@@ -1,4 +1,4 @@
-describe=$(cd ../.. && git describe --long)
+describe=$(cd ../.. && git describe --long --abbrev=9)
 echo "$describe" >$3
 redo-always
 redo-stamp <$3

version/version.go

@@ -7,5 +7,5 @@
 // Package version provides the version that the binary was built at.
 package version
 
-const LONG = "date.20200703"
+const LONG = "date.20200727"
 const SHORT = LONG

wgengine/magicsock/magicsock.go

@@ -49,6 +49,7 @@ import (
 	"tailscale.com/tailcfg"
 	"tailscale.com/types/key"
 	"tailscale.com/types/logger"
+	"tailscale.com/types/nettype"
 	"tailscale.com/types/opt"
 	"tailscale.com/types/structs"
 	"tailscale.com/version"
@@ -57,14 +58,15 @@ import (
 // A Conn routes UDP packets and actively manages a list of its endpoints.
 // It implements wireguard/conn.Bind.
 type Conn struct {
-	pconnPort    uint16 // the preferred port from opts.Port; 0 means auto
-	pconn4       *RebindingUDPConn
-	pconn6       *RebindingUDPConn // non-nil if IPv6 available
-	epFunc       func(endpoints []string)
-	logf         logger.Logf
-	sendLogLimit *rate.Limiter
-	netChecker   *netcheck.Client
-	idleFunc     func() time.Duration // nil means unknown
+	pconnPort        uint16 // the preferred port from opts.Port; 0 means auto
+	pconn4           *RebindingUDPConn
+	pconn6           *RebindingUDPConn // non-nil if IPv6 available
+	epFunc           func(endpoints []string)
+	logf             logger.Logf
+	sendLogLimit     *rate.Limiter
+	netChecker       *netcheck.Client
+	idleFunc         func() time.Duration   // nil means unknown
+	noteRecvActivity func(tailcfg.DiscoKey) // or nil, see Options.NoteRecvActivity
 
 	// bufferedIPv4From and bufferedIPv4Packet are owned by
 	// ReceiveIPv4, and used when both a DERP and IPv4 packet arrive
@@ -82,9 +84,19 @@ type Conn struct {
 	udpRecvCh  chan udpReadResult
 	derpRecvCh chan derpReadResult
 
+	// packetListener optionally specifies a test hook to open a PacketConn.
+	packetListener nettype.PacketListener
+
 	// ============================================================
 	mu sync.Mutex // guards all following fields
 
+	// canCreateEPUnlocked tracks at one place whether mu is
+	// already held. It's then checked in CreateEndpoint to avoid
+	// double-locking mu and thus deadlocking. mu should be held
+	// while setting this; but can be read without mu held.
+	// TODO(bradfitz): delete this shameful hack; refactor the one use
+	canCreateEPUnlocked syncs.AtomicBool
+
 	started bool // Start was called
 	closed  bool // Close was called
 
@@ -100,8 +112,8 @@ type Conn struct {
 	nodeOfDisco     map[tailcfg.DiscoKey]*tailcfg.Node
 	discoOfNode     map[tailcfg.NodeKey]tailcfg.DiscoKey
 	discoOfAddr     map[netaddr.IPPort]tailcfg.DiscoKey // validated non-DERP paths only
-	endpointOfDisco map[tailcfg.DiscoKey]*discoEndpoint
-	sharedDiscoKey  map[tailcfg.DiscoKey]*[32]byte // nacl/box precomputed key
+	endpointOfDisco map[tailcfg.DiscoKey]*discoEndpoint // those with activity only
+	sharedDiscoKey  map[tailcfg.DiscoKey]*[32]byte      // nacl/box precomputed key
 
 	// addrsByUDP is a map of every remote ip:port to a priority
 	// list of endpoint addresses for a peer.
@@ -210,8 +222,6 @@ type activeDerp struct {
 // The current default (zero) means to auto-select a random free port.
 const DefaultPort = 0
 
-var DisableSTUNForTesting bool
-
 // Options contains options for Listen.
 type Options struct {
 	// Logf optionally provides a log function to use.
@@ -229,6 +239,21 @@ type Options struct {
 	// IdleFunc optionally provides a func to return how long
 	// it's been since a TUN packet was sent or received.
 	IdleFunc func() time.Duration
+
+	// PacketListener optionally specifies how to create PacketConns.
+	// It's meant for testing.
+	PacketListener nettype.PacketListener
+
+	// NoteRecvActivity, if provided, is a func for magicsock to
+	// call whenever it receives a packet from a a
+	// discovery-capable peer if it's been more than ~10 seconds
+	// since the last one. (10 seconds is somewhat arbitrary; the
+	// sole user just doesn't need or want it called on every
+	// packet, just every minute or two for Wireguard timeouts,
+	// and 10 seconds seems like a good trade-off between often
+	// enough and not too often.) The provided func likely calls
+	// Conn.CreateEndpoint, which acquires Conn.mu.
+	NoteRecvActivity func(tailcfg.DiscoKey)
 }
 
 func (o *Options) logf() logger.Logf {
@@ -275,6 +300,8 @@ func NewConn(opts Options) (*Conn, error) {
 	c.logf = opts.logf()
 	c.epFunc = opts.endpointsFunc()
 	c.idleFunc = opts.IdleFunc
+	c.packetListener = opts.PacketListener
+	c.noteRecvActivity = opts.NoteRecvActivity
 
 	if err := c.initialBind(); err != nil {
 		return nil, err
@@ -367,7 +394,7 @@ func (c *Conn) updateNetInfo(ctx context.Context) (*netcheck.Report, error) {
 	dm := c.derpMap
 	c.mu.Unlock()
 
-	if DisableSTUNForTesting || dm == nil {
+	if dm == nil {
 		return new(netcheck.Report), nil
 	}
 
@@ -389,6 +416,9 @@ func (c *Conn) updateNetInfo(ctx context.Context) (*netcheck.Report, error) {
 		DERPLatency:           map[string]float64{},
 		MappingVariesByDestIP: report.MappingVariesByDestIP,
 		HairPinning:           report.HairPinning,
+		UPnP:                  report.UPnP,
+		PMP:                   report.PMP,
+		PCP:                   report.PCP,
 	}
 	for rid, d := range report.RegionV4Latency {
 		ni.DERPLatency[fmt.Sprintf("%d-v4", rid)] = d.Seconds()
@@ -504,19 +534,26 @@ func (c *Conn) SetNetInfoCallback(fn func(*tailcfg.NetInfo)) {
 	}
 }
 
-// SetDiscoPrivateKey sets the discovery key.
-func (c *Conn) SetDiscoPrivateKey(k key.Private) {
+// DiscoPublicKey returns the discovery public key.
+func (c *Conn) DiscoPublicKey() tailcfg.DiscoKey {
 	c.mu.Lock()
 	defer c.mu.Unlock()
-	if !c.discoPrivate.IsZero() && c.discoPrivate != k {
-		// TODO: support changing a key at runtime; need to
-		// clear a bunch of maps at least
-		panic("unsupported")
+	if c.discoPrivate.IsZero() {
+		priv := key.NewPrivate()
+		c.discoPrivate = priv
+		c.discoPublic = tailcfg.DiscoKey(priv.Public())
+		c.discoShort = c.discoPublic.ShortString()
+		c.logf("magicsock: disco key = %v", c.discoShort)
 	}
-	c.discoPrivate = k
-	c.discoPublic = tailcfg.DiscoKey(k.Public())
-	c.discoShort = c.discoPublic.ShortString()
-	c.logf("magicsock: set disco key = %v", c.discoShort)
+	return c.discoPublic
+}
+
+// PeerHasDiscoKey reports whether peer k supports discovery keys (client version 0.100.0+).
+func (c *Conn) PeerHasDiscoKey(k tailcfg.NodeKey) bool {
+	c.mu.Lock()
+	defer c.mu.Unlock()
+	_, ok := c.discoOfNode[k]
+	return ok
 }
 
 // c.mu must NOT be held.
@@ -670,7 +707,7 @@ func shouldSprayPacket(b []byte) bool {
 
 var logPacketDests, _ = strconv.ParseBool(os.Getenv("TS_DEBUG_LOG_PACKET_DESTS"))
 
-var logDisco, _ = strconv.ParseBool(os.Getenv("TS_DEBUG_DISCO"))
+var debugDisco, _ = strconv.ParseBool(os.Getenv("TS_DEBUG_DISCO"))
 
 const sprayPeriod = 3 * time.Second
 
@@ -1117,7 +1154,15 @@ func (c *Conn) runDerpReader(ctx context.Context, derpFakeAddr netaddr.IPPort, d
 			}
 			c.ReSTUN("derp-close")
 			c.logf("magicsock: [%p] derp.Recv(derp-%d): %v", dc, regionID, err)
-			time.Sleep(250 * time.Millisecond)
+
+			// Avoid excessive spinning.
+			// TODO: use a backoff timer, perhaps between 10ms and 500ms?
+			// Don't want to sleep too long. For now 250ms seems fine.
+			select {
+			case <-ctx.Done():
+				return
+			case <-time.After(250 * time.Millisecond):
+			}
 			continue
 		}
 		switch m := msg.(type) {
@@ -1143,7 +1188,12 @@ func (c *Conn) runDerpReader(ctx context.Context, derpFakeAddr netaddr.IPPort, d
 		case <-ctx.Done():
 			return
 		case c.derpRecvCh <- res:
-			<-didCopy
+			select {
+			case <-ctx.Done():
+				return
+			case <-didCopy:
+				continue
+			}
 		}
 	}
 }
@@ -1270,6 +1320,16 @@ func wgRecvAddr(e conn.Endpoint, ipp netaddr.IPPort, addr *net.UDPAddr) *net.UDP
 	return ipp.UDPAddr()
 }
 
+// noteRecvActivity calls the magicsock.Conn.noteRecvActivity hook if
+// e is a discovery-capable peer.
+//
+// This should be called whenever a packet arrives from e.
+func noteRecvActivity(e conn.Endpoint) {
+	if de, ok := e.(*discoEndpoint); ok {
+		de.onRecvActivity()
+	}
+}
+
 func (c *Conn) ReceiveIPv4(b []byte) (n int, ep conn.Endpoint, addr *net.UDPAddr, err error) {
 Top:
 	// First, process any buffered packet from earlier.
@@ -1277,6 +1337,7 @@ Top:
 		c.bufferedIPv4From = netaddr.IPPort{}
 		addr = from.UDPAddr()
 		ep := c.findEndpoint(from, addr)
+		noteRecvActivity(ep)
 		return copy(b, c.bufferedIPv4Packet), ep, wgRecvAddr(ep, from, addr), nil
 	}
 
@@ -1289,6 +1350,7 @@ Top:
 	var addrSet *AddrSet
 	var discoEp *discoEndpoint
 	var ipp netaddr.IPPort
+	var didNoteRecvActivity bool
 
 	select {
 	case dm := <-c.derpRecvCh:
@@ -1330,6 +1392,24 @@ Top:
 		c.mu.Lock()
 		if dk, ok := c.discoOfNode[tailcfg.NodeKey(dm.src)]; ok {
 			discoEp = c.endpointOfDisco[dk]
+			// If we know about the node (it's in discoOfNode) but don't know about the
+			// endpoint, that's because it's an idle peer that doesn't yet exist in the
+			// wireguard config. So run the receive hook, if defined, which should
+			// create the wireguard peer.
+			if discoEp == nil && c.noteRecvActivity != nil {
+				didNoteRecvActivity = true
+				c.mu.Unlock()          // release lock before calling noteRecvActivity
+				c.noteRecvActivity(dk) // (calls back into CreateEndpoint)
+				// Now require the lock. No invariants need to be rechecked; just
+				// 1-2 map lookups follow that are harmless if, say, the peer has
+				// been deleted during this time.  In that case we'll treate it as a
+				// legacy pre-disco UDP receive and hand it to wireguard which'll
+				// likely just drop it.
+				c.mu.Lock()
+
+				discoEp = c.endpointOfDisco[dk]
+				c.logf("magicsock: DERP packet received from idle peer %v; created=%v", dm.src.ShortString(), discoEp != nil)
+			}
 		}
 		if discoEp == nil {
 			addrSet = c.addrsByKey[dm.src]
@@ -1368,6 +1448,9 @@ Top:
 	} else {
 		ep = c.findEndpoint(ipp, addr)
 	}
+	if !didNoteRecvActivity {
+		noteRecvActivity(ep)
+	}
 	return n, ep, wgRecvAddr(ep, ipp, addr), nil
 }
 
@@ -1394,12 +1477,29 @@ func (c *Conn) ReceiveIPv6(b []byte) (int, conn.Endpoint, *net.UDPAddr, error) {
 		}
 
 		ep := c.findEndpoint(ipp, addr)
+		noteRecvActivity(ep)
 		return n, ep, wgRecvAddr(ep, ipp, addr), nil
 	}
 }
 
-func (c *Conn) sendDiscoMessage(dst netaddr.IPPort, dstKey key.Public, dstDisco tailcfg.DiscoKey, m disco.Message) (sent bool, err error) {
+// discoLogLevel controls the verbosity of discovery log messages.
+type discoLogLevel int
+
+const (
+	// discoLog means that a message should be logged.
+	discoLog discoLogLevel = iota
+
+	// discoVerboseLog means that a message should only be logged
+	// in TS_DEBUG_DISCO mode.
+	discoVerboseLog
+)
+
+func (c *Conn) sendDiscoMessage(dst netaddr.IPPort, dstKey tailcfg.NodeKey, dstDisco tailcfg.DiscoKey, m disco.Message, logLevel discoLogLevel) (sent bool, err error) {
 	c.mu.Lock()
+	if c.closed {
+		c.mu.Unlock()
+		return false, errClosed
+	}
 	var nonce [disco.NonceLen]byte
 	if _, err := crand.Read(nonce[:]); err != nil {
 		panic(err) // worth dying for
@@ -1412,9 +1512,11 @@ func (c *Conn) sendDiscoMessage(dst netaddr.IPPort, dstKey key.Public, dstDisco
 	c.mu.Unlock()
 
 	pkt = box.SealAfterPrecomputation(pkt, m.AppendMarshal(nil), &nonce, sharedKey)
-	sent, err = c.sendAddr(dst, dstKey, pkt)
+	sent, err = c.sendAddr(dst, key.Public(dstKey), pkt)
 	if sent {
-		c.logf("magicsock: disco: %v->%v (%v, %v) sent %v", c.discoShort, dstDisco.ShortString(), dstKey.ShortString(), derpStr(dst.String()), disco.MessageSummary(m))
+		if logLevel == discoLog || (logLevel == discoVerboseLog && debugDisco) {
+			c.logf("magicsock: disco: %v->%v (%v, %v) sent %v", c.discoShort, dstDisco.ShortString(), dstKey.ShortString(), derpStr(dst.String()), disco.MessageSummary(m))
+		}
 	} else if err == nil {
 		// Can't send. (e.g. no IPv6 locally)
 	} else {
@@ -1446,26 +1548,58 @@ func (c *Conn) handleDiscoMessage(msg []byte, src netaddr.IPPort) bool {
 	c.mu.Lock()
 	defer c.mu.Unlock()
 
-	if logDisco {
+	if c.closed {
+		return true
+	}
+	if debugDisco {
 		c.logf("magicsock: disco: got disco-looking frame from %v", sender.ShortString())
 	}
 	if c.discoPrivate.IsZero() {
-		if logDisco {
+		if debugDisco {
 			c.logf("magicsock: disco: ignoring disco-looking frame, no local key")
 		}
 		return false
 	}
 
-	de, ok := c.endpointOfDisco[sender]
+	peerNode, ok := c.nodeOfDisco[sender]
 	if !ok {
-		if logDisco {
-			c.logf("magicsock: disco: ignoring disco-looking frame, don't know about %v", sender.ShortString())
+		if debugDisco {
+			c.logf("magicsock: disco: ignoring disco-looking frame, don't know node for %v", sender.ShortString())
 		}
 		// Returning false keeps passing it down, to WireGuard.
 		// WireGuard will almost surely reject it, but give it a chance.
 		return false
 	}
 
+	de, ok := c.endpointOfDisco[sender]
+	if !ok {
+		// We don't have an active endpoint for this sender but we knew about the node, so
+		// it's an idle endpoint that doesn't yet exist in the wireguard config. We now have
+		// to notify the userspace engine (via noteRecvActivity) so wireguard-go can create
+		// an Endpoint (ultimately calling our CreateEndpoint).
+		if debugDisco {
+			c.logf("magicsock: disco: got message from inactive peer %v", sender.ShortString())
+		}
+		if c.noteRecvActivity == nil {
+			c.logf("magicsock: [unexpected] have node without endpoint, without c.noteRecvActivity hook")
+			return false
+		}
+		// noteRecvActivity calls back into CreateEndpoint, which we can't easily control,
+		// and CreateEndpoint expects to be called with c.mu held, but we hold it here, and
+		// it's too invasive for now to release it here and recheck invariants.  So instead,
+		// use this unfortunate hack: set canCreateEPUnlocked which CreateEndpoint then
+		// checks to conditionally acquire the mutex. I'm so sorry.
+		c.canCreateEPUnlocked.Set(true)
+		c.noteRecvActivity(sender)
+		c.canCreateEPUnlocked.Set(false)
+		de, ok = c.endpointOfDisco[sender]
+		if !ok {
+			c.logf("magicsock: [unexpected] lazy endpoint not created for %v, %v", peerNode.Key.ShortString(), sender.ShortString())
+			return false
+		}
+		c.logf("magicsock: lazy endpoint created via disco message for %v, %v", peerNode.Key.ShortString(), sender.ShortString())
+	}
+
 	// First, do we even know (and thus care) about this sender? If not,
 	// don't bother decrypting it.
 
@@ -1482,7 +1616,7 @@ func (c *Conn) handleDiscoMessage(msg []byte, src netaddr.IPPort) bool {
 		// Don't log in normal case. Pass on to wireguard, in case
 		// it's actually a a wireguard packet (super unlikely,
 		// but).
-		if logDisco {
+		if debugDisco {
 			c.logf("magicsock: disco: failed to open naclbox from %v (wrong rcpt?)", sender)
 		}
 		// TODO(bradfitz): add some counter for this that logs rarely
@@ -1490,7 +1624,7 @@ func (c *Conn) handleDiscoMessage(msg []byte, src netaddr.IPPort) bool {
 	}
 
 	dm, err := disco.Parse(payload)
-	if logDisco {
+	if debugDisco {
 		c.logf("magicsock: disco: disco.Parse = %T, %v", dm, err)
 	}
 	if err != nil {
@@ -1505,8 +1639,11 @@ func (c *Conn) handleDiscoMessage(msg []byte, src netaddr.IPPort) bool {
 
 	switch dm := dm.(type) {
 	case *disco.Ping:
-		c.handlePingLocked(dm, de, src)
+		c.handlePingLocked(dm, de, src, sender, peerNode)
 	case *disco.Pong:
+		if de == nil {
+			return true
+		}
 		de.handlePongConnLocked(dm, src)
 	case disco.CallMeMaybe:
 		if src.IP != derpMagicIPAddr {
@@ -1514,24 +1651,43 @@ func (c *Conn) handleDiscoMessage(msg []byte, src netaddr.IPPort) bool {
 			c.logf("[unexpected] CallMeMaybe packets should only come via DERP")
 			return true
 		}
-		c.logf("magicsock: disco: %v<-%v (%v, %v)  got call-me-maybe", c.discoShort, de.discoShort, de.publicKey.ShortString(), derpStr(src.String()))
-		go de.handleCallMeMaybe()
+		if de != nil {
+			c.logf("magicsock: disco: %v<-%v (%v, %v)  got call-me-maybe", c.discoShort, de.discoShort, de.publicKey.ShortString(), derpStr(src.String()))
+			go de.handleCallMeMaybe()
+		}
 	}
 
 	return true
 }
 
-func (c *Conn) handlePingLocked(dm *disco.Ping, de *discoEndpoint, src netaddr.IPPort) {
-	c.logf("magicsock: disco: %v<-%v (%v, %v)  got ping tx=%x", c.discoShort, de.discoShort, de.publicKey.ShortString(), src, dm.TxID[:6])
+// de may be nil
+func (c *Conn) handlePingLocked(dm *disco.Ping, de *discoEndpoint, src netaddr.IPPort, sender tailcfg.DiscoKey, peerNode *tailcfg.Node) {
+	if peerNode == nil {
+		c.logf("magicsock: disco: [unexpected] ignoring ping from unknown peer Node")
+		return
+	}
+	likelyHeartBeat := de != nil && src == de.lastPingFrom && time.Since(de.lastPingTime) < 5*time.Second
+	var discoShort string
+	if de != nil {
+		discoShort = de.discoShort
+		de.lastPingFrom = src
+		de.lastPingTime = time.Now()
+	} else {
+		discoShort = sender.ShortString()
+	}
+	if !likelyHeartBeat || debugDisco {
+		c.logf("magicsock: disco: %v<-%v (%v, %v)  got ping tx=%x", c.discoShort, discoShort, peerNode.Key.ShortString(), src, dm.TxID[:6])
+	}
 
 	// Remember this route if not present.
-	c.setAddrToDiscoLocked(src, de.discoKey, nil)
+	c.setAddrToDiscoLocked(src, sender, nil)
 
-	pongDst := src
-	go de.sendDiscoMessage(pongDst, &disco.Pong{
+	ipDst := src
+	discoDest := sender
+	go c.sendDiscoMessage(ipDst, peerNode.Key, discoDest, &disco.Pong{
 		TxID: dm.TxID,
 		Src:  src,
-	})
+	}, discoVerboseLog)
 }
 
 // setAddrToDiscoLocked records that newk is at src.
@@ -1632,13 +1788,16 @@ func (c *Conn) SetPrivateKey(privateKey wgcfg.PrivateKey) error {
 	if oldKey.IsZero() {
 		c.logf("magicsock: SetPrivateKey called (init)")
 		go c.ReSTUN("set-private-key")
+	} else if newKey.IsZero() {
+		c.logf("magicsock: SetPrivateKey called (zeroed)")
+		c.closeAllDerpLocked("zero-private-key")
 	} else {
-		c.logf("magicsock: SetPrivateKey called (changed")
+		c.logf("magicsock: SetPrivateKey called (changed)")
+		c.closeAllDerpLocked("new-private-key")
 	}
-	c.closeAllDerpLocked("new-private-key")
 
 	// Key changed. Close existing DERP connections and reconnect to home.
-	if c.myDerp != 0 {
+	if c.myDerp != 0 && !newKey.IsZero() {
 		c.logf("magicsock: private key changed, reconnecting to home derp-%d", c.myDerp)
 		c.goDerpConnect(c.myDerp)
 	}
@@ -1688,11 +1847,25 @@ func (c *Conn) SetDERPMap(dm *tailcfg.DERPMap) {
 		return
 	}
 
-	go c.ReSTUN("derp-map-update")
+	if c.started {
+		go c.ReSTUN("derp-map-update")
+	}
+}
+
+func nodesEqual(x, y []*tailcfg.Node) bool {
+	if len(x) != len(y) {
+		return false
+	}
+	for i := range x {
+		if !x[i].Equal(y[i]) {
+			return false
+		}
+	}
+	return true
 }
 
 // SetNetworkMap is called when the control client gets a new network
-// map from the control server.
+// map from the control server. It must always be non-nil.
 //
 // It should not use the DERPMap field of NetworkMap; that's
 // conditionally sent to SetDERPMap instead.
@@ -1700,7 +1873,7 @@ func (c *Conn) SetNetworkMap(nm *controlclient.NetworkMap) {
 	c.mu.Lock()
 	defer c.mu.Unlock()
 
-	if reflect.DeepEqual(nm, c.netMap) {
+	if c.netMap != nil && nodesEqual(c.netMap.Peers, nm.Peers) {
 		return
 	}
 
@@ -1860,6 +2033,10 @@ func (c *Conn) Close() error {
 	}
 	defer c.mu.Unlock()
 
+	for _, ep := range c.endpointOfDisco {
+		ep.cleanup()
+	}
+
 	c.closed = true
 	c.connCtxCancel()
 	c.closeAllDerpLocked("conn-close")
@@ -1867,6 +2044,17 @@ func (c *Conn) Close() error {
 		c.pconn6.Close()
 	}
 	err := c.pconn4.Close()
+	// The goroutine running dc.Connect in derpWriteChanOfAddr may linger
+	// and appear to leak, as observed in https://github.com/tailscale/tailscale/issues/554.
+	// This is despite the underlying context being cancelled by connCtxCancel above.
+	// To avoid this condition, we must wait on derpStarted here
+	// to ensure that this goroutine has exited by the time Close returns.
+	// We only do this if derpWriteChanOfAddr has executed at least once:
+	// on the first run, it sets firstDerp := true and spawns the aforementioned goroutine.
+	// To detect this, we check activeDerp, which is initialized to non-nil on the first run.
+	if c.activeDerp != nil {
+		<-c.derpStarted
+	}
 	// Wait on endpoints updating right at the end, once everything is
 	// already closed. We want everything else in the Conn to be
 	// consistently in the closed state before we release mu to wait
@@ -1969,6 +2157,10 @@ func (c *Conn) ReSTUN(why string) {
 		// raced with a shutdown.
 		return
 	}
+	if c.privateKey.IsZero() {
+		c.logf("magicsock: ReSTUN(%q) ignored; no private key", why)
+		return
+	}
 
 	if c.endpointsUpdateActive {
 		if c.wantEndpointsUpdate != why {
@@ -1991,6 +2183,13 @@ func (c *Conn) initialBind() error {
 	return nil
 }
 
+func (c *Conn) listenPacket(ctx context.Context, network, addr string) (net.PacketConn, error) {
+	if c.packetListener != nil {
+		return c.packetListener.ListenPacket(ctx, network, addr)
+	}
+	return netns.Listener().ListenPacket(ctx, network, addr)
+}
+
 func (c *Conn) bind1(ruc **RebindingUDPConn, which string) error {
 	host := ""
 	if v, _ := strconv.ParseBool(os.Getenv("IN_TS_TEST")); v {
@@ -2000,13 +2199,13 @@ func (c *Conn) bind1(ruc **RebindingUDPConn, which string) error {
 	var err error
 	listenCtx := context.Background() // unused without DNS name to resolve
 	if c.pconnPort == 0 && DefaultPort != 0 {
-		pc, err = netns.Listener().ListenPacket(listenCtx, which, fmt.Sprintf("%s:%d", host, DefaultPort))
+		pc, err = c.listenPacket(listenCtx, which, fmt.Sprintf("%s:%d", host, DefaultPort))
 		if err != nil {
 			c.logf("magicsock: bind: default port %s/%v unavailable; picking random", which, DefaultPort)
 		}
 	}
 	if pc == nil {
-		pc, err = netns.Listener().ListenPacket(listenCtx, which, fmt.Sprintf("%s:%d", host, c.pconnPort))
+		pc, err = c.listenPacket(listenCtx, which, fmt.Sprintf("%s:%d", host, c.pconnPort))
 	}
 	if err != nil {
 		c.logf("magicsock: bind(%s/%v): %v", which, c.pconnPort, err)
@@ -2015,7 +2214,7 @@ func (c *Conn) bind1(ruc **RebindingUDPConn, which string) error {
 	if *ruc == nil {
 		*ruc = new(RebindingUDPConn)
 	}
-	(*ruc).Reset(pc.(*net.UDPConn))
+	(*ruc).Reset(pc)
 	return nil
 }
 
@@ -2032,7 +2231,7 @@ func (c *Conn) Rebind() {
 		if err := c.pconn4.pconn.Close(); err != nil {
 			c.logf("magicsock: link change close failed: %v", err)
 		}
-		packetConn, err := netns.Listener().ListenPacket(listenCtx, "udp4", fmt.Sprintf("%s:%d", host, c.pconnPort))
+		packetConn, err := c.listenPacket(listenCtx, "udp4", fmt.Sprintf("%s:%d", host, c.pconnPort))
 		if err == nil {
 			c.logf("magicsock: link change rebound port: %d", c.pconnPort)
 			c.pconn4.pconn = packetConn.(*net.UDPConn)
@@ -2043,7 +2242,7 @@ func (c *Conn) Rebind() {
 		c.pconn4.mu.Unlock()
 	}
 	c.logf("magicsock: link change, binding new port")
-	packetConn, err := netns.Listener().ListenPacket(listenCtx, "udp4", host+":0")
+	packetConn, err := c.listenPacket(listenCtx, "udp4", host+":0")
 	if err != nil {
 		c.logf("magicsock: link change failed to bind new port: %v", err)
 		return
@@ -2052,8 +2251,12 @@ func (c *Conn) Rebind() {
 
 	c.mu.Lock()
 	c.closeAllDerpLocked("rebind")
+	haveKey := !c.privateKey.IsZero()
 	c.mu.Unlock()
-	c.goDerpConnect(c.myDerp)
+
+	if haveKey {
+		c.goDerpConnect(c.myDerp)
+	}
 	c.resetAddrSetStates()
 }
 
@@ -2377,17 +2580,30 @@ func (c *Conn) CreateEndpoint(pubKey [32]byte, addrs string) (conn.Endpoint, err
 		if err != nil {
 			return nil, fmt.Errorf("magicsock: invalid discokey endpoint %q for %v: %w", addrs, pk.ShortString(), err)
 		}
-		c.mu.Lock()
-		defer c.mu.Unlock()
+		if !c.canCreateEPUnlocked.Get() { // sorry
+			c.mu.Lock()
+			defer c.mu.Unlock()
+		}
 		de := &discoEndpoint{
 			c:                  c,
-			publicKey:          pk,                         // peer public key (for WireGuard + DERP)
+			publicKey:          tailcfg.NodeKey(pk),        // peer public key (for WireGuard + DERP)
 			discoKey:           tailcfg.DiscoKey(discoKey), // for discovery mesages
 			discoShort:         tailcfg.DiscoKey(discoKey).ShortString(),
 			wgEndpointHostPort: addrs,
 			sentPing:           map[stun.TxID]sentPing{},
 			endpointState:      map[netaddr.IPPort]*endpointState{},
 		}
+		lastRecvTime := new(int64) // atomic
+		de.onRecvActivity = func() {
+			now := time.Now().Unix()
+			old := atomic.LoadInt64(lastRecvTime)
+			if old == 0 || old <= now-10 {
+				atomic.StoreInt64(lastRecvTime, now)
+				if c.noteRecvActivity != nil {
+					c.noteRecvActivity(de.discoKey)
+				}
+			}
+		}
 		de.initFakeUDPAddr()
 		de.updateFromNode(c.nodeOfDisco[de.discoKey])
 		c.endpointOfDisco[de.discoKey] = de
@@ -2470,10 +2686,10 @@ type RebindingUDPConn struct {
 	ippCache ippCache
 
 	mu    sync.Mutex
-	pconn *net.UDPConn
+	pconn net.PacketConn
 }
 
-func (c *RebindingUDPConn) Reset(pconn *net.UDPConn) {
+func (c *RebindingUDPConn) Reset(pconn net.PacketConn) {
 	c.mu.Lock()
 	old := c.pconn
 	c.pconn = pconn
@@ -2528,7 +2744,7 @@ func (c *RebindingUDPConn) WriteToUDP(b []byte, addr *net.UDPAddr) (int, error)
 		pconn := c.pconn
 		c.mu.Unlock()
 
-		n, err := pconn.WriteToUDP(b, addr)
+		n, err := pconn.WriteTo(b, addr)
 		if err != nil {
 			c.mu.Lock()
 			pconn2 := c.pconn
@@ -2616,14 +2832,14 @@ func (c *Conn) UpdateStatus(sb *ipnstate.StatusBuilder) {
 	c.mu.Lock()
 	defer c.mu.Unlock()
 
-	for dk, de := range c.endpointOfDisco {
+	for dk, n := range c.nodeOfDisco {
 		ps := &ipnstate.PeerStatus{InMagicSock: true}
-		if node, ok := c.nodeOfDisco[dk]; ok {
-			ps.Addrs = append(ps.Addrs, node.Endpoints...)
-			ps.Relay = c.derpRegionCodeOfAddrLocked(node.DERP)
+		ps.Addrs = append(ps.Addrs, n.Endpoints...)
+		ps.Relay = c.derpRegionCodeOfAddrLocked(n.DERP)
+		if de, ok := c.endpointOfDisco[dk]; ok {
+			de.populatePeerStatus(ps)
 		}
-		de.populatePeerStatus(ps)
-		sb.AddPeer(de.publicKey, ps)
+		sb.AddPeer(key.Public(n.Key), ps)
 	}
 	// Old-style (pre-disco) peers:
 	for k, as := range c.addrsByKey {
@@ -2653,12 +2869,17 @@ func udpAddrDebugString(ua net.UDPAddr) string {
 type discoEndpoint struct {
 	// These fields are initialized once and never modified.
 	c                  *Conn
-	publicKey          key.Public       // peer public key (for WireGuard + DERP)
+	publicKey          tailcfg.NodeKey  // peer public key (for WireGuard + DERP)
 	discoKey           tailcfg.DiscoKey // for discovery mesages
 	discoShort         string           // ShortString of discoKey
 	fakeWGAddr         netaddr.IPPort   // the UDP address we tell wireguard-go we're using
 	fakeWGAddrStd      *net.UDPAddr     // the *net.UDPAddr form of fakeWGAddr
 	wgEndpointHostPort string           // string from CreateEndpoint: "<hex-discovery-key>.disco.tailscale:12345"
+	onRecvActivity     func()
+
+	// Owned by Conn.mu:
+	lastPingFrom netaddr.IPPort
+	lastPingTime time.Time
 
 	// mu protects all following fields.
 	mu sync.Mutex // Lock ordering: Conn.mu, then discoEndpoint.mu
@@ -2730,9 +2951,10 @@ type pongReply struct {
 }
 
 type sentPing struct {
-	to    netaddr.IPPort
-	at    time.Time
-	timer *time.Timer // timeout timer
+	to      netaddr.IPPort
+	at      time.Time
+	timer   *time.Timer // timeout timer
+	purpose discoPingPurpose
 }
 
 // initFakeUDPAddr populates fakeWGAddr with a globally unique fake UDPAddr.
@@ -2750,6 +2972,13 @@ func (de *discoEndpoint) initFakeUDPAddr() {
 	de.fakeWGAddrStd = de.fakeWGAddr.UDPAddr()
 }
 
+// String exists purely so wireguard-go internals can log.Printf("%v")
+// its internal conn.Endpoints and we don't end up with data races
+// from fmt (via log) reading mutex fields and such.
+func (de *discoEndpoint) String() string {
+	return fmt.Sprintf("magicsock.discoEndpoint{%v, %v}", de.publicKey.ShortString(), de.discoShort)
+}
+
 func (de *discoEndpoint) Addrs() []wgcfg.Endpoint {
 	// This has to be the same string that was passed to
 	// CreateEndpoint, otherwise Reconfig will end up recreating
@@ -2762,7 +2991,7 @@ func (de *discoEndpoint) Addrs() []wgcfg.Endpoint {
 	if err != nil {
 		panic(err)
 	}
-	return []wgcfg.Endpoint{{host, uint16(port)}}
+	return []wgcfg.Endpoint{{Host: host, Port: uint16(port)}}
 }
 
 func (de *discoEndpoint) ClearSrc()           {}
@@ -2815,7 +3044,7 @@ func (de *discoEndpoint) heartbeat() {
 	udpAddr, _ := de.addrForSendLocked(now)
 	if !udpAddr.IsZero() {
 		// We have a preferred path. Ping that every 2 seconds.
-		de.startPingLocked(udpAddr, now)
+		de.startPingLocked(udpAddr, now, pingHeartbeat)
 	}
 
 	if de.wantFullPingLocked(now) {
@@ -2868,10 +3097,10 @@ func (de *discoEndpoint) send(b []byte) error {
 	}
 	var err error
 	if !udpAddr.IsZero() {
-		_, err = de.c.sendAddr(udpAddr, de.publicKey, b)
+		_, err = de.c.sendAddr(udpAddr, key.Public(de.publicKey), b)
 	}
 	if !derpAddr.IsZero() {
-		if ok, _ := de.c.sendAddr(derpAddr, de.publicKey, b); ok && err != nil {
+		if ok, _ := de.c.sendAddr(derpAddr, key.Public(de.publicKey), b); ok && err != nil {
 			// UDP failed but DERP worked, so good enough:
 			return nil
 		}
@@ -2879,6 +3108,19 @@ func (de *discoEndpoint) send(b []byte) error {
 	return err
 }
 
+func (de *discoEndpoint) pingTimeout(txid stun.TxID) {
+	de.mu.Lock()
+	defer de.mu.Unlock()
+	sp, ok := de.sentPing[txid]
+	if !ok {
+		return
+	}
+	if debugDisco || de.bestAddr.IsZero() || time.Now().After(de.trustBestAddrUntil) {
+		de.c.logf("magicsock: disco: timeout waiting for pong %x from %v (%v, %v)", txid[:6], sp.to, de.publicKey.ShortString(), de.discoShort)
+	}
+	de.removeSentPingLocked(txid, sp)
+}
+
 // forgetPing is called by a timer when a ping either fails to send or
 // has taken too long to get a pong reply.
 func (de *discoEndpoint) forgetPing(txid stun.TxID) {
@@ -2900,14 +3142,27 @@ func (de *discoEndpoint) removeSentPingLocked(txid stun.TxID, sp sentPing) {
 //
 // The caller (startPingLocked) should've already been recorded the ping in
 // sentPing and set up the timer.
-func (de *discoEndpoint) sendDiscoPing(ep netaddr.IPPort, txid stun.TxID) {
-	sent, _ := de.sendDiscoMessage(ep, &disco.Ping{TxID: [12]byte(txid)})
+func (de *discoEndpoint) sendDiscoPing(ep netaddr.IPPort, txid stun.TxID, logLevel discoLogLevel) {
+	sent, _ := de.sendDiscoMessage(ep, &disco.Ping{TxID: [12]byte(txid)}, logLevel)
 	if !sent {
 		de.forgetPing(txid)
 	}
 }
 
-func (de *discoEndpoint) startPingLocked(ep netaddr.IPPort, now time.Time) {
+// discoPingPurpose is the reason why a discovery ping message was sent.
+type discoPingPurpose int
+
+const (
+	// pingDiscovery means that purpose of a ping was to see if a
+	// path was valid.
+	pingDiscovery discoPingPurpose = iota
+
+	// pingHeartbeat means that purpose of a ping was whether a
+	// peer was still there.
+	pingHeartbeat
+)
+
+func (de *discoEndpoint) startPingLocked(ep netaddr.IPPort, now time.Time, purpose discoPingPurpose) {
 	st, ok := de.endpointState[ep]
 	if !ok {
 		// Shouldn't happen. But don't ping an endpoint that's
@@ -2919,14 +3174,16 @@ func (de *discoEndpoint) startPingLocked(ep netaddr.IPPort, now time.Time) {
 
 	txid := stun.NewTxID()
 	de.sentPing[txid] = sentPing{
-		to: ep,
-		at: now,
-		timer: time.AfterFunc(pingTimeoutDuration, func() {
-			de.c.logf("magicsock: disco: timeout waiting for pong %x from %v (%v, %v)", txid[:6], ep, de.publicKey.ShortString(), de.discoShort)
-			de.forgetPing(txid)
-		}),
+		to:      ep,
+		at:      now,
+		timer:   time.AfterFunc(pingTimeoutDuration, func() { de.pingTimeout(txid) }),
+		purpose: purpose,
 	}
-	go de.sendDiscoPing(ep, txid)
+	logLevel := discoLog
+	if purpose == pingHeartbeat {
+		logLevel = discoVerboseLog
+	}
+	go de.sendDiscoPing(ep, txid, logLevel)
 }
 
 func (de *discoEndpoint) sendPingsLocked(now time.Time, sendCallMeMaybe bool) {
@@ -2945,7 +3202,7 @@ func (de *discoEndpoint) sendPingsLocked(now time.Time, sendCallMeMaybe bool) {
 			de.c.logf("magicsock: disco: send, starting discovery for %v (%v)", de.publicKey.ShortString(), de.discoShort)
 		}
 
-		de.startPingLocked(ep, now)
+		de.startPingLocked(ep, now, pingDiscovery)
 	}
 	derpAddr := de.derpAddr
 	if sentAny && sendCallMeMaybe && !derpAddr.IsZero() {
@@ -2954,13 +3211,13 @@ func (de *discoEndpoint) sendPingsLocked(now time.Time, sendCallMeMaybe bool) {
 		// so our firewall ports are probably open and now would be a good time
 		// for them to connect.
 		time.AfterFunc(5*time.Millisecond, func() {
-			de.sendDiscoMessage(derpAddr, disco.CallMeMaybe{})
+			de.sendDiscoMessage(derpAddr, disco.CallMeMaybe{}, discoLog)
 		})
 	}
 }
 
-func (de *discoEndpoint) sendDiscoMessage(dst netaddr.IPPort, dm disco.Message) (sent bool, err error) {
-	return de.c.sendDiscoMessage(dst, de.publicKey, de.discoKey, dm)
+func (de *discoEndpoint) sendDiscoMessage(dst netaddr.IPPort, dm disco.Message, logLevel discoLogLevel) (sent bool, err error) {
+	return de.c.sendDiscoMessage(dst, de.publicKey, de.discoKey, dm, logLevel)
 }
 
 func (de *discoEndpoint) updateFromNode(n *tailcfg.Node) {
@@ -3057,11 +3314,13 @@ func (de *discoEndpoint) handlePongConnLocked(m *disco.Pong, src netaddr.IPPort)
 		pongSrc: m.Src,
 	})
 
-	de.c.logf("magicsock: disco: %v<-%v (%v, %v)  got pong tx=%x latency=%v pong.src=%v%v", de.c.discoShort, de.discoShort, de.publicKey.ShortString(), src, m.TxID[:6], latency.Round(time.Millisecond), m.Src, logger.ArgWriter(func(bw *bufio.Writer) {
-		if sp.to != src {
-			fmt.Fprintf(bw, " ping.to=%v", sp.to)
-		}
-	}))
+	if sp.purpose != pingHeartbeat {
+		de.c.logf("magicsock: disco: %v<-%v (%v, %v)  got pong tx=%x latency=%v pong.src=%v%v", de.c.discoShort, de.discoShort, de.publicKey.ShortString(), src, m.TxID[:6], latency.Round(time.Millisecond), m.Src, logger.ArgWriter(func(bw *bufio.Writer) {
+			if sp.to != src {
+				fmt.Fprintf(bw, " ping.to=%v", sp.to)
+			}
+		}))
+	}
 
 	// Promote this pong response to our current best address if it's lower latency.
 	// TODO(bradfitz): decide how latency vs. preference order affects decision
@@ -3184,3 +3443,5 @@ type ippCacheKey struct {
 
 // derpStr replaces DERP IPs in s with "derp-".
 func derpStr(s string) string { return strings.ReplaceAll(s, "127.3.3.40:", "derp-") }
+
+var errClosed = errors.New("conn is closed")

wgengine/magicsock/magicsock_test.go

@@ -10,6 +10,7 @@ import (
 	"crypto/tls"
 	"encoding/binary"
 	"fmt"
+	"io/ioutil"
 	"net"
 	"net/http"
 	"net/http/httptest"
@@ -31,8 +32,10 @@ import (
 	"tailscale.com/net/stun/stuntest"
 	"tailscale.com/tailcfg"
 	"tailscale.com/tstest"
+	"tailscale.com/tstest/natlab"
 	"tailscale.com/types/key"
 	"tailscale.com/types/logger"
+	"tailscale.com/types/nettype"
 	"tailscale.com/wgengine/filter"
 	"tailscale.com/wgengine/tstun"
 )
@@ -57,6 +60,136 @@ func (c *Conn) WaitReady(t *testing.T) {
 	}
 }
 
+func runDERPAndStun(t *testing.T, logf logger.Logf, l nettype.PacketListener, stunIP netaddr.IP) (derpMap *tailcfg.DERPMap, cleanup func()) {
+	var serverPrivateKey key.Private
+	if _, err := crand.Read(serverPrivateKey[:]); err != nil {
+		t.Fatal(err)
+	}
+	d := derp.NewServer(serverPrivateKey, logf)
+	if l != (nettype.Std{}) {
+		// When using virtual networking, only allow DERP to forward
+		// discovery traffic, not actual packets.
+		d.OnlyDisco = true
+	}
+
+	httpsrv := httptest.NewUnstartedServer(derphttp.Handler(d))
+	httpsrv.Config.ErrorLog = logger.StdLogger(logf)
+	httpsrv.Config.TLSNextProto = make(map[string]func(*http.Server, *tls.Conn, http.Handler))
+	httpsrv.StartTLS()
+
+	stunAddr, stunCleanup := stuntest.ServeWithPacketListener(t, l)
+
+	m := &tailcfg.DERPMap{
+		Regions: map[int]*tailcfg.DERPRegion{
+			1: &tailcfg.DERPRegion{
+				RegionID:   1,
+				RegionCode: "test",
+				Nodes: []*tailcfg.DERPNode{
+					{
+						Name:         "t1",
+						RegionID:     1,
+						HostName:     "test-node.unused",
+						IPv4:         "127.0.0.1",
+						IPv6:         "none",
+						STUNPort:     stunAddr.Port,
+						DERPTestPort: httpsrv.Listener.Addr().(*net.TCPAddr).Port,
+						STUNTestIP:   stunIP.String(),
+					},
+				},
+			},
+		},
+	}
+
+	cleanup = func() {
+		httpsrv.CloseClientConnections()
+		httpsrv.Close()
+		d.Close()
+		stunCleanup()
+	}
+
+	return m, cleanup
+}
+
+// magicStack is a magicsock, plus all the stuff around it that's
+// necessary to send and receive packets to test e2e wireguard
+// happiness.
+type magicStack struct {
+	privateKey wgcfg.PrivateKey
+	epCh       chan []string       // endpoint updates produced by this peer
+	conn       *Conn               // the magicsock itself
+	tun        *tuntest.ChannelTUN // tuntap device to send/receive packets
+	tsTun      *tstun.TUN          // wrapped tun that implements filtering and wgengine hooks
+	dev        *device.Device      // the wireguard-go Device that connects the previous things
+}
+
+// newMagicStack builds and initializes an idle magicsock and
+// friends. You need to call conn.SetNetworkMap and dev.Reconfig
+// before anything interesting happens.
+func newMagicStack(t *testing.T, logf logger.Logf, l nettype.PacketListener, derpMap *tailcfg.DERPMap) *magicStack {
+	t.Helper()
+
+	privateKey, err := wgcfg.NewPrivateKey()
+	if err != nil {
+		t.Fatalf("generating private key: %v", err)
+	}
+
+	epCh := make(chan []string, 100) // arbitrary
+	conn, err := NewConn(Options{
+		Logf:           logf,
+		PacketListener: l,
+		EndpointsFunc: func(eps []string) {
+			epCh <- eps
+		},
+	})
+	if err != nil {
+		t.Fatalf("constructing magicsock: %v", err)
+	}
+	conn.Start()
+	conn.SetDERPMap(derpMap)
+	if err := conn.SetPrivateKey(privateKey); err != nil {
+		t.Fatalf("setting private key in magicsock: %v", err)
+	}
+
+	tun := tuntest.NewChannelTUN()
+	tsTun := tstun.WrapTUN(logf, tun.TUN())
+	tsTun.SetFilter(filter.NewAllowAll([]filter.Net{filter.NetAny}, logf))
+
+	dev := device.NewDevice(tsTun, &device.DeviceOptions{
+		Logger: &device.Logger{
+			Debug: logger.StdLogger(logf),
+			Info:  logger.StdLogger(logf),
+			Error: logger.StdLogger(logf),
+		},
+		CreateEndpoint: conn.CreateEndpoint,
+		CreateBind:     conn.CreateBind,
+		SkipBindUpdate: true,
+	})
+	dev.Up()
+
+	// Wait for magicsock to connect up to DERP.
+	conn.WaitReady(t)
+
+	// Wait for first endpoint update to be available
+	deadline := time.Now().Add(2 * time.Second)
+	for len(epCh) == 0 && time.Now().Before(deadline) {
+		time.Sleep(10 * time.Millisecond)
+	}
+
+	return &magicStack{
+		privateKey: privateKey,
+		epCh:       epCh,
+		conn:       conn,
+		tun:        tun,
+		tsTun:      tsTun,
+		dev:        dev,
+	}
+}
+
+func (s *magicStack) Close() {
+	s.dev.Close()
+	s.conn.Close()
+}
+
 func TestNewConn(t *testing.T) {
 	tstest.PanicOnLog()
 	rc := tstest.NewResourceCheck()
@@ -82,8 +215,9 @@ func TestNewConn(t *testing.T) {
 		t.Fatal(err)
 	}
 	defer conn.Close()
-	conn.Start()
 	conn.SetDERPMap(stuntest.DERPMapOf(stunAddr.String()))
+	conn.SetPrivateKey(wgcfg.PrivateKey(key.NewPrivate()))
+	conn.Start()
 
 	go func() {
 		var pkt [64 << 10]byte
@@ -185,13 +319,13 @@ func TestPickDERPFallback(t *testing.T) {
 	}
 }
 
-func makeConfigs(t *testing.T, ports []uint16) []wgcfg.Config {
+func makeConfigs(t *testing.T, addrs []netaddr.IPPort) []wgcfg.Config {
 	t.Helper()
 
 	var privKeys []wgcfg.PrivateKey
 	var addresses [][]wgcfg.CIDR
 
-	for i := range ports {
+	for i := range addrs {
 		privKey, err := wgcfg.NewPrivateKey()
 		if err != nil {
 			t.Fatal(err)
@@ -204,14 +338,14 @@ func makeConfigs(t *testing.T, ports []uint16) []wgcfg.Config {
 	}
 
 	var cfgs []wgcfg.Config
-	for i, port := range ports {
+	for i, addr := range addrs {
 		cfg := wgcfg.Config{
 			Name:       fmt.Sprintf("peer%d", i+1),
 			PrivateKey: privKeys[i],
 			Addresses:  addresses[i],
-			ListenPort: port,
+			ListenPort: addr.Port,
 		}
-		for peerNum, port := range ports {
+		for peerNum, addr := range addrs {
 			if peerNum == i {
 				continue
 			}
@@ -219,8 +353,8 @@ func makeConfigs(t *testing.T, ports []uint16) []wgcfg.Config {
 				PublicKey:  privKeys[peerNum].Public(),
 				AllowedIPs: addresses[peerNum],
 				Endpoints: []wgcfg.Endpoint{{
-					Host: "127.0.0.1",
-					Port: port,
+					Host: addr.IP.String(),
+					Port: addr.Port,
 				}},
 				PersistentKeepalive: 25,
 			}
@@ -240,44 +374,6 @@ func parseCIDR(t *testing.T, addr string) wgcfg.CIDR {
 	return cidr
 }
 
-func runDERP(t *testing.T, logf logger.Logf) (s *derp.Server, addr *net.TCPAddr, cleanupFn func()) {
-	var serverPrivateKey key.Private
-	if _, err := crand.Read(serverPrivateKey[:]); err != nil {
-		t.Fatal(err)
-	}
-
-	s = derp.NewServer(serverPrivateKey, logf)
-
-	httpsrv := httptest.NewUnstartedServer(derphttp.Handler(s))
-	httpsrv.Config.ErrorLog = logger.StdLogger(logf)
-	httpsrv.Config.TLSNextProto = make(map[string]func(*http.Server, *tls.Conn, http.Handler))
-	httpsrv.StartTLS()
-	logf("DERP server URL: %s", httpsrv.URL)
-
-	cleanupFn = func() {
-		httpsrv.CloseClientConnections()
-		httpsrv.Close()
-		s.Close()
-	}
-
-	return s, httpsrv.Listener.Addr().(*net.TCPAddr), cleanupFn
-}
-
-// devLogger returns a wireguard-go device.Logger that writes
-// wireguard logs to the test logger.
-func devLogger(t *testing.T, prefix string, logfx logger.Logf) *device.Logger {
-	pfx := []interface{}{prefix}
-	logf := func(format string, args ...interface{}) {
-		t.Helper()
-		logfx("%s: "+format, append(pfx, args...)...)
-	}
-	return &device.Logger{
-		Debug: logger.StdLogger(logf),
-		Info:  logger.StdLogger(logf),
-		Error: logger.StdLogger(logf),
-	}
-}
-
 // TestDeviceStartStop exercises the startup and shutdown logic of
 // wireguard-go, which is intimately intertwined with magicsock's own
 // lifecycle. We seem to be good at generating deadlocks here, so if
@@ -301,7 +397,11 @@ func TestDeviceStartStop(t *testing.T) {
 
 	tun := tuntest.NewChannelTUN()
 	dev := device.NewDevice(tun.TUN(), &device.DeviceOptions{
-		Logger:         devLogger(t, "dev", t.Logf),
+		Logger: &device.Logger{
+			Debug: logger.StdLogger(t.Logf),
+			Info:  logger.StdLogger(t.Logf),
+			Error: logger.StdLogger(t.Logf),
+		},
 		CreateEndpoint: conn.CreateEndpoint,
 		CreateBind:     conn.CreateBind,
 		SkipBindUpdate: true,
@@ -333,6 +433,79 @@ func makeNestable(t *testing.T) (logf logger.Logf, setT func(t *testing.T)) {
 }
 
 func TestTwoDevicePing(t *testing.T) {
+	t.Run("real", func(t *testing.T) {
+		l, ip := nettype.Std{}, netaddr.IPv4(127, 0, 0, 1)
+		n := &devices{
+			m1:     l,
+			m1IP:   ip,
+			m2:     l,
+			m2IP:   ip,
+			stun:   l,
+			stunIP: ip,
+		}
+		testTwoDevicePing(t, n)
+	})
+	t.Run("natlab", func(t *testing.T) {
+		t.Run("simple internet", func(t *testing.T) {
+			mstun := &natlab.Machine{Name: "stun"}
+			m1 := &natlab.Machine{Name: "m1"}
+			m2 := &natlab.Machine{Name: "m2"}
+			inet := natlab.NewInternet()
+			sif := mstun.Attach("eth0", inet)
+			m1if := m1.Attach("eth0", inet)
+			m2if := m2.Attach("eth0", inet)
+
+			n := &devices{
+				m1:     m1,
+				m1IP:   m1if.V4(),
+				m2:     m2,
+				m2IP:   m2if.V4(),
+				stun:   mstun,
+				stunIP: sif.V4(),
+			}
+			testTwoDevicePing(t, n)
+		})
+
+		t.Run("facing firewalls", func(t *testing.T) {
+			mstun := &natlab.Machine{Name: "stun"}
+			m1 := &natlab.Machine{
+				Name:          "m1",
+				PacketHandler: &natlab.Firewall{},
+			}
+			m2 := &natlab.Machine{
+				Name:          "m2",
+				PacketHandler: &natlab.Firewall{},
+			}
+			inet := natlab.NewInternet()
+			sif := mstun.Attach("eth0", inet)
+			m1if := m1.Attach("eth0", inet)
+			m2if := m2.Attach("eth0", inet)
+
+			n := &devices{
+				m1:     m1,
+				m1IP:   m1if.V4(),
+				m2:     m2,
+				m2IP:   m2if.V4(),
+				stun:   mstun,
+				stunIP: sif.V4(),
+			}
+			testTwoDevicePing(t, n)
+		})
+	})
+}
+
+type devices struct {
+	m1   nettype.PacketListener
+	m1IP netaddr.IP
+
+	m2   nettype.PacketListener
+	m2IP netaddr.IP
+
+	stun   nettype.PacketListener
+	stunIP netaddr.IP
+}
+
+func testTwoDevicePing(t *testing.T, d *devices) {
 	tstest.PanicOnLog()
 	rc := tstest.NewResourceCheck()
 	defer rc.Assert(t)
@@ -341,114 +514,33 @@ func TestTwoDevicePing(t *testing.T) {
 	// all log using the "current" t.Logf function. Sigh.
 	logf, setT := makeNestable(t)
 
-	derpServer, derpAddr, derpCleanupFn := runDERP(t, logf)
-	defer derpCleanupFn()
-	stunAddr, stunCleanupFn := stuntest.Serve(t)
-	defer stunCleanupFn()
+	derpMap, cleanup := runDERPAndStun(t, logf, d.stun, d.stunIP)
+	defer cleanup()
 
-	derpMap := &tailcfg.DERPMap{
-		Regions: map[int]*tailcfg.DERPRegion{
-			1: &tailcfg.DERPRegion{
-				RegionID:   1,
-				RegionCode: "test",
-				Nodes: []*tailcfg.DERPNode{
-					{
-						Name:         "t1",
-						RegionID:     1,
-						HostName:     "test-node.unused",
-						IPv4:         "127.0.0.1",
-						IPv6:         "none",
-						STUNPort:     stunAddr.Port,
-						DERPTestPort: derpAddr.Port,
-					},
-				},
-			},
-		},
+	m1 := newMagicStack(t, logf, d.m1, derpMap)
+	defer m1.Close()
+	m2 := newMagicStack(t, logf, d.m2, derpMap)
+	defer m2.Close()
+
+	addrs := []netaddr.IPPort{
+		{IP: d.m1IP, Port: m1.conn.LocalPort()},
+		{IP: d.m2IP, Port: m2.conn.LocalPort()},
 	}
+	cfgs := makeConfigs(t, addrs)
 
-	epCh1 := make(chan []string, 16)
-	conn1, err := NewConn(Options{
-		Logf: logger.WithPrefix(logf, "conn1: "),
-		EndpointsFunc: func(eps []string) {
-			epCh1 <- eps
-		},
-	})
-	if err != nil {
+	if err := m1.dev.Reconfig(&cfgs[0]); err != nil {
 		t.Fatal(err)
 	}
-	defer conn1.Close()
-	conn1.Start()
-	conn1.SetDERPMap(derpMap)
-
-	epCh2 := make(chan []string, 16)
-	conn2, err := NewConn(Options{
-		Logf: logger.WithPrefix(logf, "conn2: "),
-		EndpointsFunc: func(eps []string) {
-			epCh2 <- eps
-		},
-	})
-	if err != nil {
+	if err := m2.dev.Reconfig(&cfgs[1]); err != nil {
 		t.Fatal(err)
 	}
-	defer conn2.Close()
-	conn2.Start()
-	conn2.SetDERPMap(derpMap)
-
-	ports := []uint16{conn1.LocalPort(), conn2.LocalPort()}
-	cfgs := makeConfigs(t, ports)
-
-	if err := conn1.SetPrivateKey(cfgs[0].PrivateKey); err != nil {
-		t.Fatal(err)
-	}
-	if err := conn2.SetPrivateKey(cfgs[1].PrivateKey); err != nil {
-		t.Fatal(err)
-	}
-
-	//uapi1, _ := cfgs[0].ToUAPI()
-	//logf("cfg0: %v", uapi1)
-	//uapi2, _ := cfgs[1].ToUAPI()
-	//logf("cfg1: %v", uapi2)
-
-	tun1 := tuntest.NewChannelTUN()
-	tstun1 := tstun.WrapTUN(logf, tun1.TUN())
-	tstun1.SetFilter(filter.NewAllowAll([]filter.Net{filter.NetAny}, logf))
-	dev1 := device.NewDevice(tstun1, &device.DeviceOptions{
-		Logger:         devLogger(t, "dev1", logf),
-		CreateEndpoint: conn1.CreateEndpoint,
-		CreateBind:     conn1.CreateBind,
-		SkipBindUpdate: true,
-	})
-	dev1.Up()
-	if err := dev1.Reconfig(&cfgs[0]); err != nil {
-		t.Fatal(err)
-	}
-	defer dev1.Close()
-
-	tun2 := tuntest.NewChannelTUN()
-	tstun2 := tstun.WrapTUN(logf, tun2.TUN())
-	tstun2.SetFilter(filter.NewAllowAll([]filter.Net{filter.NetAny}, logf))
-	dev2 := device.NewDevice(tstun2, &device.DeviceOptions{
-		Logger:         devLogger(t, "dev2", logf),
-		CreateEndpoint: conn2.CreateEndpoint,
-		CreateBind:     conn2.CreateBind,
-		SkipBindUpdate: true,
-	})
-	dev2.Up()
-	defer dev2.Close()
-
-	if err := dev2.Reconfig(&cfgs[1]); err != nil {
-		t.Fatal(err)
-	}
-
-	conn1.WaitReady(t)
-	conn2.WaitReady(t)
 
 	ping1 := func(t *testing.T) {
 		msg2to1 := tuntest.Ping(net.ParseIP("1.0.0.1"), net.ParseIP("1.0.0.2"))
-		tun2.Outbound <- msg2to1
+		m2.tun.Outbound <- msg2to1
 		t.Log("ping1 sent")
 		select {
-		case msgRecv := <-tun1.Inbound:
+		case msgRecv := <-m1.tun.Inbound:
 			if !bytes.Equal(msg2to1, msgRecv) {
 				t.Error("ping did not transit correctly")
 			}
@@ -458,10 +550,10 @@ func TestTwoDevicePing(t *testing.T) {
 	}
 	ping2 := func(t *testing.T) {
 		msg1to2 := tuntest.Ping(net.ParseIP("1.0.0.2"), net.ParseIP("1.0.0.1"))
-		tun1.Outbound <- msg1to2
+		m1.tun.Outbound <- msg1to2
 		t.Log("ping2 sent")
 		select {
-		case msgRecv := <-tun2.Inbound:
+		case msgRecv := <-m2.tun.Inbound:
 			if !bytes.Equal(msg1to2, msgRecv) {
 				t.Error("return ping did not transit correctly")
 			}
@@ -487,12 +579,12 @@ func TestTwoDevicePing(t *testing.T) {
 		setT(t)
 		defer setT(outerT)
 		msg1to2 := tuntest.Ping(net.ParseIP("1.0.0.2"), net.ParseIP("1.0.0.1"))
-		if err := tstun1.InjectOutbound(msg1to2); err != nil {
+		if err := m1.tsTun.InjectOutbound(msg1to2); err != nil {
 			t.Fatal(err)
 		}
 		t.Log("SendPacket sent")
 		select {
-		case msgRecv := <-tun2.Inbound:
+		case msgRecv := <-m2.tun.Inbound:
 			if !bytes.Equal(msg1to2, msgRecv) {
 				t.Error("return ping did not transit correctly")
 			}
@@ -504,7 +596,7 @@ func TestTwoDevicePing(t *testing.T) {
 	t.Run("no-op dev1 reconfig", func(t *testing.T) {
 		setT(t)
 		defer setT(outerT)
-		if err := dev1.Reconfig(&cfgs[0]); err != nil {
+		if err := m1.dev.Reconfig(&cfgs[0]); err != nil {
 			t.Fatal(err)
 		}
 		ping1(t)
@@ -546,14 +638,14 @@ func TestTwoDevicePing(t *testing.T) {
 
 		for i := 0; i < count; i++ {
 			b := msg(i)
-			tun1.Outbound <- b
+			m1.tun.Outbound <- b
 			time.Sleep(interPacketGap)
 		}
 
 		for i := 0; i < count; i++ {
 			b := msg(i)
 			select {
-			case msgRecv := <-tun2.Inbound:
+			case msgRecv := <-m2.tun.Inbound:
 				if !bytes.Equal(b, msgRecv) {
 					if strict {
 						t.Errorf("return ping %d did not transit correctly: %s", i, cmp.Diff(b, msgRecv))
@@ -565,7 +657,6 @@ func TestTwoDevicePing(t *testing.T) {
 				}
 			}
 		}
-
 	}
 
 	t.Run("ping 1.0.0.1 x50", func(t *testing.T) {
@@ -582,29 +673,26 @@ func TestTwoDevicePing(t *testing.T) {
 	ep1 := cfgs[1].Peers[0].Endpoints
 	ep1 = append([]wgcfg.Endpoint{derpEp}, ep1...)
 	cfgs[1].Peers[0].Endpoints = ep1
-	if err := dev1.Reconfig(&cfgs[0]); err != nil {
+	if err := m1.dev.Reconfig(&cfgs[0]); err != nil {
 		t.Fatal(err)
 	}
-	if err := dev2.Reconfig(&cfgs[1]); err != nil {
+	if err := m2.dev.Reconfig(&cfgs[1]); err != nil {
 		t.Fatal(err)
 	}
 
 	t.Run("add DERP", func(t *testing.T) {
 		setT(t)
 		defer setT(outerT)
-		defer func() {
-			logf("DERP vars: %s", derpServer.ExpVar().String())
-		}()
 		pingSeq(t, 20, 0, true)
 	})
 
 	// Disable real route.
 	cfgs[0].Peers[0].Endpoints = []wgcfg.Endpoint{derpEp}
 	cfgs[1].Peers[0].Endpoints = []wgcfg.Endpoint{derpEp}
-	if err := dev1.Reconfig(&cfgs[0]); err != nil {
+	if err := m1.dev.Reconfig(&cfgs[0]); err != nil {
 		t.Fatal(err)
 	}
-	if err := dev2.Reconfig(&cfgs[1]); err != nil {
+	if err := m2.dev.Reconfig(&cfgs[1]); err != nil {
 		t.Fatal(err)
 	}
 	time.Sleep(250 * time.Millisecond) // TODO remove
@@ -613,7 +701,6 @@ func TestTwoDevicePing(t *testing.T) {
 		setT(t)
 		defer setT(outerT)
 		defer func() {
-			logf("DERP vars: %s", derpServer.ExpVar().String())
 			if t.Failed() || true {
 				uapi1, _ := cfgs[0].ToUAPI()
 				logf("cfg0: %v", uapi1)
@@ -624,8 +711,8 @@ func TestTwoDevicePing(t *testing.T) {
 		pingSeq(t, 20, 0, true)
 	})
 
-	dev1.RemoveAllPeers()
-	dev2.RemoveAllPeers()
+	m1.dev.RemoveAllPeers()
+	m2.dev.RemoveAllPeers()
 
 	// Give one peer a non-DERP endpoint. We expect the other to
 	// accept it via roamAddr.
@@ -633,10 +720,10 @@ func TestTwoDevicePing(t *testing.T) {
 	if ep2 := cfgs[1].Peers[0].Endpoints; len(ep2) != 1 {
 		t.Errorf("unexpected peer endpoints in dev2: %v", ep2)
 	}
-	if err := dev2.Reconfig(&cfgs[1]); err != nil {
+	if err := m2.dev.Reconfig(&cfgs[1]); err != nil {
 		t.Fatal(err)
 	}
-	if err := dev1.Reconfig(&cfgs[0]); err != nil {
+	if err := m1.dev.Reconfig(&cfgs[0]); err != nil {
 		t.Fatal(err)
 	}
 	// Dear future human debugging a test failure here: this test is
@@ -650,7 +737,7 @@ func TestTwoDevicePing(t *testing.T) {
 		defer setT(outerT)
 		pingSeq(t, 50, 700*time.Millisecond, false)
 
-		ep2 := dev2.Config().Peers[0].Endpoints
+		ep2 := m2.dev.Config().Peers[0].Endpoints
 		if len(ep2) != 2 {
 			t.Error("handshake spray failed to find real route")
 		}
@@ -854,15 +941,19 @@ func initAddrSet(as *AddrSet) {
 }
 
 func TestDiscoMessage(t *testing.T) {
-	peer1Priv := key.NewPrivate()
-	peer1Pub := peer1Priv.Public()
-
 	c := newConn()
 	c.logf = t.Logf
-	c.SetDiscoPrivateKey(key.NewPrivate())
+
+	peer1Pub := c.DiscoPublicKey()
+	peer1Priv := c.discoPrivate
 	c.endpointOfDisco = map[tailcfg.DiscoKey]*discoEndpoint{
 		tailcfg.DiscoKey(peer1Pub): &discoEndpoint{
-			// ...
+			// ... (enough for this test)
+		},
+	}
+	c.nodeOfDisco = map[tailcfg.DiscoKey]*tailcfg.Node{
+		tailcfg.DiscoKey(peer1Pub): &tailcfg.Node{
+			// ... (enough for this test)
 		},
 	}
 
@@ -880,3 +971,22 @@ func TestDiscoMessage(t *testing.T) {
 		t.Error("failed to open it")
 	}
 }
+
+// tests that having a discoEndpoint.String prevents wireguard-go's
+// log.Printf("%v") of its conn.Endpoint values from using reflect to
+// walk into read mutex while they're being used and then causing data
+// races.
+func TestDiscoStringLogRace(t *testing.T) {
+	de := new(discoEndpoint)
+	var wg sync.WaitGroup
+	wg.Add(2)
+	go func() {
+		defer wg.Done()
+		fmt.Fprintf(ioutil.Discard, "%v", de)
+	}()
+	go func() {
+		defer wg.Done()
+		de.mu.Lock()
+	}()
+	wg.Wait()
+}

wgengine/monitor/monitor.go

@@ -3,7 +3,8 @@
 // license that can be found in the LICENSE file.
 
 // Package monitor provides facilities for monitoring network
-// interface changes.
+// interface and route changes. It primarily exists to know when
+// portable devices move between different networks.
 package monitor
 
 import (
@@ -14,10 +15,10 @@ import (
 )
 
 // message represents a message returned from an osMon.
-//
-// TODO: currently messages are being discarded, so the properties of
-// the message haven't been defined.
-type message interface{}
+type message interface {
+	// Ignore is whether we should ignore this message.
+	ignore() bool
+}
 
 // osMon is the interface that each operating system-specific
 // implementation of the link monitor must implement.
@@ -52,7 +53,8 @@ type Mon struct {
 // are propagated to the callback function.
 // The returned monitor is inactive until it's started by the Start method.
 func New(logf logger.Logf, callback ChangeFunc) (*Mon, error) {
-	om, err := newOSMon()
+	logf = logger.WithPrefix(logf, "monitor: ")
+	om, err := newOSMon(logf)
 	if err != nil {
 		return nil, err
 	}
@@ -100,7 +102,7 @@ func (m *Mon) Close() error {
 func (m *Mon) pump() {
 	defer m.goroutines.Done()
 	for {
-		_, err := m.om.Receive()
+		msg, err := m.om.Receive()
 		if err != nil {
 			select {
 			case <-m.stop:
@@ -108,10 +110,13 @@ func (m *Mon) pump() {
 			default:
 			}
 			// Keep retrying while we're not closed.
-			m.logf("Error receiving from connection: %v", err)
+			m.logf("error receiving from connection: %v", err)
 			time.Sleep(time.Second)
 			continue
 		}
+		if msg.ignore() {
+			continue
+		}
 		select {
 		case m.change <- struct{}{}:
 		case <-m.stop:

wgengine/monitor/monitor_freebsd.go

@@ -9,14 +9,23 @@ import (
 	"fmt"
 	"net"
 	"strings"
+
+	"tailscale.com/types/logger"
 )
 
+// unspecifiedMessage is a minimal message implementation that should not
+// be ignored. In general, OS-specific implementations should use better
+// types and avoid this if they can.
+type unspecifiedMessage struct{}
+
+func (unspecifiedMessage) ignore() bool { return false }
+
 // devdConn implements osMon using devd(8).
 type devdConn struct {
 	conn net.Conn
 }
 
-func newOSMon() (osMon, error) {
+func newOSMon(logf logger.Logf) (osMon, error) {
 	conn, err := net.Dial("unixpacket", "/var/run/devd.seqpacket.pipe")
 	if err != nil {
 		return nil, fmt.Errorf("devd dial error: %v", err)
@@ -41,8 +50,8 @@ func (c *devdConn) Receive() (message, error) {
 		if !strings.Contains(msg, "system=IFNET") {
 			continue
 		}
-		// TODO(]|[): this is where the devd-specific message would
+		// TODO: this is where the devd-specific message would
 		// get converted into a "standard" event message and returned.
-		return nil, nil
+		return unspecifiedMessage{}, nil
 	}
 }

wgengine/monitor/monitor_linux.go

@@ -8,22 +8,36 @@ package monitor
 
 import (
 	"fmt"
+	"net"
 	"time"
 
+	"github.com/jsimonetti/rtnetlink"
 	"github.com/mdlayher/netlink"
 	"golang.org/x/sys/unix"
+	"inet.af/netaddr"
+	"tailscale.com/net/tsaddr"
+	"tailscale.com/types/logger"
 )
 
+// unspecifiedMessage is a minimal message implementation that should not
+// be ignored. In general, OS-specific implementations should use better
+// types and avoid this if they can.
+type unspecifiedMessage struct{}
+
+func (unspecifiedMessage) ignore() bool { return false }
+
 // nlConn wraps a *netlink.Conn and returns a monitor.Message
 // instead of a netlink.Message. Currently, messages are discarded,
 // but down the line, when messages trigger different logic depending
 // on the type of event, this provides the capability of handling
 // each architecture-specific message in a generic fashion.
 type nlConn struct {
-	conn *netlink.Conn
+	logf     logger.Logf
+	conn     *netlink.Conn
+	buffered []netlink.Message
 }
 
-func newOSMon() (osMon, error) {
+func newOSMon(logf logger.Logf) (osMon, error) {
 	conn, err := netlink.Dial(unix.NETLINK_ROUTE, &netlink.Config{
 		// IPv4 address and route changes. Routes get us most of the
 		// events of interest, but we need address as well to cover
@@ -35,7 +49,7 @@ func newOSMon() (osMon, error) {
 	if err != nil {
 		return nil, fmt.Errorf("dialing netlink socket: %v", err)
 	}
-	return &nlConn{conn}, nil
+	return &nlConn{logf: logf, conn: conn}, nil
 }
 
 func (c *nlConn) Close() error {
@@ -44,12 +58,79 @@ func (c *nlConn) Close() error {
 }
 
 func (c *nlConn) Receive() (message, error) {
-	// currently ignoring the message
-	_, err := c.conn.Receive()
-	if err != nil {
-		return nil, err
+	if len(c.buffered) == 0 {
+		var err error
+		c.buffered, err = c.conn.Receive()
+		if err != nil {
+			return nil, err
+		}
+		if len(c.buffered) == 0 {
+			// Unexpected. Not seen in wild, but sleep defensively.
+			time.Sleep(time.Second)
+			return ignoreMessage{}, nil
+		}
+	}
+	msg := c.buffered[0]
+	c.buffered = c.buffered[1:]
+
+	// See https://github.com/torvalds/linux/blob/master/include/uapi/linux/rtnetlink.h
+	// And https://man7.org/linux/man-pages/man7/rtnetlink.7.html
+	switch msg.Header.Type {
+	case unix.RTM_NEWADDR, unix.RTM_DELADDR:
+		var rmsg rtnetlink.AddressMessage
+		if err := rmsg.UnmarshalBinary(msg.Data); err != nil {
+			c.logf("failed to parse type 0x%x: %v", msg.Header.Type, err)
+			return unspecifiedMessage{}, nil
+		}
+		return &newAddrMessage{
+			Label:  rmsg.Attributes.Label,
+			Addr:   netaddrIP(rmsg.Attributes.Local),
+			Delete: msg.Header.Type == unix.RTM_DELADDR,
+		}, nil
+	case unix.RTM_NEWROUTE:
+		var rmsg rtnetlink.RouteMessage
+		if err := rmsg.UnmarshalBinary(msg.Data); err != nil {
+			c.logf("RTM_NEWROUTE: failed to parse: %v", err)
+			return unspecifiedMessage{}, nil
+		}
+		return &newRouteMessage{
+			Table:   rmsg.Table,
+			Src:     netaddrIP(rmsg.Attributes.Src),
+			Dst:     netaddrIP(rmsg.Attributes.Dst),
+			Gateway: netaddrIP(rmsg.Attributes.Gateway),
+		}, nil
+	default:
+		c.logf("unhandled netlink msg type 0x%x: %+v, %q", msg.Header.Type, msg.Header, msg.Data)
+		return unspecifiedMessage{}, nil
 	}
-	// TODO(]|[): this is where the NetLink-specific message would
-	// get converted into a "standard" event message and returned.
-	return nil, nil
 }
+
+func netaddrIP(std net.IP) netaddr.IP {
+	ip, _ := netaddr.FromStdIP(std)
+	return ip
+}
+
+// newRouteMessage is a message for a new route being added.
+type newRouteMessage struct {
+	Src, Dst, Gateway netaddr.IP
+	Table             uint8
+}
+
+func (m *newRouteMessage) ignore() bool {
+	return m.Table == 88 || tsaddr.IsTailscaleIP(m.Dst)
+}
+
+// newAddrMessage is a message for a new address being added.
+type newAddrMessage struct {
+	Delete bool
+	Addr   netaddr.IP
+	Label  string // netlink Label attribute (e.g. "tailscale0")
+}
+
+func (m *newAddrMessage) ignore() bool {
+	return tsaddr.IsTailscaleIP(m.Addr)
+}
+
+type ignoreMessage struct{}
+
+func (ignoreMessage) ignore() bool { return true }

wgengine/monitor/monitor_unsupported.go

@@ -6,4 +6,6 @@
 
 package monitor
 
-func newOSMon() (osMon, error) { return nil, nil }
+import "tailscale.com/types/logger"
+
+func newOSMon(logger.Logf) (osMon, error) { return nil, nil }

wgengine/packet/ip.go

@@ -7,6 +7,8 @@ package packet
 import (
 	"fmt"
 	"net"
+
+	"inet.af/netaddr"
 )
 
 // IP is an IPv4 address.
@@ -22,6 +24,17 @@ func NewIP(b net.IP) IP {
 	return IP(get32(b4))
 }
 
+// IPFromNetaddr converts a netaddr.IP to an IP.
+func IPFromNetaddr(ip netaddr.IP) IP {
+	ipbytes := ip.As4()
+	return IP(get32(ipbytes[:]))
+}
+
+// Netaddr converts an IP to a netaddr.IP.
+func (ip IP) Netaddr() netaddr.IP {
+	return netaddr.IPv4(byte(ip>>24), byte(ip>>16), byte(ip>>8), byte(ip))
+}
+
 func (ip IP) String() string {
 	return fmt.Sprintf("%d.%d.%d.%d", byte(ip>>24), byte(ip>>16), byte(ip>>8), byte(ip))
 }

wgengine/packet/packet_test.go

@@ -206,9 +206,11 @@ func TestParsedPacket(t *testing.T) {
 		})
 	}
 
+	var sink string
 	allocs := testing.AllocsPerRun(1000, func() {
-		tests[0].qdecode.String()
+		sink = tests[0].qdecode.String()
 	})
+	_ = sink
 	if allocs != 1 {
 		t.Errorf("allocs = %v; want 1", allocs)
 	}

wgengine/router/dns.go

@@ -0,0 +1,74 @@
+// Copyright (c) 2020 Tailscale Inc & AUTHORS All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package router
+
+import (
+	"inet.af/netaddr"
+)
+
+// DNSConfig is the subset of Config that contains DNS parameters.
+type DNSConfig struct {
+	// Nameservers are the IP addresses of the nameservers to use.
+	Nameservers []netaddr.IP
+	// Domains are the search domains to use.
+	Domains []string
+}
+
+// EquivalentTo determines whether its argument and receiver
+// represent equivalent DNS configurations (then DNS reconfig is a no-op).
+func (lhs DNSConfig) EquivalentTo(rhs DNSConfig) bool {
+	if len(lhs.Nameservers) != len(rhs.Nameservers) {
+		return false
+	}
+
+	if len(lhs.Domains) != len(rhs.Domains) {
+		return false
+	}
+
+	// With how we perform resolution order shouldn't matter,
+	// but it is unlikely that we will encounter different orders.
+	for i, server := range lhs.Nameservers {
+		if rhs.Nameservers[i] != server {
+			return false
+		}
+	}
+
+	for i, domain := range lhs.Domains {
+		if rhs.Domains[i] != domain {
+			return false
+		}
+	}
+
+	return true
+}
+
+// dnsMode determines how DNS settings are managed.
+type dnsMode uint8
+
+const (
+	// dnsDirect indicates that /etc/resolv.conf is edited directly.
+	dnsDirect dnsMode = iota
+	// dnsResolvconf indicates that a resolvconf binary is used.
+	dnsResolvconf
+	// dnsNetworkManager indicates that the NetworkManaer DBus API is used.
+	dnsNetworkManager
+	// dnsResolved indicates that the systemd-resolved DBus API is used.
+	dnsResolved
+)
+
+func (m dnsMode) String() string {
+	switch m {
+	case dnsDirect:
+		return "direct"
+	case dnsResolvconf:
+		return "resolvconf"
+	case dnsNetworkManager:
+		return "networkmanager"
+	case dnsResolved:
+		return "resolved"
+	default:
+		return "???"
+	}
+}

wgengine/router/dns_direct.go

@@ -0,0 +1,151 @@
+// Copyright (c) 2020 Tailscale Inc & AUTHORS All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+// +build linux freebsd openbsd
+
+package router
+
+import (
+	"bufio"
+	"bytes"
+	"fmt"
+	"io"
+	"io/ioutil"
+	"os"
+	"strings"
+
+	"inet.af/netaddr"
+	"tailscale.com/atomicfile"
+)
+
+const (
+	tsConf     = "/etc/resolv.tailscale.conf"
+	backupConf = "/etc/resolv.pre-tailscale-backup.conf"
+	resolvConf = "/etc/resolv.conf"
+)
+
+// dnsWriteConfig writes DNS configuration in resolv.conf format to the given writer.
+func dnsWriteConfig(w io.Writer, servers []netaddr.IP, domains []string) {
+	io.WriteString(w, "# resolv.conf(5) file generated by tailscale\n")
+	io.WriteString(w, "# DO NOT EDIT THIS FILE BY HAND -- CHANGES WILL BE OVERWRITTEN\n\n")
+	for _, ns := range servers {
+		io.WriteString(w, "nameserver ")
+		io.WriteString(w, ns.String())
+		io.WriteString(w, "\n")
+	}
+	if len(domains) > 0 {
+		io.WriteString(w, "search")
+		for _, domain := range domains {
+			io.WriteString(w, " ")
+			io.WriteString(w, domain)
+		}
+		io.WriteString(w, "\n")
+	}
+}
+
+// dnsReadConfig reads DNS configuration from /etc/resolv.conf.
+func dnsReadConfig() (DNSConfig, error) {
+	var config DNSConfig
+
+	f, err := os.Open("/etc/resolv.conf")
+	if err != nil {
+		return config, err
+	}
+
+	scanner := bufio.NewScanner(f)
+	for scanner.Scan() {
+		line := strings.TrimSpace(scanner.Text())
+
+		if strings.HasPrefix(line, "nameserver") {
+			nameserver := strings.TrimPrefix(line, "nameserver")
+			nameserver = strings.TrimSpace(nameserver)
+			ip, err := netaddr.ParseIP(nameserver)
+			if err != nil {
+				return config, err
+			}
+			config.Nameservers = append(config.Nameservers, ip)
+			continue
+		}
+
+		if strings.HasPrefix(line, "search") {
+			domain := strings.TrimPrefix(line, "search")
+			domain = strings.TrimSpace(domain)
+			config.Domains = append(config.Domains, domain)
+			continue
+		}
+	}
+
+	return config, nil
+}
+
+// dnsDirectUp replaces /etc/resolv.conf with a file generated
+// from the given configuration, creating a backup of its old state.
+//
+// This way of configuring DNS is precarious, since it does not react
+// to the disappearance of the Tailscale interface.
+// The caller must call dnsDirectDown before program shutdown
+// and ensure that router.Cleanup is run if the program terminates unexpectedly.
+func dnsDirectUp(config DNSConfig) error {
+	// Write the tsConf file.
+	buf := new(bytes.Buffer)
+	dnsWriteConfig(buf, config.Nameservers, config.Domains)
+	if err := atomicfile.WriteFile(tsConf, buf.Bytes(), 0644); err != nil {
+		return err
+	}
+
+	if linkPath, err := os.Readlink(resolvConf); err != nil {
+		// Remove any old backup that may exist.
+		os.Remove(backupConf)
+
+		// Backup the existing /etc/resolv.conf file.
+		contents, err := ioutil.ReadFile(resolvConf)
+		// If the original did not exist, still back up an empty file.
+		// The presence of a backup file is the way we know that Up ran.
+		if err != nil && !os.IsNotExist(err) {
+			return err
+		}
+		if err := atomicfile.WriteFile(backupConf, contents, 0644); err != nil {
+			return err
+		}
+	} else if linkPath != tsConf {
+		// Backup the existing symlink.
+		os.Remove(backupConf)
+		if err := os.Symlink(linkPath, backupConf); err != nil {
+			return err
+		}
+	} else {
+		// Nothing to do, resolvConf already points to tsConf.
+		return nil
+	}
+
+	os.Remove(resolvConf)
+	if err := os.Symlink(tsConf, resolvConf); err != nil {
+		return err
+	}
+
+	return nil
+}
+
+// dnsDirectDown restores /etc/resolv.conf to its state before dnsDirectUp.
+// It is idempotent and behaves correctly even if dnsDirectUp has never been run.
+func dnsDirectDown() error {
+	if _, err := os.Stat(backupConf); err != nil {
+		// If the backup file does not exist, then Up never ran successfully.
+		if os.IsNotExist(err) {
+			return nil
+		}
+		return err
+	}
+
+	if ln, err := os.Readlink(resolvConf); err != nil {
+		return err
+	} else if ln != tsConf {
+		return fmt.Errorf("resolv.conf is not a symlink to %s", tsConf)
+	}
+	if err := os.Rename(backupConf, resolvConf); err != nil {
+		return err
+	}
+	os.Remove(tsConf)
+	return nil
+}

wgengine/router/dns_networkmanager.go

@@ -0,0 +1,195 @@
+// Copyright (c) 2020 Tailscale Inc & AUTHORS All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+// +build linux
+
+package router
+
+import (
+	"bufio"
+	"bytes"
+	"context"
+	"encoding/binary"
+	"fmt"
+	"os"
+	"os/exec"
+
+	"github.com/godbus/dbus/v5"
+)
+
+type nmConnectionSettings map[string]map[string]dbus.Variant
+
+// nmIsActive determines if NetworkManager is currently managing system DNS settings.
+func nmIsActive() bool {
+	// This is somewhat tricky because NetworkManager supports a number
+	// of DNS configuration modes. In all cases, we expect it to be installed
+	// and /etc/resolv.conf to contain a mention of NetworkManager in the comments.
+	_, err := exec.LookPath("NetworkManager")
+	if err != nil {
+		return false
+	}
+
+	f, err := os.Open("/etc/resolv.conf")
+	if err != nil {
+		return false
+	}
+	defer f.Close()
+
+	scanner := bufio.NewScanner(f)
+	for scanner.Scan() {
+		line := scanner.Bytes()
+		// Look for the word "NetworkManager" until comments end.
+		if len(line) > 0 && line[0] != '#' {
+			return false
+		}
+		if bytes.Contains(line, []byte("NetworkManager")) {
+			return true
+		}
+	}
+	return false
+}
+
+// dnsNetworkManagerUp updates the DNS config for the Tailscale interface
+// through the NetworkManager DBus API.
+func dnsNetworkManagerUp(config DNSConfig, interfaceName string) error {
+	ctx, cancel := context.WithTimeout(context.Background(), dnsReconfigTimeout)
+	defer cancel()
+
+	// conn is a shared connection whose lifecycle is managed by the dbus package.
+	// We should not interfere with that by closing it.
+	conn, err := dbus.SystemBus()
+	if err != nil {
+		return fmt.Errorf("connecting to system bus: %w", err)
+	}
+
+	// This is how we get at the DNS settings:
+	//
+	//               org.freedesktop.NetworkManager
+	//                              |
+	//                    [GetDeviceByIpIface]
+	//                              |
+	//                              v
+	//           org.freedesktop.NetworkManager.Device <--------\
+	//              (describes a network interface)             |
+	//                              |                           |
+	//                   [GetAppliedConnection]             [Reapply]
+	//                              |                           |
+	//                              v                           |
+	//          org.freedesktop.NetworkManager.Connection       |
+	//                   (connection settings)            ------/
+	//          contains {dns, dns-priority, dns-search}
+	//
+	// Ref: https://developer.gnome.org/NetworkManager/stable/settings-ipv4.html.
+
+	nm := conn.Object(
+		"org.freedesktop.NetworkManager",
+		dbus.ObjectPath("/org/freedesktop/NetworkManager"),
+	)
+
+	var devicePath dbus.ObjectPath
+	err = nm.CallWithContext(
+		ctx, "org.freedesktop.NetworkManager.GetDeviceByIpIface", 0,
+		interfaceName,
+	).Store(&devicePath)
+	if err != nil {
+		return fmt.Errorf("getDeviceByIpIface: %w", err)
+	}
+	device := conn.Object("org.freedesktop.NetworkManager", devicePath)
+
+	var (
+		settings nmConnectionSettings
+		version  uint64
+	)
+	err = device.CallWithContext(
+		ctx, "org.freedesktop.NetworkManager.Device.GetAppliedConnection", 0,
+		uint32(0),
+	).Store(&settings, &version)
+	if err != nil {
+		return fmt.Errorf("getAppliedConnection: %w", err)
+	}
+
+	// Frustratingly, NetworkManager represents IPv4 addresses as uint32s,
+	// although IPv6 addresses are represented as byte arrays.
+	// Perform the conversion here.
+	var (
+		dnsv4 []uint32
+		dnsv6 [][]byte
+	)
+	for _, ip := range config.Nameservers {
+		b := ip.As16()
+		if ip.Is4() {
+			dnsv4 = append(dnsv4, binary.LittleEndian.Uint32(b[12:]))
+		} else {
+			dnsv6 = append(dnsv6, b[:])
+		}
+	}
+
+	ipv4Map := settings["ipv4"]
+	ipv4Map["dns"] = dbus.MakeVariant(dnsv4)
+	ipv4Map["dns-search"] = dbus.MakeVariant(config.Domains)
+	// We should only request priority if we have nameservers to set.
+	if len(dnsv4) == 0 {
+		ipv4Map["dns-priority"] = dbus.MakeVariant(100)
+	} else {
+		// dns-priority = -1 ensures that we have priority
+		// over other interfaces, except those exploiting this same trick.
+		// Ref: https://bugs.launchpad.net/ubuntu/+source/network-manager/+bug/1211110/comments/92.
+		ipv4Map["dns-priority"] = dbus.MakeVariant(-1)
+	}
+	// In principle, we should not need set this to true,
+	// as our interface does not configure any automatic DNS settings (presumably via DHCP).
+	// All the same, better to be safe.
+	ipv4Map["ignore-auto-dns"] = dbus.MakeVariant(true)
+
+	ipv6Map := settings["ipv6"]
+	// This is a hack.
+	// Methods "disabled", "ignore", "link-local" (IPv6 default) prevent us from setting DNS.
+	// It seems that our only recourse is "manual" or "auto".
+	// "manual" requires addresses, so we use "auto", which will assign us a random IPv6 /64.
+	ipv6Map["method"] = dbus.MakeVariant("auto")
+	// Our IPv6 config is a fake, so it should never become the default route.
+	ipv6Map["never-default"] = dbus.MakeVariant(true)
+	// Moreover, we should ignore all autoconfigured routes (hopefully none), as they are bogus.
+	ipv6Map["ignore-auto-routes"] = dbus.MakeVariant(true)
+
+	// Finally, set the actual DNS config.
+	ipv6Map["dns"] = dbus.MakeVariant(dnsv6)
+	ipv6Map["dns-search"] = dbus.MakeVariant(config.Domains)
+	if len(dnsv6) == 0 {
+		ipv6Map["dns-priority"] = dbus.MakeVariant(100)
+	} else {
+		ipv6Map["dns-priority"] = dbus.MakeVariant(-1)
+	}
+	ipv6Map["ignore-auto-dns"] = dbus.MakeVariant(true)
+
+	// deprecatedProperties are the properties in interface settings
+	// that are deprecated by NetworkManager.
+	//
+	// In practice, this means that they are returned for reading,
+	// but submitting a settings object with them present fails
+	// with hard-to-diagnose errors. They must be removed.
+	deprecatedProperties := []string{
+		"addresses", "routes",
+	}
+
+	for _, property := range deprecatedProperties {
+		delete(ipv4Map, property)
+		delete(ipv6Map, property)
+	}
+
+	err = device.CallWithContext(
+		ctx, "org.freedesktop.NetworkManager.Device.Reapply", 0,
+		settings, version, uint32(0),
+	).Store()
+	if err != nil {
+		return fmt.Errorf("reapply: %w", err)
+	}
+
+	return nil
+}
+
+// dnsNetworkManagerDown undoes the changes made by dnsNetworkManagerUp.
+func dnsNetworkManagerDown(interfaceName string) error {
+	return dnsNetworkManagerUp(DNSConfig{Nameservers: nil, Domains: nil}, interfaceName)
+}

wgengine/router/dns_resolvconf.go

@@ -0,0 +1,139 @@
+// Copyright (c) 2020 Tailscale Inc & AUTHORS All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+// +build linux freebsd
+
+package router
+
+import (
+	"bufio"
+	"bytes"
+	"fmt"
+	"os"
+	"os/exec"
+)
+
+// resolvconfIsActive indicates whether the system appears to be using resolvconf.
+// If this is true, then dnsManualUp should be avoided:
+// resolvconf has exclusive ownership of /etc/resolv.conf.
+func resolvconfIsActive() bool {
+	// Sanity-check first: if there is no resolvconf binary, then this is fruitless.
+	//
+	// However, this binary may be a shim like the one systemd-resolved provides.
+	// Such a shim may not behave as expected: in particular, systemd-resolved
+	// does not seem to respect the exclusive mode -x, saying:
+	//   -x            Send DNS traffic preferably over this interface
+	// whereas e.g. openresolv sends DNS traffix _exclusively_ over that interface,
+	// or not at all (in case of another exclusive-mode request later in time).
+	//
+	// Moreover, resolvconf may be installed but unused, in which case we should
+	// not use it either, lest we clobber existing configuration.
+	//
+	// To handle all the above correctly, we scan the comments in /etc/resolv.conf
+	// to ensure that it was generated by a resolvconf implementation.
+	_, err := exec.LookPath("resolvconf")
+	if err != nil {
+		return false
+	}
+
+	f, err := os.Open("/etc/resolv.conf")
+	if err != nil {
+		return false
+	}
+	defer f.Close()
+
+	scanner := bufio.NewScanner(f)
+	for scanner.Scan() {
+		line := scanner.Bytes()
+		// Look for the word "resolvconf" until comments end.
+		if len(line) > 0 && line[0] != '#' {
+			return false
+		}
+		if bytes.Contains(line, []byte("resolvconf")) {
+			return true
+		}
+	}
+	return false
+}
+
+// resolvconfImplementation enumerates supported implementations of the resolvconf CLI.
+type resolvconfImplementation uint8
+
+const (
+	// resolvconfOpenresolv is the implementation packaged as "openresolv" on Ubuntu.
+	// It supports exclusive mode and interface metrics.
+	resolvconfOpenresolv resolvconfImplementation = iota
+	// resolvconfLegacy is the implementation by Thomas Hood packaged as "resolvconf" on Ubuntu.
+	// It does not support exclusive mode or interface metrics.
+	resolvconfLegacy
+)
+
+// getResolvconfImplementation returns the implementation of resolvconf
+// that appears to be in use.
+func getResolvconfImplementation() resolvconfImplementation {
+	err := exec.Command("resolvconf", "-v").Run()
+	if err != nil {
+		if exitErr, ok := err.(*exec.ExitError); ok {
+			// Thomas Hood's resolvconf has a minimal flag set
+			// and exits with code 99 when passed an unknown flag.
+			if exitErr.ExitCode() == 99 {
+				return resolvconfLegacy
+			}
+		}
+	}
+	return resolvconfOpenresolv
+}
+
+// resolvconfConfigName is the name of the config submitted to resolvconf.
+// It has this form to match the "tun*" rule in interface-order
+// when running resolvconfLegacy, hopefully placing our config first.
+const resolvconfConfigName = "tun-tailscale.inet"
+
+// dnsResolvconfUp invokes the resolvconf binary to associate
+// the given DNS configuration the Tailscale interface.
+func dnsResolvconfUp(config DNSConfig, interfaceName string) error {
+	implementation := getResolvconfImplementation()
+
+	stdin := new(bytes.Buffer)
+	dnsWriteConfig(stdin, config.Nameservers, config.Domains) // dns_direct.go
+
+	var cmd *exec.Cmd
+	switch implementation {
+	case resolvconfOpenresolv:
+		// Request maximal priority (metric 0) and exclusive mode.
+		cmd = exec.Command("resolvconf", "-m", "0", "-x", "-a", resolvconfConfigName)
+	case resolvconfLegacy:
+		// This does not quite give us the desired behavior (queries leak),
+		// but there is nothing else we can do without messing with other interfaces' settings.
+		cmd = exec.Command("resolvconf", "-a", resolvconfConfigName)
+	}
+	cmd.Stdin = stdin
+	out, err := cmd.CombinedOutput()
+	if err != nil {
+		return fmt.Errorf("running %s: %s", cmd, out)
+	}
+
+	return nil
+}
+
+// dnsResolvconfDown undoes the action of dnsResolvconfUp.
+func dnsResolvconfDown(interfaceName string) error {
+	implementation := getResolvconfImplementation()
+
+	var cmd *exec.Cmd
+	switch implementation {
+	case resolvconfOpenresolv:
+		cmd = exec.Command("resolvconf", "-f", "-d", resolvconfConfigName)
+	case resolvconfLegacy:
+		// resolvconfLegacy lacks the -f flag.
+		// Instead, it succeeds even when the config does not exist.
+		cmd = exec.Command("resolvconf", "-d", resolvconfConfigName)
+	}
+	out, err := cmd.CombinedOutput()
+	if err != nil {
+		return fmt.Errorf("running %s: %s", cmd, out)
+	}
+
+	return nil
+}

wgengine/router/dns_resolved.go

@@ -0,0 +1,188 @@
+// Copyright (c) 2020 Tailscale Inc & AUTHORS All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+// +build linux
+
+package router
+
+import (
+	"context"
+	"errors"
+	"fmt"
+	"os/exec"
+	"time"
+
+	"github.com/godbus/dbus/v5"
+	"golang.org/x/sys/unix"
+	"inet.af/netaddr"
+	"tailscale.com/net/interfaces"
+)
+
+// resolvedListenAddr is the listen address of the resolved stub resolver.
+//
+// We only consider resolved to be the system resolver if the stub resolver is;
+// that is, if this address is the sole nameserver in /etc/resolved.conf.
+// In other cases, resolved may still be managing the system DNS configuration directly.
+// Then the nameserver list will be a concatenation of those for all
+// the interfaces that register their interest in being a default resolver with
+//   SetLinkDomains([]{{"~.", true}, ...})
+// which includes at least the interface with the default route, i.e. not us.
+// This does not work for us: there is a possibility of getting NXDOMAIN
+// from the other nameservers before we are asked or get a chance to respond.
+// We consider this case as lacking resolved support and fall through to dnsDirect.
+//
+// While it may seem that we need to read a config option to get at this,
+// this address is, in fact, hard-coded into resolved.
+var resolvedListenAddr = netaddr.IPv4(127, 0, 0, 53)
+
+// dnsReconfigTimeout is the timeout for DNS reconfiguration.
+//
+// This is useful because certain conditions can cause indefinite hangs
+// (such as improper dbus auth followed by contextless dbus.Object.Call).
+// Such operations should be wrapped in a timeout context.
+const dnsReconfigTimeout = time.Second
+
+var errNotReady = errors.New("interface not ready")
+
+type resolvedLinkNameserver struct {
+	Family  int32
+	Address []byte
+}
+
+type resolvedLinkDomain struct {
+	Domain      string
+	RoutingOnly bool
+}
+
+// resolvedIsActive determines if resolved is currently managing system DNS settings.
+func resolvedIsActive() bool {
+	// systemd-resolved is never installed without systemd.
+	_, err := exec.LookPath("systemctl")
+	if err != nil {
+		return false
+	}
+
+	// is-active exits with code 3 if the service is not active.
+	err = exec.Command("systemctl", "is-active", "systemd-resolved").Run()
+	if err != nil {
+		return false
+	}
+
+	config, err := dnsReadConfig()
+	if err != nil {
+		return false
+	}
+
+	// The sole nameserver must be the systemd-resolved stub.
+	if len(config.Nameservers) == 1 && config.Nameservers[0] == resolvedListenAddr {
+		return true
+	}
+
+	return false
+}
+
+// dnsResolvedUp sets the DNS parameters for the Tailscale interface
+// to given nameservers and search domains using the resolved DBus API.
+func dnsResolvedUp(config DNSConfig) error {
+	ctx, cancel := context.WithTimeout(context.Background(), dnsReconfigTimeout)
+	defer cancel()
+
+	// conn is a shared connection whose lifecycle is managed by the dbus package.
+	// We should not interfere with that by closing it.
+	conn, err := dbus.SystemBus()
+	if err != nil {
+		return fmt.Errorf("connecting to system bus: %w", err)
+	}
+
+	resolved := conn.Object(
+		"org.freedesktop.resolve1",
+		dbus.ObjectPath("/org/freedesktop/resolve1"),
+	)
+
+	_, iface, err := interfaces.Tailscale()
+	if err != nil {
+		return fmt.Errorf("getting interface index: %w", err)
+	}
+	if iface == nil {
+		return errNotReady
+	}
+
+	var linkNameservers = make([]resolvedLinkNameserver, len(config.Nameservers))
+	for i, server := range config.Nameservers {
+		ip := server.As16()
+		if server.Is4() {
+			linkNameservers[i] = resolvedLinkNameserver{
+				Family:  unix.AF_INET,
+				Address: ip[12:],
+			}
+		} else {
+			linkNameservers[i] = resolvedLinkNameserver{
+				Family:  unix.AF_INET6,
+				Address: ip[:],
+			}
+		}
+	}
+
+	err = resolved.CallWithContext(
+		ctx, "org.freedesktop.resolve1.Manager.SetLinkDNS", 0,
+		iface.Index, linkNameservers,
+	).Store()
+	if err != nil {
+		return fmt.Errorf("SetLinkDNS: %w", err)
+	}
+
+	var linkDomains = make([]resolvedLinkDomain, len(config.Domains))
+	for i, domain := range config.Domains {
+		linkDomains[i] = resolvedLinkDomain{
+			Domain:      domain,
+			RoutingOnly: false,
+		}
+	}
+
+	err = resolved.CallWithContext(
+		ctx, "org.freedesktop.resolve1.Manager.SetLinkDomains", 0,
+		iface.Index, linkDomains,
+	).Store()
+	if err != nil {
+		return fmt.Errorf("SetLinkDomains: %w", err)
+	}
+
+	return nil
+}
+
+// dnsResolvedDown undoes the changes made by dnsResolvedUp.
+func dnsResolvedDown() error {
+	ctx, cancel := context.WithTimeout(context.Background(), dnsReconfigTimeout)
+	defer cancel()
+
+	// conn is a shared connection whose lifecycle is managed by the dbus package.
+	// We should not interfere with that by closing it.
+	conn, err := dbus.SystemBus()
+	if err != nil {
+		return fmt.Errorf("connecting to system bus: %w", err)
+	}
+
+	resolved := conn.Object(
+		"org.freedesktop.resolve1",
+		dbus.ObjectPath("/org/freedesktop/resolve1"),
+	)
+
+	_, iface, err := interfaces.Tailscale()
+	if err != nil {
+		return fmt.Errorf("getting interface index: %w", err)
+	}
+	if iface == nil {
+		return errNotReady
+	}
+
+	err = resolved.CallWithContext(
+		ctx, "org.freedesktop.resolve1.Manager.RevertLink", 0,
+		iface.Index,
+	).Store()
+	if err != nil {
+		return fmt.Errorf("RevertLink: %w", err)
+	}
+
+	return nil
+}

wgengine/router/ifconfig_windows.go

@@ -262,7 +262,7 @@ func configureInterface(cfg *Config, tun *tun.NativeTun) error {
 		}
 	}()
 
-	setDNSDomains(guid, cfg.DNSDomains)
+	setDNSDomains(guid, cfg.Domains)
 
 	routes := []winipcfg.RouteData{}
 	var firstGateway4 *net.IP
@@ -359,7 +359,7 @@ func configureInterface(cfg *Config, tun *tun.NativeTun) error {
 	}
 
 	var dnsIPs []net.IP
-	for _, ip := range cfg.DNS {
+	for _, ip := range cfg.Nameservers {
 		dnsIPs = append(dnsIPs, ip.IPAddr().IP)
 	}
 	err = iface.SetDNS(dnsIPs)

wgengine/router/router.go

@@ -32,9 +32,17 @@ type Router interface {
 // New returns a new Router for the current platform, using the
 // provided tun device.
 func New(logf logger.Logf, wgdev *device.Device, tundev tun.Device) (Router, error) {
+	logf = logger.WithPrefix(logf, "router: ")
 	return newUserspaceRouter(logf, wgdev, tundev)
 }
 
+// Cleanup restores the system network configuration to its original state
+// in case the Tailscale daemon terminated without closing the router.
+// No other state needs to be instantiated before this runs.
+func Cleanup(logf logger.Logf, interfaceName string) {
+	cleanup(logf, interfaceName)
+}
+
 // NetfilterMode is the firewall management mode to use when
 // programming the Linux network stack.
 type NetfilterMode int
@@ -62,10 +70,10 @@ func (m NetfilterMode) String() string {
 // the OS's network stack.
 type Config struct {
 	LocalAddrs []netaddr.IPPrefix
-	DNS        []netaddr.IP
-	DNSDomains []string
 	Routes     []netaddr.IPPrefix // routes to point into the Tailscale interface
 
+	DNSConfig
+
 	// Linux-only things below, ignored on other platforms.
 
 	SubnetRoutes     []netaddr.IPPrefix // subnets being advertised to other Tailscale nodes

wgengine/router/router_darwin.go

@@ -52,3 +52,13 @@ func (r *darwinRouter) Up() error {
 	}
 	return r.Router.Up()
 }
+
+func upDNS(config DNSConfig, interfaceName string) error {
+	// Handled by IPNExtension
+	return nil
+}
+
+func downDNS(interfaceName string) error {
+	// Handled by IPNExtension
+	return nil
+}

wgengine/router/router_default.go

@@ -15,3 +15,7 @@ import (
 func newUserspaceRouter(logf logger.Logf, tunname string, dev *device.Device, tuntap tun.Device, netChanged func()) Router {
 	return NewFakeRouter(logf, tunname, dev, tuntap, netChanged)
 }
+
+func cleanup() error {
+	return nil
+}

wgengine/router/router_freebsd.go

@@ -5,6 +5,8 @@
 package router
 
 import (
+	"fmt"
+
 	"github.com/tailscale/wireguard-go/device"
 	"github.com/tailscale/wireguard-go/tun"
 	"tailscale.com/types/logger"
@@ -18,3 +20,35 @@ import (
 func newUserspaceRouter(logf logger.Logf, _ *device.Device, tundev tun.Device) (Router, error) {
 	return newUserspaceBSDRouter(logf, nil, tundev)
 }
+
+func upDNS(config DNSConfig, interfaceName string) error {
+	if len(config.Nameservers) == 0 {
+		return downDNS(interfaceName)
+	}
+
+	if resolvconfIsActive() {
+		if err := dnsResolvconfUp(config, interfaceName); err != nil {
+			return fmt.Errorf("resolvconf: %w")
+		}
+		return nil
+	}
+
+	if err := dnsDirectUp(config); err != nil {
+		return fmt.Errorf("direct: %w")
+	}
+	return nil
+}
+
+func downDNS(interfaceName string) error {
+	if resolvconfIsActive() {
+		if err := dnsResolvconfDown(interfaceName); err != nil {
+			return fmt.Errorf("resolvconf: %w")
+		}
+		return nil
+	}
+
+	if err := dnsDirectDown(); err != nil {
+		return fmt.Errorf("direct: %w")
+	}
+	return nil
+}

wgengine/router/router_linux.go

@@ -5,19 +5,15 @@
 package router
 
 import (
-	"bytes"
 	"fmt"
-	"io/ioutil"
-	"os"
 	"os/exec"
-	"path/filepath"
 	"strings"
 
 	"github.com/coreos/go-iptables/iptables"
 	"github.com/tailscale/wireguard-go/device"
 	"github.com/tailscale/wireguard-go/tun"
 	"inet.af/netaddr"
-	"tailscale.com/atomicfile"
+	"tailscale.com/net/tsaddr"
 	"tailscale.com/types/logger"
 )
 
@@ -41,19 +37,30 @@ import (
 const (
 	// Packet is from Tailscale and to a subnet route destination, so
 	// is allowed to be routed through this machine.
-	tailscaleSubnetRouteMark = "0x10000"
+	tailscaleSubnetRouteMark = "0x40000"
 	// Packet was originated by tailscaled itself, and must not be
 	// routed over the Tailscale network.
 	//
 	// Keep this in sync with tailscaleBypassMark in
 	// net/netns/netns_linux.go.
-	tailscaleBypassMark = "0x20000"
+	tailscaleBypassMark = "0x80000"
 )
 
-// chromeOSVMRange is the subset of the CGNAT IPv4 range used by
-// ChromeOS to interconnect the host OS to containers and VMs. We
-// avoid allocating Tailscale IPs from it, to avoid conflicts.
-const chromeOSVMRange = "100.115.92.0/23"
+// tailscaleRouteTable is the routing table number for Tailscale
+// network routes. See addIPRules for the detailed policy routing
+// logic that ends up doing lookups within that table.
+//
+// NOTE(danderson): We chose 52 because those are the digits above the
+// letters "TS" on a qwerty keyboard, and 52 is sufficiently unlikely
+// to be picked by other software.
+//
+// NOTE(danderson): You might wonder why we didn't pick some high
+// table number like 5252, to further avoid the potential for
+// collisions with other software. Unfortunately, Busybox's `ip`
+// implementation believes that table numbers are 8-bit integers, so
+// for maximum compatibility we have to stay in the 0-255 range even
+// though linux itself supports larger numbers.
+const tailscaleRouteTable = "52"
 
 // netfilterRunner abstracts helpers to run netfilter commands. It
 // exists purely to swap out go-iptables for a fake implementation in
@@ -77,6 +84,9 @@ type linuxRouter struct {
 	snatSubnetRoutes bool
 	netfilterMode    NetfilterMode
 
+	dnsMode   dnsMode
+	dnsConfig DNSConfig
+
 	ipt4 netfilterRunner
 	cmd  commandRunner
 }
@@ -123,10 +133,27 @@ func (r *linuxRouter) Up() error {
 		return err
 	}
 
+	switch {
+	// TODO(dmytro): enable resolved when per-domain resolvers are desired.
+	case resolvedIsActive():
+		r.dnsMode = dnsDirect
+	//	r.dnsMode = dnsResolved
+	case nmIsActive():
+		r.dnsMode = dnsNetworkManager
+	case resolvconfIsActive():
+		r.dnsMode = dnsResolvconf
+	default:
+		r.dnsMode = dnsDirect
+	}
+	r.logf("dns mode: %v", r.dnsMode)
+
 	return nil
 }
 
-func (r *linuxRouter) down() error {
+func (r *linuxRouter) Close() error {
+	if err := r.downDNS(); err != nil {
+		return err
+	}
 	if err := r.downInterface(); err != nil {
 		return err
 	}
@@ -143,20 +170,6 @@ func (r *linuxRouter) down() error {
 	return nil
 }
 
-func (r *linuxRouter) Close() error {
-	var ret error
-	if ret = r.restoreResolvConf(); ret != nil {
-		r.logf("failed to restore system resolv.conf: %v", ret)
-	}
-	if err := r.down(); err != nil {
-		if ret == nil {
-			ret = err
-		}
-	}
-
-	return ret
-}
-
 // Set implements the Router interface.
 func (r *linuxRouter) Set(cfg *Config) error {
 	if cfg == nil {
@@ -193,12 +206,14 @@ func (r *linuxRouter) Set(cfg *Config) error {
 	}
 	r.snatSubnetRoutes = cfg.SNATSubnetRoutes
 
-	// TODO: this:
-	if false {
-		if err := r.replaceResolvConf(cfg.DNS, cfg.DNSDomains); err != nil {
-			return fmt.Errorf("replacing resolv.conf failed: %w", err)
+	if !r.dnsConfig.EquivalentTo(cfg.DNSConfig) {
+		if err := r.upDNS(cfg.DNSConfig); err != nil {
+			r.logf("dns up: %v", err)
+		} else {
+			r.dnsConfig = cfg.DNSConfig
 		}
 	}
+
 	return nil
 }
 
@@ -319,102 +334,6 @@ func (r *linuxRouter) setNetfilterMode(mode NetfilterMode) error {
 	return nil
 }
 
-const (
-	tsConf     = "/etc/resolv.tailscale.conf"
-	backupConf = "/etc/resolv.pre-tailscale-backup.conf"
-	resolvConf = "/etc/resolv.conf"
-)
-
-func (r *linuxRouter) replaceResolvConf(servers []netaddr.IP, domains []string) error {
-	if len(servers) == 0 {
-		return r.restoreResolvConf()
-	}
-
-	// First write the tsConf file.
-	buf := new(bytes.Buffer)
-	fmt.Fprintf(buf, "# resolv.conf(5) file generated by tailscale\n")
-	fmt.Fprintf(buf, "#     DO NOT EDIT THIS FILE BY HAND -- CHANGES WILL BE OVERWRITTEN\n\n")
-	for _, ns := range servers {
-		fmt.Fprintf(buf, "nameserver %s\n", ns)
-	}
-	if len(domains) > 0 {
-		fmt.Fprintf(buf, "search "+strings.Join(domains, " ")+"\n")
-	}
-	f, err := ioutil.TempFile(filepath.Dir(tsConf), filepath.Base(tsConf)+".*")
-	if err != nil {
-		return err
-	}
-	f.Close()
-	if err := atomicfile.WriteFile(f.Name(), buf.Bytes(), 0644); err != nil {
-		return err
-	}
-	os.Chmod(f.Name(), 0644) // ioutil.TempFile creates the file with 0600
-	if err := os.Rename(f.Name(), tsConf); err != nil {
-		return err
-	}
-
-	if linkPath, err := os.Readlink(resolvConf); err != nil {
-		// Remove any old backup that may exist.
-		os.Remove(backupConf)
-
-		// Backup the existing /etc/resolv.conf file.
-		contents, err := ioutil.ReadFile(resolvConf)
-		if os.IsNotExist(err) {
-			// No existing /etc/resolv.conf file to backup.
-			// Nothing to do.
-			return nil
-		} else if err != nil {
-			return err
-		}
-		if err := atomicfile.WriteFile(backupConf, contents, 0644); err != nil {
-			return err
-		}
-	} else if linkPath != tsConf {
-		// Backup the existing symlink.
-		os.Remove(backupConf)
-		if err := os.Symlink(linkPath, backupConf); err != nil {
-			return err
-		}
-	} else {
-		// Nothing to do, resolvConf already points to tsConf.
-		return nil
-	}
-
-	os.Remove(resolvConf)
-	if err := os.Symlink(tsConf, resolvConf); err != nil {
-		return nil
-	}
-
-	out, _ := exec.Command("service", "systemd-resolved", "restart").CombinedOutput()
-	if len(out) > 0 {
-		r.logf("service systemd-resolved restart: %s", out)
-	}
-	return nil
-}
-
-func (r *linuxRouter) restoreResolvConf() error {
-	if _, err := os.Stat(backupConf); err != nil {
-		if os.IsNotExist(err) {
-			return nil // no backup resolv.conf to restore
-		}
-		return err
-	}
-	if ln, err := os.Readlink(resolvConf); err != nil {
-		return err
-	} else if ln != tsConf {
-		return fmt.Errorf("resolv.conf is not a symlink to %s", tsConf)
-	}
-	if err := os.Rename(backupConf, resolvConf); err != nil {
-		return err
-	}
-	os.Remove(tsConf) // best effort removal of tsConf file
-	out, _ := exec.Command("service", "systemd-resolved", "restart").CombinedOutput()
-	if len(out) > 0 {
-		r.logf("service systemd-resolved restart: %s", out)
-	}
-	return nil
-}
-
 // addAddress adds an IP/mask to the tunnel interface. Fails if the
 // address is already assigned to the interface, or if the addition
 // fails.
@@ -475,7 +394,7 @@ func (r *linuxRouter) addRoute(cidr netaddr.IPPrefix) error {
 		"dev", r.tunname,
 	}
 	if r.ipRuleAvailable {
-		args = append(args, "table", "88")
+		args = append(args, "table", tailscaleRouteTable)
 	}
 	return r.cmd.run(args...)
 }
@@ -490,7 +409,7 @@ func (r *linuxRouter) delRoute(cidr netaddr.IPPrefix) error {
 		"dev", r.tunname,
 	}
 	if r.ipRuleAvailable {
-		args = append(args, "table", "88")
+		args = append(args, "table", tailscaleRouteTable)
 	}
 	return r.cmd.run(args...)
 }
@@ -528,7 +447,7 @@ func (r *linuxRouter) addIPRules() error {
 	// NOTE(apenwarr): This sequence seems complicated, right?
 	// If we could simply have a rule that said "match packets that
 	// *don't* have this fwmark", then we would only need to add one
-	// link to table 88 and we'd be done. Unfortunately, older kernels
+	// link to table 52 and we'd be done. Unfortunately, older kernels
 	// and 'ip rule' implementations (including busybox), don't support
 	// checking for the lack of a fwmark, only the presence. The technique
 	// below works even on very old kernels.
@@ -537,7 +456,7 @@ func (r *linuxRouter) addIPRules() error {
 	// main routing table.
 	rg.Run(
 		"ip", "rule", "add",
-		"pref", "8810",
+		"pref", tailscaleRouteTable+"10",
 		"fwmark", tailscaleBypassMark,
 		"table", "main",
 	)
@@ -545,7 +464,7 @@ func (r *linuxRouter) addIPRules() error {
 	// even though it's been empty on every Linux system I've ever seen.
 	rg.Run(
 		"ip", "rule", "add",
-		"pref", "8830",
+		"pref", tailscaleRouteTable+"30",
 		"fwmark", tailscaleBypassMark,
 		"table", "default",
 	)
@@ -554,23 +473,22 @@ func (r *linuxRouter) addIPRules() error {
 	// to the tailscale routes, because that would create routing loops.
 	rg.Run(
 		"ip", "rule", "add",
-		"pref", "8850",
+		"pref", tailscaleRouteTable+"50",
 		"fwmark", tailscaleBypassMark,
 		"type", "unreachable",
 	)
 	// If we get to this point, capture all packets and send them
-	// through to table 88, the set of tailscale routes.
-	// For apps other than us (ie. with no fwmark set), this is the
-	// first routing table, so it takes precedence over all the others,
-	// ie. VPN routes always beat non-VPN routes.
+	// through to the tailscale route table. For apps other than us
+	// (ie. with no fwmark set), this is the first routing table, so
+	// it takes precedence over all the others, ie. VPN routes always
+	// beat non-VPN routes.
 	//
-	// NOTE(apenwarr): tables >255 are not supported in busybox.
-	// I really wanted to use table 8888 here for symmetry, but no luck
-	// with busybox alas.
+	// NOTE(apenwarr): tables >255 are not supported in busybox, so we
+	// can't use a table number that aligns with the rule preferences.
 	rg.Run(
 		"ip", "rule", "add",
-		"pref", "8888",
-		"table", "88",
+		"pref", tailscaleRouteTable+"70",
+		"table", tailscaleRouteTable,
 	)
 	// If that didn't match, then non-fwmark packets fall through to the
 	// usual rules (pref 32766 and 32767, ie. main and default).
@@ -611,23 +529,23 @@ func (r *linuxRouter) delIPRules() error {
 	// Delete new-style tailscale rules.
 	rg.Run(
 		"ip", "rule", "del",
-		"pref", "8810",
+		"pref", tailscaleRouteTable+"10",
 		"table", "main",
 	)
 	rg.Run(
 		"ip", "rule", "del",
-		"pref", "8830",
+		"pref", tailscaleRouteTable+"30",
 		"table", "default",
 	)
 	rg.Run(
 		"ip", "rule", "del",
-		"pref", "8850",
+		"pref", tailscaleRouteTable+"50",
 		"type", "unreachable",
 	)
 	rg.Run(
 		"ip", "rule", "del",
-		"pref", "8888",
-		"table", "88",
+		"pref", tailscaleRouteTable+"70",
+		"table", tailscaleRouteTable,
 	)
 	return rg.ErrAcc
 }
@@ -666,11 +584,11 @@ func (r *linuxRouter) addNetfilterBase() error {
 	//
 	// Note, this will definitely break nodes that end up using the
 	// CGNAT range for other purposes :(.
-	args := []string{"!", "-i", r.tunname, "-s", chromeOSVMRange, "-j", "RETURN"}
+	args := []string{"!", "-i", r.tunname, "-s", tsaddr.ChromeOSVMRange().String(), "-j", "RETURN"}
 	if err := r.ipt4.Append("filter", "ts-input", args...); err != nil {
 		return fmt.Errorf("adding %v in filter/ts-input: %w", args, err)
 	}
-	args = []string{"!", "-i", r.tunname, "-s", "100.64.0.0/10", "-j", "DROP"}
+	args = []string{"!", "-i", r.tunname, "-s", tsaddr.CGNATRange().String(), "-j", "DROP"}
 	if err := r.ipt4.Append("filter", "ts-input", args...); err != nil {
 		return fmt.Errorf("adding %v in filter/ts-input: %w", args, err)
 	}
@@ -694,7 +612,7 @@ func (r *linuxRouter) addNetfilterBase() error {
 	if err := r.ipt4.Append("filter", "ts-forward", args...); err != nil {
 		return fmt.Errorf("adding %v in filter/ts-forward: %w", args, err)
 	}
-	args = []string{"-o", r.tunname, "-s", "100.64.0.0/10", "-j", "DROP"}
+	args = []string{"-o", r.tunname, "-s", tsaddr.CGNATRange().String(), "-j", "DROP"}
 	if err := r.ipt4.Append("filter", "ts-forward", args...); err != nil {
 		return fmt.Errorf("adding %v in filter/ts-forward: %w", args, err)
 	}
@@ -936,3 +854,69 @@ func normalizeCIDR(cidr netaddr.IPPrefix) string {
 	nip := ncidr.IP.Mask(ncidr.Mask)
 	return fmt.Sprintf("%s/%d", nip, cidr.Bits)
 }
+
+// upDNS updates the system DNS configuration to the given one.
+func (r *linuxRouter) upDNS(config DNSConfig) error {
+	if len(config.Nameservers) == 0 {
+		return r.downDNS()
+	}
+
+	switch r.dnsMode {
+	case dnsResolved:
+		if err := dnsResolvedUp(config); err != nil {
+			return fmt.Errorf("resolved: %w", err)
+		}
+	case dnsResolvconf:
+		if err := dnsResolvconfUp(config, r.tunname); err != nil {
+			return fmt.Errorf("resolvconf: %w", err)
+		}
+	case dnsNetworkManager:
+		if err := dnsNetworkManagerUp(config, r.tunname); err != nil {
+			return fmt.Errorf("network manager: %w", err)
+		}
+	case dnsDirect:
+		if err := dnsDirectUp(config); err != nil {
+			return fmt.Errorf("direct: %w", err)
+		}
+	}
+	return nil
+}
+
+// downDNS restores system DNS configuration to its state before upDNS.
+// It is idempotent (in particular, it does nothing if upDNS was never run).
+func (r *linuxRouter) downDNS() error {
+	switch r.dnsMode {
+	case dnsResolved:
+		if err := dnsResolvedDown(); err != nil {
+			return fmt.Errorf("resolved: %w", err)
+		}
+	case dnsResolvconf:
+		if err := dnsResolvconfDown(r.tunname); err != nil {
+			return fmt.Errorf("resolvconf: %w", err)
+		}
+	case dnsNetworkManager:
+		if err := dnsNetworkManagerDown(r.tunname); err != nil {
+			return fmt.Errorf("network manager: %w", err)
+		}
+	case dnsDirect:
+		if err := dnsDirectDown(); err != nil {
+			return fmt.Errorf("direct: %w", err)
+		}
+	}
+	return nil
+}
+
+func cleanup(logf logger.Logf, interfaceName string) {
+	// Note: we need not do anything for dnsResolved,
+	// as its settings are interface-bound and get cleaned up for us.
+	switch {
+	case resolvconfIsActive():
+		if err := dnsResolvconfDown(interfaceName); err != nil {
+			logf("down down: resolvconf: %v", err)
+		}
+	default:
+		if err := dnsDirectDown(); err != nil {
+			logf("dns down: direct: %v", err)
+		}
+	}
+}

wgengine/router/router_linux_test.go

@@ -34,10 +34,10 @@ func mustCIDRs(ss ...string) []netaddr.IPPrefix {
 
 func TestRouterStates(t *testing.T) {
 	basic := `
-ip rule add pref 8810 fwmark 0x20000 table main
-ip rule add pref 8830 fwmark 0x20000 table default
-ip rule add pref 8850 fwmark 0x20000 type unreachable
-ip rule add pref 8888 table 88
+ip rule add pref 5210 fwmark 0x80000 table main
+ip rule add pref 5230 fwmark 0x80000 table default
+ip rule add pref 5250 fwmark 0x80000 type unreachable
+ip rule add pref 5270 table 52
 `
 	states := []struct {
 		name string
@@ -71,8 +71,8 @@ ip addr add 100.101.102.103/10 dev tailscale0` + basic,
 			want: `
 up
 ip addr add 100.101.102.103/10 dev tailscale0
-ip route add 100.100.100.100/32 dev tailscale0 table 88
-ip route add 192.168.16.0/24 dev tailscale0 table 88` + basic,
+ip route add 100.100.100.100/32 dev tailscale0 table 52
+ip route add 192.168.16.0/24 dev tailscale0 table 52` + basic,
 		},
 
 		{
@@ -86,8 +86,8 @@ ip route add 192.168.16.0/24 dev tailscale0 table 88` + basic,
 			want: `
 up
 ip addr add 100.101.102.103/10 dev tailscale0
-ip route add 100.100.100.100/32 dev tailscale0 table 88
-ip route add 192.168.16.0/24 dev tailscale0 table 88` + basic,
+ip route add 100.100.100.100/32 dev tailscale0 table 52
+ip route add 192.168.16.0/24 dev tailscale0 table 52` + basic,
 		},
 
 		{
@@ -102,19 +102,19 @@ ip route add 192.168.16.0/24 dev tailscale0 table 88` + basic,
 			want: `
 up
 ip addr add 100.101.102.104/10 dev tailscale0
-ip route add 10.0.0.0/8 dev tailscale0 table 88
-ip route add 100.100.100.100/32 dev tailscale0 table 88` + basic +
+ip route add 10.0.0.0/8 dev tailscale0 table 52
+ip route add 100.100.100.100/32 dev tailscale0 table 52` + basic +
 				`filter/FORWARD -j ts-forward
 filter/INPUT -j ts-input
-filter/ts-forward -i tailscale0 -j MARK --set-mark 0x10000
-filter/ts-forward -m mark --mark 0x10000 -j ACCEPT
+filter/ts-forward -i tailscale0 -j MARK --set-mark 0x40000
+filter/ts-forward -m mark --mark 0x40000 -j ACCEPT
 filter/ts-forward -o tailscale0 -s 100.64.0.0/10 -j DROP
 filter/ts-forward -o tailscale0 -j ACCEPT
 filter/ts-input -i lo -s 100.101.102.104 -j ACCEPT
 filter/ts-input ! -i tailscale0 -s 100.115.92.0/23 -j RETURN
 filter/ts-input ! -i tailscale0 -s 100.64.0.0/10 -j DROP
 nat/POSTROUTING -j ts-postrouting
-nat/ts-postrouting -m mark --mark 0x10000 -j MASQUERADE
+nat/ts-postrouting -m mark --mark 0x40000 -j MASQUERADE
 `,
 		},
 		{
@@ -127,12 +127,12 @@ nat/ts-postrouting -m mark --mark 0x10000 -j MASQUERADE
 			want: `
 up
 ip addr add 100.101.102.104/10 dev tailscale0
-ip route add 10.0.0.0/8 dev tailscale0 table 88
-ip route add 100.100.100.100/32 dev tailscale0 table 88` + basic +
+ip route add 10.0.0.0/8 dev tailscale0 table 52
+ip route add 100.100.100.100/32 dev tailscale0 table 52` + basic +
 				`filter/FORWARD -j ts-forward
 filter/INPUT -j ts-input
-filter/ts-forward -i tailscale0 -j MARK --set-mark 0x10000
-filter/ts-forward -m mark --mark 0x10000 -j ACCEPT
+filter/ts-forward -i tailscale0 -j MARK --set-mark 0x40000
+filter/ts-forward -m mark --mark 0x40000 -j ACCEPT
 filter/ts-forward -o tailscale0 -s 100.64.0.0/10 -j DROP
 filter/ts-forward -o tailscale0 -j ACCEPT
 filter/ts-input -i lo -s 100.101.102.104 -j ACCEPT
@@ -154,12 +154,12 @@ nat/POSTROUTING -j ts-postrouting
 			want: `
 up
 ip addr add 100.101.102.104/10 dev tailscale0
-ip route add 10.0.0.0/8 dev tailscale0 table 88
-ip route add 100.100.100.100/32 dev tailscale0 table 88` + basic +
+ip route add 10.0.0.0/8 dev tailscale0 table 52
+ip route add 100.100.100.100/32 dev tailscale0 table 52` + basic +
 				`filter/FORWARD -j ts-forward
 filter/INPUT -j ts-input
-filter/ts-forward -i tailscale0 -j MARK --set-mark 0x10000
-filter/ts-forward -m mark --mark 0x10000 -j ACCEPT
+filter/ts-forward -i tailscale0 -j MARK --set-mark 0x40000
+filter/ts-forward -m mark --mark 0x40000 -j ACCEPT
 filter/ts-forward -o tailscale0 -s 100.64.0.0/10 -j DROP
 filter/ts-forward -o tailscale0 -j ACCEPT
 filter/ts-input -i lo -s 100.101.102.104 -j ACCEPT
@@ -178,12 +178,12 @@ nat/POSTROUTING -j ts-postrouting
 			want: `
 up
 ip addr add 100.101.102.104/10 dev tailscale0
-ip route add 10.0.0.0/8 dev tailscale0 table 88
-ip route add 100.100.100.100/32 dev tailscale0 table 88` + basic +
+ip route add 10.0.0.0/8 dev tailscale0 table 52
+ip route add 100.100.100.100/32 dev tailscale0 table 52` + basic +
 				`filter/FORWARD -j ts-forward
 filter/INPUT -j ts-input
-filter/ts-forward -i tailscale0 -j MARK --set-mark 0x10000
-filter/ts-forward -m mark --mark 0x10000 -j ACCEPT
+filter/ts-forward -i tailscale0 -j MARK --set-mark 0x40000
+filter/ts-forward -m mark --mark 0x40000 -j ACCEPT
 filter/ts-forward -o tailscale0 -s 100.64.0.0/10 -j DROP
 filter/ts-forward -o tailscale0 -j ACCEPT
 filter/ts-input -i lo -s 100.101.102.104 -j ACCEPT
@@ -203,10 +203,10 @@ nat/POSTROUTING -j ts-postrouting
 			want: `
 up
 ip addr add 100.101.102.104/10 dev tailscale0
-ip route add 10.0.0.0/8 dev tailscale0 table 88
-ip route add 100.100.100.100/32 dev tailscale0 table 88` + basic +
-				`filter/ts-forward -i tailscale0 -j MARK --set-mark 0x10000
-filter/ts-forward -m mark --mark 0x10000 -j ACCEPT
+ip route add 10.0.0.0/8 dev tailscale0 table 52
+ip route add 100.100.100.100/32 dev tailscale0 table 52` + basic +
+				`filter/ts-forward -i tailscale0 -j MARK --set-mark 0x40000
+filter/ts-forward -m mark --mark 0x40000 -j ACCEPT
 filter/ts-forward -o tailscale0 -s 100.64.0.0/10 -j DROP
 filter/ts-forward -o tailscale0 -j ACCEPT
 filter/ts-input -i lo -s 100.101.102.104 -j ACCEPT
@@ -224,12 +224,12 @@ filter/ts-input ! -i tailscale0 -s 100.64.0.0/10 -j DROP
 			want: `
 up
 ip addr add 100.101.102.104/10 dev tailscale0
-ip route add 10.0.0.0/8 dev tailscale0 table 88
-ip route add 100.100.100.100/32 dev tailscale0 table 88` + basic +
+ip route add 10.0.0.0/8 dev tailscale0 table 52
+ip route add 100.100.100.100/32 dev tailscale0 table 52` + basic +
 				`filter/FORWARD -j ts-forward
 filter/INPUT -j ts-input
-filter/ts-forward -i tailscale0 -j MARK --set-mark 0x10000
-filter/ts-forward -m mark --mark 0x10000 -j ACCEPT
+filter/ts-forward -i tailscale0 -j MARK --set-mark 0x40000
+filter/ts-forward -m mark --mark 0x40000 -j ACCEPT
 filter/ts-forward -o tailscale0 -s 100.64.0.0/10 -j DROP
 filter/ts-forward -o tailscale0 -j ACCEPT
 filter/ts-input -i lo -s 100.101.102.104 -j ACCEPT

wgengine/router/router_openbsd.go

@@ -5,20 +5,14 @@
 package router
 
 import (
-	"bytes"
 	"errors"
 	"fmt"
-	"io/ioutil"
 	"log"
-	"os"
 	"os/exec"
-	"path/filepath"
-	"strings"
 
 	"github.com/tailscale/wireguard-go/device"
 	"github.com/tailscale/wireguard-go/tun"
 	"inet.af/netaddr"
-	"tailscale.com/atomicfile"
 	"tailscale.com/types/logger"
 )
 
@@ -31,6 +25,8 @@ type openbsdRouter struct {
 	tunname string
 	local   netaddr.IPPrefix
 	routes  map[netaddr.IPPrefix]struct{}
+
+	dnsConfig DNSConfig
 }
 
 func newUserspaceRouter(logf logger.Logf, _ *device.Device, tundev tun.Device) (Router, error) {
@@ -159,112 +155,28 @@ func (r *openbsdRouter) Set(cfg *Config) error {
 	r.local = localAddr
 	r.routes = newRoutes
 
-	if err := r.replaceResolvConf(cfg.DNS, cfg.DNSDomains); err != nil {
-		errq = fmt.Errorf("replacing resolv.conf failed: %v", err)
+	if !r.dnsConfig.EquivalentTo(cfg.DNSConfig) {
+		if err := dnsDirectUp(cfg.DNSConfig); err != nil {
+			errq = fmt.Errorf("dns up: direct: %v", err)
+		} else {
+			r.dnsConfig = cfg.DNSConfig
+		}
 	}
 
 	return errq
 }
 
 func (r *openbsdRouter) Close() error {
-	out, err := cmd("ifconfig", r.tunname, "down").CombinedOutput()
+	cleanup(r.logf, r.tunname)
+	return nil
+}
+
+func cleanup(logf logger.Logf, interfaceName string) {
+	if err := dnsDirectDown(); err != nil {
+		logf("dns down: direct: %v", err)
+	}
+	out, err := cmd("ifconfig", interfaceName, "down").CombinedOutput()
 	if err != nil {
-		r.logf("running ifconfig failed: %v\n%s", err, out)
+		logf("ifconfig down: %v\n%s", err, out)
 	}
-
-	if err := r.restoreResolvConf(); err != nil {
-		r.logf("failed to restore system resolv.conf: %v", err)
-	}
-
-	return nil
-}
-
-const (
-	tsConf     = "/etc/resolv.tailscale.conf"
-	backupConf = "/etc/resolv.pre-tailscale-backup.conf"
-	resolvConf = "/etc/resolv.conf"
-)
-
-func (r *openbsdRouter) replaceResolvConf(servers []netaddr.IP, domains []string) error {
-	if len(servers) == 0 {
-		return r.restoreResolvConf()
-	}
-
-	// Write the tsConf file.
-	buf := new(bytes.Buffer)
-	fmt.Fprintf(buf, "# resolv.conf(5) file generated by tailscale\n")
-	fmt.Fprintf(buf, "# DO NOT EDIT THIS FILE BY HAND -- CHANGES WILL BE OVERWRITTEN\n\n")
-	for _, ns := range servers {
-		fmt.Fprintf(buf, "nameserver %s\n", ns)
-	}
-	if len(domains) > 0 {
-		fmt.Fprintf(buf, "search "+strings.Join(domains, " ")+"\n")
-	}
-	tf, err := ioutil.TempFile(filepath.Dir(tsConf), filepath.Base(tsConf)+".*")
-	if err != nil {
-		return err
-	}
-	tempName := tf.Name()
-	tf.Close()
-
-	if err := atomicfile.WriteFile(tempName, buf.Bytes(), 0644); err != nil {
-		return err
-	}
-	if err := os.Rename(tempName, tsConf); err != nil {
-		return err
-	}
-
-	if linkPath, err := os.Readlink(resolvConf); err != nil {
-		// Remove any old backup that may exist.
-		os.Remove(backupConf)
-
-		// Backup the existing /etc/resolv.conf file.
-		contents, err := ioutil.ReadFile(resolvConf)
-		if os.IsNotExist(err) {
-			// No existing /etc/resolv.conf file to backup.
-			// Nothing to do.
-			return nil
-		} else if err != nil {
-			return err
-		}
-		if err := atomicfile.WriteFile(backupConf, contents, 0644); err != nil {
-			return err
-		}
-	} else if linkPath != tsConf {
-		// Backup the existing symlink.
-		os.Remove(backupConf)
-		if err := os.Symlink(linkPath, backupConf); err != nil {
-			return err
-		}
-	} else {
-		// Nothing to do, resolvConf already points to tsConf.
-		return nil
-	}
-
-	os.Remove(resolvConf)
-	if err := os.Symlink(tsConf, resolvConf); err != nil {
-		return nil
-	}
-
-	return nil
-}
-
-func (r *openbsdRouter) restoreResolvConf() error {
-	if _, err := os.Stat(backupConf); err != nil {
-		if os.IsNotExist(err) {
-			return nil // No backup resolv.conf to restore.
-		}
-		return err
-	}
-	if ln, err := os.Readlink(resolvConf); err != nil {
-		return err
-	} else if ln != tsConf {
-		return fmt.Errorf("resolv.conf is not a symlink to %s", tsConf)
-	}
-	if err := os.Rename(backupConf, resolvConf); err != nil {
-		return err
-	}
-	os.Remove(tsConf) // Best effort removal.
-
-	return nil
 }

wgengine/router/router_userspace_bsd.go

@@ -23,6 +23,8 @@ type userspaceBSDRouter struct {
 	tunname string
 	local   netaddr.IPPrefix
 	routes  map[netaddr.IPPrefix]struct{}
+
+	dnsConfig DNSConfig
 }
 
 func newUserspaceBSDRouter(logf logger.Logf, _ *device.Device, tundev tun.Device) (Router, error) {
@@ -36,7 +38,7 @@ func newUserspaceBSDRouter(logf logger.Logf, _ *device.Device, tundev tun.Device
 	}, nil
 }
 
-func (r *userspaceBSDRouter) cmd(args ...string) *exec.Cmd {
+func cmd(args ...string) *exec.Cmd {
 	if len(args) == 0 {
 		log.Fatalf("exec.Cmd(%#v) invalid; need argv[0]\n", args)
 	}
@@ -45,7 +47,7 @@ func (r *userspaceBSDRouter) cmd(args ...string) *exec.Cmd {
 
 func (r *userspaceBSDRouter) Up() error {
 	ifup := []string{"ifconfig", r.tunname, "up"}
-	if out, err := r.cmd(ifup...).CombinedOutput(); err != nil {
+	if out, err := cmd(ifup...).CombinedOutput(); err != nil {
 		r.logf("running ifconfig failed: %v\n%s", err, out)
 		return err
 	}
@@ -73,7 +75,7 @@ func (r *userspaceBSDRouter) Set(cfg *Config) error {
 		if r.local != (netaddr.IPPrefix{}) {
 			addrdel := []string{"ifconfig", r.tunname,
 				"inet", r.local.String(), "-alias"}
-			out, err := r.cmd(addrdel...).CombinedOutput()
+			out, err := cmd(addrdel...).CombinedOutput()
 			if err != nil {
 				r.logf("addr del failed: %v: %v\n%s", addrdel, err, out)
 				if errq == nil {
@@ -85,7 +87,7 @@ func (r *userspaceBSDRouter) Set(cfg *Config) error {
 		// Add the interface.
 		addradd := []string{"ifconfig", r.tunname,
 			"inet", localAddr.String(), localAddr.IP.String()}
-		out, err := r.cmd(addradd...).CombinedOutput()
+		out, err := cmd(addradd...).CombinedOutput()
 		if err != nil {
 			r.logf("addr add failed: %v: %v\n%s", addradd, err, out)
 			if errq == nil {
@@ -107,7 +109,7 @@ func (r *userspaceBSDRouter) Set(cfg *Config) error {
 			routedel := []string{"route", "-q", "-n",
 				"del", "-inet", nstr,
 				"-iface", r.tunname}
-			out, err := r.cmd(routedel...).CombinedOutput()
+			out, err := cmd(routedel...).CombinedOutput()
 			if err != nil {
 				r.logf("route del failed: %v: %v\n%s", routedel, err, out)
 				if errq == nil {
@@ -125,7 +127,7 @@ func (r *userspaceBSDRouter) Set(cfg *Config) error {
 			routeadd := []string{"route", "-q", "-n",
 				"add", "-inet", nstr,
 				"-iface", r.tunname}
-			out, err := r.cmd(routeadd...).CombinedOutput()
+			out, err := cmd(routeadd...).CombinedOutput()
 			if err != nil {
 				r.logf("addr add failed: %v: %v\n%s", routeadd, err, out)
 				if errq == nil {
@@ -139,18 +141,29 @@ func (r *userspaceBSDRouter) Set(cfg *Config) error {
 	r.local = localAddr
 	r.routes = newRoutes
 
-	if err := r.replaceResolvConf(cfg.DNS, cfg.DNSDomains); err != nil {
-		errq = fmt.Errorf("replacing resolv.conf failed: %v", err)
+	if !r.dnsConfig.EquivalentTo(cfg.DNSConfig) {
+		if err := upDNS(cfg.DNSConfig, r.tunname); err != nil {
+			errq = fmt.Errorf("dns up: %v", err)
+		} else {
+			r.dnsConfig = cfg.DNSConfig
+		}
 	}
 
 	return errq
 }
 
 func (r *userspaceBSDRouter) Close() error {
+	cleanup(r.logf, r.tunname)
 	return nil
 }
 
-// TODO(mbaillie): these are no-ops for now. They could re-use the Linux funcs
-// (sans systemd parts), but I note Linux DNS is disabled(?) so leaving for now.
-func (r *userspaceBSDRouter) replaceResolvConf(_ []netaddr.IP, _ []string) error { return nil }
-func (r *userspaceBSDRouter) restoreResolvConf() error                           { return nil }
+func cleanup(logf logger.Logf, interfaceName string) {
+	if err := downDNS(interfaceName); err != nil {
+		logf("dns down: %v", err)
+	}
+
+	ifup := []string{"ifconfig", interfaceName, "down"}
+	if out, err := cmd(ifup...).CombinedOutput(); err != nil {
+		logf("ifconfig down: %v\n%s", err, out)
+	}
+}

wgengine/router/router_windows.go

@@ -64,3 +64,7 @@ func (r *winRouter) Close() error {
 	}
 	return nil
 }
+
+func cleanup(logf logger.Logf, interfaceName string) {
+	// DNS is interface-bound, so nothing to do here.
+}

wgengine/tsdns/tsdns.go

@@ -7,128 +7,319 @@
 package tsdns
 
 import (
-	"encoding/binary"
+	"bytes"
+	"context"
 	"errors"
-	"strings"
 	"sync"
 	"time"
 
 	dns "golang.org/x/net/dns/dnsmessage"
 	"inet.af/netaddr"
+	"tailscale.com/net/netns"
 	"tailscale.com/types/logger"
-	"tailscale.com/wgengine/packet"
 )
 
+// maxResponseSize is the maximum size of a response from a Resolver.
+const maxResponseSize = 512
+
+// queueSize is the maximal number of DNS requests that can be pending at a time.
+// If EnqueueRequest is called when this many requests are already pending,
+// the request will be dropped to avoid blocking the caller.
+const queueSize = 8
+
+// delegateTimeout is the maximal amount of time Resolver will wait
+// for upstream nameservers to process a query.
+const delegateTimeout = 5 * time.Second
+
 // defaultTTL is the TTL of all responses from Resolver.
 const defaultTTL = 600 * time.Second
 
+// ErrClosed indicates that the resolver has been closed and readers should exit.
+var ErrClosed = errors.New("closed")
+
 var (
+	errAllFailed      = errors.New("all upstream nameservers failed")
+	errFullQueue      = errors.New("request queue full")
 	errMapNotSet      = errors.New("domain map not set")
-	errNoSuchDomain   = errors.New("domain does not exist")
 	errNotImplemented = errors.New("query type not implemented")
-	errNotOurName     = errors.New("not an *.ipn.dev domain")
-	errNotOurQuery    = errors.New("query not for this resolver")
 	errNotQuery       = errors.New("not a DNS query")
-	errSmallBuffer    = errors.New("response buffer too small")
 )
 
-var (
-	defaultIP   = packet.IP(binary.BigEndian.Uint32([]byte{100, 100, 100, 100}))
-	defaultPort = uint16(53)
-)
-
-// Map is all the data Resolver needs to resolve DNS queries.
+// Map is all the data Resolver needs to resolve DNS queries within the Tailscale network.
 type Map struct {
 	// domainToIP is a mapping of Tailscale domains to their IP addresses.
-	// For example, monitoring.ipn.dev -> 100.64.0.1.
+	// For example, monitoring.tailscale.us -> 100.64.0.1.
 	domainToIP map[string]netaddr.IP
 }
 
 // NewMap returns a new Map with domain to address mapping given by domainToIP.
-// It takes ownership of the provided map.
 func NewMap(domainToIP map[string]netaddr.IP) *Map {
-	return &Map{
-		domainToIP: domainToIP,
-	}
+	return &Map{domainToIP: domainToIP}
 }
 
-// Resolver is a DNS resolver for domain names of the form *.ipn.dev.
+// Packet represents a DNS payload together with the address of its origin.
+type Packet struct {
+	// Payload is the application layer DNS payload.
+	// Resolver assumes ownership of the request payload when it is enqueued
+	// and cedes ownership of the response payload when it is returned from NextResponse.
+	Payload []byte
+	// Addr is the source address for a request and the destination address for a response.
+	Addr netaddr.IPPort
+}
+
+// Resolver is a DNS resolver for nodes on the Tailscale network,
+// associating them with domain names of the form <mynode>.<mydomain>.<root>.
+// If it is asked to resolve a domain that is not of that form,
+// it delegates to upstream nameservers if any are set.
 type Resolver struct {
 	logf logger.Logf
 
-	// ip is the IP on which the resolver is listening.
-	ip packet.IP
-	// port is the port on which the resolver is listening.
-	port uint16
+	// The asynchronous interface is due to the fact that resolution may potentially
+	// block for a long time (if the upstream nameserver is slow to reach).
+
+	// queue is a buffered channel holding DNS requests queued for resolution.
+	queue chan Packet
+	// responses is an unbuffered channel to which responses are sent.
+	responses chan Packet
+	// errors is an unbuffered channel to which errors are sent.
+	errors chan error
+	// closed notifies the poll goroutines to stop.
+	closed chan struct{}
+	// pollGroup signals when all poll goroutines have stopped.
+	pollGroup sync.WaitGroup
+
+	// rootDomain is <root> in <mynode>.<mydomain>.<root>.
+	rootDomain []byte
+
+	// dialer is the netns.Dialer used for delegation.
+	dialer netns.Dialer
 
 	// mu guards the following fields from being updated while used.
-	mu sync.Mutex
+	mu sync.RWMutex
 	// dnsMap is the map most recently received from the control server.
 	dnsMap *Map
+	// nameservers is the list of nameserver addresses that should be used
+	// if the received query is not for a Tailscale node.
+	// The addresses are strings of the form ip:port, as expected by Dial.
+	nameservers []string
 }
 
-// NewResolver constructs a resolver with default parameters.
-func NewResolver(logf logger.Logf) *Resolver {
+// NewResolver constructs a resolver associated with the given root domain.
+func NewResolver(logf logger.Logf, rootDomain string) *Resolver {
 	r := &Resolver{
-		logf: logf,
-		ip:   defaultIP,
-		port: defaultPort,
+		logf:      logger.WithPrefix(logf, "tsdns: "),
+		queue:     make(chan Packet, queueSize),
+		responses: make(chan Packet),
+		errors:    make(chan error),
+		closed:    make(chan struct{}),
+		// Conform to the name format dnsmessage uses (trailing period, bytes).
+		rootDomain: []byte(rootDomain + "."),
+		dialer:     netns.NewDialer(),
 	}
 
 	return r
 }
 
-// AcceptsPacket determines if the given packet is
-// directed to this resolver (by ip and port).
-// We also require that UDP be used to simplify things for now.
-func (r *Resolver) AcceptsPacket(in *packet.ParsedPacket) bool {
-	return in.DstIP == r.ip && in.DstPort == r.port && in.IPProto == packet.UDP
+func (r *Resolver) Start() {
+	// TODO(dmytro): spawn more than one goroutine? They block on delegation.
+	r.pollGroup.Add(1)
+	go r.poll()
 }
 
-// SetMap sets the resolver's DNS map.
+// Close shuts down the resolver and ensures poll goroutines have exited.
+// The Resolver cannot be used again after Close is called.
+func (r *Resolver) Close() {
+	select {
+	case <-r.closed:
+		return
+	default:
+		// continue
+	}
+	close(r.closed)
+	r.pollGroup.Wait()
+}
+
+// SetMap sets the resolver's DNS map, taking ownership of it.
 func (r *Resolver) SetMap(m *Map) {
 	r.mu.Lock()
 	r.dnsMap = m
 	r.mu.Unlock()
 }
 
-// Resolve maps a given domain name to the IP address of the host that owns it.
-func (r *Resolver) Resolve(domain string) (netaddr.IP, dns.RCode, error) {
-	// If not a subdomain of ipn.dev, then we must refuse this query.
-	// We do this before checking the map to distinguish beween nonexistent domains
-	// and misdirected queries.
-	if !strings.HasSuffix(domain, ".ipn.dev") {
-		return netaddr.IP{}, dns.RCodeRefused, errNotOurName
-	}
-
+// SetUpstreamNameservers sets the addresses of the resolver's
+// upstream nameservers, taking ownership of the argument.
+// The addresses should be strings of the form ip:port,
+// matching what Dial("udp", addr) expects as addr.
+func (r *Resolver) SetNameservers(nameservers []string) {
 	r.mu.Lock()
+	r.nameservers = nameservers
+	r.mu.Unlock()
+}
+
+// EnqueueRequest places the given DNS request in the resolver's queue.
+// It takes ownership of the payload and does not block.
+// If the queue is full, the request will be dropped and an error will be returned.
+func (r *Resolver) EnqueueRequest(request Packet) error {
+	select {
+	case r.queue <- request:
+		return nil
+	default:
+		return errFullQueue
+	}
+}
+
+// NextResponse returns a DNS response to a previously enqueued request.
+// It blocks until a response is available and gives up ownership of the response payload.
+func (r *Resolver) NextResponse() (Packet, error) {
+	select {
+	case resp := <-r.responses:
+		return resp, nil
+	case err := <-r.errors:
+		return Packet{}, err
+	case <-r.closed:
+		return Packet{}, ErrClosed
+	}
+}
+
+// Resolve maps a given domain name to the IP address of the host that owns it.
+// The domain name must not have a trailing period.
+func (r *Resolver) Resolve(domain string) (netaddr.IP, dns.RCode, error) {
+	r.mu.RLock()
 	if r.dnsMap == nil {
-		r.mu.Unlock()
+		r.mu.RUnlock()
 		return netaddr.IP{}, dns.RCodeServerFailure, errMapNotSet
 	}
 	addr, found := r.dnsMap.domainToIP[domain]
-	r.mu.Unlock()
+	r.mu.RUnlock()
 
 	if !found {
-		return netaddr.IP{}, dns.RCodeNameError, errNoSuchDomain
+		return netaddr.IP{}, dns.RCodeNameError, nil
 	}
 	return addr, dns.RCodeSuccess, nil
 }
 
+func (r *Resolver) poll() {
+	defer r.pollGroup.Done()
+
+	var (
+		packet Packet
+		err    error
+	)
+	for {
+		select {
+		case packet = <-r.queue:
+			// continue
+		case <-r.closed:
+			return
+		}
+
+		packet.Payload, err = r.respond(packet.Payload)
+		if err != nil {
+			select {
+			case r.errors <- err:
+				// continue
+			case <-r.closed:
+				return
+			}
+		} else {
+			select {
+			case r.responses <- packet:
+				// continue
+			case <-r.closed:
+				return
+			}
+		}
+	}
+}
+
+// queryServer obtains a DNS response by querying the given server.
+func (r *Resolver) queryServer(ctx context.Context, server string, query []byte) ([]byte, error) {
+	conn, err := r.dialer.DialContext(ctx, "udp", server)
+	if err != nil {
+		return nil, err
+	}
+	defer conn.Close()
+
+	// Interrupt the current operation when the context is cancelled.
+	go func() {
+		<-ctx.Done()
+		conn.SetDeadline(time.Unix(1, 0))
+	}()
+
+	_, err = conn.Write(query)
+	if err != nil {
+		return nil, err
+	}
+
+	out := make([]byte, maxResponseSize)
+	n, err := conn.Read(out)
+	if err != nil {
+		return nil, err
+	}
+
+	return out[:n], nil
+}
+
+// delegate forwards the query to all upstream nameservers and returns the first response.
+func (r *Resolver) delegate(query []byte) ([]byte, error) {
+	r.mu.RLock()
+	nameservers := r.nameservers
+	r.mu.RUnlock()
+
+	if len(nameservers) == 0 {
+		return nil, errAllFailed
+	}
+
+	ctx, cancel := context.WithTimeout(context.Background(), delegateTimeout)
+	defer cancel()
+
+	// Common case, don't spawn goroutines.
+	if len(nameservers) == 1 {
+		return r.queryServer(ctx, nameservers[0], query)
+	}
+
+	datach := make(chan []byte)
+	for _, server := range nameservers {
+		go func(s string) {
+			resp, err := r.queryServer(ctx, s, query)
+			// Only print errors not due to cancelation after first response.
+			if err != nil && ctx.Err() != context.Canceled {
+				r.logf("querying %s: %v", s, err)
+			}
+
+			datach <- resp
+		}(server)
+	}
+
+	var response []byte
+	for range nameservers {
+		cur := <-datach
+		if cur != nil && response == nil {
+			// Received first successful response
+			response = cur
+			cancel()
+		}
+	}
+
+	if response == nil {
+		return nil, errAllFailed
+	}
+	return response, nil
+}
+
 type response struct {
-	Header         dns.Header
-	ResourceHeader dns.ResourceHeader
-	Question       dns.Question
-	// TODO(dmytro): support IPv6.
-	IP netaddr.IP
+	Header   dns.Header
+	Question dns.Question
+	Name     string
+	IP       netaddr.IP
 }
 
 // parseQuery parses the query in given packet into a response struct.
-func (r *Resolver) parseQuery(query *packet.ParsedPacket, resp *response) error {
+func (r *Resolver) parseQuery(query []byte, resp *response) error {
 	var parser dns.Parser
 	var err error
 
-	resp.Header, err = parser.Start(query.Payload())
+	resp.Header, err = parser.Start(query)
 	if err != nil {
 		return err
 	}
@@ -145,146 +336,123 @@ func (r *Resolver) parseQuery(query *packet.ParsedPacket, resp *response) error
 	return nil
 }
 
-// makeResponse resolves the question stored in resp and sets the answer fields.
-func (r *Resolver) makeResponse(resp *response) error {
-	var err error
-
-	name := resp.Question.Name.String()
-	if len(name) > 0 {
-		name = name[:len(name)-1]
-	}
-
-	if resp.Question.Type == dns.TypeA {
-		// Remove final dot from name: *.ipn.dev. -> *.ipn.dev
-		resp.IP, resp.Header.RCode, err = r.Resolve(name)
-	} else {
-		resp.Header.RCode = dns.RCodeNotImplemented
-		err = errNotImplemented
-	}
-
-	return err
-}
-
-// marshalAnswer serializes the answer record into an active builder.
+// marshalARecord serializes an A record into an active builder.
 // The caller may continue using the builder following the call.
-func marshalAnswer(resp *response, builder *dns.Builder) error {
+func marshalARecord(name dns.Name, ip netaddr.IP, builder *dns.Builder) error {
 	var answer dns.AResource
 
-	err := builder.StartAnswers()
-	if err != nil {
-		return err
-	}
-
 	answerHeader := dns.ResourceHeader{
-		Name:  resp.Question.Name,
+		Name:  name,
 		Type:  dns.TypeA,
 		Class: dns.ClassINET,
 		TTL:   uint32(defaultTTL / time.Second),
 	}
-	ip := resp.IP.As16()
-	copy(answer.A[:], ip[12:])
+	ipbytes := ip.As4()
+	copy(answer.A[:], ipbytes[:])
 	return builder.AResource(answerHeader, answer)
 }
 
-// marshalResponse serializes the DNS response into an active builder.
+// marshalAAAARecord serializes an AAAA record into an active builder.
 // The caller may continue using the builder following the call.
-func marshalResponse(resp *response, builder *dns.Builder) error {
-	err := builder.StartQuestions()
-	if err != nil {
-		return err
-	}
+func marshalAAAARecord(name dns.Name, ip netaddr.IP, builder *dns.Builder) error {
+	var answer dns.AAAAResource
 
-	err = builder.Question(resp.Question)
-	if err != nil {
-		return err
+	answerHeader := dns.ResourceHeader{
+		Name:  name,
+		Type:  dns.TypeAAAA,
+		Class: dns.ClassINET,
+		TTL:   uint32(defaultTTL / time.Second),
 	}
-
-	if resp.Header.RCode == dns.RCodeSuccess {
-		err = marshalAnswer(resp, builder)
-		if err != nil {
-			return err
-		}
-	}
-
-	return nil
+	ipbytes := ip.As16()
+	copy(answer.AAAA[:], ipbytes[:])
+	return builder.AAAAResource(answerHeader, answer)
 }
 
-// marshalReponsePacket marshals a full DNS packet (including headers)
-// representing resp, which is a response to query, into buf.
-// It returns buf trimmed to the length of the response packet.
-func marshalResponsePacket(query *packet.ParsedPacket, resp *response, buf []byte) ([]byte, error) {
-	udpHeader := query.UDPHeader()
-	udpHeader.ToResponse()
-	offset := udpHeader.Len()
-
+// marshalResponse serializes the DNS response into a new buffer.
+func marshalResponse(resp *response) ([]byte, error) {
 	resp.Header.Response = true
 	resp.Header.Authoritative = true
 	if resp.Header.RecursionDesired {
 		resp.Header.RecursionAvailable = true
 	}
 
-	// dns.Builder appends to the passed buffer (without reallocation when possible),
-	// so we pass in a zero-length slice starting at the point it should start writing.
-	builder := dns.NewBuilder(buf[offset:offset], resp.Header)
+	builder := dns.NewBuilder(nil, resp.Header)
 
-	err := marshalResponse(resp, &builder)
+	err := builder.StartQuestions()
 	if err != nil {
 		return nil, err
 	}
 
-	// rbuf is the response slice with the correct length starting at offset.
-	rbuf, err := builder.Finish()
+	err = builder.Question(resp.Question)
 	if err != nil {
 		return nil, err
 	}
 
-	end := offset + len(rbuf)
-	err = udpHeader.Marshal(buf[:end])
+	// Only successful responses contain answers.
+	if resp.Header.RCode != dns.RCodeSuccess {
+		return builder.Finish()
+	}
+
+	err = builder.StartAnswers()
 	if err != nil {
 		return nil, err
 	}
 
-	return buf[:end], nil
+	if resp.IP.Is4() {
+		err = marshalARecord(resp.Question.Name, resp.IP, &builder)
+	} else {
+		err = marshalAAAARecord(resp.Question.Name, resp.IP, &builder)
+	}
+	if err != nil {
+		return nil, err
+	}
+
+	return builder.Finish()
 }
 
-// Respond writes a response to query into buf and returns buf trimmed to the response length.
-// It is assumed that r.AcceptsPacket(query) is true.
-func (r *Resolver) Respond(query *packet.ParsedPacket, buf []byte) ([]byte, error) {
-	var resp response
-	var err error
+// respond returns a DNS response to query.
+func (r *Resolver) respond(query []byte) ([]byte, error) {
+	resp := new(response)
 
-	// 0. Verify that contract is upheld.
-	if !r.AcceptsPacket(query) {
-		return nil, errNotOurQuery
-	}
-	// A DNS response is at least as long as the query
-	if len(buf) < len(query.Buffer()) {
-		return nil, errSmallBuffer
-	}
-
-	// 1. Parse query packet.
-	err = r.parseQuery(query, &resp)
+	// ParseQuery is sufficiently fast to run on every DNS packet.
+	// This is considerably simpler than extracting the name by hand
+	// to shave off microseconds in case of delegation.
+	err := r.parseQuery(query, resp)
 	// We will not return this error: it is the sender's fault.
 	if err != nil {
-		r.logf("tsdns: error during query parsing: %v", err)
+		r.logf("parsing query: %v", err)
 		resp.Header.RCode = dns.RCodeFormatError
-		return marshalResponsePacket(query, &resp, buf)
+		return marshalResponse(resp)
 	}
 
-	// 2. Service the query.
-	err = r.makeResponse(&resp)
+	// Delegate only when not a subdomain of rootDomain.
+	// We do this on bytes because Name.String() allocates.
+	rawName := resp.Question.Name.Data[:resp.Question.Name.Length]
+	if !bytes.HasSuffix(rawName, r.rootDomain) {
+		out, err := r.delegate(query)
+		if err != nil {
+			r.logf("delegating: %v", err)
+			resp.Header.RCode = dns.RCodeServerFailure
+			return marshalResponse(resp)
+		}
+		return out, nil
+	}
+
+	switch resp.Question.Type {
+	case dns.TypeA, dns.TypeAAAA, dns.TypeALL:
+		domain := resp.Question.Name.String()
+		// Strip off the trailing period.
+		// This is safe: Name is guaranteed to have a trailing period by construction.
+		domain = domain[:len(domain)-1]
+		resp.IP, resp.Header.RCode, err = r.Resolve(domain)
+	default:
+		resp.Header.RCode = dns.RCodeNotImplemented
+		err = errNotImplemented
+	}
 	// We will not return this error: it is the sender's fault.
 	if err != nil {
-		r.logf("tsdns: error during name resolution: %v", err)
-		return marshalResponsePacket(query, &resp, buf)
-	}
-	// For now, we require IPv4 in all cases.
-	// If we somehow came up with a non-IPv4 address, it's our fault.
-	if !resp.IP.Is4() {
-		resp.Header.RCode = dns.RCodeServerFailure
-		r.logf("tsdns: error during name resolution: IPv6 address: %v", resp.IP)
+		r.logf("resolving: %v", err)
 	}
 
-	// 3. Serialize the response.
-	return marshalResponsePacket(query, &resp, buf)
+	return marshalResponse(resp)
 }

wgengine/tsdns/tsdns_server_test.go

@@ -0,0 +1,77 @@
+// Copyright (c) 2020 Tailscale Inc & AUTHORS All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package tsdns
+
+import (
+	"github.com/miekg/dns"
+	"inet.af/netaddr"
+)
+
+// This file exists to isolate the test infrastructure
+// that depends on github.com/miekg/dns
+// from the rest, which only depends on dnsmessage.
+
+var dnsHandleFunc = dns.HandleFunc
+
+// resolveToIP returns a handler function which responds
+// to queries of type A it receives with an A record containing ipv4
+// and to queries of type AAAA with an AAAA records containing ipv6.
+func resolveToIP(ipv4, ipv6 netaddr.IP) dns.HandlerFunc {
+	return func(w dns.ResponseWriter, req *dns.Msg) {
+		m := new(dns.Msg)
+		m.SetReply(req)
+
+		if len(req.Question) != 1 {
+			panic("not a single-question request")
+		}
+		question := req.Question[0]
+
+		var ans dns.RR
+		if question.Qtype == dns.TypeA {
+			ans = &dns.A{
+				Hdr: dns.RR_Header{
+					Name:   question.Name,
+					Rrtype: dns.TypeA,
+					Class:  dns.ClassINET,
+				},
+				A: ipv4.IPAddr().IP,
+			}
+		} else {
+			ans = &dns.AAAA{
+				Hdr: dns.RR_Header{
+					Name:   question.Name,
+					Rrtype: dns.TypeAAAA,
+					Class:  dns.ClassINET,
+				},
+				AAAA: ipv6.IPAddr().IP,
+			}
+		}
+		m.Answer = append(m.Answer, ans)
+
+		w.WriteMsg(m)
+	}
+}
+
+func resolveToNXDOMAIN(w dns.ResponseWriter, req *dns.Msg) {
+	m := new(dns.Msg)
+	m.SetRcode(req, dns.RcodeNameError)
+	w.WriteMsg(m)
+}
+
+func serveDNS(addr string) (*dns.Server, chan error) {
+	server := &dns.Server{Addr: addr, Net: "udp"}
+
+	waitch := make(chan struct{})
+	server.NotifyStartedFunc = func() { close(waitch) }
+
+	errch := make(chan error, 1)
+	go func() {
+		errch <- server.ListenAndServe()
+		close(errch)
+	}()
+
+	<-waitch
+	return server, errch
+}

wgengine/tsdns/tsdns_test.go

@@ -6,113 +6,202 @@ package tsdns
 
 import (
 	"bytes"
+	"errors"
 	"sync"
 	"testing"
 
 	dns "golang.org/x/net/dns/dnsmessage"
 	"inet.af/netaddr"
-	"tailscale.com/wgengine/packet"
 )
 
+var testipv4 = netaddr.IPv4(1, 2, 3, 4)
+var testipv6 = netaddr.IPv6Raw([16]byte{
+	0x00, 0x01, 0x02, 0x03,
+	0x04, 0x05, 0x06, 0x07,
+	0x08, 0x09, 0x0a, 0x0b,
+	0x0c, 0x0d, 0x0e, 0x0f,
+})
+
 var dnsMap = &Map{
 	domainToIP: map[string]netaddr.IP{
-		"test1.ipn.dev": netaddr.IPv4(1, 2, 3, 4),
-		"test2.ipn.dev": netaddr.IPv4(5, 6, 7, 8),
+		"test1.ipn.dev": testipv4,
+		"test2.ipn.dev": testipv6,
 	},
 }
 
-func dnspacket(srcip, dstip packet.IP, domain string, tp dns.Type, response bool) *packet.ParsedPacket {
-	dnsHeader := dns.Header{Response: response}
+func dnspacket(domain string, tp dns.Type) []byte {
+	var dnsHeader dns.Header
 	question := dns.Question{
 		Name:  dns.MustNewName(domain),
 		Type:  tp,
 		Class: dns.ClassINET,
 	}
-	udpHeader := &packet.UDPHeader{
-		IPHeader: packet.IPHeader{
-			SrcIP:   srcip,
-			DstIP:   dstip,
-			IPProto: packet.UDP,
-		},
-		SrcPort: 1234,
-		DstPort: 53,
-	}
 
 	builder := dns.NewBuilder(nil, dnsHeader)
 	builder.StartQuestions()
 	builder.Question(question)
 	payload, _ := builder.Finish()
 
-	buf := packet.Generate(udpHeader, payload)
-
-	pp := new(packet.ParsedPacket)
-	pp.Decode(buf)
-
-	return pp
+	return payload
 }
 
-func TestAcceptsPacket(t *testing.T) {
-	r := NewResolver(t.Logf)
-	r.SetMap(dnsMap)
+func extractipcode(response []byte) (netaddr.IP, dns.RCode, error) {
+	var ip netaddr.IP
+	var parser dns.Parser
 
-	src := packet.IP(0x64656667) // 100.101.102.103
-	dst := packet.IP(0x64646464) // 100.100.100.100
-	tests := []struct {
-		name    string
-		request *packet.ParsedPacket
-		want    bool
-	}{
-		{"valid", dnspacket(src, dst, "test1.ipn.dev.", dns.TypeA, false), true},
-		{"invalid", dnspacket(dst, src, "test1.ipn.dev.", dns.TypeA, false), false},
+	h, err := parser.Start(response)
+	if err != nil {
+		return ip, 0, err
 	}
 
-	for _, tt := range tests {
-		t.Run(tt.name, func(t *testing.T) {
-			accepts := r.AcceptsPacket(tt.request)
-			if accepts != tt.want {
-				t.Errorf("accepts = %v; want %v", accepts, tt.want)
-			}
-		})
+	if !h.Response {
+		return ip, 0, errors.New("not a response")
 	}
+	if h.RCode != dns.RCodeSuccess {
+		return ip, h.RCode, nil
+	}
+
+	err = parser.SkipAllQuestions()
+	if err != nil {
+		return ip, 0, err
+	}
+
+	ah, err := parser.AnswerHeader()
+	if err != nil {
+		return ip, 0, err
+	}
+	switch ah.Type {
+	case dns.TypeA:
+		res, err := parser.AResource()
+		if err != nil {
+			return ip, 0, err
+		}
+		ip = netaddr.IPv4(res.A[0], res.A[1], res.A[2], res.A[3])
+	case dns.TypeAAAA:
+		res, err := parser.AAAAResource()
+		if err != nil {
+			return ip, 0, err
+		}
+		ip = netaddr.IPv6Raw(res.AAAA)
+	default:
+		return ip, 0, errors.New("type not in {A, AAAA}")
+	}
+
+	return ip, h.RCode, nil
+}
+
+func syncRespond(r *Resolver, query []byte) ([]byte, error) {
+	request := Packet{Payload: query}
+	r.EnqueueRequest(request)
+	resp, err := r.NextResponse()
+	return resp.Payload, err
 }
 
 func TestResolve(t *testing.T) {
-	r := NewResolver(t.Logf)
+	r := NewResolver(t.Logf, "ipn.dev")
 	r.SetMap(dnsMap)
+	r.Start()
 
 	tests := []struct {
 		name   string
 		domain string
 		ip     netaddr.IP
 		code   dns.RCode
-		iserr  bool
 	}{
-		{"valid", "test1.ipn.dev", netaddr.IPv4(1, 2, 3, 4), dns.RCodeSuccess, false},
-		{"nxdomain", "test3.ipn.dev", netaddr.IP{}, dns.RCodeNameError, true},
-		{"not our domain", "google.com", netaddr.IP{}, dns.RCodeRefused, true},
+		{"ipv4", "test1.ipn.dev", testipv4, dns.RCodeSuccess},
+		{"ipv6", "test2.ipn.dev", testipv6, dns.RCodeSuccess},
+		{"nxdomain", "test3.ipn.dev", netaddr.IP{}, dns.RCodeNameError},
+		{"foreign domain", "google.com", netaddr.IP{}, dns.RCodeNameError},
 	}
 
 	for _, tt := range tests {
 		t.Run(tt.name, func(t *testing.T) {
 			ip, code, err := r.Resolve(tt.domain)
-			if err != nil && !tt.iserr {
+			if err != nil {
 				t.Errorf("err = %v; want nil", err)
-			} else if err == nil && tt.iserr {
-				t.Errorf("err = nil; want non-nil")
 			}
 			if code != tt.code {
 				t.Errorf("code = %v; want %v", code, tt.code)
 			}
 			// Only check ip for non-err
-			if !tt.iserr && ip != tt.ip {
+			if ip != tt.ip {
 				t.Errorf("ip = %v; want %v", ip, tt.ip)
 			}
 		})
 	}
 }
 
-func TestConcurrentSet(t *testing.T) {
-	r := NewResolver(t.Logf)
+func TestDelegate(t *testing.T) {
+	dnsHandleFunc("test.site.", resolveToIP(testipv4, testipv6))
+	dnsHandleFunc("nxdomain.site.", resolveToNXDOMAIN)
+
+	v4server, v4errch := serveDNS("127.0.0.1:0")
+	v6server, v6errch := serveDNS("[::1]:0")
+
+	defer func() {
+		if err := <-v4errch; err != nil {
+			t.Errorf("v4 server error: %v", err)
+		}
+		if err := <-v6errch; err != nil {
+			t.Errorf("v6 server error: %v", err)
+		}
+	}()
+	if v4server != nil {
+		defer v4server.Shutdown()
+	}
+	if v6server != nil {
+		defer v6server.Shutdown()
+	}
+
+	if v4server == nil || v6server == nil {
+		// There is an error in at least one of the channels
+		// and we cannot proceed; return to see it.
+		return
+	}
+
+	r := NewResolver(t.Logf, "ipn.dev")
+	r.SetNameservers([]string{
+		v4server.PacketConn.LocalAddr().String(),
+		v6server.PacketConn.LocalAddr().String(),
+	})
+	r.Start()
+
+	tests := []struct {
+		name  string
+		query []byte
+		ip    netaddr.IP
+		code  dns.RCode
+	}{
+		{"ipv4", dnspacket("test.site.", dns.TypeA), testipv4, dns.RCodeSuccess},
+		{"ipv6", dnspacket("test.site.", dns.TypeAAAA), testipv6, dns.RCodeSuccess},
+		{"nxdomain", dnspacket("nxdomain.site.", dns.TypeA), netaddr.IP{}, dns.RCodeNameError},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			resp, err := syncRespond(r, tt.query)
+			if err != nil {
+				t.Errorf("err = %v; want nil", err)
+				return
+			}
+			ip, code, err := extractipcode(resp)
+			if err != nil {
+				t.Errorf("extract: err = %v; want nil (in %x)", err, resp)
+				return
+			}
+			if code != tt.code {
+				t.Errorf("code = %v; want %v", code, tt.code)
+			}
+			if ip != tt.ip {
+				t.Errorf("ip = %v; want %v", ip, tt.ip)
+			}
+		})
+	}
+}
+
+func TestConcurrentSetMap(t *testing.T) {
+	r := NewResolver(t.Logf, "ipn.dev")
+	r.Start()
 
 	// This is purely to ensure that Resolve does not race with SetMap.
 	var wg sync.WaitGroup
@@ -128,16 +217,26 @@ func TestConcurrentSet(t *testing.T) {
 	wg.Wait()
 }
 
-var validResponse = []byte{
-	// IP header
-	0x45, 0x00, 0x00, 0x58, 0xff, 0xff, 0x00, 0x00, 0x40, 0x11, 0xe7, 0x00,
-	// Source IP
-	0x64, 0x64, 0x64, 0x64,
-	// Destination IP
-	0x64, 0x65, 0x66, 0x67,
-	// UDP header
-	0x00, 0x35, 0x04, 0xd2, 0x00, 0x44, 0x53, 0xdd,
-	// DNS payload
+func TestConcurrentSetNameservers(t *testing.T) {
+	r := NewResolver(t.Logf, "ipn.dev")
+	r.Start()
+	packet := dnspacket("google.com.", dns.TypeA)
+
+	// This is purely to ensure that delegation does not race with SetNameservers.
+	var wg sync.WaitGroup
+	wg.Add(2)
+	go func() {
+		defer wg.Done()
+		r.SetNameservers([]string{"9.9.9.9:53"})
+	}()
+	go func() {
+		defer wg.Done()
+		syncRespond(r, packet)
+	}()
+	wg.Wait()
+}
+
+var validIPv4Response = []byte{
 	0x00, 0x00, // transaction id: 0
 	0x84, 0x00, // flags: response, authoritative, no error
 	0x00, 0x01, // one question
@@ -154,16 +253,25 @@ var validResponse = []byte{
 	0x01, 0x02, 0x03, 0x04, // A: 1.2.3.4
 }
 
+var validIPv6Response = []byte{
+	0x00, 0x00, // transaction id: 0
+	0x84, 0x00, // flags: response, authoritative, no error
+	0x00, 0x01, // one question
+	0x00, 0x01, // one answer
+	0x00, 0x00, 0x00, 0x00, // no authority or additional RRs
+	// Question:
+	0x05, 0x74, 0x65, 0x73, 0x74, 0x32, 0x03, 0x69, 0x70, 0x6e, 0x03, 0x64, 0x65, 0x76, 0x00, // name
+	0x00, 0x1c, 0x00, 0x01, // type AAAA, class IN
+	// Answer:
+	0x05, 0x74, 0x65, 0x73, 0x74, 0x32, 0x03, 0x69, 0x70, 0x6e, 0x03, 0x64, 0x65, 0x76, 0x00, // name
+	0x00, 0x1c, 0x00, 0x01, // type AAAA, class IN
+	0x00, 0x00, 0x02, 0x58, // TTL: 600
+	0x00, 0x10, // length: 16 bytes
+	// AAAA: 0001:0203:0405:0607:0809:0A0B:0C0D:0E0F
+	0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0x09, 0x0a, 0xb, 0xc, 0xd, 0xe, 0xf,
+}
+
 var nxdomainResponse = []byte{
-	// IP header
-	0x45, 0x00, 0x00, 0x3b, 0xff, 0xff, 0x00, 0x00, 0x40, 0x11, 0xe7, 0x1d,
-	// Source IP
-	0x64, 0x64, 0x64, 0x64,
-	// Destination IP
-	0x64, 0x65, 0x66, 0x67,
-	// UDP header
-	0x00, 0x35, 0x04, 0xd2, 0x00, 0x27, 0x25, 0x33,
-	// DNS payload
 	0x00, 0x00, // transaction id: 0
 	0x84, 0x03, // flags: response, authoritative, error: nxdomain
 	0x00, 0x01, // one question
@@ -175,25 +283,24 @@ var nxdomainResponse = []byte{
 }
 
 func TestFull(t *testing.T) {
-	r := NewResolver(t.Logf)
+	r := NewResolver(t.Logf, "ipn.dev")
 	r.SetMap(dnsMap)
+	r.Start()
 
-	src := packet.IP(0x64656667) // 100.101.102.103
-	dst := packet.IP(0x64646464) // 100.100.100.100
 	// One full packet and one error packet
 	tests := []struct {
 		name     string
-		request  *packet.ParsedPacket
+		request  []byte
 		response []byte
 	}{
-		{"valid", dnspacket(src, dst, "test1.ipn.dev.", dns.TypeA, false), validResponse},
-		{"error", dnspacket(src, dst, "test3.ipn.dev.", dns.TypeA, false), nxdomainResponse},
+		{"ipv4", dnspacket("test1.ipn.dev.", dns.TypeA), validIPv4Response},
+		{"ipv6", dnspacket("test2.ipn.dev.", dns.TypeAAAA), validIPv6Response},
+		{"error", dnspacket("test3.ipn.dev.", dns.TypeA), nxdomainResponse},
 	}
 
 	for _, tt := range tests {
 		t.Run(tt.name, func(t *testing.T) {
-			buf := make([]byte, 512)
-			response, err := r.Respond(tt.request, buf)
+			response, err := syncRespond(r, tt.request)
 			if err != nil {
 				t.Errorf("err = %v; want nil", err)
 			}
@@ -205,43 +312,41 @@ func TestFull(t *testing.T) {
 }
 
 func TestAllocs(t *testing.T) {
-	r := NewResolver(t.Logf)
+	r := NewResolver(t.Logf, "ipn.dev")
 	r.SetMap(dnsMap)
+	r.Start()
 
-	src := packet.IP(0x64656667) // 100.101.102.103
-	dst := packet.IP(0x64646464) // 100.100.100.100
-	query := dnspacket(src, dst, "test1.ipn.dev.", dns.TypeA, false)
+	// It is seemingly pointless to test allocs in the delegate path,
+	// as dialer.Dial -> Read -> Write alone comprise 12 allocs.
+	query := dnspacket("test1.ipn.dev.", dns.TypeA)
 
-	buf := make([]byte, 512)
 	allocs := testing.AllocsPerRun(100, func() {
-		r.Respond(query, buf)
+		syncRespond(r, query)
 	})
 
-	if allocs > 0 {
-		t.Errorf("allocs = %v; want 0", allocs)
+	if allocs > 1 {
+		t.Errorf("allocs = %v; want 1", allocs)
 	}
 }
 
 func BenchmarkFull(b *testing.B) {
-	r := NewResolver(b.Logf)
+	r := NewResolver(b.Logf, "ipn.dev")
 	r.SetMap(dnsMap)
+	r.Start()
 
-	src := packet.IP(0x64656667) // 100.101.102.103
-	dst := packet.IP(0x64646464) // 100.100.100.100
 	// One full packet and one error packet
 	tests := []struct {
 		name    string
-		request *packet.ParsedPacket
+		request []byte
 	}{
-		{"valid", dnspacket(src, dst, "test1.ipn.dev.", dns.TypeA, false)},
-		{"nxdomain", dnspacket(src, dst, "test3.ipn.dev.", dns.TypeA, false)},
+		{"valid", dnspacket("test1.ipn.dev.", dns.TypeA)},
+		{"nxdomain", dnspacket("test3.ipn.dev.", dns.TypeA)},
 	}
 
-	buf := make([]byte, 512)
 	for _, tt := range tests {
 		b.Run(tt.name, func(b *testing.B) {
 			for i := 0; i < b.N; i++ {
-				r.Respond(tt.request, buf)
+				syncRespond(r, tt.request)
 			}
 		})
 	}

wgengine/tstun/tun.go

@@ -45,6 +45,11 @@ var (
 	errOffsetTooSmall = errors.New("offset smaller than PacketStartOffset")
 )
 
+// parsedPacketPool holds a pool of ParsedPacket structs for use in filtering.
+// This is needed because escape analysis cannot see that parsed packets
+// do not escape through {Pre,Post}Filter{In,Out}.
+var parsedPacketPool = sync.Pool{New: func() interface{} { return new(packet.ParsedPacket) }}
+
 // FilterFunc is a packet-filtering function with access to the TUN device.
 // It must not hold onto the packet struct, as its backing storage will be reused.
 type FilterFunc func(*packet.ParsedPacket, *TUN) filter.Response
@@ -58,17 +63,16 @@ type TUN struct {
 	// tdev is the underlying TUN device.
 	tdev tun.Device
 
-	lastActivityAtomic int64 // unix seconds of last send or receive
+	_                  [4]byte // force 64-bit alignment of following field on 32-bit
+	lastActivityAtomic int64   // unix seconds of last send or receive
+
+	destIPActivity atomic.Value // of map[packet.IP]func()
 
 	// buffer stores the oldest unconsumed packet from tdev.
 	// It is made a static buffer in order to avoid allocations.
 	buffer [maxBufferSize]byte
 	// bufferConsumed synchronizes access to buffer (shared by Read and poll).
 	bufferConsumed chan struct{}
-	// parsedPacketPool holds a pool of ParsedPacket structs for use in filtering.
-	// This is needed because escape analysis cannot see that parsed packets
-	// do not escape through {Pre,Post}Filter{In,Out}.
-	parsedPacketPool sync.Pool // of *packet.ParsedPacket
 
 	// closed signals poll (by closing) when the device is closed.
 	closed chan struct{}
@@ -108,7 +112,7 @@ type TUN struct {
 
 func WrapTUN(logf logger.Logf, tdev tun.Device) *TUN {
 	tun := &TUN{
-		logf: logf,
+		logf: logger.WithPrefix(logf, "tstun: "),
 		tdev: tdev,
 		// bufferConsumed is conceptually a condition variable:
 		// a goroutine should not block when setting it, even with no listeners.
@@ -120,10 +124,6 @@ func WrapTUN(logf logger.Logf, tdev tun.Device) *TUN {
 		filterFlags: filter.LogAccepts | filter.LogDrops,
 	}
 
-	tun.parsedPacketPool.New = func() interface{} {
-		return new(packet.ParsedPacket)
-	}
-
 	go tun.poll()
 	// The buffer starts out consumed.
 	tun.bufferConsumed <- struct{}{}
@@ -131,6 +131,14 @@ func WrapTUN(logf logger.Logf, tdev tun.Device) *TUN {
 	return tun
 }
 
+// SetDestIPActivityFuncs sets a map of funcs to run per packet
+// destination (the map keys).
+//
+// The map ownership passes to the TUN. It must be non-nil.
+func (t *TUN) SetDestIPActivityFuncs(m map[packet.IP]func()) {
+	t.destIPActivity.Store(m)
+}
+
 func (t *TUN) Close() error {
 	select {
 	case <-t.closed:
@@ -206,10 +214,7 @@ func (t *TUN) poll() {
 	}
 }
 
-func (t *TUN) filterOut(buf []byte) filter.Response {
-	p := t.parsedPacketPool.Get().(*packet.ParsedPacket)
-	defer t.parsedPacketPool.Put(p)
-	p.Decode(buf)
+func (t *TUN) filterOut(p *packet.ParsedPacket) filter.Response {
 
 	if t.PreFilterOut != nil {
 		if t.PreFilterOut(p, t) == filter.Drop {
@@ -220,7 +225,6 @@ func (t *TUN) filterOut(buf []byte) filter.Response {
 	filt, _ := t.filter.Load().(*filter.Filter)
 
 	if filt == nil {
-		t.logf("tstun: warning: you forgot to use SetFilter()! Packet dropped.")
 		return filter.Drop
 	}
 
@@ -274,8 +278,18 @@ func (t *TUN) Read(buf []byte, offset int) (int, error) {
 		}
 	}
 
+	p := parsedPacketPool.Get().(*packet.ParsedPacket)
+	defer parsedPacketPool.Put(p)
+	p.Decode(buf[offset : offset+n])
+
+	if m, ok := t.destIPActivity.Load().(map[packet.IP]func()); ok {
+		if fn := m[p.DstIP]; fn != nil {
+			fn()
+		}
+	}
+
 	if !t.disableFilter {
-		response := t.filterOut(buf[offset : offset+n])
+		response := t.filterOut(p)
 		if response != filter.Accept {
 			// Wireguard considers read errors fatal; pretend nothing was read
 			return 0, nil
@@ -287,8 +301,8 @@ func (t *TUN) Read(buf []byte, offset int) (int, error) {
 }
 
 func (t *TUN) filterIn(buf []byte) filter.Response {
-	p := t.parsedPacketPool.Get().(*packet.ParsedPacket)
-	defer t.parsedPacketPool.Put(p)
+	p := parsedPacketPool.Get().(*packet.ParsedPacket)
+	defer parsedPacketPool.Put(p)
 	p.Decode(buf)
 
 	if t.PreFilterIn != nil {
@@ -300,7 +314,6 @@ func (t *TUN) filterIn(buf []byte) filter.Response {
 	filt, _ := t.filter.Load().(*filter.Filter)
 
 	if filt == nil {
-		t.logf("tstun: warning: you forgot to use SetFilter()! Packet dropped.")
 		return filter.Drop
 	}
 

wgengine/tstun/tun_test.go

@@ -29,17 +29,24 @@ func udp(src, dst packet.IP, sport, dport uint16) []byte {
 	return packet.Generate(header, []byte("udp_payload"))
 }
 
+func filterNet(ip, mask packet.IP) filter.Net {
+	return filter.Net{IP: ip, Mask: mask}
+}
+
 func nets(ips []packet.IP) []filter.Net {
 	out := make([]filter.Net, 0, len(ips))
 	for _, ip := range ips {
-		out = append(out, filter.Net{ip, filter.Netmask(32)})
+		out = append(out, filterNet(ip, filter.Netmask(32)))
 	}
 	return out
 }
 
 func ippr(ip packet.IP, start, end uint16) []filter.NetPortRange {
 	return []filter.NetPortRange{
-		filter.NetPortRange{filter.Net{ip, filter.Netmask(32)}, filter.PortRange{start, end}},
+		filter.NetPortRange{
+			Net:   filterNet(ip, filter.Netmask(32)),
+			Ports: filter.PortRange{First: start, Last: end},
+		},
 	}
 }
 
@@ -49,7 +56,7 @@ func setfilter(logf logger.Logf, tun *TUN) {
 		{Srcs: nets([]packet.IP{0x01020304}), Dsts: ippr(0x05060708, 98, 98)},
 	}
 	localNets := []filter.Net{
-		{packet.IP(0x01020304), filter.Netmask(16)},
+		filterNet(packet.IP(0x01020304), filter.Netmask(16)),
 	}
 	tun.SetFilter(filter.New(matches, localNets, nil, logf))
 }

wgengine/userspace.go

@@ -8,11 +8,11 @@ import (
 	"bufio"
 	"bytes"
 	"context"
+	"encoding/binary"
 	"errors"
 	"fmt"
 	"io"
 	"log"
-	"net"
 	"os"
 	"os/exec"
 	"runtime"
@@ -25,6 +25,7 @@ import (
 	"github.com/tailscale/wireguard-go/tun"
 	"github.com/tailscale/wireguard-go/wgcfg"
 	"go4.org/mem"
+	"inet.af/netaddr"
 	"tailscale.com/control/controlclient"
 	"tailscale.com/internal/deepprint"
 	"tailscale.com/ipn/ipnstate"
@@ -51,6 +52,23 @@ import (
 // discovery.
 const minimalMTU = 1280
 
+const (
+	magicDNSIP   = 0x64646464 // 100.100.100.100
+	magicDNSPort = 53
+)
+
+// magicDNSDomain is the parent domain for Tailscale nodes.
+const magicDNSDomain = "b.tailscale.net"
+
+// Lazy wireguard-go configuration parameters.
+const (
+	// lazyPeerIdleThreshold is the idle duration after
+	// which we remove a peer from the wireguard configuration.
+	// (This includes peers that have never been idle, which
+	// effectively have infinite idleness)
+	lazyPeerIdleThreshold = 5 * time.Minute
+)
+
 type userspaceEngine struct {
 	logf            logger.Logf
 	reqCh           chan struct{}
@@ -68,17 +86,21 @@ type userspaceEngine struct {
 	// incorrectly sent to us.
 	localAddrs atomic.Value // of map[packet.IP]bool
 
-	wgLock        sync.Mutex // serializes all wgdev operations; see lock order comment below
-	lastEngineSig string
-	lastRouterSig string
-	lastCfg       wgcfg.Config
+	wgLock              sync.Mutex // serializes all wgdev operations; see lock order comment below
+	lastCfgFull         wgcfg.Config
+	lastRouterSig       string // of router.Config
+	lastEngineSigFull   string // of full wireguard config
+	lastEngineSigTrim   string // of trimmed wireguard config
+	recvActivityAt      map[tailcfg.DiscoKey]time.Time
+	sentActivityAt      map[packet.IP]*int64 // value is atomic int64 of unixtime
+	destIPActivityFuncs map[packet.IP]func()
 
 	mu             sync.Mutex // guards following; see lock order comment below
 	closing        bool       // Close was called (even if we're still closing)
 	statusCallback StatusCallback
 	peerSequence   []wgcfg.Key
 	endpoints      []string
-	pingers        map[wgcfg.Key]*pinger
+	pingers        map[wgcfg.Key]*pinger // legacy pingers for pre-discovery peers
 	linkState      *interfaces.State
 
 	// Lock ordering: wgLock, then mu.
@@ -100,7 +122,7 @@ type EngineConfig struct {
 	// EchoRespondToAll determines whether ICMP Echo requests incoming from Tailscale peers
 	// will be intercepted and responded to, regardless of the source host.
 	EchoRespondToAll bool
-	// UseTailscaleDNS determines whether DNS requests for names of the form *.ipn.dev
+	// UseTailscaleDNS determines whether DNS requests for names of the form <mynode>.<mydomain>.<root>
 	// directed to the designated Taislcale DNS address (see wgengine/tsdns)
 	// will be intercepted and resolved by a tsdns.Resolver.
 	UseTailscaleDNS bool
@@ -174,7 +196,7 @@ func newUserspaceEngineAdvanced(conf EngineConfig) (_ Engine, reterr error) {
 		reqCh:           make(chan struct{}, 1),
 		waitCh:          make(chan struct{}),
 		tundev:          tstun.WrapTUN(logf, conf.TUN),
-		resolver:        tsdns.NewResolver(logf),
+		resolver:        tsdns.NewResolver(logf, magicDNSDomain),
 		useTailscaleDNS: conf.UseTailscaleDNS,
 		pingers:         make(map[wgcfg.Key]*pinger),
 	}
@@ -202,10 +224,11 @@ func newUserspaceEngineAdvanced(conf EngineConfig) (_ Engine, reterr error) {
 		e.RequestStatus()
 	}
 	magicsockOpts := magicsock.Options{
-		Logf:          logf,
-		Port:          conf.ListenPort,
-		EndpointsFunc: endpointsFn,
-		IdleFunc:      e.tundev.IdleDuration,
+		Logf:             logf,
+		Port:             conf.ListenPort,
+		EndpointsFunc:    endpointsFn,
+		IdleFunc:         e.tundev.IdleDuration,
+		NoteRecvActivity: e.noteReceiveActivity,
 	}
 	e.magicConn, err = magicsock.NewConn(magicsockOpts)
 	if err != nil {
@@ -224,7 +247,7 @@ func newUserspaceEngineAdvanced(conf EngineConfig) (_ Engine, reterr error) {
 
 	opts := &device.DeviceOptions{
 		Logger: &logger,
-		HandshakeDone: func(peerKey wgcfg.Key, allowedIPs []net.IPNet) {
+		HandshakeDone: func(peerKey wgcfg.Key, peer *device.Peer, deviceAllowedIPs *device.AllowedIPs) {
 			// Send an unsolicited status event every time a
 			// handshake completes. This makes sure our UI can
 			// update quickly as soon as it connects to a peer.
@@ -235,9 +258,18 @@ func newUserspaceEngineAdvanced(conf EngineConfig) (_ Engine, reterr error) {
 			// here.
 			go e.RequestStatus()
 
+			if e.magicConn.PeerHasDiscoKey(tailcfg.NodeKey(peerKey)) {
+				e.logf("wireguard handshake complete for %v", peerKey.ShortString())
+				// This is a modern peer with discovery support. No need to send pings.
+				return
+			}
+
+			e.logf("wireguard handshake complete for %v; sending legacy pings", peerKey.ShortString())
+
 			// Ping every single-IP that peer routes.
 			// These synthetic packets are used to traverse NATs.
 			var ips []wgcfg.IP
+			allowedIPs := deviceAllowedIPs.EntriesForPeer(peer)
 			for _, ipNet := range allowedIPs {
 				if ones, bits := ipNet.Mask.Size(); ones == bits && ones != 0 {
 					var ip wgcfg.IP
@@ -308,6 +340,9 @@ func newUserspaceEngineAdvanced(conf EngineConfig) (_ Engine, reterr error) {
 	e.linkMon.Start()
 	e.magicConn.Start()
 
+	e.resolver.Start()
+	go e.pollResolver()
+
 	return e, nil
 }
 
@@ -360,26 +395,59 @@ func (e *userspaceEngine) isLocalAddr(ip packet.IP) bool {
 
 // handleDNS is an outbound pre-filter resolving Tailscale domains.
 func (e *userspaceEngine) handleDNS(p *packet.ParsedPacket, t *tstun.TUN) filter.Response {
-	if e.resolver.AcceptsPacket(p) {
-		// TODO(dmytro): avoid this allocation without having tsdns know tstun quirks.
-		buf := make([]byte, tstun.MaxPacketSize)
-		offset := tstun.PacketStartOffset
-		response, err := e.resolver.Respond(p, buf[offset:])
-		if err != nil {
-			e.logf("DNS resolver error: %v", err)
-		} else {
-			t.InjectInboundDirect(buf[:offset+len(response)], offset)
+	if p.DstIP == magicDNSIP && p.DstPort == magicDNSPort && p.IPProto == packet.UDP {
+		request := tsdns.Packet{
+			Payload: p.Payload(),
+			Addr:    netaddr.IPPort{IP: p.SrcIP.Netaddr(), Port: p.SrcPort},
+		}
+		err := e.resolver.EnqueueRequest(request)
+		if err != nil {
+			e.logf("tsdns: enqueue: %v", err)
 		}
-		// We already handled it, stop.
 		return filter.Drop
 	}
 	return filter.Accept
 }
 
+// pollResolver reads responses from the DNS resolver and injects them inbound.
+func (e *userspaceEngine) pollResolver() {
+	for {
+		resp, err := e.resolver.NextResponse()
+		if err == tsdns.ErrClosed {
+			return
+		}
+		if err != nil {
+			e.logf("tsdns: error: %v", err)
+			continue
+		}
+
+		h := packet.UDPHeader{
+			IPHeader: packet.IPHeader{
+				SrcIP: packet.IP(magicDNSIP),
+				DstIP: packet.IPFromNetaddr(resp.Addr.IP),
+			},
+			SrcPort: magicDNSPort,
+			DstPort: resp.Addr.Port,
+		}
+		hlen := h.Len()
+
+		// TODO(dmytro): avoid this allocation without importing tstun quirks into tsdns.
+		const offset = tstun.PacketStartOffset
+		buf := make([]byte, offset+hlen+len(resp.Payload))
+		copy(buf[offset+hlen:], resp.Payload)
+		h.Marshal(buf[offset:])
+
+		e.tundev.InjectInboundDirect(buf, offset)
+	}
+}
+
 // pinger sends ping packets for a few seconds.
 //
 // These generated packets are used to ensure we trigger the spray logic in
 // the magicsock package for NAT traversal.
+//
+// These are only used with legacy peers (before 0.100.0) that don't
+// have advertised discovery keys.
 type pinger struct {
 	e      *userspaceEngine
 	done   chan struct{} // closed after shutdown (not the ctx.Done() chan)
@@ -452,13 +520,16 @@ func (p *pinger) run(ctx context.Context, peerKey wgcfg.Key, ips []wgcfg.IP, src
 //
 // These generated packets are used to ensure we trigger the spray logic in
 // the magicsock package for NAT traversal.
+//
+// This is only used with legacy peers (before 0.100.0) that don't
+// have advertised discovery keys.
 func (e *userspaceEngine) pinger(peerKey wgcfg.Key, ips []wgcfg.IP) {
 	e.logf("generating initial ping traffic to %s (%v)", peerKey.ShortString(), ips)
 	var srcIP packet.IP
 
 	e.wgLock.Lock()
-	if len(e.lastCfg.Addresses) > 0 {
-		srcIP = packet.NewIP(e.lastCfg.Addresses[0].IP.IP())
+	if len(e.lastCfgFull.Addresses) > 0 {
+		srcIP = packet.NewIP(e.lastCfgFull.Addresses[0].IP.IP())
 	}
 	e.wgLock.Unlock()
 
@@ -498,6 +569,198 @@ func updateSig(last *string, v interface{}) (changed bool) {
 	return false
 }
 
+// isTrimmablePeer reports whether p is a peer that we can trim out of the
+// network map.
+//
+// We can only trim peers that both a) support discovery (because we
+// know who they are when we receive their data and don't need to rely
+// on wireguard-go figuring it out) and b) for implementation
+// simplicity, have only one IP address (an IPv4 /32), which is the
+// common case for most peers. Subnet router nodes will just always be
+// created in the wireguard-go config.
+func isTrimmablePeer(p *wgcfg.Peer) bool {
+	if len(p.AllowedIPs) != 1 || len(p.Endpoints) != 1 {
+		return false
+	}
+	if !strings.HasSuffix(p.Endpoints[0].Host, ".disco.tailscale") {
+		return false
+	}
+	aip := p.AllowedIPs[0]
+	// TODO: IPv6 support, once we support IPv6 within the tunnel. In that case,
+	// len(p.AllowedIPs) probably will be more than 1.
+	if aip.Mask != 32 || !aip.IP.Is4() {
+		return false
+	}
+	return true
+}
+
+// noteReceiveActivity is called by magicsock when a packet has been received
+// by the peer using discovery key dk. Magicsock calls this no more than
+// every 10 seconds for a given peer.
+func (e *userspaceEngine) noteReceiveActivity(dk tailcfg.DiscoKey) {
+	e.wgLock.Lock()
+	defer e.wgLock.Unlock()
+
+	now := time.Now()
+	was, ok := e.recvActivityAt[dk]
+	if !ok {
+		// Not a trimmable peer we care about tracking. (See isTrimmablePeer)
+		return
+	}
+	e.recvActivityAt[dk] = now
+
+	// If the last activity time jumped a bunch (say, at least
+	// half the idle timeout) then see if we need to reprogram
+	// Wireguard. This could probably be just
+	// lazyPeerIdleThreshold without the divide by 2, but
+	// maybeReconfigWireguardLocked is cheap enough to call every
+	// couple minutes (just not on every packet).
+	if was.IsZero() || now.Sub(was) < -lazyPeerIdleThreshold/2 {
+		e.maybeReconfigWireguardLocked()
+	}
+}
+
+// isActiveSince reports whether the peer identified by (dk, ip) has
+// had a packet sent to or received from it since t.
+//
+// e.wgLock must be held.
+func (e *userspaceEngine) isActiveSince(dk tailcfg.DiscoKey, ip wgcfg.IP, t time.Time) bool {
+	if e.recvActivityAt[dk].After(t) {
+		return true
+	}
+	pip := packet.IP(binary.BigEndian.Uint32(ip.Addr[12:]))
+	timePtr, ok := e.sentActivityAt[pip]
+	if !ok {
+		return false
+	}
+	unixTime := atomic.LoadInt64(timePtr)
+	return unixTime >= t.Unix()
+}
+
+// discoKeyFromPeer returns the DiscoKey for a wireguard config's Peer.
+//
+// Invariant: isTrimmablePeer(p) == true, so it should have 1 endpoint with
+// Host of form "<64-hex-digits>.disco.tailscale". If invariant is violated,
+// we return the zero value.
+func discoKeyFromPeer(p *wgcfg.Peer) tailcfg.DiscoKey {
+	host := p.Endpoints[0].Host
+	if len(host) < 64 {
+		return tailcfg.DiscoKey{}
+	}
+	k, err := key.NewPublicFromHexMem(mem.S(host[:64]))
+	if err != nil {
+		return tailcfg.DiscoKey{}
+	}
+	return tailcfg.DiscoKey(k)
+}
+
+// e.wgLock must be held.
+func (e *userspaceEngine) maybeReconfigWireguardLocked() error {
+	full := e.lastCfgFull
+
+	// Compute a minimal config to pass to wireguard-go
+	// based on the full config. Prune off all the peers
+	// and only add the active ones back.
+	min := full
+	min.Peers = nil
+
+	// We'll only keep a peer around if it's been active in
+	// the past 5 minutes. That's more than WireGuard's key
+	// rotation time anyway so it's no harm if we remove it
+	// later if it's been inactive.
+	activeCutoff := time.Now().Add(-lazyPeerIdleThreshold)
+
+	// Not all peers can be trimmed from the network map (see
+	// isTrimmablePeer).  For those are are trimmable, keep track
+	// of their DiscoKey and Tailscale IPs.  These are the ones
+	// we'll need to install tracking hooks for to watch their
+	// send/receive activity.
+	trackDisco := make([]tailcfg.DiscoKey, 0, len(full.Peers))
+	trackIPs := make([]wgcfg.IP, 0, len(full.Peers))
+
+	for i := range full.Peers {
+		p := &full.Peers[i]
+		if !isTrimmablePeer(p) {
+			min.Peers = append(min.Peers, *p)
+			continue
+		}
+		tsIP := p.AllowedIPs[0].IP
+		dk := discoKeyFromPeer(p)
+		trackDisco = append(trackDisco, dk)
+		trackIPs = append(trackIPs, tsIP)
+		if e.isActiveSince(dk, tsIP, activeCutoff) {
+			min.Peers = append(min.Peers, *p)
+		}
+	}
+
+	if !updateSig(&e.lastEngineSigTrim, min) {
+		// No changes
+		return nil
+	}
+
+	e.updateActivityMapsLocked(trackDisco, trackIPs)
+
+	e.logf("wgengine: Reconfig: configuring userspace wireguard config (with %d/%d peers)", len(min.Peers), len(full.Peers))
+	if err := e.wgdev.Reconfig(&min); err != nil {
+		e.logf("wgdev.Reconfig: %v", err)
+		return err
+	}
+	return nil
+}
+
+// updateActivityMapsLocked updates the data structures used for tracking the activity
+// of wireguard peers that we might add/remove dynamically from the real config
+// as given to wireguard-go.
+//
+// e.wgLock must be held.
+func (e *userspaceEngine) updateActivityMapsLocked(trackDisco []tailcfg.DiscoKey, trackIPs []wgcfg.IP) {
+	// Generate the new map of which discokeys we want to track
+	// receive times for.
+	mr := map[tailcfg.DiscoKey]time.Time{} // TODO: only recreate this if set of keys changed
+	for _, dk := range trackDisco {
+		// Preserve old times in the new map, but also
+		// populate map entries for new trackDisco values with
+		// time.Time{} zero values. (Only entries in this map
+		// are tracked, so the Time zero values allow it to be
+		// tracked later)
+		mr[dk] = e.recvActivityAt[dk]
+	}
+	e.recvActivityAt = mr
+
+	oldTime := e.sentActivityAt
+	e.sentActivityAt = make(map[packet.IP]*int64, len(oldTime))
+	oldFunc := e.destIPActivityFuncs
+	e.destIPActivityFuncs = make(map[packet.IP]func(), len(oldFunc))
+
+	for _, wip := range trackIPs {
+		pip := packet.IP(binary.BigEndian.Uint32(wip.Addr[12:]))
+		timePtr := oldTime[pip]
+		if timePtr == nil {
+			timePtr = new(int64)
+		}
+		e.sentActivityAt[pip] = timePtr
+
+		fn := oldFunc[pip]
+		if fn == nil {
+			// This is the func that gets run on every outgoing packet for tracked IPs:
+			fn = func() {
+				now, old := time.Now().Unix(), atomic.LoadInt64(timePtr)
+				if old > now-10 {
+					return
+				}
+				atomic.StoreInt64(timePtr, now)
+				if old == 0 || (now-old) <= 60 {
+					e.wgLock.Lock()
+					defer e.wgLock.Unlock()
+					e.maybeReconfigWireguardLocked()
+				}
+			}
+		}
+		e.destIPActivityFuncs[pip] = fn
+	}
+	e.tundev.SetDestIPActivityFuncs(e.destIPActivityFuncs)
+}
+
 func (e *userspaceEngine) Reconfig(cfg *wgcfg.Config, routerCfg *router.Config) error {
 	if routerCfg == nil {
 		panic("routerCfg must not be nil")
@@ -509,8 +772,7 @@ func (e *userspaceEngine) Reconfig(cfg *wgcfg.Config, routerCfg *router.Config)
 		if !addr.IP.Is4() {
 			continue
 		}
-		bs := addr.IP.As16()
-		localAddrs[packet.NewIP(net.IP(bs[12:16]))] = true
+		localAddrs[packet.IPFromNetaddr(addr.IP)] = true
 	}
 	e.localAddrs.Store(localAddrs)
 
@@ -526,29 +788,31 @@ func (e *userspaceEngine) Reconfig(cfg *wgcfg.Config, routerCfg *router.Config)
 	}
 	e.mu.Unlock()
 
-	engineChanged := updateSig(&e.lastEngineSig, cfg)
+	// If the only nameserver is quad 100 (Magic DNS), set up the resolver appropriately.
+	if len(routerCfg.Nameservers) == 1 && routerCfg.Nameservers[0] == packet.IP(magicDNSIP).Netaddr() {
+		// TODO(dmytro): plumb dnsReadConfig here instead of hardcoding this.
+		e.resolver.SetNameservers([]string{"8.8.8.8:53"})
+		routerCfg.Domains = append([]string{magicDNSDomain}, routerCfg.Domains...)
+	}
+
+	engineChanged := updateSig(&e.lastEngineSigFull, cfg)
 	routerChanged := updateSig(&e.lastRouterSig, routerCfg)
 	if !engineChanged && !routerChanged {
 		return ErrNoChanges
 	}
-	e.lastCfg = cfg.Copy()
+	e.lastCfgFull = cfg.Copy()
 
-	if engineChanged {
-		e.logf("wgengine: Reconfig: configuring userspace wireguard config")
-		// Tell magicsock about the new (or initial) private key
-		// (which is needed by DERP) before wgdev gets it, as wgdev
-		// will start trying to handshake, which we want to be able to
-		// go over DERP.
-		if err := e.magicConn.SetPrivateKey(cfg.PrivateKey); err != nil {
-			e.logf("wgengine: Reconfig: SetPrivateKey: %v", err)
-		}
+	// Tell magicsock about the new (or initial) private key
+	// (which is needed by DERP) before wgdev gets it, as wgdev
+	// will start trying to handshake, which we want to be able to
+	// go over DERP.
+	if err := e.magicConn.SetPrivateKey(cfg.PrivateKey); err != nil {
+		e.logf("wgengine: Reconfig: SetPrivateKey: %v", err)
+	}
+	e.magicConn.UpdatePeers(peerSet)
 
-		if err := e.wgdev.Reconfig(cfg); err != nil {
-			e.logf("wgdev.Reconfig: %v", err)
-			return err
-		}
-
-		e.magicConn.UpdatePeers(peerSet)
+	if err := e.maybeReconfigWireguardLocked(); err != nil {
+		return err
 	}
 
 	if routerChanged {
@@ -618,7 +882,12 @@ func (e *userspaceEngine) getStatus() (*Status, error) {
 		bw := bufio.NewWriterSize(pw, lineLen)
 		// TODO(apenwarr): get rid of silly uapi stuff for in-process comms
 		// FIXME: get notified of status changes instead of polling.
-		if err := e.wgdev.IpcGetOperation(bw); err != nil {
+		filter := device.IPCGetFilter{
+			// The allowed_ips are somewhat expensive to compute and they're
+			// unused below; request that they not be sent instead.
+			FilterAllowedIPs: true,
+		}
+		if err := e.wgdev.IpcGetOperationFiltered(bw, filter); err != nil {
 			errc <- fmt.Errorf("IpcGetOperation: %w", err)
 			return
 		}
@@ -691,15 +960,9 @@ func (e *userspaceEngine) getStatus() (*Status, error) {
 
 	var peers []PeerStatus
 	for _, pk := range e.peerSequence {
-		p := pp[pk]
-		if p == nil {
-			p = &PeerStatus{}
+		if p, ok := pp[pk]; ok { // ignore idle ones not in wireguard-go's config
+			peers = append(peers, *p)
 		}
-		peers = append(peers, *p)
-	}
-
-	if len(pp) != len(e.peerSequence) {
-		e.logf("wg status returned %v peers, expected %v", len(pp), len(e.peerSequence))
 	}
 
 	return &Status{
@@ -759,6 +1022,7 @@ func (e *userspaceEngine) Close() {
 
 	r := bufio.NewReader(strings.NewReader(""))
 	e.wgdev.IpcSetOperation(r)
+	e.resolver.Close()
 	e.magicConn.Close()
 	e.linkMon.Close()
 	e.router.Close()
@@ -827,8 +1091,8 @@ func (e *userspaceEngine) SetNetworkMap(nm *controlclient.NetworkMap) {
 	e.magicConn.SetNetworkMap(nm)
 }
 
-func (e *userspaceEngine) SetDiscoPrivateKey(k key.Private) {
-	e.magicConn.SetDiscoPrivateKey(k)
+func (e *userspaceEngine) DiscoPublicKey() tailcfg.DiscoKey {
+	return e.magicConn.DiscoPublicKey()
 }
 
 func (e *userspaceEngine) UpdateStatus(sb *ipnstate.StatusBuilder) {

wgengine/watchdog.go

@@ -6,7 +6,9 @@ package wgengine
 
 import (
 	"log"
+	"os"
 	"runtime/pprof"
+	"strconv"
 	"strings"
 	"time"
 
@@ -14,7 +16,6 @@ import (
 	"tailscale.com/control/controlclient"
 	"tailscale.com/ipn/ipnstate"
 	"tailscale.com/tailcfg"
-	"tailscale.com/types/key"
 	"tailscale.com/wgengine/filter"
 	"tailscale.com/wgengine/router"
 	"tailscale.com/wgengine/tsdns"
@@ -25,6 +26,9 @@ import (
 //
 // If they do not, the watchdog crashes the process.
 func NewWatchdog(e Engine) Engine {
+	if v, _ := strconv.ParseBool(os.Getenv("TS_DEBUG_DISABLE_WATCHDOG")); v {
+		return e
+	}
 	return &watchdogEngine{
 		wrap:    e,
 		logf:    log.Printf,
@@ -101,8 +105,9 @@ func (e *watchdogEngine) SetDERPMap(m *tailcfg.DERPMap) {
 func (e *watchdogEngine) SetNetworkMap(nm *controlclient.NetworkMap) {
 	e.watchdog("SetNetworkMap", func() { e.wrap.SetNetworkMap(nm) })
 }
-func (e *watchdogEngine) SetDiscoPrivateKey(k key.Private) {
-	e.watchdog("SetDiscoPrivateKey", func() { e.wrap.SetDiscoPrivateKey(k) })
+func (e *watchdogEngine) DiscoPublicKey() (k tailcfg.DiscoKey) {
+	e.watchdog("DiscoPublicKey", func() { k = e.wrap.DiscoPublicKey() })
+	return k
 }
 func (e *watchdogEngine) Close() {
 	e.watchdog("Close", e.wrap.Close)

wgengine/wgengine.go

@@ -12,7 +12,6 @@ import (
 	"tailscale.com/control/controlclient"
 	"tailscale.com/ipn/ipnstate"
 	"tailscale.com/tailcfg"
-	"tailscale.com/types/key"
 	"tailscale.com/wgengine/filter"
 	"tailscale.com/wgengine/router"
 	"tailscale.com/wgengine/tsdns"
@@ -117,9 +116,9 @@ type Engine interface {
 	// new NetInfo summary is available.
 	SetNetInfoCallback(NetInfoCallback)
 
-	// SetDiscoPrivateKey sets the private key used for path discovery
+	// DiscoPublicKey gets the public key used for path discovery
 	// messages.
-	SetDiscoPrivateKey(key.Private)
+	DiscoPublicKey() tailcfg.DiscoKey
 
 	// UpdateStatus populates the network state using the provided
 	// status builder.