net/netns, net/interfaces: move defaultRouteInterface, add Android fallback

Signed-off-by: Brad Fitzpatrick <bradfitz@tailscale.com>

.github/workflows/cross-darwin.yml

@@ -3,7 +3,7 @@ name: Darwin-Cross
 on:
   push:
     branches:
-      - master
+      - main
   pull_request:
     branches:
       - '*'

.github/workflows/cross-freebsd.yml

@@ -3,7 +3,7 @@ name: FreeBSD-Cross
 on:
   push:
     branches:
-      - master
+      - main
   pull_request:
     branches:
       - '*'

.github/workflows/cross-openbsd.yml

@@ -3,7 +3,7 @@ name: OpenBSD-Cross
 on:
   push:
     branches:
-      - master
+      - main
   pull_request:
     branches:
       - '*'

.github/workflows/cross-windows.yml

@@ -3,7 +3,7 @@ name: Windows-Cross
 on:
   push:
     branches:
-      - master
+      - main
   pull_request:
     branches:
       - '*'

.github/workflows/license.yml

@@ -3,7 +3,7 @@ name: license
 on:
   push:
     branches:
-      - master
+      - main
   pull_request:
     branches:
       - '*'

.github/workflows/linux.yml

@@ -3,7 +3,7 @@ name: Linux
 on:
   push:
     branches:
-      - master
+      - main
   pull_request:
     branches:
       - '*'

.github/workflows/linux32.yml

@@ -0,0 +1,48 @@
+name: Linux 32-bit
+
+on:
+  push:
+    branches:
+      - main
+  pull_request:
+    branches:
+      - '*'
+
+jobs:
+  build:
+    runs-on: ubuntu-latest
+
+    if: "!contains(github.event.head_commit.message, '[ci skip]')"
+
+    steps:
+
+    - name: Set up Go
+      uses: actions/setup-go@v1
+      with:
+        go-version: 1.14
+      id: go
+
+    - name: Check out code into the Go module directory
+      uses: actions/checkout@v1
+
+    - name: Basic build
+      run: GOARCH=386 go build ./cmd/...
+
+    - name: Run tests on linux
+      run: GOARCH=386 go test ./...
+
+    - uses: k0kubun/action-slack@v2.0.0
+      with:
+        payload: |
+          {
+            "attachments": [{
+              "text": "${{ job.status }}: ${{ github.workflow }} <https://github.com/${{ github.repository }}/commit/${{ github.sha }}/checks|${{ env.COMMIT_DATE }} #${{ env.COMMIT_NUMBER_OF_DAY }}> " +
+                      "(<https://github.com/${{ github.repository }}/commit/${{ github.sha }}|" + "${{ github.sha }}".substring(0, 10) + ">) " +
+                      "of ${{ github.repository }}@" + "${{ github.ref }}".split('/').reverse()[0] + " by ${{ github.event.head_commit.committer.name }}",
+              "color": "danger"
+            }]
+          }
+      env:
+        SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK_URL }}
+      if: failure() && github.event_name == 'push'
+

.github/workflows/staticcheck.yml

@@ -3,7 +3,7 @@ name: staticcheck
 on:
   push:
     branches:
-      - master
+      - main
   pull_request:
     branches:
       - '*'
@@ -21,6 +21,9 @@ jobs:
     - name: Check out code
       uses: actions/checkout@v1
 
+    - name: Run go vet
+      run: go vet ./...
+
     - name: Print staticcheck version
       run: go run honnef.co/go/tools/cmd/staticcheck -version
 

Makefile

@@ -0,0 +1,7 @@
+usage:
+	echo "See Makefile"
+
+check: staticcheck
+
+staticcheck:
+	go run honnef.co/go/tools/cmd/staticcheck -- $$(go list ./... | grep -v tempfork)

README.md

@@ -6,17 +6,24 @@ Private WireGuard® networks made easy
 
 ## Overview
 
-This repository contains all the open source Tailscale code.
-It currently includes the Linux client.
+This repository contains all the open source Tailscale client code and
+the `tailscaled` daemon and `tailscale` CLI tool. The `tailscaled`
+daemon runs primarily on Linux; it also works to varying degrees on
+FreeBSD, OpenBSD, Darwin, and Windows.
 
-The Linux client is currently `cmd/relaynode`, but will
-soon be replaced by `cmd/tailscaled`.
+The Android app is at https://github.com/tailscale/tailscale-android
 
 ## Using
 
 We serve packages for a variety of distros at
 https://pkgs.tailscale.com .
 
+## Other clients
+
+The [macOS, iOS, and Windows clients](https://tailscale.com/download)
+use the code in this repository but additionally include small GUI
+wrappers that are not open source.
+
 ## Building
 
 ```
@@ -35,10 +42,8 @@ Please file any issues about this code or the hosted service on
 
 ## Contributing
 
-`under_construction.gif`
-
-PRs welcome, but we are still working out our contribution process and
-tooling.
+PRs welcome! But please file bugs. Commit messages should [reference
+bugs](https://docs.github.com/en/github/writing-on-github/autolinked-references-and-urls).
 
 We require [Developer Certificate of
 Origin](https://en.wikipedia.org/wiki/Developer_Certificate_of_Origin)
@@ -46,7 +51,7 @@ Origin](https://en.wikipedia.org/wiki/Developer_Certificate_of_Origin)
 
 ## About Us
 
-We are apenwarr, bradfitz, crawshaw, danderson, dfcarney,
+We are apenwarr, bradfitz, crawshaw, danderson, dfcarney, josharian
 from Tailscale Inc.
 You can learn more about us from [our website](https://tailscale.com).
 

atomicfile/atomicfile.go

@@ -9,20 +9,39 @@
 package atomicfile // import "tailscale.com/atomicfile"
 
 import (
-	"fmt"
 	"io/ioutil"
 	"os"
+	"path/filepath"
+	"runtime"
 )
 
 // WriteFile writes data to filename+some suffix, then renames it
 // into filename.
-func WriteFile(filename string, data []byte, perm os.FileMode) error {
-	tmpname := filename + ".new.tmp"
-	if err := ioutil.WriteFile(tmpname, data, perm); err != nil {
-		return fmt.Errorf("%#v: %v", tmpname, err)
+func WriteFile(filename string, data []byte, perm os.FileMode) (err error) {
+	f, err := ioutil.TempFile(filepath.Dir(filename), filepath.Base(filename)+".tmp")
+	if err != nil {
+		return err
 	}
-	if err := os.Rename(tmpname, filename); err != nil {
-		return fmt.Errorf("%#v->%#v: %v", tmpname, filename, err)
+	tmpName := f.Name()
+	defer func() {
+		if err != nil {
+			f.Close()
+			os.Remove(tmpName)
+		}
+	}()
+	if _, err := f.Write(data); err != nil {
+		return err
 	}
-	return nil
+	if runtime.GOOS != "windows" {
+		if err := f.Chmod(perm); err != nil {
+			return err
+		}
+	}
+	if err := f.Sync(); err != nil {
+		return err
+	}
+	if err := f.Close(); err != nil {
+		return err
+	}
+	return os.Rename(tmpName, filename)
 }

cmd/cloner/cloner.go

@@ -0,0 +1,264 @@
+// Copyright (c) 2020 Tailscale Inc & AUTHORS All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+// Cloner is a tool to automate the creation of a Clone method.
+//
+// The result of the Clone method aliases no memory that can be edited
+// with the original.
+//
+// This tool makes lots of implicit assumptions about the types you feed it.
+// In particular, it can only write relatively "shallow" Clone methods.
+// That is, if a type contains another named struct type, cloner assumes that
+// named type will also have a Clone method.
+package main
+
+import (
+	"bytes"
+	"flag"
+	"fmt"
+	"go/ast"
+	"go/format"
+	"go/token"
+	"go/types"
+	"io/ioutil"
+	"log"
+	"os"
+	"strings"
+
+	"golang.org/x/tools/go/packages"
+)
+
+var (
+	flagTypes     = flag.String("type", "", "comma-separated list of types; required")
+	flagOutput    = flag.String("output", "", "output file; required")
+	flagBuildTags = flag.String("tags", "", "compiler build tags to apply")
+)
+
+func main() {
+	log.SetFlags(0)
+	log.SetPrefix("cloner: ")
+	flag.Parse()
+	if len(*flagTypes) == 0 {
+		flag.Usage()
+		os.Exit(2)
+	}
+	typeNames := strings.Split(*flagTypes, ",")
+
+	cfg := &packages.Config{
+		Mode:  packages.NeedTypes | packages.NeedTypesInfo | packages.NeedSyntax | packages.NeedName,
+		Tests: false,
+	}
+	if *flagBuildTags != "" {
+		cfg.BuildFlags = []string{"-tags=" + *flagBuildTags}
+	}
+	pkgs, err := packages.Load(cfg, ".")
+	if err != nil {
+		log.Fatal(err)
+	}
+	if len(pkgs) != 1 {
+		log.Fatalf("wrong number of packages: %d", len(pkgs))
+	}
+	pkg := pkgs[0]
+	buf := new(bytes.Buffer)
+	imports := make(map[string]struct{})
+	for _, typeName := range typeNames {
+		found := false
+		for _, file := range pkg.Syntax {
+			//var fbuf bytes.Buffer
+			//ast.Fprint(&fbuf, pkg.Fset, file, nil)
+			//fmt.Println(fbuf.String())
+
+			for _, d := range file.Decls {
+				decl, ok := d.(*ast.GenDecl)
+				if !ok || decl.Tok != token.TYPE {
+					continue
+				}
+				for _, s := range decl.Specs {
+					spec, ok := s.(*ast.TypeSpec)
+					if !ok || spec.Name.Name != typeName {
+						continue
+					}
+					typeNameObj := pkg.TypesInfo.Defs[spec.Name]
+					typ, ok := typeNameObj.Type().(*types.Named)
+					if !ok {
+						continue
+					}
+					pkg := typeNameObj.Pkg()
+					gen(buf, imports, typeName, typ, pkg)
+				}
+				found = true
+			}
+		}
+		if !found {
+			log.Fatalf("could not find type %s", typeName)
+		}
+	}
+
+	contents := new(bytes.Buffer)
+	fmt.Fprintf(contents, header, *flagTypes, pkg.Name)
+	fmt.Fprintf(contents, "import (\n")
+	for s := range imports {
+		fmt.Fprintf(contents, "\t%q\n", s)
+	}
+	fmt.Fprintf(contents, ")\n\n")
+	contents.Write(buf.Bytes())
+
+	out, err := format.Source(contents.Bytes())
+	if err != nil {
+		log.Fatalf("%s, in source:\n%s", err, contents.Bytes())
+	}
+
+	output := *flagOutput
+	if output == "" {
+		flag.Usage()
+		os.Exit(2)
+	}
+	if err := ioutil.WriteFile(output, out, 0666); err != nil {
+		log.Fatal(err)
+	}
+}
+
+const header = `// Copyright (c) 2020 Tailscale Inc & AUTHORS All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+// Code generated by tailscale.com/cmd/cloner -type %s; DO NOT EDIT.
+
+package %s
+
+`
+
+func gen(buf *bytes.Buffer, imports map[string]struct{}, name string, typ *types.Named, thisPkg *types.Package) {
+	pkgQual := func(pkg *types.Package) string {
+		if thisPkg == pkg {
+			return ""
+		}
+		imports[pkg.Path()] = struct{}{}
+		return pkg.Name()
+	}
+	importedName := func(t types.Type) string {
+		return types.TypeString(t, pkgQual)
+	}
+
+	switch t := typ.Underlying().(type) {
+	case *types.Struct:
+		_ = t
+		name := typ.Obj().Name()
+		fmt.Fprintf(buf, "// Clone makes a deep copy of %s.\n", name)
+		fmt.Fprintf(buf, "// The result aliases no memory with the original.\n")
+		fmt.Fprintf(buf, "func (src *%s) Clone() *%s {\n", name, name)
+		writef := func(format string, args ...interface{}) {
+			fmt.Fprintf(buf, "\t"+format+"\n", args...)
+		}
+		writef("if src == nil {")
+		writef("\treturn nil")
+		writef("}")
+		writef("dst := new(%s)", name)
+		writef("*dst = *src")
+		for i := 0; i < t.NumFields(); i++ {
+			fname := t.Field(i).Name()
+			ft := t.Field(i).Type()
+			if !containsPointers(ft) {
+				continue
+			}
+			if named, _ := ft.(*types.Named); named != nil && !hasBasicUnderlying(ft) {
+				writef("dst.%s = *src.%s.Clone()", fname, fname)
+				continue
+			}
+			switch ft := ft.Underlying().(type) {
+			case *types.Slice:
+				if containsPointers(ft.Elem()) {
+					n := importedName(ft.Elem())
+					writef("dst.%s = make([]%s, len(src.%s))", fname, n, fname)
+					writef("for i := range dst.%s {", fname)
+					if _, isPtr := ft.Elem().(*types.Pointer); isPtr {
+						writef("\tdst.%s[i] = src.%s[i].Clone()", fname, fname)
+					} else {
+						writef("\tdst.%s[i] = *src.%s[i].Clone()", fname, fname)
+					}
+					writef("}")
+				} else {
+					writef("dst.%s = append(src.%s[:0:0], src.%s...)", fname, fname, fname)
+				}
+			case *types.Pointer:
+				if named, _ := ft.Elem().(*types.Named); named != nil && containsPointers(ft.Elem()) {
+					writef("dst.%s = src.%s.Clone()", fname, fname)
+					continue
+				}
+				n := importedName(ft.Elem())
+				writef("if dst.%s != nil {", fname)
+				writef("\tdst.%s = new(%s)", fname, n)
+				writef("\t*dst.%s = *src.%s", fname, fname)
+				if containsPointers(ft.Elem()) {
+					writef("\t" + `panic("TODO pointers in pointers")`)
+				}
+				writef("}")
+			case *types.Map:
+				writef("if dst.%s != nil {", fname)
+				writef("\tdst.%s = map[%s]%s{}", fname, importedName(ft.Key()), importedName(ft.Elem()))
+				if sliceType, isSlice := ft.Elem().(*types.Slice); isSlice {
+					n := importedName(sliceType.Elem())
+					writef("\tfor k := range src.%s {", fname)
+					// use zero-length slice instead of nil to ensure
+					// the key is always copied.
+					writef("\t\tdst.%s[k] = append([]%s{}, src.%s[k]...)", fname, n, fname)
+					writef("\t}")
+				} else if containsPointers(ft.Elem()) {
+					writef("\t\t" + `panic("TODO map value pointers")`)
+				} else {
+					writef("\tfor k, v := range src.%s {", fname)
+					writef("\t\tdst.%s[k] = v", fname)
+					writef("\t}")
+				}
+				writef("}")
+			case *types.Struct:
+				writef(`panic("TODO struct %s")`, fname)
+			default:
+				writef(`panic(fmt.Sprintf("TODO: %T", ft))`)
+			}
+		}
+		writef("return dst")
+		fmt.Fprintf(buf, "}\n\n")
+	}
+}
+
+func hasBasicUnderlying(typ types.Type) bool {
+	switch typ.Underlying().(type) {
+	case *types.Slice, *types.Map:
+		return true
+	default:
+		return false
+	}
+}
+
+func containsPointers(typ types.Type) bool {
+	switch typ.String() {
+	case "time.Time":
+		// time.Time contains a pointer that does not need copying
+		return false
+	case "inet.af/netaddr.IP":
+		return false
+	}
+	switch ft := typ.Underlying().(type) {
+	case *types.Array:
+		return containsPointers(ft.Elem())
+	case *types.Chan:
+		return true
+	case *types.Interface:
+		return true // a little too broad
+	case *types.Map:
+		return true
+	case *types.Pointer:
+		return true
+	case *types.Slice:
+		return true
+	case *types.Struct:
+		for i := 0; i < ft.NumFields(); i++ {
+			if containsPointers(ft.Field(i).Type()) {
+				return true
+			}
+		}
+	}
+	return false
+}

cmd/derper/derper.go

@@ -20,6 +20,7 @@ import (
 	"os"
 	"path/filepath"
 	"regexp"
+	"strings"
 	"time"
 
 	"github.com/tailscale/wireguard-go/wgcfg"
@@ -32,6 +33,7 @@ import (
 	"tailscale.com/net/stun"
 	"tailscale.com/tsweb"
 	"tailscale.com/types/key"
+	"tailscale.com/version"
 )
 
 var (
@@ -42,6 +44,8 @@ var (
 	hostname      = flag.String("hostname", "derp.tailscale.com", "LetsEncrypt host name, if addr's port is :443")
 	logCollection = flag.String("logcollection", "", "If non-empty, logtail collection to log to")
 	runSTUN       = flag.Bool("stun", false, "also run a STUN server")
+	meshPSKFile   = flag.String("mesh-psk-file", defaultMeshPSKFile(), "if non-empty, path to file containing the mesh pre-shared key file. It should contain some hex string; whitespace is trimmed.")
+	meshWith      = flag.String("mesh-with", "", "optional comma-separated list of hostnames to mesh with; the server's own hostname can be in the list")
 )
 
 type config struct {
@@ -118,6 +122,22 @@ func main() {
 	letsEncrypt := tsweb.IsProd443(*addr)
 
 	s := derp.NewServer(key.Private(cfg.PrivateKey), log.Printf)
+
+	if *meshPSKFile != "" {
+		b, err := ioutil.ReadFile(*meshPSKFile)
+		if err != nil {
+			log.Fatal(err)
+		}
+		key := strings.TrimSpace(string(b))
+		if matched, _ := regexp.MatchString(`(?i)^[0-9a-f]{64,}$`, key); !matched {
+			log.Fatalf("key in %s must contain 64+ hex digits", *meshPSKFile)
+		}
+		s.SetMeshKey(key)
+		log.Printf("DERP mesh key configured")
+	}
+	if err := startMesh(s); err != nil {
+		log.Fatalf("startMesh: %v", err)
+	}
 	expvar.Publish("derp", s.ExpVar())
 
 	// Create our own mux so we don't expose /debug/ stuff to the world.
@@ -166,7 +186,7 @@ func main() {
 		}
 		httpsrv.TLSConfig = certManager.TLSConfig()
 		go func() {
-			err := http.ListenAndServe(":80", certManager.HTTPHandler(tsweb.Port80Handler{mux}))
+			err := http.ListenAndServe(":80", certManager.HTTPHandler(tsweb.Port80Handler{Main: mux}))
 			if err != nil {
 				if err != http.ErrServerClosed {
 					log.Fatal(err)
@@ -185,6 +205,15 @@ func main() {
 
 func debugHandler(s *derp.Server) http.Handler {
 	return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		if r.RequestURI == "/debug/check" {
+			err := s.ConsistencyCheck()
+			if err != nil {
+				http.Error(w, err.Error(), 500)
+			} else {
+				io.WriteString(w, "derp.Server ConsistencyCheck okay")
+			}
+			return
+		}
 		f := func(format string, args ...interface{}) { fmt.Fprintf(w, format, args...) }
 		f(`<html><body>
 <h1>DERP debug</h1>
@@ -192,12 +221,15 @@ func debugHandler(s *derp.Server) http.Handler {
 `)
 		f("<li><b>Hostname:</b> %v</li>\n", *hostname)
 		f("<li><b>Uptime:</b> %v</li>\n", tsweb.Uptime())
+		f("<li><b>Mesh Key:</b> %v</li>\n", s.HasMeshKey())
+		f("<li><b>Version:</b> %v</li>\n", version.LONG)
 
 		f(`<li><a href="/debug/vars">/debug/vars</a> (Go)</li>
    <li><a href="/debug/varz">/debug/varz</a> (Prometheus)</li>
    <li><a href="/debug/pprof/">/debug/pprof/</a></li>
    <li><a href="/debug/pprof/goroutine?debug=1">/debug/pprof/goroutine</a> (collapsed)</li>
    <li><a href="/debug/pprof/goroutine?debug=2">/debug/pprof/goroutine</a> (full)</li>
+   <li><a href="/debug/check">/debug/check</a> internal consistency check</li>
 <ul>
 </html>
 `)
@@ -268,7 +300,7 @@ func serveSTUN() {
 	}
 }
 
-var validProdHostname = regexp.MustCompile(`^derp(\d+|\-\w+)?\.tailscale\.com\.?$`)
+var validProdHostname = regexp.MustCompile(`^derp([^.]*)\.tailscale\.com\.?$`)
 
 func prodAutocertHostPolicy(_ context.Context, host string) error {
 	if validProdHostname.MatchString(host) {
@@ -276,3 +308,16 @@ func prodAutocertHostPolicy(_ context.Context, host string) error {
 	}
 	return errors.New("invalid hostname")
 }
+
+func defaultMeshPSKFile() string {
+	try := []string{
+		"/home/derp/keys/derp-mesh.key",
+		filepath.Join(os.Getenv("HOME"), "keys", "derp-mesh.key"),
+	}
+	for _, p := range try {
+		if _, err := os.Stat(p); err == nil {
+			return p
+		}
+	}
+	return ""
+}

cmd/derper/derper_test.go

@@ -17,10 +17,11 @@ func TestProdAutocertHostPolicy(t *testing.T) {
 		{"derp.tailscale.com", true},
 		{"derp.tailscale.com.", true},
 		{"derp1.tailscale.com", true},
+		{"derp1b.tailscale.com", true},
 		{"derp2.tailscale.com", true},
 		{"derp02.tailscale.com", true},
 		{"derp-nyc.tailscale.com", true},
-		{"derpfoo.tailscale.com", false},
+		{"derpfoo.tailscale.com", true},
 		{"derp02.bar.tailscale.com", false},
 		{"example.net", false},
 	}

cmd/derper/mesh.go

@@ -0,0 +1,45 @@
+// Copyright (c) 2020 Tailscale Inc & AUTHORS All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package main
+
+import (
+	"errors"
+	"fmt"
+	"log"
+	"strings"
+
+	"tailscale.com/derp"
+	"tailscale.com/derp/derphttp"
+	"tailscale.com/types/key"
+	"tailscale.com/types/logger"
+)
+
+func startMesh(s *derp.Server) error {
+	if *meshWith == "" {
+		return nil
+	}
+	if !s.HasMeshKey() {
+		return errors.New("--mesh-with requires --mesh-psk-file")
+	}
+	for _, host := range strings.Split(*meshWith, ",") {
+		if err := startMeshWithHost(s, host); err != nil {
+			return err
+		}
+	}
+	return nil
+}
+
+func startMeshWithHost(s *derp.Server, host string) error {
+	logf := logger.WithPrefix(log.Printf, fmt.Sprintf("mesh(%q): ", host))
+	c, err := derphttp.NewClient(s.PrivateKey(), "https://"+host+"/derp", logf)
+	if err != nil {
+		return err
+	}
+	c.MeshKey = s.MeshKey()
+	add := func(k key.Public) { s.AddPacketForwarder(k, c) }
+	remove := func(k key.Public) { s.RemovePacketForwarder(k, c) }
+	go c.RunWatchConnectionLoop(s.PublicKey(), add, remove)
+	return nil
+}

cmd/relaynode/relaynode.od

@@ -1 +0,0 @@
-# placeholder to work around redo bug

cmd/tailscale/cli/cli.go

@@ -0,0 +1,128 @@
+// Copyright (c) 2020 Tailscale Inc & AUTHORS All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+// Package cli contains the cmd/tailscale CLI code in a package that can be included
+// in other wrapper binaries such as the Mac and Windows clients.
+package cli
+
+import (
+	"context"
+	"flag"
+	"log"
+	"net"
+	"os"
+	"os/signal"
+	"runtime"
+	"strings"
+	"syscall"
+
+	"github.com/peterbourgon/ff/v2/ffcli"
+	"tailscale.com/ipn"
+	"tailscale.com/paths"
+	"tailscale.com/safesocket"
+)
+
+// ActLikeCLI reports whether a GUI application should act like the
+// CLI based on os.Args, GOOS, the context the process is running in
+// (pty, parent PID), etc.
+func ActLikeCLI() bool {
+	if len(os.Args) < 2 {
+		return false
+	}
+	switch os.Args[1] {
+	case "up", "status", "netcheck", "version",
+		"-V", "--version", "-h", "--help":
+		return true
+	}
+	return false
+}
+
+// Run runs the CLI. The args do not include the binary name.
+func Run(args []string) error {
+	if len(args) == 1 && (args[0] == "-V" || args[0] == "--version") {
+		args = []string{"version"}
+	}
+
+	rootfs := flag.NewFlagSet("tailscale", flag.ExitOnError)
+	rootfs.StringVar(&rootArgs.socket, "socket", paths.DefaultTailscaledSocket(), "path to tailscaled's unix socket")
+
+	rootCmd := &ffcli.Command{
+		Name:       "tailscale",
+		ShortUsage: "tailscale subcommand [flags]",
+		ShortHelp:  "The easiest, most secure way to use WireGuard.",
+		LongHelp: strings.TrimSpace(`
+This CLI is still under active development. Commands and flags will
+change in the future.
+`),
+		Subcommands: []*ffcli.Command{
+			upCmd,
+			netcheckCmd,
+			statusCmd,
+			versionCmd,
+		},
+		FlagSet: rootfs,
+		Exec:    func(context.Context, []string) error { return flag.ErrHelp },
+	}
+
+	if err := rootCmd.Parse(args); err != nil {
+		return err
+	}
+
+	err := rootCmd.Run(context.Background())
+	if err == flag.ErrHelp {
+		return nil
+	}
+	return err
+}
+
+func fatalf(format string, a ...interface{}) {
+	log.SetFlags(0)
+	log.Fatalf(format, a...)
+}
+
+var rootArgs struct {
+	socket string
+}
+
+func connect(ctx context.Context) (net.Conn, *ipn.BackendClient, context.Context, context.CancelFunc) {
+	c, err := safesocket.Connect(rootArgs.socket, 41112)
+	if err != nil {
+		if runtime.GOOS != "windows" && rootArgs.socket == "" {
+			fatalf("--socket cannot be empty")
+		}
+		fatalf("Failed to connect to connect to tailscaled. (safesocket.Connect: %v)\n", err)
+	}
+	clientToServer := func(b []byte) {
+		ipn.WriteMsg(c, b)
+	}
+
+	ctx, cancel := context.WithCancel(ctx)
+
+	go func() {
+		interrupt := make(chan os.Signal, 1)
+		signal.Notify(interrupt, syscall.SIGINT, syscall.SIGTERM)
+		<-interrupt
+		c.Close()
+		cancel()
+	}()
+
+	bc := ipn.NewBackendClient(log.Printf, clientToServer)
+	return c, bc, ctx, cancel
+}
+
+// pump receives backend messages on conn and pushes them into bc.
+func pump(ctx context.Context, bc *ipn.BackendClient, conn net.Conn) {
+	defer conn.Close()
+	for ctx.Err() == nil {
+		msg, err := ipn.ReadMsg(conn)
+		if err != nil {
+			if ctx.Err() != nil {
+				return
+			}
+			log.Printf("ReadMsg: %v\n", err)
+			break
+		}
+		bc.GotNotifyMsg(msg)
+	}
+}

cmd/tailscale/cli/netcheck.go

@@ -2,7 +2,7 @@
 // Use of this source code is governed by a BSD-style
 // license that can be found in the LICENSE file.
 
-package main
+package cli
 
 import (
 	"context"
@@ -117,6 +117,7 @@ func printReport(dm *tailcfg.DERPMap, report *netcheck.Report) error {
 	}
 	fmt.Printf("\t* MappingVariesByDestIP: %v\n", report.MappingVariesByDestIP)
 	fmt.Printf("\t* HairPinning: %v\n", report.HairPinning)
+	fmt.Printf("\t* PortMapping: %v\n", portMapping(report))
 
 	// When DERP latency checking failed,
 	// magicsock will try to pick the DERP server that
@@ -142,3 +143,20 @@ func printReport(dm *tailcfg.DERPMap, report *netcheck.Report) error {
 	}
 	return nil
 }
+
+func portMapping(r *netcheck.Report) string {
+	if !r.AnyPortMappingChecked() {
+		return "not checked"
+	}
+	var got []string
+	if r.UPnP.EqualBool(true) {
+		got = append(got, "UPnP")
+	}
+	if r.PMP.EqualBool(true) {
+		got = append(got, "NAT-PMP")
+	}
+	if r.PCP.EqualBool(true) {
+		got = append(got, "PCP")
+	}
+	return strings.Join(got, ", ")
+}

cmd/tailscale/cli/status.go

@@ -2,7 +2,7 @@
 // Use of this source code is governed by a BSD-style
 // license that can be found in the LICENSE file.
 
-package main
+package cli
 
 import (
 	"bytes"
@@ -14,6 +14,7 @@ import (
 	"net"
 	"net/http"
 	"os"
+	"time"
 
 	"github.com/peterbourgon/ff/v2/ffcli"
 	"github.com/toqueteos/webbrowser"
@@ -24,13 +25,14 @@ import (
 
 var statusCmd = &ffcli.Command{
 	Name:       "status",
-	ShortUsage: "status [-web] [-json]",
+	ShortUsage: "status [-active] [-web] [-json]",
 	ShortHelp:  "Show state of tailscaled and its connections",
 	Exec:       runStatus,
 	FlagSet: (func() *flag.FlagSet {
 		fs := flag.NewFlagSet("status", flag.ExitOnError)
 		fs.BoolVar(&statusArgs.json, "json", false, "output in JSON format (WARNING: format subject to change)")
 		fs.BoolVar(&statusArgs.web, "web", false, "run webserver with HTML showing status")
+		fs.BoolVar(&statusArgs.active, "active", false, "filter output to only peers with active sessions (not applicable to web mode)")
 		fs.StringVar(&statusArgs.listen, "listen", "127.0.0.1:8384", "listen address; use port 0 for automatic")
 		fs.BoolVar(&statusArgs.browser, "browser", true, "Open a browser in web mode")
 		return fs
@@ -42,6 +44,7 @@ var statusArgs struct {
 	web     bool   // run webserver
 	listen  string // in web mode, webserver address to listen on, empty means auto
 	browser bool   // in web mode, whether to open browser
+	active  bool   // in CLI mode, filter output to only peers with active sessions
 }
 
 func runStatus(ctx context.Context, args []string) error {
@@ -75,6 +78,13 @@ func runStatus(ctx context.Context, args []string) error {
 		return err
 	}
 	if statusArgs.json {
+		if statusArgs.active {
+			for peer, ps := range st.Peer {
+				if !peerActive(ps) {
+					delete(st.Peer, peer)
+				}
+			}
+		}
 		j, err := json.MarshalIndent(st, "", "  ")
 		if err != nil {
 			return err
@@ -119,6 +129,10 @@ func runStatus(ctx context.Context, args []string) error {
 	f := func(format string, a ...interface{}) { fmt.Fprintf(&buf, format, a...) }
 	for _, peer := range st.Peers() {
 		ps := st.Peer[peer]
+		active := peerActive(ps)
+		if statusArgs.active && !active {
+			continue
+		}
 		f("%s %-7s %-15s %-18s tx=%8d rx=%8d ",
 			peer.ShortString(),
 			ps.OS,
@@ -127,6 +141,13 @@ func runStatus(ctx context.Context, args []string) error {
 			ps.TxBytes,
 			ps.RxBytes,
 		)
+		relay := ps.Relay
+		if active && relay != "" && ps.CurAddr == "" {
+			relay = "*" + relay + "*"
+		} else {
+			relay = " " + relay
+		}
+		f("%-6s", relay)
 		for i, addr := range ps.Addrs {
 			if i != 0 {
 				f(", ")
@@ -142,3 +163,10 @@ func runStatus(ctx context.Context, args []string) error {
 	os.Stdout.Write(buf.Bytes())
 	return nil
 }
+
+// peerActive reports whether ps has recent activity.
+//
+// TODO: have the server report this bool instead.
+func peerActive(ps *ipnstate.PeerStatus) bool {
+	return !ps.LastWrite.IsZero() && time.Since(ps.LastWrite) < 2*time.Minute
+}

cmd/tailscale/cli/up.go

@@ -0,0 +1,258 @@
+// Copyright (c) 2020 Tailscale Inc & AUTHORS All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package cli
+
+import (
+	"bytes"
+	"context"
+	"flag"
+	"fmt"
+	"log"
+	"os"
+	"os/exec"
+	"runtime"
+	"strconv"
+	"strings"
+
+	"github.com/peterbourgon/ff/v2/ffcli"
+	"github.com/tailscale/wireguard-go/wgcfg"
+	"inet.af/netaddr"
+	"tailscale.com/ipn"
+	"tailscale.com/tailcfg"
+	"tailscale.com/version"
+	"tailscale.com/wgengine/router"
+)
+
+// globalStateKey is the ipn.StateKey that tailscaled loads on
+// startup.
+//
+// We have to support multiple state keys for other OSes (Windows in
+// particular), but right now Unix daemons run with a single
+// node-global state. To keep open the option of having per-user state
+// later, the global state key doesn't look like a username.
+const globalStateKey = "_daemon"
+
+var upCmd = &ffcli.Command{
+	Name:       "up",
+	ShortUsage: "up [flags]",
+	ShortHelp:  "Connect to your Tailscale network",
+
+	LongHelp: strings.TrimSpace(`
+"tailscale up" connects this machine to your Tailscale network,
+triggering authentication if necessary.
+
+The flags passed to this command are specific to this machine. If you don't
+specify any flags, options are reset to their default.
+`),
+	FlagSet: (func() *flag.FlagSet {
+		upf := flag.NewFlagSet("up", flag.ExitOnError)
+		upf.StringVar(&upArgs.server, "login-server", "https://login.tailscale.com", "base URL of control server")
+		upf.BoolVar(&upArgs.acceptRoutes, "accept-routes", false, "accept routes advertised by other Tailscale nodes")
+		upf.BoolVar(&upArgs.acceptDNS, "accept-dns", true, "accept DNS configuration from the admin panel")
+		upf.BoolVar(&upArgs.singleRoutes, "host-routes", true, "install host routes to other Tailscale nodes")
+		upf.BoolVar(&upArgs.shieldsUp, "shields-up", false, "don't allow incoming connections")
+		upf.StringVar(&upArgs.advertiseTags, "advertise-tags", "", "ACL tags to request (comma-separated, e.g. eng,montreal,ssh)")
+		upf.StringVar(&upArgs.authKey, "authkey", "", "node authorization key")
+		upf.StringVar(&upArgs.hostname, "hostname", "", "hostname to use instead of the one provided by the OS")
+		upf.BoolVar(&upArgs.enableDERP, "enable-derp", true, "enable the use of DERP servers")
+		if runtime.GOOS == "linux" || isBSD(runtime.GOOS) || version.OS() == "macOS" {
+			upf.StringVar(&upArgs.advertiseRoutes, "advertise-routes", "", "routes to advertise to other nodes (comma-separated, e.g. 10.0.0.0/8,192.168.0.0/24)")
+		}
+		if runtime.GOOS == "linux" {
+			upf.BoolVar(&upArgs.snat, "snat-subnet-routes", true, "source NAT traffic to local routes advertised with -advertise-routes")
+			upf.StringVar(&upArgs.netfilterMode, "netfilter-mode", "on", "netfilter mode (one of on, nodivert, off)")
+		}
+		return upf
+	})(),
+	Exec: runUp,
+}
+
+var upArgs struct {
+	server          string
+	acceptRoutes    bool
+	acceptDNS       bool
+	singleRoutes    bool
+	shieldsUp       bool
+	advertiseRoutes string
+	advertiseTags   string
+	enableDERP      bool
+	snat            bool
+	netfilterMode   string
+	authKey         string
+	hostname        string
+}
+
+// parseIPOrCIDR parses an IP address or a CIDR prefix. If the input
+// is an IP address, it is returned in CIDR form with a /32 mask for
+// IPv4 or a /128 mask for IPv6.
+func parseIPOrCIDR(s string) (wgcfg.CIDR, bool) {
+	if strings.Contains(s, "/") {
+		ret, err := wgcfg.ParseCIDR(s)
+		if err != nil {
+			return wgcfg.CIDR{}, false
+		}
+		return ret, true
+	}
+
+	ip, ok := wgcfg.ParseIP(s)
+	if !ok {
+		return wgcfg.CIDR{}, false
+	}
+	if ip.Is4() {
+		return wgcfg.CIDR{IP: ip, Mask: 32}, true
+	} else {
+		return wgcfg.CIDR{IP: ip, Mask: 128}, true
+	}
+}
+
+func isBSD(s string) bool {
+	return s == "dragonfly" || s == "freebsd" || s == "netbsd" || s == "openbsd"
+}
+
+func warnf(format string, args ...interface{}) {
+	fmt.Printf("Warning: "+format+"\n", args...)
+}
+
+// checkIPForwarding prints warnings if IP forwarding is not
+// enabled, or if we were unable to verify the state of IP forwarding.
+func checkIPForwarding() {
+	var key string
+
+	if runtime.GOOS == "linux" {
+		key = "net.ipv4.ip_forward"
+	} else if isBSD(runtime.GOOS) || version.OS() == "macOS" {
+		key = "net.inet.ip.forwarding"
+	} else {
+		return
+	}
+
+	bs, err := exec.Command("sysctl", "-n", key).Output()
+	if err != nil {
+		warnf("couldn't check %s (%v).\nSubnet routes won't work without IP forwarding.", key, err)
+		return
+	}
+	on, err := strconv.ParseBool(string(bytes.TrimSpace(bs)))
+	if err != nil {
+		warnf("couldn't parse %s (%v).\nSubnet routes won't work without IP forwarding.", key, err)
+		return
+	}
+	if !on {
+		warnf("%s is disabled. Subnet routes won't work.", key)
+	}
+}
+
+func runUp(ctx context.Context, args []string) error {
+	if len(args) > 0 {
+		log.Fatalf("too many non-flag arguments: %q", args)
+	}
+
+	var routes []wgcfg.CIDR
+	if upArgs.advertiseRoutes != "" {
+		advroutes := strings.Split(upArgs.advertiseRoutes, ",")
+		for _, s := range advroutes {
+			cidr, ok := parseIPOrCIDR(s)
+			ipp, err := netaddr.ParseIPPrefix(s) // parse it with other pawith both packages
+			if !ok || err != nil {
+				fatalf("%q is not a valid IP address or CIDR prefix", s)
+			}
+			if ipp != ipp.Masked() {
+				fatalf("%s has non-address bits set; expected %s", ipp, ipp.Masked())
+			}
+			routes = append(routes, cidr)
+		}
+		checkIPForwarding()
+	}
+
+	var tags []string
+	if upArgs.advertiseTags != "" {
+		tags = strings.Split(upArgs.advertiseTags, ",")
+		for _, tag := range tags {
+			err := tailcfg.CheckTag(tag)
+			if err != nil {
+				fatalf("tag: %q: %s", tag, err)
+			}
+		}
+	}
+
+	if len(upArgs.hostname) > 256 {
+		fatalf("hostname too long: %d bytes (max 256)", len(upArgs.hostname))
+	}
+
+	// TODO(apenwarr): fix different semantics between prefs and uflags
+	// TODO(apenwarr): allow setting/using CorpDNS
+	prefs := ipn.NewPrefs()
+	prefs.ControlURL = upArgs.server
+	prefs.WantRunning = true
+	prefs.RouteAll = upArgs.acceptRoutes
+	prefs.CorpDNS = upArgs.acceptDNS
+	prefs.AllowSingleHosts = upArgs.singleRoutes
+	prefs.ShieldsUp = upArgs.shieldsUp
+	prefs.AdvertiseRoutes = routes
+	prefs.AdvertiseTags = tags
+	prefs.NoSNAT = !upArgs.snat
+	prefs.DisableDERP = !upArgs.enableDERP
+	prefs.Hostname = upArgs.hostname
+	if runtime.GOOS == "linux" {
+		switch upArgs.netfilterMode {
+		case "on":
+			prefs.NetfilterMode = router.NetfilterOn
+		case "nodivert":
+			prefs.NetfilterMode = router.NetfilterNoDivert
+			warnf("netfilter=nodivert; add iptables calls to ts-* chains manually.")
+		case "off":
+			prefs.NetfilterMode = router.NetfilterOff
+			warnf("netfilter=off; configure iptables yourself.")
+		default:
+			fatalf("invalid value --netfilter-mode: %q", upArgs.netfilterMode)
+		}
+	}
+
+	c, bc, ctx, cancel := connect(ctx)
+	defer cancel()
+
+	var printed bool
+
+	bc.SetPrefs(prefs)
+	opts := ipn.Options{
+		StateKey: globalStateKey,
+		AuthKey:  upArgs.authKey,
+		Notify: func(n ipn.Notify) {
+			if n.ErrMessage != nil {
+				fatalf("backend error: %v\n", *n.ErrMessage)
+			}
+			if s := n.State; s != nil {
+				switch *s {
+				case ipn.NeedsLogin:
+					printed = true
+					bc.StartLoginInteractive()
+				case ipn.NeedsMachineAuth:
+					printed = true
+					fmt.Fprintf(os.Stderr, "\nTo authorize your machine, visit (as admin):\n\n\t%s/admin/machines\n\n", upArgs.server)
+				case ipn.Starting, ipn.Running:
+					// Done full authentication process
+					if printed {
+						// Only need to print an update if we printed the "please click" message earlier.
+						fmt.Fprintf(os.Stderr, "Success.\n")
+					}
+					cancel()
+				}
+			}
+			if url := n.BrowseToURL; url != nil {
+				fmt.Fprintf(os.Stderr, "\nTo authenticate, visit:\n\n\t%s\n\n", *url)
+			}
+		},
+	}
+	// We still have to Start right now because it's the only way to
+	// set up notifications and whatnot. This causes a bunch of churn
+	// every time the CLI touches anything.
+	//
+	// TODO(danderson): redo the frontend/backend API to assume
+	// ephemeral frontends that read/modify/write state, once
+	// Windows/Mac state is moved into backend.
+	bc.Start(opts)
+	pump(ctx, bc, c)
+
+	return nil
+}

cmd/tailscale/cli/version.go

@@ -0,0 +1,69 @@
+// Copyright (c) 2020 Tailscale Inc & AUTHORS All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package cli
+
+import (
+	"context"
+	"flag"
+	"fmt"
+	"log"
+
+	"github.com/peterbourgon/ff/v2/ffcli"
+	"tailscale.com/ipn"
+	"tailscale.com/version"
+)
+
+var versionCmd = &ffcli.Command{
+	Name:       "version",
+	ShortUsage: "version [flags]",
+	ShortHelp:  "Print Tailscale version",
+	FlagSet: (func() *flag.FlagSet {
+		fs := flag.NewFlagSet("version", flag.ExitOnError)
+		fs.BoolVar(&versionArgs.daemon, "daemon", false, "also print local node's daemon version")
+		return fs
+	})(),
+	Exec: runVersion,
+}
+
+var versionArgs struct {
+	daemon bool // also check local node's daemon version
+}
+
+func runVersion(ctx context.Context, args []string) error {
+	if len(args) > 0 {
+		log.Fatalf("too many non-flag arguments: %q", args)
+	}
+	if !versionArgs.daemon {
+		fmt.Println(version.LONG)
+		return nil
+	}
+	fmt.Printf("Client: %s\n", version.LONG)
+
+	c, bc, ctx, cancel := connect(ctx)
+	defer cancel()
+
+	bc.AllowVersionSkew = true
+
+	done := make(chan struct{})
+
+	bc.SetNotifyCallback(func(n ipn.Notify) {
+		if n.ErrMessage != nil {
+			log.Fatal(*n.ErrMessage)
+		}
+		if n.Status != nil {
+			fmt.Printf("Daemon: %s\n", n.Version)
+			close(done)
+		}
+	})
+	go pump(ctx, bc, c)
+
+	bc.RequestStatus()
+	select {
+	case <-done:
+		return nil
+	case <-ctx.Done():
+		return ctx.Err()
+	}
+}

cmd/tailscale/tailscale.go

@@ -7,306 +7,22 @@
 package main // import "tailscale.com/cmd/tailscale"
 
 import (
-	"bytes"
-	"context"
-	"flag"
 	"fmt"
-	"io/ioutil"
 	"log"
-	"net"
 	"os"
-	"os/signal"
-	"runtime"
-	"strconv"
-	"strings"
-	"syscall"
 
 	"github.com/apenwarr/fixconsole"
-	"github.com/peterbourgon/ff/v2/ffcli"
-	"github.com/tailscale/wireguard-go/wgcfg"
-	"tailscale.com/ipn"
-	"tailscale.com/paths"
-	"tailscale.com/safesocket"
-	"tailscale.com/tailcfg"
-	"tailscale.com/wgengine/router"
+	"tailscale.com/cmd/tailscale/cli"
 )
 
-// globalStateKey is the ipn.StateKey that tailscaled loads on
-// startup.
-//
-// We have to support multiple state keys for other OSes (Windows in
-// particular), but right now Unix daemons run with a single
-// node-global state. To keep open the option of having per-user state
-// later, the global state key doesn't look like a username.
-const globalStateKey = "_daemon"
-
-var rootArgs struct {
-	socket string
-}
-
 func main() {
 	err := fixconsole.FixConsoleIfNeeded()
 	if err != nil {
 		log.Printf("fixConsoleOutput: %v\n", err)
 	}
 
-	upf := flag.NewFlagSet("up", flag.ExitOnError)
-	upf.StringVar(&upArgs.server, "login-server", "https://login.tailscale.com", "base URL of control server")
-	upf.BoolVar(&upArgs.acceptRoutes, "accept-routes", false, "accept routes advertised by other Tailscale nodes")
-	upf.BoolVar(&upArgs.singleRoutes, "host-routes", true, "install host routes to other Tailscale nodes")
-	upf.BoolVar(&upArgs.shieldsUp, "shields-up", false, "don't allow incoming connections")
-	upf.StringVar(&upArgs.advertiseTags, "advertise-tags", "", "ACL tags to request (comma-separated, e.g. eng,montreal,ssh)")
-	upf.StringVar(&upArgs.authKey, "authkey", "", "node authorization key")
-	upf.BoolVar(&upArgs.enableDERP, "enable-derp", true, "enable the use of DERP servers")
-	if runtime.GOOS == "linux" {
-		upf.StringVar(&upArgs.advertiseRoutes, "advertise-routes", "", "routes to advertise to other nodes (comma-separated, e.g. 10.0.0.0/8,192.168.0.0/24)")
-		upf.BoolVar(&upArgs.snat, "snat-subnet-routes", true, "source NAT traffic to local routes advertised with -advertise-routes")
-		upf.StringVar(&upArgs.netfilterMode, "netfilter-mode", "on", "netfilter mode (one of on, nodivert, off)")
-	}
-	upCmd := &ffcli.Command{
-		Name:       "up",
-		ShortUsage: "up [flags]",
-		ShortHelp:  "Connect to your Tailscale network",
-
-		LongHelp: strings.TrimSpace(`
-"tailscale up" connects this machine to your Tailscale network,
-triggering authentication if necessary.
-
-The flags passed to this command are specific to this machine. If you don't
-specify any flags, options are reset to their default.
-`),
-		FlagSet: upf,
-		Exec:    runUp,
-	}
-
-	rootfs := flag.NewFlagSet("tailscale", flag.ExitOnError)
-	rootfs.StringVar(&rootArgs.socket, "socket", paths.DefaultTailscaledSocket(), "path to tailscaled's unix socket")
-
-	rootCmd := &ffcli.Command{
-		Name:       "tailscale",
-		ShortUsage: "tailscale subcommand [flags]",
-		ShortHelp:  "The easiest, most secure way to use WireGuard.",
-		LongHelp: strings.TrimSpace(`
-This CLI is still under active development. Commands and flags will
-change in the future.
-`),
-		Subcommands: []*ffcli.Command{
-			upCmd,
-			netcheckCmd,
-			statusCmd,
-		},
-		FlagSet: rootfs,
-		Exec:    func(context.Context, []string) error { return flag.ErrHelp },
-	}
-
-	if err := rootCmd.ParseAndRun(context.Background(), os.Args[1:]); err != nil && err != flag.ErrHelp {
-		log.Fatal(err)
-	}
-}
-
-var upArgs struct {
-	server          string
-	acceptRoutes    bool
-	singleRoutes    bool
-	shieldsUp       bool
-	advertiseRoutes string
-	advertiseTags   string
-	enableDERP      bool
-	snat            bool
-	netfilterMode   string
-	authKey         string
-}
-
-// parseIPOrCIDR parses an IP address or a CIDR prefix. If the input
-// is an IP address, it is returned in CIDR form with a /32 mask for
-// IPv4 or a /128 mask for IPv6.
-func parseIPOrCIDR(s string) (wgcfg.CIDR, bool) {
-	if strings.Contains(s, "/") {
-		ret, err := wgcfg.ParseCIDR(s)
-		if err != nil {
-			return wgcfg.CIDR{}, false
-		}
-		return ret, true
-	}
-
-	ip, ok := wgcfg.ParseIP(s)
-	if !ok {
-		return wgcfg.CIDR{}, false
-	}
-	if ip.Is4() {
-		return wgcfg.CIDR{ip, 32}, true
-	} else {
-		return wgcfg.CIDR{ip, 128}, true
-	}
-}
-
-func warning(format string, args ...interface{}) {
-	fmt.Printf("Warning: "+format+"\n", args...)
-}
-
-// checkIPForwarding prints warnings on linux if IP forwarding is not
-// enabled, or if we were unable to verify the state of IP forwarding.
-func checkIPForwarding() {
-	if runtime.GOOS != "linux" {
-		return
-	}
-	bs, err := ioutil.ReadFile("/proc/sys/net/ipv4/ip_forward")
-	if err != nil {
-		warning("couldn't check /proc/sys/net/ipv4/ip_forward (%v).\nSubnet routes won't work without IP forwarding.", err)
-		return
-	}
-	on, err := strconv.ParseBool(string(bytes.TrimSpace(bs)))
-	if err != nil {
-		warning("couldn't parse /proc/sys/net/ipv4/ip_forward (%v).\nSubnet routes won't work without IP forwarding.", err)
-		return
-	}
-	if !on {
-		warning("/proc/sys/net/ipv4/ip_forward is disabled. Subnet routes won't work.")
-	}
-}
-
-func runUp(ctx context.Context, args []string) error {
-	if len(args) > 0 {
-		log.Fatalf("too many non-flag arguments: %q", args)
-	}
-
-	var routes []wgcfg.CIDR
-	if upArgs.advertiseRoutes != "" {
-		checkIPForwarding()
-		advroutes := strings.Split(upArgs.advertiseRoutes, ",")
-		for _, s := range advroutes {
-			cidr, ok := parseIPOrCIDR(s)
-			if !ok {
-				log.Fatalf("%q is not a valid IP address or CIDR prefix", s)
-			}
-			routes = append(routes, cidr)
-		}
-	}
-
-	var tags []string
-	if upArgs.advertiseTags != "" {
-		tags = strings.Split(upArgs.advertiseTags, ",")
-		for _, tag := range tags {
-			err := tailcfg.CheckTag(tag)
-			if err != nil {
-				log.Fatalf("tag: %q: %s", tag, err)
-			}
-		}
-	}
-
-	// TODO(apenwarr): fix different semantics between prefs and uflags
-	// TODO(apenwarr): allow setting/using CorpDNS
-	prefs := ipn.NewPrefs()
-	prefs.ControlURL = upArgs.server
-	prefs.WantRunning = true
-	prefs.RouteAll = upArgs.acceptRoutes
-	prefs.AllowSingleHosts = upArgs.singleRoutes
-	prefs.ShieldsUp = upArgs.shieldsUp
-	prefs.AdvertiseRoutes = routes
-	prefs.AdvertiseTags = tags
-	prefs.NoSNAT = !upArgs.snat
-	prefs.DisableDERP = !upArgs.enableDERP
-	if runtime.GOOS == "linux" {
-		switch upArgs.netfilterMode {
-		case "on":
-			prefs.NetfilterMode = router.NetfilterOn
-		case "nodivert":
-			prefs.NetfilterMode = router.NetfilterNoDivert
-			warning("netfilter=nodivert; add iptables calls to ts-* chains manually.")
-		case "off":
-			prefs.NetfilterMode = router.NetfilterOff
-			warning("netfilter=off; configure iptables yourself.")
-		default:
-			log.Fatalf("invalid value --netfilter-mode: %q", upArgs.netfilterMode)
-		}
-	}
-
-	c, bc, ctx, cancel := connect(ctx)
-	defer cancel()
-
-	var printed bool
-
-	bc.SetPrefs(prefs)
-	opts := ipn.Options{
-		StateKey: globalStateKey,
-		AuthKey:  upArgs.authKey,
-		Notify: func(n ipn.Notify) {
-			if n.ErrMessage != nil {
-				log.Fatalf("backend error: %v\n", *n.ErrMessage)
-			}
-			if s := n.State; s != nil {
-				switch *s {
-				case ipn.NeedsLogin:
-					printed = true
-					bc.StartLoginInteractive()
-				case ipn.NeedsMachineAuth:
-					printed = true
-					fmt.Fprintf(os.Stderr, "\nTo authorize your machine, visit (as admin):\n\n\t%s/admin/machines\n\n", upArgs.server)
-				case ipn.Starting, ipn.Running:
-					// Done full authentication process
-					if printed {
-						// Only need to print an update if we printed the "please click" message earlier.
-						fmt.Fprintf(os.Stderr, "Success.\n")
-					}
-					cancel()
-				}
-			}
-			if url := n.BrowseToURL; url != nil {
-				fmt.Fprintf(os.Stderr, "\nTo authenticate, visit:\n\n\t%s\n\n", *url)
-			}
-		},
-	}
-	// We still have to Start right now because it's the only way to
-	// set up notifications and whatnot. This causes a bunch of churn
-	// every time the CLI touches anything.
-	//
-	// TODO(danderson): redo the frontend/backend API to assume
-	// ephemeral frontends that read/modify/write state, once
-	// Windows/Mac state is moved into backend.
-	bc.Start(opts)
-	pump(ctx, bc, c)
-
-	return nil
-}
-
-func connect(ctx context.Context) (net.Conn, *ipn.BackendClient, context.Context, context.CancelFunc) {
-	c, err := safesocket.Connect(rootArgs.socket, 41112)
-	if err != nil {
-		if runtime.GOOS != "windows" && rootArgs.socket == "" {
-			log.Fatalf("--socket cannot be empty")
-		}
-		log.Fatalf("Failed to connect to connect to tailscaled. (safesocket.Connect: %v)\n", err)
-	}
-	clientToServer := func(b []byte) {
-		ipn.WriteMsg(c, b)
-	}
-
-	ctx, cancel := context.WithCancel(ctx)
-
-	go func() {
-		interrupt := make(chan os.Signal, 1)
-		signal.Notify(interrupt, syscall.SIGINT, syscall.SIGTERM)
-		<-interrupt
-		c.Close()
-		cancel()
-	}()
-
-	bc := ipn.NewBackendClient(log.Printf, clientToServer)
-	return c, bc, ctx, cancel
-}
-
-// pump receives backend messages on conn and pushes them into bc.
-func pump(ctx context.Context, bc *ipn.BackendClient, conn net.Conn) {
-	defer conn.Close()
-	for ctx.Err() == nil {
-		msg, err := ipn.ReadMsg(conn)
-		if err != nil {
-			if ctx.Err() != nil {
-				return
-			}
-			log.Printf("ReadMsg: %v\n", err)
-			break
-		}
-		bc.GotNotifyMsg(msg)
+	if err := cli.Run(os.Args[1:]); err != nil {
+		fmt.Fprintln(os.Stderr, err)
+		os.Exit(1)
 	}
 }

cmd/tailscaled/tailscaled.go

@@ -15,8 +15,10 @@ import (
 	"net/http"
 	"net/http/pprof"
 	"os"
+	"os/signal"
 	"runtime"
 	"runtime/debug"
+	"syscall"
 	"time"
 
 	"github.com/apenwarr/fixconsole"
@@ -27,6 +29,7 @@ import (
 	"tailscale.com/types/logger"
 	"tailscale.com/wgengine"
 	"tailscale.com/wgengine/magicsock"
+	"tailscale.com/wgengine/router"
 )
 
 // globalStateKey is the ipn.StateKey that tailscaled loads on
@@ -38,6 +41,27 @@ import (
 // later, the global state key doesn't look like a username.
 const globalStateKey = "_daemon"
 
+// defaultTunName returns the default tun device name for the platform.
+func defaultTunName() string {
+	switch runtime.GOOS {
+	case "openbsd":
+		return "tun"
+	case "windows":
+		return "Tailscale"
+	}
+	return "tailscale0"
+}
+
+var args struct {
+	cleanup    bool
+	fake       bool
+	debug      string
+	tunname    string
+	port       uint16
+	statepath  string
+	socketpath string
+}
+
 func main() {
 	// We aren't very performance sensitive, and the parts that are
 	// performance sensitive (wireguard) try hard not to do any memory
@@ -47,77 +71,117 @@ func main() {
 		debug.SetGCPercent(10)
 	}
 
-	defaultTunName := "tailscale0"
-	if runtime.GOOS == "openbsd" {
-		defaultTunName = "tun"
-	}
+	// Set default values for getopt.
+	args.tunname = defaultTunName()
+	args.port = magicsock.DefaultPort
+	args.statepath = paths.DefaultTailscaledStateFile()
+	args.socketpath = paths.DefaultTailscaledSocket()
 
-	fake := getopt.BoolLong("fake", 0, "fake tunnel+routing instead of tuntap")
-	debug := getopt.StringLong("debug", 0, "", "Address of debug server")
-	tunname := getopt.StringLong("tun", 0, defaultTunName, "tunnel interface name")
-	listenport := getopt.Uint16Long("port", 'p', magicsock.DefaultPort, "WireGuard port (0=autoselect)")
-	statepath := getopt.StringLong("state", 0, paths.DefaultTailscaledStateFile(), "Path of state file")
-	socketpath := getopt.StringLong("socket", 's', paths.DefaultTailscaledSocket(), "Path of the service unix socket")
-
-	logf := wgengine.RusagePrefixLog(log.Printf)
-	logf = logger.RateLimitedFn(logf, 5*time.Second, 5, 100)
+	getopt.FlagLong(&args.cleanup, "cleanup", 0, "clean up system state and exit")
+	getopt.FlagLong(&args.fake, "fake", 0, "fake tunnel+routing instead of tuntap")
+	getopt.FlagLong(&args.debug, "debug", 0, "address of debug server")
+	getopt.FlagLong(&args.tunname, "tun", 0, "tunnel interface name")
+	getopt.FlagLong(&args.port, "port", 'p', "WireGuard port (0=autoselect)")
+	getopt.FlagLong(&args.statepath, "state", 0, "path of state file")
+	getopt.FlagLong(&args.socketpath, "socket", 's', "path of the service unix socket")
 
 	err := fixconsole.FixConsoleIfNeeded()
 	if err != nil {
-		logf("fixConsoleOutput: %v", err)
+		log.Fatalf("fixConsoleOutput: %v", err)
 	}
-	pol := logpolicy.New("tailnode.log.tailscale.io")
 
 	getopt.Parse()
 	if len(getopt.Args()) > 0 {
 		log.Fatalf("too many non-flag arguments: %#v", getopt.Args()[0])
 	}
 
-	if *statepath == "" {
+	if args.statepath == "" {
 		log.Fatalf("--state is required")
 	}
 
-	if *socketpath == "" {
+	if args.socketpath == "" && runtime.GOOS != "windows" {
 		log.Fatalf("--socket is required")
 	}
 
+	if err := run(); err != nil {
+		// No need to log; the func already did
+		os.Exit(1)
+	}
+}
+
+func run() error {
+	var err error
+
+	pol := logpolicy.New("tailnode.log.tailscale.io")
+	defer func() {
+		// Finish uploading logs after closing everything else.
+		ctx, cancel := context.WithTimeout(context.Background(), time.Second)
+		defer cancel()
+		pol.Shutdown(ctx)
+	}()
+
+	logf := wgengine.RusagePrefixLog(log.Printf)
+	logf = logger.RateLimitedFn(logf, 5*time.Second, 5, 100)
+
+	if args.cleanup {
+		router.Cleanup(logf, args.tunname)
+		return nil
+	}
+
 	var debugMux *http.ServeMux
-	if *debug != "" {
+	if args.debug != "" {
 		debugMux = newDebugMux()
-		go runDebugServer(debugMux, *debug)
+		go runDebugServer(debugMux, args.debug)
 	}
 
 	var e wgengine.Engine
-	if *fake {
+	if args.fake {
 		e, err = wgengine.NewFakeUserspaceEngine(logf, 0)
 	} else {
-		e, err = wgengine.NewUserspaceEngine(logf, *tunname, *listenport)
+		e, err = wgengine.NewUserspaceEngine(logf, args.tunname, args.port)
 	}
 	if err != nil {
-		log.Fatalf("wgengine.New: %v", err)
+		logf("wgengine.New: %v", err)
+		return err
 	}
 	e = wgengine.NewWatchdog(e)
 
+	ctx, cancel := context.WithCancel(context.Background())
+	// Exit gracefully by cancelling the ipnserver context in most common cases:
+	// interrupted from the TTY or killed by a service manager.
+	interrupt := make(chan os.Signal, 1)
+	signal.Notify(interrupt, syscall.SIGINT, syscall.SIGTERM)
+	// SIGPIPE sometimes gets generated when CLIs disconnect from
+	// tailscaled. The default action is to terminate the process, we
+	// want to keep running.
+	signal.Ignore(syscall.SIGPIPE)
+	go func() {
+		select {
+		case s := <-interrupt:
+			logf("tailscaled got signal %v; shutting down", s)
+			cancel()
+		case <-ctx.Done():
+			// continue
+		}
+	}()
+
 	opts := ipnserver.Options{
-		SocketPath:         *socketpath,
+		SocketPath:         args.socketpath,
 		Port:               41112,
-		StatePath:          *statepath,
+		StatePath:          args.statepath,
 		AutostartStateKey:  globalStateKey,
-		LegacyConfigPath:   paths.LegacyConfigPath,
+		LegacyConfigPath:   paths.LegacyConfigPath(),
 		SurviveDisconnects: true,
 		DebugMux:           debugMux,
 	}
-	err = ipnserver.Run(context.Background(), logf, pol.PublicID.String(), opts, e)
-	if err != nil {
-		log.Fatalf("tailscaled: %v", err)
+	err = ipnserver.Run(ctx, logf, pol.PublicID.String(), ipnserver.FixedEngine(e), opts)
+	// Cancelation is not an error: it is the only way to stop ipnserver.
+	if err != nil && err != context.Canceled {
+		logf("ipnserver.Run: %v", err)
+		return err
 	}
 
-	// TODO(crawshaw): It would be nice to start a timeout context the moment a signal
-	// is received and use that timeout to give us a moment to finish uploading logs
-	// here. But the signal is handled inside ipnserver.Run, so some plumbing is needed.
-	ctx, cancel := context.WithCancel(context.Background())
-	cancel()
-	pol.Shutdown(ctx)
+	return nil
 }
 
 func newDebugMux() *http.ServeMux {

cmd/tailscaled/tailscaled.service

@@ -3,12 +3,11 @@ Description=Tailscale node agent
 Documentation=https://tailscale.com/kb/
 Wants=network-pre.target
 After=network-pre.target
-StartLimitIntervalSec=0
-StartLimitBurst=0
 
 [Service]
 EnvironmentFile=/etc/default/tailscaled
 ExecStart=/usr/sbin/tailscaled --state=/var/lib/tailscale/tailscaled.state --socket=/run/tailscale/tailscaled.sock --port $PORT $FLAGS
+ExecStopPost=/usr/sbin/tailscaled --cleanup
 
 Restart=on-failure
 

control/controlclient/auto.go

@@ -117,13 +117,15 @@ type Client struct {
 	mu         sync.Mutex   // mutex guards the following fields
 	statusFunc func(Status) // called to update Client status
 
-	loggedIn     bool       // true if currently logged in
-	loginGoal    *LoginGoal // non-nil if some login activity is desired
-	synced       bool       // true if our netmap is up-to-date
-	hostinfo     *tailcfg.Hostinfo
-	inPollNetMap bool // true if currently running a PollNetMap
-	inSendStatus int  // number of sendStatus calls currently in progress
-	state        State
+	paused         bool // whether we should stop making HTTP requests
+	unpauseWaiters []chan struct{}
+	loggedIn       bool       // true if currently logged in
+	loginGoal      *LoginGoal // non-nil if some login activity is desired
+	synced         bool       // true if our netmap is up-to-date
+	hostinfo       *tailcfg.Hostinfo
+	inPollNetMap   bool // true if currently running a PollNetMap
+	inSendStatus   int  // number of sendStatus calls currently in progress
+	state          State
 
 	authCtx    context.Context // context used for auth requests
 	mapCtx     context.Context // context used for netmap requests
@@ -169,6 +171,27 @@ func NewNoStart(opts Options) (*Client, error) {
 	return c, nil
 }
 
+// SetPaused controls whether HTTP activity should be paused.
+//
+// The client can be paused and unpaused repeatedly, unlike Start and Shutdown, which can only be used once.
+func (c *Client) SetPaused(paused bool) {
+	c.mu.Lock()
+	defer c.mu.Unlock()
+	if paused == c.paused {
+		return
+	}
+	c.paused = paused
+	if paused {
+		// Just cancel the map routine. The auth routine isn't expensive.
+		c.cancelMapLocked()
+	} else {
+		for _, ch := range c.unpauseWaiters {
+			close(ch)
+		}
+		c.unpauseWaiters = nil
+	}
+}
+
 // Start starts the client's goroutines.
 //
 // It should only be called for clients created by NewNoStart.
@@ -241,7 +264,7 @@ func (c *Client) cancelMapSafely() {
 
 func (c *Client) authRoutine() {
 	defer close(c.authDone)
-	bo := backoff.NewBackoff("authRoutine", c.logf)
+	bo := backoff.NewBackoff("authRoutine", c.logf, 30*time.Second)
 
 	for {
 		c.mu.Lock()
@@ -272,6 +295,7 @@ func (c *Client) authRoutine() {
 		if goal == nil {
 			// Wait for something interesting to happen
 			var exp <-chan time.Time
+			var expTimer *time.Timer
 			if expiry != nil && !expiry.IsZero() {
 				// if expiry is in the future, don't delay
 				// past that time.
@@ -284,11 +308,15 @@ func (c *Client) authRoutine() {
 					if delay > 5*time.Second {
 						delay = time.Second
 					}
-					exp = time.After(delay)
+					expTimer = time.NewTimer(delay)
+					exp = expTimer.C
 				}
 			}
 			select {
 			case <-ctx.Done():
+				if expTimer != nil {
+					expTimer.Stop()
+				}
 				c.logf("authRoutine: context done.")
 			case <-exp:
 				// Unfortunately the key expiry isn't provided
@@ -310,7 +338,7 @@ func (c *Client) authRoutine() {
 				}
 			}
 		} else if !goal.wantLoggedIn {
-			err := c.direct.TryLogout(c.authCtx)
+			err := c.direct.TryLogout(ctx)
 			if err != nil {
 				report(err, "TryLogout")
 				bo.BackOff(ctx, err)
@@ -355,12 +383,13 @@ func (c *Client) authRoutine() {
 					err = fmt.Errorf("weird: server required a new url?")
 					report(err, "WaitLoginURL")
 				}
-				goal.url = url
-				goal.token = nil
-				goal.flags = LoginDefault
 
 				c.mu.Lock()
-				c.loginGoal = goal
+				c.loginGoal = &LoginGoal{
+					wantLoggedIn: true,
+					flags:        LoginDefault,
+					url:          url,
+				}
 				c.state = StateURLVisitRequired
 				c.synced = false
 				c.mu.Unlock()
@@ -398,12 +427,35 @@ func (c *Client) Direct() *Direct {
 	return c.direct
 }
 
+// unpausedChanLocked returns a new channel that is closed when the
+// current Client pause is unpaused.
+//
+// c.mu must be held
+func (c *Client) unpausedChanLocked() <-chan struct{} {
+	unpaused := make(chan struct{})
+	c.unpauseWaiters = append(c.unpauseWaiters, unpaused)
+	return unpaused
+}
+
 func (c *Client) mapRoutine() {
 	defer close(c.mapDone)
-	bo := backoff.NewBackoff("mapRoutine", c.logf)
+	bo := backoff.NewBackoff("mapRoutine", c.logf, 30*time.Second)
 
 	for {
 		c.mu.Lock()
+		if c.paused {
+			unpaused := c.unpausedChanLocked()
+			c.mu.Unlock()
+			c.logf("mapRoutine: awaiting unpause")
+			select {
+			case <-unpaused:
+				c.logf("mapRoutine: unpaused")
+			case <-c.quit:
+				c.logf("mapRoutine: quit")
+				return
+			}
+			continue
+		}
 		c.logf("mapRoutine: %s", c.state)
 		loggedIn := c.loggedIn
 		ctx := c.mapCtx
@@ -486,8 +538,14 @@ func (c *Client) mapRoutine() {
 			if c.state == StateSynchronized {
 				c.state = StateAuthenticated
 			}
+			paused := c.paused
 			c.mu.Unlock()
 
+			if paused {
+				c.logf("mapRoutine: paused")
+				continue
+			}
+
 			if err != nil {
 				report(err, "PollNetMap")
 				bo.BackOff(ctx, err)
@@ -516,7 +574,7 @@ func (c *Client) SetHostinfo(hi *tailcfg.Hostinfo) {
 		panic("nil Hostinfo")
 	}
 	if !c.direct.SetHostinfo(hi) {
-		c.logf("[unexpected] duplicate Hostinfo: %v", hi)
+		// No changes. Don't log.
 		return
 	}
 	c.logf("Hostinfo: %v", hi)

control/controlclient/controlclient_test.go

@@ -70,3 +70,10 @@ func TestStatusEqual(t *testing.T) {
 		}
 	}
 }
+
+func TestOSVersion(t *testing.T) {
+	if osVersion == nil {
+		t.Skip("not available for OS")
+	}
+	t.Logf("Got: %#q", osVersion())
+}

control/controlclient/direct.go

@@ -4,6 +4,8 @@
 
 package controlclient
 
+//go:generate go run tailscale.com/cmd/cloner -type=Persist -output=direct_clone.go
+
 import (
 	"bytes"
 	"context"
@@ -19,6 +21,8 @@ import (
 	"net/url"
 	"os"
 	"reflect"
+	"runtime"
+	"sort"
 	"strconv"
 	"strings"
 	"sync"
@@ -27,6 +31,7 @@ import (
 	"github.com/tailscale/wireguard-go/wgcfg"
 	"golang.org/x/crypto/nacl/box"
 	"golang.org/x/oauth2"
+	"inet.af/netaddr"
 	"tailscale.com/log/logheap"
 	"tailscale.com/net/netns"
 	"tailscale.com/net/tlsdial"
@@ -85,6 +90,7 @@ type Direct struct {
 	newDecompressor func() (Decompressor, error)
 	keepAlive       bool
 	logf            logger.Logf
+	discoPubKey     tailcfg.DiscoKey
 
 	mu           sync.Mutex // mutex guards the following fields
 	serverKey    wgcfg.Key
@@ -92,9 +98,10 @@ type Direct struct {
 	authKey      string
 	tryingNewKey wgcfg.PrivateKey
 	expiry       *time.Time
-	hostinfo     *tailcfg.Hostinfo // always non-nil
-	endpoints    []string
-	localPort    uint16 // or zero to mean auto
+	// hostinfo is mutated in-place while mu is held.
+	hostinfo  *tailcfg.Hostinfo // always non-nil
+	endpoints []string
+	localPort uint16 // or zero to mean auto
 }
 
 type Options struct {
@@ -103,6 +110,7 @@ type Options struct {
 	AuthKey         string            // optional node auth key for auto registration
 	TimeNow         func() time.Time  // time.Now implementation used by Client
 	Hostinfo        *tailcfg.Hostinfo // non-nil passes ownership, nil means to use default using os.Hostname, etc
+	DiscoPublicKey  tailcfg.DiscoKey
 	NewDecompressor func() (Decompressor, error)
 	KeepAlive       bool
 	Logf            logger.Logf
@@ -152,6 +160,7 @@ func NewDirect(opts Options) (*Direct, error) {
 		keepAlive:       opts.KeepAlive,
 		persist:         opts.Persist,
 		authKey:         opts.AuthKey,
+		discoPubKey:     opts.DiscoPublicKey,
 	}
 	if opts.Hostinfo == nil {
 		c.SetHostinfo(NewHostinfo())
@@ -161,12 +170,20 @@ func NewDirect(opts Options) (*Direct, error) {
 	return c, nil
 }
 
+var osVersion func() string // non-nil on some platforms
+
 func NewHostinfo() *tailcfg.Hostinfo {
 	hostname, _ := os.Hostname()
+	var osv string
+	if osVersion != nil {
+		osv = osVersion()
+	}
 	return &tailcfg.Hostinfo{
 		IPNVersion: version.LONG,
 		Hostname:   hostname,
 		OS:         version.OS(),
+		OSVersion:  osv,
+		GoArch:     runtime.GOARCH,
 	}
 }
 
@@ -262,6 +279,8 @@ func (c *Direct) doLogin(ctx context.Context, t *oauth2.Token, flags LoginFlags,
 	tryingNewKey := c.tryingNewKey
 	serverKey := c.serverKey
 	authKey := c.authKey
+	hostinfo := c.hostinfo.Clone()
+	backendLogID := hostinfo.BackendLogID
 	expired := c.expiry != nil && !c.expiry.IsZero() && c.expiry.Before(c.timeNow())
 	c.mu.Unlock()
 
@@ -318,7 +337,7 @@ func (c *Direct) doLogin(ctx context.Context, t *oauth2.Token, flags LoginFlags,
 	if tryingNewKey == (wgcfg.PrivateKey{}) {
 		log.Fatalf("tryingNewKey is empty, give up")
 	}
-	if c.hostinfo.BackendLogID == "" {
+	if backendLogID == "" {
 		err = errors.New("hostinfo: BackendLogID missing")
 		return regen, url, err
 	}
@@ -326,7 +345,7 @@ func (c *Direct) doLogin(ctx context.Context, t *oauth2.Token, flags LoginFlags,
 		Version:    1,
 		OldNodeKey: tailcfg.NodeKey(oldNodeKey),
 		NodeKey:    tailcfg.NodeKey(tryingNewKey.Public()),
-		Hostinfo:   c.hostinfo,
+		Hostinfo:   hostinfo,
 		Followup:   url,
 	}
 	c.logf("RegisterReq: onode=%v node=%v fup=%v",
@@ -381,7 +400,7 @@ func (c *Direct) doLogin(ctx context.Context, t *oauth2.Token, flags LoginFlags,
 	//	- user is disabled
 
 	if resp.AuthURL != "" {
-		c.logf("AuthURL is %.20v...", resp.AuthURL)
+		c.logf("AuthURL is %v", resp.AuthURL)
 	} else {
 		c.logf("No AuthURL")
 	}
@@ -445,19 +464,18 @@ func (c *Direct) SetEndpoints(localPort uint16, endpoints []string) (changed boo
 	return c.newEndpoints(localPort, endpoints)
 }
 
-var debugNetmap, _ = strconv.ParseBool(os.Getenv("TS_DEBUG_NETMAP"))
-
 func (c *Direct) PollNetMap(ctx context.Context, maxPolls int, cb func(*NetworkMap)) error {
 	c.mu.Lock()
 	persist := c.persist
 	serverURL := c.serverURL
 	serverKey := c.serverKey
-	hostinfo := c.hostinfo
+	hostinfo := c.hostinfo.Clone()
+	backendLogID := hostinfo.BackendLogID
 	localPort := c.localPort
 	ep := append([]string(nil), c.endpoints...)
 	c.mu.Unlock()
 
-	if hostinfo.BackendLogID == "" {
+	if backendLogID == "" {
 		return errors.New("hostinfo: BackendLogID missing")
 	}
 
@@ -465,18 +483,21 @@ func (c *Direct) PollNetMap(ctx context.Context, maxPolls int, cb func(*NetworkM
 	c.logf("PollNetMap: stream=%v :%v %v", maxPolls, localPort, ep)
 
 	vlogf := logger.Discard
-	if debugNetmap {
+	if Debug.NetMap {
 		vlogf = c.logf
 	}
 
 	request := tailcfg.MapRequest{
-		Version:     4,
-		IncludeIPv6: includeIPv6(),
-		KeepAlive:   c.keepAlive,
-		NodeKey:     tailcfg.NodeKey(persist.PrivateNodeKey.Public()),
-		Endpoints:   ep,
-		Stream:      allowStream,
-		Hostinfo:    hostinfo,
+		Version:         4,
+		IncludeIPv6:     true,
+		DeltaPeers:      true,
+		KeepAlive:       c.keepAlive,
+		NodeKey:         tailcfg.NodeKey(persist.PrivateNodeKey.Public()),
+		DiscoKey:        c.discoPubKey,
+		Endpoints:       ep,
+		Stream:          allowStream,
+		Hostinfo:        hostinfo,
+		DebugForceDisco: Debug.ForceDisco,
 	}
 	if c.newDecompressor != nil {
 		request.Compress = "zstd"
@@ -554,6 +575,7 @@ func (c *Direct) PollNetMap(ctx context.Context, maxPolls int, cb func(*NetworkM
 	// the same format before just closing the connection.
 	// We can use this same read loop either way.
 	var msg []byte
+	var previousPeers []*tailcfg.Node // for delta-purposes
 	for i := 0; i < maxPolls || maxPolls < 0; i++ {
 		vlogf("netmap: starting size read after %v (poll %v)", time.Since(t0).Round(time.Millisecond), i)
 		var siz [4]byte
@@ -575,31 +597,51 @@ func (c *Direct) PollNetMap(ctx context.Context, maxPolls int, cb func(*NetworkM
 			vlogf("netmap: decode error: %v")
 			return err
 		}
+
 		if resp.KeepAlive {
 			vlogf("netmap: got keep-alive")
-			select {
-			case timeoutReset <- struct{}{}:
-				vlogf("netmap: sent keep-alive timer reset")
-			case <-ctx.Done():
-				c.logf("netmap: not resetting timer for keep-alive due to: %v", ctx.Err())
-				return ctx.Err()
-			}
+		} else {
+			vlogf("netmap: got new map")
+		}
+		select {
+		case timeoutReset <- struct{}{}:
+			vlogf("netmap: sent timer reset")
+		case <-ctx.Done():
+			c.logf("netmap: not resetting timer; context done: %v", ctx.Err())
+			return ctx.Err()
+		}
+		if resp.KeepAlive {
 			continue
 		}
-		vlogf("netmap: got new map")
+
+		undeltaPeers(&resp, previousPeers)
+		previousPeers = cloneNodes(resp.Peers) // defensive/lazy clone, since this escapes to who knows where
 
 		if resp.DERPMap != nil {
 			vlogf("netmap: new map contains DERP map")
 			lastDERPMap = resp.DERPMap
 		}
 		if resp.Debug != nil && resp.Debug.LogHeapPprof {
-			logheap.LogHeap()
+			go logheap.LogHeap(resp.Debug.LogHeapURL)
+		}
+		// Temporarily (2020-06-29) support removing all but
+		// discovery-supporting nodes during development, for
+		// less noise.
+		if Debug.OnlyDisco {
+			filtered := resp.Peers[:0]
+			for _, p := range resp.Peers {
+				if !p.DiscoKey.IsZero() {
+					filtered = append(filtered, p)
+				}
+			}
+			resp.Peers = filtered
 		}
 
 		nm := &NetworkMap{
 			NodeKey:      tailcfg.NodeKey(persist.PrivateNodeKey.Public()),
 			PrivateKey:   persist.PrivateNodeKey,
 			Expiry:       resp.Node.KeyExpiry,
+			Name:         resp.Node.Name,
 			Addresses:    resp.Node.Addresses,
 			Peers:        resp.Peers,
 			LocalPort:    localPort,
@@ -607,11 +649,11 @@ func (c *Direct) PollNetMap(ctx context.Context, maxPolls int, cb func(*NetworkM
 			UserProfiles: make(map[tailcfg.UserID]tailcfg.UserProfile),
 			Domain:       resp.Domain,
 			Roles:        resp.Roles,
-			DNS:          resp.DNS,
-			DNSDomains:   resp.SearchPaths,
+			DNS:          resp.DNSConfig,
 			Hostinfo:     resp.Node.Hostinfo,
 			PacketFilter: c.parsePacketFilter(resp.PacketFilter),
 			DERPMap:      lastDERPMap,
+			Debug:        resp.Debug,
 		}
 		for _, profile := range resp.UserProfiles {
 			nm.UserProfiles[profile.ID] = profile
@@ -621,6 +663,15 @@ func (c *Direct) PollNetMap(ctx context.Context, maxPolls int, cb func(*NetworkM
 		} else {
 			nm.MachineStatus = tailcfg.MachineUnauthorized
 		}
+		if len(resp.DNS) > 0 {
+			nm.DNS.Nameservers = wgIPToNetaddr(resp.DNS)
+		}
+		if len(resp.SearchPaths) > 0 {
+			nm.DNS.Domains = resp.SearchPaths
+		}
+		if Debug.ProxyDNS {
+			nm.DNS.Proxied = true
+		}
 
 		// Printing the netmap can be extremely verbose, but is very
 		// handy for debugging. Let's limit how often we do it.
@@ -658,8 +709,10 @@ func decode(res *http.Response, v interface{}, serverKey *wgcfg.Key, mkey *wgcfg
 }
 
 func (c *Direct) decodeMsg(msg []byte, v interface{}) error {
+	c.mu.Lock()
 	mkey := c.persist.PrivateMachineKey
 	serverKey := c.serverKey
+	c.mu.Unlock()
 
 	decrypted, err := decryptMsg(msg, &serverKey, &mkey)
 	if err != nil {
@@ -758,13 +811,145 @@ func loadServerKey(ctx context.Context, httpc *http.Client, serverURL string) (w
 	return key, nil
 }
 
-// includeIPv6 reports whether we should enable IPv6 for magicsock
-// connections. This is only here temporarily (2020-03-26) as a
-// opt-out in case there are problems.
-func includeIPv6() bool {
-	if e := os.Getenv("DEBUG_INCLUDE_IPV6"); e != "" {
-		v, _ := strconv.ParseBool(e)
-		return v
+func wgIPToNetaddr(ips []wgcfg.IP) (ret []netaddr.IP) {
+	for _, ip := range ips {
+		nip, ok := netaddr.FromStdIP(ip.IP())
+		if !ok {
+			panic(fmt.Sprintf("conversion of %s from wgcfg to netaddr IP failed", ip))
+		}
+		ret = append(ret, nip.Unmap())
+	}
+	return ret
+}
+
+// Debug contains temporary internal-only debug knobs.
+// They're unexported to not draw attention to them.
+var Debug = initDebug()
+
+type debug struct {
+	NetMap     bool
+	ProxyDNS   bool
+	OnlyDisco  bool
+	Disco      bool
+	ForceDisco bool // ask control server to not filter out our disco key
+}
+
+func initDebug() debug {
+	d := debug{
+		NetMap:     envBool("TS_DEBUG_NETMAP"),
+		ProxyDNS:   envBool("TS_DEBUG_PROXY_DNS"),
+		OnlyDisco:  os.Getenv("TS_DEBUG_USE_DISCO") == "only",
+		ForceDisco: os.Getenv("TS_DEBUG_USE_DISCO") == "only" || envBool("TS_DEBUG_USE_DISCO"),
+	}
+	if d.ForceDisco || os.Getenv("TS_DEBUG_USE_DISCO") == "" {
+		// This is now defaults to on.
+		d.Disco = true
+	}
+	return d
+}
+
+func envBool(k string) bool {
+	e := os.Getenv(k)
+	if e == "" {
+		return false
+	}
+	v, err := strconv.ParseBool(e)
+	if err != nil {
+		panic(fmt.Sprintf("invalid non-bool %q for env var %q", e, k))
+	}
+	return v
+}
+
+// undeltaPeers updates mapRes.Peers to be complete based on the provided previous peer list
+// and the PeersRemoved and PeersChanged fields in mapRes.
+// It then also nils out the delta fields.
+func undeltaPeers(mapRes *tailcfg.MapResponse, prev []*tailcfg.Node) {
+	if len(mapRes.Peers) > 0 {
+		// Not delta encoded.
+		if !nodesSorted(mapRes.Peers) {
+			log.Printf("netmap: undeltaPeers: MapResponse.Peers not sorted; sorting")
+			sortNodes(mapRes.Peers)
+		}
+		return
+	}
+
+	var removed map[tailcfg.NodeID]bool
+	if pr := mapRes.PeersRemoved; len(pr) > 0 {
+		removed = make(map[tailcfg.NodeID]bool, len(pr))
+		for _, id := range pr {
+			removed[id] = true
+		}
+	}
+	changed := mapRes.PeersChanged
+
+	if len(removed) == 0 && len(changed) == 0 {
+		// No changes fast path.
+		mapRes.Peers = prev
+		return
+	}
+
+	if !nodesSorted(changed) {
+		log.Printf("netmap: undeltaPeers: MapResponse.PeersChanged not sorted; sorting")
+		sortNodes(changed)
+	}
+	if !nodesSorted(prev) {
+		// Internal error (unrelated to the network) if we get here.
+		log.Printf("netmap: undeltaPeers: [unexpected] prev not sorted; sorting")
+		sortNodes(prev)
+	}
+
+	newFull := make([]*tailcfg.Node, 0, len(prev)-len(removed))
+	for len(prev) > 0 && len(changed) > 0 {
+		pID := prev[0].ID
+		cID := changed[0].ID
+		if removed[pID] {
+			prev = prev[1:]
+			continue
+		}
+		switch {
+		case pID < cID:
+			newFull = append(newFull, prev[0])
+			prev = prev[1:]
+		case pID == cID:
+			newFull = append(newFull, changed[0])
+			prev, changed = prev[1:], changed[1:]
+		case cID < pID:
+			newFull = append(newFull, changed[0])
+			changed = changed[1:]
+		}
+	}
+	newFull = append(newFull, changed...)
+	for _, n := range prev {
+		if !removed[n.ID] {
+			newFull = append(newFull, n)
+		}
+	}
+	sortNodes(newFull)
+	mapRes.Peers = newFull
+	mapRes.PeersChanged = nil
+	mapRes.PeersRemoved = nil
+}
+
+func nodesSorted(v []*tailcfg.Node) bool {
+	for i, n := range v {
+		if i > 0 && n.ID <= v[i-1].ID {
+			return false
+		}
 	}
 	return true
 }
+
+func sortNodes(v []*tailcfg.Node) {
+	sort.Slice(v, func(i, j int) bool { return v[i].ID < v[j].ID })
+}
+
+func cloneNodes(v1 []*tailcfg.Node) []*tailcfg.Node {
+	if v1 == nil {
+		return nil
+	}
+	v2 := make([]*tailcfg.Node, len(v1))
+	for i, n := range v1 {
+		v2[i] = n.Clone()
+	}
+	return v2
+}

control/controlclient/direct_clone.go

@@ -0,0 +1,20 @@
+// Copyright (c) 2020 Tailscale Inc & AUTHORS All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+// Code generated by tailscale.com/cmd/cloner -type Persist; DO NOT EDIT.
+
+package controlclient
+
+import ()
+
+// Clone makes a deep copy of Persist.
+// The result aliases no memory with the original.
+func (src *Persist) Clone() *Persist {
+	if src == nil {
+		return nil
+	}
+	dst := new(Persist)
+	*dst = *src
+	return dst
+}

control/controlclient/direct_test.go

@@ -0,0 +1,93 @@
+// Copyright (c) 2020 Tailscale Inc & AUTHORS All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package controlclient
+
+import (
+	"fmt"
+	"reflect"
+	"strings"
+	"testing"
+
+	"tailscale.com/tailcfg"
+)
+
+func TestUndeltaPeers(t *testing.T) {
+	n := func(id tailcfg.NodeID, name string) *tailcfg.Node {
+		return &tailcfg.Node{ID: id, Name: name}
+	}
+	peers := func(nv ...*tailcfg.Node) []*tailcfg.Node { return nv }
+	tests := []struct {
+		name   string
+		mapRes *tailcfg.MapResponse
+		prev   []*tailcfg.Node
+		want   []*tailcfg.Node
+	}{
+		{
+			name: "full_peers",
+			mapRes: &tailcfg.MapResponse{
+				Peers: peers(n(1, "foo"), n(2, "bar")),
+			},
+			want: peers(n(1, "foo"), n(2, "bar")),
+		},
+		{
+			name: "full_peers_ignores_deltas",
+			mapRes: &tailcfg.MapResponse{
+				Peers:        peers(n(1, "foo"), n(2, "bar")),
+				PeersRemoved: []tailcfg.NodeID{2},
+			},
+			want: peers(n(1, "foo"), n(2, "bar")),
+		},
+		{
+			name: "add_and_update",
+			prev: peers(n(1, "foo"), n(2, "bar")),
+			mapRes: &tailcfg.MapResponse{
+				PeersChanged: peers(n(0, "zero"), n(2, "bar2"), n(3, "three")),
+			},
+			want: peers(n(0, "zero"), n(1, "foo"), n(2, "bar2"), n(3, "three")),
+		},
+		{
+			name: "remove",
+			prev: peers(n(1, "foo"), n(2, "bar")),
+			mapRes: &tailcfg.MapResponse{
+				PeersRemoved: []tailcfg.NodeID{1},
+			},
+			want: peers(n(2, "bar")),
+		},
+		{
+			name: "add_and_remove",
+			prev: peers(n(1, "foo"), n(2, "bar")),
+			mapRes: &tailcfg.MapResponse{
+				PeersChanged: peers(n(1, "foo2")),
+				PeersRemoved: []tailcfg.NodeID{2},
+			},
+			want: peers(n(1, "foo2")),
+		},
+		{
+			name:   "unchanged",
+			prev:   peers(n(1, "foo"), n(2, "bar")),
+			mapRes: &tailcfg.MapResponse{},
+			want:   peers(n(1, "foo"), n(2, "bar")),
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			undeltaPeers(tt.mapRes, tt.prev)
+			if !reflect.DeepEqual(tt.mapRes.Peers, tt.want) {
+				t.Errorf("wrong results\n got: %s\nwant: %s", formatNodes(tt.mapRes.Peers), formatNodes(tt.want))
+			}
+		})
+	}
+}
+
+func formatNodes(nodes []*tailcfg.Node) string {
+	var sb strings.Builder
+	for i, n := range nodes {
+		if i > 0 {
+			sb.WriteString(", ")
+		}
+		fmt.Fprintf(&sb, "(%d, %q)", n.ID, n.Name)
+	}
+	return sb.String()
+}

control/controlclient/hostinfo_linux.go

@@ -0,0 +1,87 @@
+// Copyright (c) 2020 Tailscale Inc & AUTHORS All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+// +build linux,!android
+
+package controlclient
+
+import (
+	"bytes"
+	"fmt"
+	"io"
+	"io/ioutil"
+	"strings"
+	"syscall"
+
+	"go4.org/mem"
+	"tailscale.com/util/lineread"
+)
+
+func init() {
+	osVersion = osVersionLinux
+}
+
+func osVersionLinux() string {
+	m := map[string]string{}
+	lineread.File("/etc/os-release", func(line []byte) error {
+		eq := bytes.IndexByte(line, '=')
+		if eq == -1 {
+			return nil
+		}
+		k, v := string(line[:eq]), strings.Trim(string(line[eq+1:]), `"`)
+		m[k] = v
+		return nil
+	})
+
+	var un syscall.Utsname
+	syscall.Uname(&un)
+
+	var attrBuf strings.Builder
+	attrBuf.WriteString("; kernel=")
+	for _, b := range un.Release {
+		if b == 0 {
+			break
+		}
+		attrBuf.WriteByte(byte(b))
+	}
+	if inContainer() {
+		attrBuf.WriteString("; container")
+	}
+	attr := attrBuf.String()
+
+	id := m["ID"]
+
+	switch id {
+	case "debian":
+		slurp, _ := ioutil.ReadFile("/etc/debian_version")
+		return fmt.Sprintf("Debian %s (%s)%s", bytes.TrimSpace(slurp), m["VERSION_CODENAME"], attr)
+	case "ubuntu":
+		return fmt.Sprintf("Ubuntu %s%s", m["VERSION"], attr)
+	case "", "centos": // CentOS 6 has no /etc/os-release, so its id is ""
+		if cr, _ := ioutil.ReadFile("/etc/centos-release"); len(cr) > 0 { // "CentOS release 6.10 (Final)
+			return fmt.Sprintf("%s%s", bytes.TrimSpace(cr), attr)
+		}
+		fallthrough
+	case "fedora", "rhel", "alpine":
+		// Their PRETTY_NAME is fine as-is for all versions I tested.
+		fallthrough
+	default:
+		if v := m["PRETTY_NAME"]; v != "" {
+			return fmt.Sprintf("%s%s", v, attr)
+		}
+	}
+	return fmt.Sprintf("Other%s", attr)
+}
+
+func inContainer() (ret bool) {
+	lineread.File("/proc/1/cgroup", func(line []byte) error {
+		if mem.Contains(mem.B(line), mem.S("/docker/")) ||
+			mem.Contains(mem.B(line), mem.S("/lxc/")) {
+			ret = true
+			return io.EOF // arbitrary non-nil error to stop loop
+		}
+		return nil
+	})
+	return
+}

control/controlclient/hostinfo_windows.go

@@ -0,0 +1,26 @@
+// Copyright (c) 2020 Tailscale Inc & AUTHORS All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package controlclient
+
+import (
+	"os/exec"
+	"strings"
+	"syscall"
+)
+
+func init() {
+	osVersion = osVersionWindows
+}
+
+func osVersionWindows() string {
+	cmd := exec.Command("cmd", "/c", "ver")
+	cmd.SysProcAttr = &syscall.SysProcAttr{HideWindow: true}
+	out, _ := cmd.Output() // "\nMicrosoft Windows [Version 10.0.19041.388]\n\n"
+	s := strings.TrimSpace(string(out))
+	s = strings.TrimPrefix(s, "Microsoft Windows [")
+	s = strings.TrimSuffix(s, "]")
+	s = strings.TrimPrefix(s, "Version ") // is this localized? do it last in case.
+	return s                              // "10.0.19041.388", ideally
+}

control/controlclient/netmap.go

@@ -5,31 +5,33 @@
 package controlclient
 
 import (
-	"bytes"
-	"encoding/base64"
 	"encoding/json"
 	"fmt"
-	"log"
+	"net"
+	"reflect"
+	"strconv"
 	"strings"
 	"time"
 
 	"github.com/tailscale/wireguard-go/wgcfg"
 	"tailscale.com/tailcfg"
+	"tailscale.com/types/logger"
 	"tailscale.com/wgengine/filter"
 )
 
 type NetworkMap struct {
 	// Core networking
 
-	NodeKey       tailcfg.NodeKey
-	PrivateKey    wgcfg.PrivateKey
-	Expiry        time.Time
+	NodeKey    tailcfg.NodeKey
+	PrivateKey wgcfg.PrivateKey
+	Expiry     time.Time
+	// Name is the DNS name assigned to this node.
+	Name          string
 	Addresses     []wgcfg.CIDR
 	LocalPort     uint16 // used for debugging
 	MachineStatus tailcfg.MachineStatus
-	Peers         []*tailcfg.Node
-	DNS           []wgcfg.IP
-	DNSDomains    []string
+	Peers         []*tailcfg.Node // sorted by Node.ID
+	DNS           tailcfg.DNSConfig
 	Hostinfo      tailcfg.Hostinfo
 	PacketFilter  filter.Matches
 
@@ -37,6 +39,9 @@ type NetworkMap struct {
 	// between updates and should not be modified.
 	DERPMap *tailcfg.DERPMap
 
+	// Debug knobs from control server for debug or feature gating.
+	Debug *tailcfg.Debug
+
 	// ACLs
 
 	User   tailcfg.UserID
@@ -49,92 +54,153 @@ type NetworkMap struct {
 	// TODO(crawshaw): Capabilities []tailcfg.Capability
 }
 
-func (n *NetworkMap) Equal(n2 *NetworkMap) bool {
-	// TODO(crawshaw): this is crude, but is an easy way to avoid bugs.
-	b, err := json.Marshal(n)
-	if err != nil {
-		panic(err)
-	}
-	b2, err := json.Marshal(n2)
-	if err != nil {
-		panic(err)
-	}
-	return bytes.Equal(b, b2)
-}
-
 func (nm NetworkMap) String() string {
 	return nm.Concise()
 }
 
 func (nm *NetworkMap) Concise() string {
 	buf := new(strings.Builder)
-	fmt.Fprintf(buf, "netmap: self: %v auth=%v :%v %v\n",
-		nm.NodeKey.ShortString(), nm.MachineStatus,
-		nm.LocalPort, nm.Addresses)
+
+	nm.printConciseHeader(buf)
 	for _, p := range nm.Peers {
-		aip := make([]string, len(p.AllowedIPs))
-		for i, a := range p.AllowedIPs {
-			s := fmt.Sprint(a)
-			if strings.HasSuffix(s, "/32") {
-				s = s[0 : len(s)-3]
-			}
-			aip[i] = s
-		}
-
-		ep := make([]string, len(p.Endpoints))
-		for i, e := range p.Endpoints {
-			// Align vertically on the ':' between IP and port
-			colon := strings.IndexByte(e, ':')
-			for colon > 0 && len(e)-colon < 6 {
-				e += " "
-				colon--
-			}
-			ep[i] = fmt.Sprintf("%21v", e)
-		}
-
-		derp := p.DERP
-		const derpPrefix = "127.3.3.40:"
-		if strings.HasPrefix(derp, derpPrefix) {
-			derp = "D" + derp[len(derpPrefix):]
-		}
-
-		// Most of the time, aip is just one element, so format the
-		// table to look good in that case. This will also make multi-
-		// subnet nodes stand out visually.
-		fmt.Fprintf(buf, " %v %-2v %-15v : %v\n",
-			p.Key.ShortString(), derp,
-			strings.Join(aip, " "),
-			strings.Join(ep, " "))
+		printPeerConcise(buf, p)
 	}
 	return buf.String()
 }
 
+// printConciseHeader prints a concise header line representing nm to buf.
+//
+// If this function is changed to access different fields of nm, keep
+// in equalConciseHeader in sync.
+func (nm *NetworkMap) printConciseHeader(buf *strings.Builder) {
+	fmt.Fprintf(buf, "netmap: self: %v auth=%v",
+		nm.NodeKey.ShortString(), nm.MachineStatus)
+	if nm.LocalPort != 0 {
+		fmt.Fprintf(buf, " port=%v", nm.LocalPort)
+	}
+	if nm.Debug != nil {
+		j, _ := json.Marshal(nm.Debug)
+		fmt.Fprintf(buf, " debug=%s", j)
+	}
+	fmt.Fprintf(buf, " %v", nm.Addresses)
+	buf.WriteByte('\n')
+}
+
+// equalConciseHeader reports whether a and b are equal for the fields
+// used by printConciseHeader.
+func (a *NetworkMap) equalConciseHeader(b *NetworkMap) bool {
+	if a.NodeKey != b.NodeKey ||
+		a.MachineStatus != b.MachineStatus ||
+		a.LocalPort != b.LocalPort ||
+		len(a.Addresses) != len(b.Addresses) {
+		return false
+	}
+	for i, a := range a.Addresses {
+		if b.Addresses[i] != a {
+			return false
+		}
+	}
+	return (a.Debug == nil && b.Debug == nil) || reflect.DeepEqual(a.Debug, b.Debug)
+}
+
+// printPeerConcise appends to buf a line repsenting the peer p.
+//
+// If this function is changed to access different fields of p, keep
+// in nodeConciseEqual in sync.
+func printPeerConcise(buf *strings.Builder, p *tailcfg.Node) {
+	aip := make([]string, len(p.AllowedIPs))
+	for i, a := range p.AllowedIPs {
+		s := strings.TrimSuffix(fmt.Sprint(a), "/32")
+		aip[i] = s
+	}
+
+	ep := make([]string, len(p.Endpoints))
+	for i, e := range p.Endpoints {
+		// Align vertically on the ':' between IP and port
+		colon := strings.IndexByte(e, ':')
+		spaces := 0
+		for colon > 0 && len(e)+spaces-colon < 6 {
+			spaces++
+			colon--
+		}
+		ep[i] = fmt.Sprintf("%21v", e+strings.Repeat(" ", spaces))
+	}
+
+	derp := p.DERP
+	const derpPrefix = "127.3.3.40:"
+	if strings.HasPrefix(derp, derpPrefix) {
+		derp = "D" + derp[len(derpPrefix):]
+	}
+	var discoShort string
+	if !p.DiscoKey.IsZero() {
+		discoShort = p.DiscoKey.ShortString() + " "
+	}
+
+	// Most of the time, aip is just one element, so format the
+	// table to look good in that case. This will also make multi-
+	// subnet nodes stand out visually.
+	fmt.Fprintf(buf, " %v %s%-2v %-15v : %v\n",
+		p.Key.ShortString(),
+		discoShort,
+		derp,
+		strings.Join(aip, " "),
+		strings.Join(ep, " "))
+}
+
+// nodeConciseEqual reports whether a and b are equal for the fields accessed by printPeerConcise.
+func nodeConciseEqual(a, b *tailcfg.Node) bool {
+	return a.Key == b.Key &&
+		a.DERP == b.DERP &&
+		a.DiscoKey == b.DiscoKey &&
+		eqCIDRsIgnoreNil(a.AllowedIPs, b.AllowedIPs) &&
+		eqStringsIgnoreNil(a.Endpoints, b.Endpoints)
+}
+
 func (b *NetworkMap) ConciseDiffFrom(a *NetworkMap) string {
-	out := []string{}
-	ra := strings.Split(a.Concise(), "\n")
-	rb := strings.Split(b.Concise(), "\n")
+	var diff strings.Builder
 
-	ma := map[string]struct{}{}
-	for _, s := range ra {
-		ma[s] = struct{}{}
+	// See if header (non-peers, "bare") part of the network map changed.
+	// If so, print its diff lines first.
+	if !a.equalConciseHeader(b) {
+		diff.WriteByte('-')
+		a.printConciseHeader(&diff)
+		diff.WriteByte('+')
+		b.printConciseHeader(&diff)
 	}
 
-	mb := map[string]struct{}{}
-	for _, s := range rb {
-		mb[s] = struct{}{}
-	}
-
-	for _, s := range ra {
-		if _, ok := mb[s]; !ok {
-			out = append(out, "-"+s)
+	aps, bps := a.Peers, b.Peers
+	for len(aps) > 0 && len(bps) > 0 {
+		pa, pb := aps[0], bps[0]
+		switch {
+		case pa.ID == pb.ID:
+			if !nodeConciseEqual(pa, pb) {
+				diff.WriteByte('-')
+				printPeerConcise(&diff, pa)
+				diff.WriteByte('+')
+				printPeerConcise(&diff, pb)
+			}
+			aps, bps = aps[1:], bps[1:]
+		case pa.ID > pb.ID:
+			// New peer in b.
+			diff.WriteByte('+')
+			printPeerConcise(&diff, pb)
+			bps = bps[1:]
+		case pb.ID > pa.ID:
+			// Deleted peer in b.
+			diff.WriteByte('-')
+			printPeerConcise(&diff, pa)
+			aps = aps[1:]
 		}
 	}
-	for _, s := range rb {
-		if _, ok := ma[s]; !ok {
-			out = append(out, "+"+s)
-		}
+	for _, pa := range aps {
+		diff.WriteByte('-')
+		printPeerConcise(&diff, pa)
 	}
-	return strings.Join(out, "\n")
+	for _, pb := range bps {
+		diff.WriteByte('+')
+		printPeerConcise(&diff, pb)
+	}
+	return diff.String()
 }
 
 func (nm *NetworkMap) JSON() string {
@@ -145,138 +211,126 @@ func (nm *NetworkMap) JSON() string {
 	return string(b)
 }
 
-const (
-	UAllowSingleHosts = 1 << iota
-	UAllowSubnetRoutes
-	UAllowDefaultRoute
-	UHackDefaultRoute
+// WGConfigFlags is a bitmask of flags to control the behavior of the
+// wireguard configuration generation done by NetMap.WGCfg.
+type WGConfigFlags int
 
-	UDefault = 0
+const (
+	AllowSingleHosts WGConfigFlags = 1 << iota
+	AllowSubnetRoutes
+	AllowDefaultRoute
+	HackDefaultRoute
 )
 
-// Several programs need to parse these arguments into uflags, so let's
-// centralize it here.
-func UFlagsHelper(uroutes, rroutes, droutes bool) int {
-	uflags := 0
-	if uroutes {
-		uflags |= UAllowSingleHosts
-	}
-	if rroutes {
-		uflags |= UAllowSubnetRoutes
-	}
-	if droutes {
-		uflags |= UAllowDefaultRoute
-	}
-	return uflags
-}
+// EndpointDiscoSuffix is appended to the hex representation of a peer's discovery key
+// and is then the sole wireguard endpoint for peers with a non-zero discovery key.
+// This form is then recognize by magicsock's CreateEndpoint.
+const EndpointDiscoSuffix = ".disco.tailscale:12345"
 
-// TODO(bradfitz): UAPI seems to only be used by the old confnode and
-// pingnode; delete this when those are deleted/rewritten?
-func (nm *NetworkMap) UAPI(uflags int, dnsOverride []wgcfg.IP) string {
-	wgcfg, err := nm.WGCfg(uflags, dnsOverride)
-	if err != nil {
-		log.Fatalf("WGCfg() failed unexpectedly: %v\n", err)
+// WGCfg returns the NetworkMaps's Wireguard configuration.
+func (nm *NetworkMap) WGCfg(logf logger.Logf, flags WGConfigFlags) (*wgcfg.Config, error) {
+	cfg := &wgcfg.Config{
+		Name:       "tailscale",
+		PrivateKey: nm.PrivateKey,
+		Addresses:  nm.Addresses,
+		ListenPort: nm.LocalPort,
+		Peers:      make([]wgcfg.Peer, 0, len(nm.Peers)),
 	}
-	s, err := wgcfg.ToUAPI()
-	if err != nil {
-		log.Fatalf("ToUAPI() failed unexpectedly: %v\n", err)
-	}
-	return s
-}
 
-func (nm *NetworkMap) WGCfg(uflags int, dnsOverride []wgcfg.IP) (*wgcfg.Config, error) {
-	s := nm._WireGuardConfig(uflags, dnsOverride, true)
-	return wgcfg.FromWgQuick(s, "tailscale")
-}
-
-// TODO(apenwarr): This mode is dangerous.
-// Discarding the extra endpoints is almost universally the wrong choice.
-// Except that plain wireguard can't handle a peer with multiple endpoints.
-// (Yet?)
-func (nm *NetworkMap) WireGuardConfigOneEndpoint(uflags int, dnsOverride []wgcfg.IP) string {
-	return nm._WireGuardConfig(uflags, dnsOverride, false)
-}
-
-func (nm *NetworkMap) _WireGuardConfig(uflags int, dnsOverride []wgcfg.IP, allEndpoints bool) string {
-	buf := new(strings.Builder)
-	fmt.Fprintf(buf, "[Interface]\n")
-	fmt.Fprintf(buf, "PrivateKey = %s\n", base64.StdEncoding.EncodeToString(nm.PrivateKey[:]))
-	if len(nm.Addresses) > 0 {
-		fmt.Fprintf(buf, "Address = ")
-		for i, cidr := range nm.Addresses {
-			if i > 0 {
-				fmt.Fprintf(buf, ", ")
-			}
-			fmt.Fprintf(buf, "%s", cidr)
-		}
-		fmt.Fprintf(buf, "\n")
-	}
-	fmt.Fprintf(buf, "ListenPort = %d\n", nm.LocalPort)
-	if len(dnsOverride) > 0 {
-		dnss := []string{}
-		for _, ip := range dnsOverride {
-			dnss = append(dnss, ip.String())
-		}
-		fmt.Fprintf(buf, "DNS = %s\n", strings.Join(dnss, ","))
-	}
-	fmt.Fprintf(buf, "\n")
-
-	for i, peer := range nm.Peers {
-		if (uflags&UAllowSingleHosts) == 0 && len(peer.AllowedIPs) < 2 {
-			log.Printf("wgcfg: %v skipping a single-host peer.\n", peer.Key.ShortString())
+	for _, peer := range nm.Peers {
+		if Debug.OnlyDisco && peer.DiscoKey.IsZero() {
 			continue
 		}
-		if i > 0 {
-			fmt.Fprintf(buf, "\n")
+		if (flags&AllowSingleHosts) == 0 && len(peer.AllowedIPs) < 2 {
+			logf("wgcfg: %v skipping a single-host peer.", peer.Key.ShortString())
+			continue
 		}
-		fmt.Fprintf(buf, "[Peer]\n")
-		fmt.Fprintf(buf, "PublicKey = %s\n", base64.StdEncoding.EncodeToString(peer.Key[:]))
-		var endpoints []string
-		if peer.DERP != "" {
-			endpoints = append(endpoints, peer.DERP)
+		cfg.Peers = append(cfg.Peers, wgcfg.Peer{
+			PublicKey: wgcfg.Key(peer.Key),
+		})
+		cpeer := &cfg.Peers[len(cfg.Peers)-1]
+		if peer.KeepAlive {
+			cpeer.PersistentKeepalive = 25 // seconds
 		}
-		endpoints = append(endpoints, peer.Endpoints...)
-		if len(endpoints) > 0 {
-			if len(endpoints) == 1 {
-				fmt.Fprintf(buf, "Endpoint = %s", endpoints[0])
-			} else if allEndpoints {
-				// TODO(apenwarr): This mode is incompatible.
-				// Normal wireguard clients don't know how to
-				// parse it (yet?)
-				fmt.Fprintf(buf, "Endpoint = %s",
-					strings.Join(endpoints, ","))
-			} else {
-				fmt.Fprintf(buf, "Endpoint = %s # other endpoints: %s",
-					endpoints[0],
-					strings.Join(endpoints[1:], ", "))
+
+		if !peer.DiscoKey.IsZero() {
+			if err := appendEndpoint(cpeer, fmt.Sprintf("%x%s", peer.DiscoKey[:], EndpointDiscoSuffix)); err != nil {
+				return nil, err
+			}
+			cpeer.Endpoints = []wgcfg.Endpoint{{Host: fmt.Sprintf("%x.disco.tailscale", peer.DiscoKey[:]), Port: 12345}}
+		} else {
+			if err := appendEndpoint(cpeer, peer.DERP); err != nil {
+				return nil, err
+			}
+			for _, ep := range peer.Endpoints {
+				if err := appendEndpoint(cpeer, ep); err != nil {
+					return nil, err
+				}
 			}
-			buf.WriteByte('\n')
 		}
-		var aips []string
 		for _, allowedIP := range peer.AllowedIPs {
-			aip := allowedIP.String()
 			if allowedIP.Mask == 0 {
-				if (uflags & UAllowDefaultRoute) == 0 {
-					log.Printf("wgcfg: %v skipping default route\n", peer.Key.ShortString())
+				if (flags & AllowDefaultRoute) == 0 {
+					logf("wgcfg: %v skipping default route", peer.Key.ShortString())
 					continue
 				}
-				if (uflags & UHackDefaultRoute) != 0 {
-					aip = "10.0.0.0/8"
-					log.Printf("wgcfg: %v converting default route => %v\n", peer.Key.ShortString(), aip)
+				if (flags & HackDefaultRoute) != 0 {
+					allowedIP = wgcfg.CIDR{IP: wgcfg.IPv4(10, 0, 0, 0), Mask: 8}
+					logf("wgcfg: %v converting default route => %v", peer.Key.ShortString(), allowedIP.String())
 				}
 			} else if allowedIP.Mask < 32 {
-				if (uflags & UAllowSubnetRoutes) == 0 {
-					log.Printf("wgcfg: %v skipping subnet route\n", peer.Key.ShortString())
+				if (flags & AllowSubnetRoutes) == 0 {
+					logf("wgcfg: %v skipping subnet route", peer.Key.ShortString())
 					continue
 				}
 			}
-			aips = append(aips, aip)
-		}
-		fmt.Fprintf(buf, "AllowedIPs = %s\n", strings.Join(aips, ", "))
-		if peer.KeepAlive {
-			fmt.Fprintf(buf, "PersistentKeepalive = 25\n")
+			cpeer.AllowedIPs = append(cpeer.AllowedIPs, allowedIP)
 		}
 	}
 
-	return buf.String()
+	return cfg, nil
+}
+
+func appendEndpoint(peer *wgcfg.Peer, epStr string) error {
+	if epStr == "" {
+		return nil
+	}
+	host, port, err := net.SplitHostPort(epStr)
+	if err != nil {
+		return fmt.Errorf("malformed endpoint %q for peer %v", epStr, peer.PublicKey.ShortString())
+	}
+	port16, err := strconv.ParseUint(port, 10, 16)
+	if err != nil {
+		return fmt.Errorf("invalid port in endpoint %q for peer %v", epStr, peer.PublicKey.ShortString())
+	}
+	peer.Endpoints = append(peer.Endpoints, wgcfg.Endpoint{Host: host, Port: uint16(port16)})
+	return nil
+}
+
+// eqStringsIgnoreNil reports whether a and b have the same length and
+// contents, but ignore whether a or b are nil.
+func eqStringsIgnoreNil(a, b []string) bool {
+	if len(a) != len(b) {
+		return false
+	}
+	for i, v := range a {
+		if v != b[i] {
+			return false
+		}
+	}
+	return true
+}
+
+// eqCIDRsIgnoreNil reports whether a and b have the same length and
+// contents, but ignore whether a or b are nil.
+func eqCIDRsIgnoreNil(a, b []wgcfg.CIDR) bool {
+	if len(a) != len(b) {
+		return false
+	}
+	for i, v := range a {
+		if v != b[i] {
+			return false
+		}
+	}
+	return true
 }

control/controlclient/netmap_test.go

@@ -0,0 +1,284 @@
+// Copyright (c) 2020 Tailscale Inc & AUTHORS All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package controlclient
+
+import (
+	"encoding/hex"
+	"testing"
+
+	"github.com/tailscale/wireguard-go/wgcfg"
+	"tailscale.com/tailcfg"
+)
+
+func testNodeKey(b byte) (ret tailcfg.NodeKey) {
+	for i := range ret {
+		ret[i] = b
+	}
+	return
+}
+
+func testDiscoKey(hexPrefix string) (ret tailcfg.DiscoKey) {
+	b, err := hex.DecodeString(hexPrefix)
+	if err != nil {
+		panic(err)
+	}
+	copy(ret[:], b)
+	return
+}
+
+func TestNetworkMapConcise(t *testing.T) {
+	for _, tt := range []struct {
+		name string
+		nm   *NetworkMap
+		want string
+	}{
+		{
+			name: "basic",
+			nm: &NetworkMap{
+				NodeKey: testNodeKey(1),
+				Peers: []*tailcfg.Node{
+					{
+						Key:       testNodeKey(2),
+						DERP:      "127.3.3.40:2",
+						Endpoints: []string{"192.168.0.100:12", "192.168.0.100:12354"},
+					},
+					{
+						Key:       testNodeKey(3),
+						DERP:      "127.3.3.40:4",
+						Endpoints: []string{"10.2.0.100:12", "10.1.0.100:12345"},
+					},
+				},
+			},
+			want: "netmap: self: [AQEBA] auth=machine-unknown []\n [AgICA] D2                 :    192.168.0.100:12     192.168.0.100:12354\n [AwMDA] D4                 :       10.2.0.100:12        10.1.0.100:12345\n",
+		},
+		{
+			name: "debug_non_nil",
+			nm: &NetworkMap{
+				NodeKey: testNodeKey(1),
+				Debug:   &tailcfg.Debug{},
+			},
+			want: "netmap: self: [AQEBA] auth=machine-unknown debug={} []\n",
+		},
+		{
+			name: "debug_values",
+			nm: &NetworkMap{
+				NodeKey: testNodeKey(1),
+				Debug:   &tailcfg.Debug{LogHeapPprof: true},
+			},
+			want: "netmap: self: [AQEBA] auth=machine-unknown debug={\"LogHeapPprof\":true} []\n",
+		},
+	} {
+		t.Run(tt.name, func(t *testing.T) {
+			var got string
+			n := int(testing.AllocsPerRun(1000, func() {
+				got = tt.nm.Concise()
+			}))
+			t.Logf("Allocs = %d", n)
+			if got != tt.want {
+				t.Errorf("Wrong output\n Got: %q\nWant: %q\n## Got (unescaped):\n%s\n## Want (unescaped):\n%s\n", got, tt.want, got, tt.want)
+			}
+		})
+	}
+}
+
+func TestConciseDiffFrom(t *testing.T) {
+	for _, tt := range []struct {
+		name string
+		a, b *NetworkMap
+		want string
+	}{
+		{
+			name: "no_change",
+			a: &NetworkMap{
+				NodeKey: testNodeKey(1),
+				Peers: []*tailcfg.Node{
+					{
+						Key:       testNodeKey(2),
+						DERP:      "127.3.3.40:2",
+						Endpoints: []string{"192.168.0.100:12", "192.168.0.100:12354"},
+					},
+				},
+			},
+			b: &NetworkMap{
+				NodeKey: testNodeKey(1),
+				Peers: []*tailcfg.Node{
+					{
+						Key:       testNodeKey(2),
+						DERP:      "127.3.3.40:2",
+						Endpoints: []string{"192.168.0.100:12", "192.168.0.100:12354"},
+					},
+				},
+			},
+			want: "",
+		},
+		{
+			name: "header_change",
+			a: &NetworkMap{
+				NodeKey: testNodeKey(1),
+				Peers: []*tailcfg.Node{
+					{
+						Key:       testNodeKey(2),
+						DERP:      "127.3.3.40:2",
+						Endpoints: []string{"192.168.0.100:12", "192.168.0.100:12354"},
+					},
+				},
+			},
+			b: &NetworkMap{
+				NodeKey: testNodeKey(2),
+				Peers: []*tailcfg.Node{
+					{
+						Key:       testNodeKey(2),
+						DERP:      "127.3.3.40:2",
+						Endpoints: []string{"192.168.0.100:12", "192.168.0.100:12354"},
+					},
+				},
+			},
+			want: "-netmap: self: [AQEBA] auth=machine-unknown []\n+netmap: self: [AgICA] auth=machine-unknown []\n",
+		},
+		{
+			name: "peer_add",
+			a: &NetworkMap{
+				NodeKey: testNodeKey(1),
+				Peers: []*tailcfg.Node{
+					{
+						ID:        2,
+						Key:       testNodeKey(2),
+						DERP:      "127.3.3.40:2",
+						Endpoints: []string{"192.168.0.100:12", "192.168.0.100:12354"},
+					},
+				},
+			},
+			b: &NetworkMap{
+				NodeKey: testNodeKey(1),
+				Peers: []*tailcfg.Node{
+					{
+						ID:        1,
+						Key:       testNodeKey(1),
+						DERP:      "127.3.3.40:1",
+						Endpoints: []string{"192.168.0.100:12", "192.168.0.100:12354"},
+					},
+					{
+						ID:        2,
+						Key:       testNodeKey(2),
+						DERP:      "127.3.3.40:2",
+						Endpoints: []string{"192.168.0.100:12", "192.168.0.100:12354"},
+					},
+					{
+						ID:        3,
+						Key:       testNodeKey(3),
+						DERP:      "127.3.3.40:3",
+						Endpoints: []string{"192.168.0.100:12", "192.168.0.100:12354"},
+					},
+				},
+			},
+			want: "+ [AQEBA] D1                 :    192.168.0.100:12     192.168.0.100:12354\n+ [AwMDA] D3                 :    192.168.0.100:12     192.168.0.100:12354\n",
+		},
+		{
+			name: "peer_remove",
+			a: &NetworkMap{
+				NodeKey: testNodeKey(1),
+				Peers: []*tailcfg.Node{
+					{
+						ID:        1,
+						Key:       testNodeKey(1),
+						DERP:      "127.3.3.40:1",
+						Endpoints: []string{"192.168.0.100:12", "192.168.0.100:12354"},
+					},
+					{
+						ID:        2,
+						Key:       testNodeKey(2),
+						DERP:      "127.3.3.40:2",
+						Endpoints: []string{"192.168.0.100:12", "192.168.0.100:12354"},
+					},
+					{
+						ID:        3,
+						Key:       testNodeKey(3),
+						DERP:      "127.3.3.40:3",
+						Endpoints: []string{"192.168.0.100:12", "192.168.0.100:12354"},
+					},
+				},
+			},
+			b: &NetworkMap{
+				NodeKey: testNodeKey(1),
+				Peers: []*tailcfg.Node{
+					{
+						ID:        2,
+						Key:       testNodeKey(2),
+						DERP:      "127.3.3.40:2",
+						Endpoints: []string{"192.168.0.100:12", "192.168.0.100:12354"},
+					},
+				},
+			},
+			want: "- [AQEBA] D1                 :    192.168.0.100:12     192.168.0.100:12354\n- [AwMDA] D3                 :    192.168.0.100:12     192.168.0.100:12354\n",
+		},
+		{
+			name: "peer_port_change",
+			a: &NetworkMap{
+				NodeKey: testNodeKey(1),
+				Peers: []*tailcfg.Node{
+					{
+						ID:        2,
+						Key:       testNodeKey(2),
+						DERP:      "127.3.3.40:2",
+						Endpoints: []string{"192.168.0.100:12", "1.1.1.1:1"},
+					},
+				},
+			},
+			b: &NetworkMap{
+				NodeKey: testNodeKey(1),
+				Peers: []*tailcfg.Node{
+					{
+						ID:        2,
+						Key:       testNodeKey(2),
+						DERP:      "127.3.3.40:2",
+						Endpoints: []string{"192.168.0.100:12", "1.1.1.1:2"},
+					},
+				},
+			},
+			want: "- [AgICA] D2                 :    192.168.0.100:12             1.1.1.1:1  \n+ [AgICA] D2                 :    192.168.0.100:12             1.1.1.1:2  \n",
+		},
+		{
+			name: "disco_key_only_change",
+			a: &NetworkMap{
+				NodeKey: testNodeKey(1),
+				Peers: []*tailcfg.Node{
+					{
+						ID:         2,
+						Key:        testNodeKey(2),
+						DERP:       "127.3.3.40:2",
+						Endpoints:  []string{"192.168.0.100:41641", "1.1.1.1:41641"},
+						DiscoKey:   testDiscoKey("f00f00f00f"),
+						AllowedIPs: []wgcfg.CIDR{{IP: wgcfg.IPv4(100, 102, 103, 104), Mask: 32}},
+					},
+				},
+			},
+			b: &NetworkMap{
+				NodeKey: testNodeKey(1),
+				Peers: []*tailcfg.Node{
+					{
+						ID:         2,
+						Key:        testNodeKey(2),
+						DERP:       "127.3.3.40:2",
+						Endpoints:  []string{"192.168.0.100:41641", "1.1.1.1:41641"},
+						DiscoKey:   testDiscoKey("ba4ba4ba4b"),
+						AllowedIPs: []wgcfg.CIDR{{IP: wgcfg.IPv4(100, 102, 103, 104), Mask: 32}},
+					},
+				},
+			},
+			want: "- [AgICA] d:f00f00f00f000000 D2 100.102.103.104 :   192.168.0.100:41641         1.1.1.1:41641\n+ [AgICA] d:ba4ba4ba4b000000 D2 100.102.103.104 :   192.168.0.100:41641         1.1.1.1:41641\n",
+		},
+	} {
+		t.Run(tt.name, func(t *testing.T) {
+			var got string
+			n := int(testing.AllocsPerRun(50, func() {
+				got = tt.b.ConciseDiffFrom(tt.a)
+			}))
+			t.Logf("Allocs = %d", n)
+			if got != tt.want {
+				t.Errorf("Wrong output\n Got: %q\nWant: %q\n## Got (unescaped):\n%s\n## Want (unescaped):\n%s\n", got, tt.want, got, tt.want)
+			}
+		})
+	}
+}

derp/derp.go

@@ -32,10 +32,11 @@ const MaxPacketSize = 64 << 10
 const magic = "DERP🔑" // 8 bytes: 0x44 45 52 50 f0 9f 94 91
 
 const (
-	nonceLen   = 24
-	keyLen     = 32
-	maxInfoLen = 1 << 20
-	keepAlive  = 60 * time.Second
+	nonceLen       = 24
+	frameHeaderLen = 1 + 4 // frameType byte + 4 byte length
+	keyLen         = 32
+	maxInfoLen     = 1 << 20
+	keepAlive      = 60 * time.Second
 )
 
 // protocolVersion is bumped whenever there's a wire-incompatible change.
@@ -71,6 +72,7 @@ const (
 	frameClientInfo    = frameType(0x02) // 32B pub key + 24B nonce + naclbox(json)
 	frameServerInfo    = frameType(0x03) // 24B nonce + naclbox(json)
 	frameSendPacket    = frameType(0x04) // 32B dest pub key + packet bytes
+	frameForwardPacket = frameType(0x0a) // 32B src pub key + 32B dst pub key + packet bytes
 	frameRecvPacket    = frameType(0x05) // v0/1: packet bytes, v2: 32B src pub key + packet bytes
 	frameKeepAlive     = frameType(0x06) // no payload, no-op (to be replaced with ping/pong)
 	frameNotePreferred = frameType(0x07) // 1 byte payload: 0x01 or 0x00 for whether this is client's home node
@@ -81,6 +83,24 @@ const (
 	// framePeerGone to B so B can forget that a reverse path
 	// exists on that connection to get back to A.
 	framePeerGone = frameType(0x08) // 32B pub key of peer that's gone
+
+	// framePeerPresent is like framePeerGone, but for other
+	// members of the DERP region when they're meshed up together.
+	framePeerPresent = frameType(0x09) // 32B pub key of peer that's connected
+
+	// frameWatchConns is how one DERP node in a regional mesh
+	// subscribes to the others in the region.
+	// There's no payload. If the sender doesn't have permission, the connection
+	// is closed. Otherwise, the client is initially flooded with
+	// framePeerPresent for all connected nodes, and then a stream of
+	// framePeerPresent & framePeerGone has peers connect and disconnect.
+	frameWatchConns = frameType(0x10)
+
+	// frameClosePeer is a privileged frame type (requires the
+	// mesh key for now) that closes the provided peer's
+	// connection. (To be used for cluster load balancing
+	// purposes, when clients end up on a non-ideal node)
+	frameClosePeer = frameType(0x11) // 32B pub key of peer to close.
 )
 
 var bin = binary.BigEndian

derp/derp_client.go

@@ -19,6 +19,7 @@ import (
 	"tailscale.com/types/logger"
 )
 
+// Client is a DERP client.
 type Client struct {
 	serverKey    key.Public // of the DERP server; not a machine or node key
 	privateKey   key.Private
@@ -27,13 +28,48 @@ type Client struct {
 	logf         logger.Logf
 	nc           Conn
 	br           *bufio.Reader
+	meshKey      string
 
-	wmu     sync.Mutex // hold while writing to bw
-	bw      *bufio.Writer
+	wmu sync.Mutex // hold while writing to bw
+	bw  *bufio.Writer
+
+	// Owned by Recv:
+	peeked  int   // bytes to discard on next Recv
 	readErr error // sticky read error
 }
 
-func NewClient(privateKey key.Private, nc Conn, brw *bufio.ReadWriter, logf logger.Logf) (*Client, error) {
+// ClientOpt is an option passed to NewClient.
+type ClientOpt interface {
+	update(*clientOpt)
+}
+
+type clientOptFunc func(*clientOpt)
+
+func (f clientOptFunc) update(o *clientOpt) { f(o) }
+
+// clientOpt are the options passed to newClient.
+type clientOpt struct {
+	MeshKey string
+}
+
+// MeshKey returns a ClientOpt to pass to the DERP server during connect to get
+// access to join the mesh.
+//
+// An empty key means to not use a mesh key.
+func MeshKey(key string) ClientOpt { return clientOptFunc(func(o *clientOpt) { o.MeshKey = key }) }
+
+func NewClient(privateKey key.Private, nc Conn, brw *bufio.ReadWriter, logf logger.Logf, opts ...ClientOpt) (*Client, error) {
+	var opt clientOpt
+	for _, o := range opts {
+		if o == nil {
+			return nil, errors.New("nil ClientOpt")
+		}
+		o.update(&opt)
+	}
+	return newClient(privateKey, nc, brw, logf, opt)
+}
+
+func newClient(privateKey key.Private, nc Conn, brw *bufio.ReadWriter, logf logger.Logf, opt clientOpt) (*Client, error) {
 	c := &Client{
 		privateKey: privateKey,
 		publicKey:  privateKey.Public(),
@@ -41,8 +77,8 @@ func NewClient(privateKey key.Private, nc Conn, brw *bufio.ReadWriter, logf logg
 		nc:         nc,
 		br:         brw.Reader,
 		bw:         brw.Writer,
+		meshKey:    opt.MeshKey,
 	}
-
 	if err := c.recvServerKey(); err != nil {
 		return nil, fmt.Errorf("derp.Client: failed to receive server key: %v", err)
 	}
@@ -109,6 +145,12 @@ func (c *Client) recvServerInfo() (*serverInfo, error) {
 
 type clientInfo struct {
 	Version int // `json:"version,omitempty"`
+
+	// MeshKey optionally specifies a pre-shared key used by
+	// trusted clients.  It's required to subscribe to the
+	// connection list & forward packets. It's empty for regular
+	// users.
+	MeshKey string // `json:"meshKey,omitempty"`
 }
 
 func (c *Client) sendClientKey() error {
@@ -116,7 +158,10 @@ func (c *Client) sendClientKey() error {
 	if _, err := crand.Read(nonce[:]); err != nil {
 		return err
 	}
-	msg, err := json.Marshal(clientInfo{Version: protocolVersion})
+	msg, err := json.Marshal(clientInfo{
+		Version: protocolVersion,
+		MeshKey: c.meshKey,
+	})
 	if err != nil {
 		return err
 	}
@@ -129,6 +174,9 @@ func (c *Client) sendClientKey() error {
 	return writeFrame(c.bw, frameClientInfo, buf)
 }
 
+// ServerPublicKey returns the server's public key.
+func (c *Client) ServerPublicKey() key.Public { return c.serverKey }
+
 // Send sends a packet to the Tailscale node identified by dstKey.
 //
 // It is an error if the packet is larger than 64KB.
@@ -160,6 +208,40 @@ func (c *Client) send(dstKey key.Public, pkt []byte) (ret error) {
 	return c.bw.Flush()
 }
 
+func (c *Client) ForwardPacket(srcKey, dstKey key.Public, pkt []byte) (err error) {
+	defer func() {
+		if err != nil {
+			err = fmt.Errorf("derp.ForwardPacket: %w", err)
+		}
+	}()
+
+	if len(pkt) > MaxPacketSize {
+		return fmt.Errorf("packet too big: %d", len(pkt))
+	}
+
+	c.wmu.Lock()
+	defer c.wmu.Unlock()
+
+	timer := time.AfterFunc(5*time.Second, c.writeTimeoutFired)
+	defer timer.Stop()
+
+	if err := writeFrameHeader(c.bw, frameForwardPacket, uint32(keyLen*2+len(pkt))); err != nil {
+		return err
+	}
+	if _, err := c.bw.Write(srcKey[:]); err != nil {
+		return err
+	}
+	if _, err := c.bw.Write(dstKey[:]); err != nil {
+		return err
+	}
+	if _, err := c.bw.Write(pkt); err != nil {
+		return err
+	}
+	return c.bw.Flush()
+}
+
+func (c *Client) writeTimeoutFired() { c.nc.Close() }
+
 // NotePreferred sends a packet that tells the server whether this
 // client is the user's preferred server. This is only used in the
 // server for stats.
@@ -186,6 +268,25 @@ func (c *Client) NotePreferred(preferred bool) (err error) {
 	return c.bw.Flush()
 }
 
+// WatchConnectionChanges sends a request to subscribe to the peer's connection list.
+// It's a fatal error if the client wasn't created using MeshKey.
+func (c *Client) WatchConnectionChanges() error {
+	c.wmu.Lock()
+	defer c.wmu.Unlock()
+	if err := writeFrameHeader(c.bw, frameWatchConns, 0); err != nil {
+		return err
+	}
+	return c.bw.Flush()
+}
+
+// ClosePeer asks the server to close target's TCP connection.
+// It's a fatal error if the client wasn't created using MeshKey.
+func (c *Client) ClosePeer(target key.Public) error {
+	c.wmu.Lock()
+	defer c.wmu.Unlock()
+	return writeFrame(c.bw, frameClosePeer, target[:])
+}
+
 // ReceivedMessage represents a type returned by Client.Recv. Unless
 // otherwise documented, the returned message aliases the byte slice
 // provided to Recv and thus the message is only as good as that
@@ -211,11 +312,23 @@ type PeerGoneMessage key.Public
 
 func (PeerGoneMessage) msg() {}
 
+// PeerPresentMessage is a ReceivedMessage that indicates that the client
+// is connected to the server. (Only used by trusted mesh clients)
+type PeerPresentMessage key.Public
+
+func (PeerPresentMessage) msg() {}
+
 // Recv reads a message from the DERP server.
-// The provided buffer must be large enough to receive a complete packet,
-// which in practice are are 1.5-4 KB, but can be up to 64 KB.
+//
+// The returned message may alias memory owned by the Client; it
+// should only be accessed until the next call to Client.
+//
 // Once Recv returns an error, the Client is dead forever.
-func (c *Client) Recv(b []byte) (m ReceivedMessage, err error) {
+func (c *Client) Recv() (m ReceivedMessage, err error) {
+	return c.recvTimeout(120 * time.Second)
+}
+
+func (c *Client) recvTimeout(timeout time.Duration) (m ReceivedMessage, err error) {
 	if c.readErr != nil {
 		return nil, c.readErr
 	}
@@ -227,11 +340,45 @@ func (c *Client) Recv(b []byte) (m ReceivedMessage, err error) {
 	}()
 
 	for {
-		c.nc.SetReadDeadline(time.Now().Add(120 * time.Second))
-		t, n, err := readFrame(c.br, 1<<20, b)
+		c.nc.SetReadDeadline(time.Now().Add(timeout))
+
+		// Discard any peeked bytes from a previous Recv call.
+		if c.peeked != 0 {
+			if n, err := c.br.Discard(c.peeked); err != nil || n != c.peeked {
+				// Documented to never fail, but might as well check.
+				return nil, fmt.Errorf("bufio.Reader.Discard(%d bytes): got %v, %v", c.peeked, n, err)
+			}
+			c.peeked = 0
+		}
+
+		t, n, err := readFrameHeader(c.br)
 		if err != nil {
 			return nil, err
 		}
+		if n > 1<<20 {
+			return nil, fmt.Errorf("unexpectedly large frame of %d bytes returned", n)
+		}
+
+		var b []byte // frame payload (past the 5 byte header)
+
+		// If the frame fits in our bufio.Reader buffer, just use it.
+		// In practice it's 4KB (from derphttp.Client's bufio.NewReader(httpConn)) and
+		// in practive, WireGuard packets (and thus DERP frames) are under 1.5KB.
+		// So This is the common path.
+		if int(n) <= c.br.Size() {
+			b, err = c.br.Peek(int(n))
+			c.peeked = int(n)
+		} else {
+			// But if for some reason we read a large DERP message (which isn't necessarily
+			// a Wireguard packet), then just allocate memory for it.
+			// TODO(bradfitz): use a pool if large frames ever happen in practice.
+			b = make([]byte, n)
+			_, err = io.ReadFull(c.br, b)
+		}
+		if err != nil {
+			return nil, err
+		}
+
 		switch t {
 		default:
 			continue
@@ -248,6 +395,15 @@ func (c *Client) Recv(b []byte) (m ReceivedMessage, err error) {
 			copy(pg[:], b[:keyLen])
 			return pg, nil
 
+		case framePeerPresent:
+			if n < keyLen {
+				c.logf("[unexpected] dropping short peerPresent frame from DERP server")
+				continue
+			}
+			var pg PeerPresentMessage
+			copy(pg[:], b[:keyLen])
+			return pg, nil
+
 		case frameRecvPacket:
 			var rp ReceivedPacket
 			if c.protoVersion < protocolSrcAddrs {

derp/derp_server.go

@@ -20,6 +20,7 @@ import (
 	"os"
 	"runtime"
 	"strconv"
+	"strings"
 	"sync"
 	"time"
 
@@ -28,6 +29,7 @@ import (
 	"tailscale.com/metrics"
 	"tailscale.com/types/key"
 	"tailscale.com/types/logger"
+	"tailscale.com/version"
 )
 
 var debug, _ = strconv.ParseBool(os.Getenv("DERP_DEBUG_LOGS"))
@@ -37,6 +39,13 @@ const (
 	writeTimeout            = 2 * time.Second
 )
 
+const host64bit = (^uint(0) >> 32) & 1 // 1 on 64-bit, 0 on 32-bit
+
+// pad32bit is 4 on 32-bit machines and 0 on 64-bit.
+// It exists so the Server struct's atomic fields can be aligned to 8
+// byte boundaries. (As tested by GOARCH=386 go test, etc)
+const pad32bit = 4 - host64bit*4 // 0 on 64-bit, 4 on 32-bit
+
 // Server is a DERP server.
 type Server struct {
 	// WriteTimeout, if non-zero, specifies how long to wait
@@ -47,31 +56,62 @@ type Server struct {
 	publicKey  key.Public
 	logf       logger.Logf
 	memSys0    uint64 // runtime.MemStats.Sys at start (or early-ish)
+	meshKey    string
 
 	// Counters:
-	packetsSent, bytesSent  expvar.Int
-	packetsRecv, bytesRecv  expvar.Int
-	packetsDropped          expvar.Int
-	packetsDroppedReason    metrics.LabelMap
-	packetsDroppedUnknown   *expvar.Int // unknown dst pubkey
-	packetsDroppedGone      *expvar.Int // dst conn shutting down
-	packetsDroppedQueueHead *expvar.Int // queue full, drop head packet
-	packetsDroppedQueueTail *expvar.Int // queue full, drop tail packet
-	packetsDroppedWrite     *expvar.Int // error writing to dst conn
-	peerGoneFrames          expvar.Int  // number of peer gone frames sent
-	accepts                 expvar.Int
-	curClients              expvar.Int
-	curHomeClients          expvar.Int // ones with preferred
-	clientsReplaced         expvar.Int
-	unknownFrames           expvar.Int
-	homeMovesIn             expvar.Int // established clients announce home server moves in
-	homeMovesOut            expvar.Int // established clients announce home server moves out
+	_                        [pad32bit]byte
+	packetsSent, bytesSent   expvar.Int
+	packetsRecv, bytesRecv   expvar.Int
+	packetsDropped           expvar.Int
+	packetsDroppedReason     metrics.LabelMap
+	packetsDroppedUnknown    *expvar.Int // unknown dst pubkey
+	packetsDroppedFwdUnknown *expvar.Int // unknown dst pubkey on forward
+	packetsDroppedGone       *expvar.Int // dst conn shutting down
+	packetsDroppedQueueHead  *expvar.Int // queue full, drop head packet
+	packetsDroppedQueueTail  *expvar.Int // queue full, drop tail packet
+	packetsDroppedWrite      *expvar.Int // error writing to dst conn
+	_                        [pad32bit]byte
+	packetsForwardedOut      expvar.Int
+	packetsForwardedIn       expvar.Int
+	peerGoneFrames           expvar.Int // number of peer gone frames sent
+	accepts                  expvar.Int
+	curClients               expvar.Int
+	curHomeClients           expvar.Int // ones with preferred
+	clientsReplaced          expvar.Int
+	unknownFrames            expvar.Int
+	homeMovesIn              expvar.Int // established clients announce home server moves in
+	homeMovesOut             expvar.Int // established clients announce home server moves out
+	multiForwarderCreated    expvar.Int
+	multiForwarderDeleted    expvar.Int
+	removePktForwardOther    expvar.Int
 
 	mu          sync.Mutex
 	closed      bool
 	netConns    map[Conn]chan struct{} // chan is closed when conn closes
 	clients     map[key.Public]*sclient
 	clientsEver map[key.Public]bool // never deleted from, for stats; fine for now
+	watchers    map[*sclient]bool   // mesh peer -> true
+	// clientsMesh tracks all clients in the cluster, both locally
+	// and to mesh peers.  If the value is nil, that means the
+	// peer is only local (and thus in the clients Map, but not
+	// remote). If the value is non-nil, it's remote (+ maybe also
+	// local).
+	clientsMesh map[key.Public]PacketForwarder
+	// sentTo tracks which peers have sent to which other peers,
+	// and at which connection number. This isn't on sclient
+	// because it includes intra-region forwarded packets as the
+	// src.
+	sentTo map[key.Public]map[key.Public]int64 // src => dst => dst's latest sclient.connNum
+}
+
+// PacketForwarder is something that can forward packets.
+//
+// It's mostly an inteface for circular dependency reasons; the
+// typical implementation is derphttp.Client. The other implementation
+// is a multiForwarder, which this package creates as needed if a
+// public key gets more than one PacketForwarder registered for it.
+type PacketForwarder interface {
+	ForwardPacket(src, dst key.Public, payload []byte) error
 }
 
 // Conn is the subset of the underlying net.Conn the DERP Server needs.
@@ -97,12 +137,16 @@ func NewServer(privateKey key.Private, logf logger.Logf) *Server {
 		publicKey:            privateKey.Public(),
 		logf:                 logf,
 		packetsDroppedReason: metrics.LabelMap{Label: "reason"},
-		clients:              make(map[key.Public]*sclient),
-		clientsEver:          make(map[key.Public]bool),
-		netConns:             make(map[Conn]chan struct{}),
+		clients:              map[key.Public]*sclient{},
+		clientsEver:          map[key.Public]bool{},
+		clientsMesh:          map[key.Public]PacketForwarder{},
+		netConns:             map[Conn]chan struct{}{},
 		memSys0:              ms.Sys,
+		watchers:             map[*sclient]bool{},
+		sentTo:               map[key.Public]map[key.Public]int64{},
 	}
 	s.packetsDroppedUnknown = s.packetsDroppedReason.Get("unknown_dest")
+	s.packetsDroppedFwdUnknown = s.packetsDroppedReason.Get("unknown_dest_on_fwd")
 	s.packetsDroppedGone = s.packetsDroppedReason.Get("gone")
 	s.packetsDroppedQueueHead = s.packetsDroppedReason.Get("queue_head")
 	s.packetsDroppedQueueTail = s.packetsDroppedReason.Get("queue_tail")
@@ -110,6 +154,26 @@ func NewServer(privateKey key.Private, logf logger.Logf) *Server {
 	return s
 }
 
+// SetMesh sets the pre-shared key that regional DERP servers used to mesh
+// amongst themselves.
+//
+// It must be called before serving begins.
+func (s *Server) SetMeshKey(v string) {
+	s.meshKey = v
+}
+
+// HasMeshKey reports whether the server is configured with a mesh key.
+func (s *Server) HasMeshKey() bool { return s.meshKey != "" }
+
+// MeshKey returns the configured mesh key, if any.
+func (s *Server) MeshKey() string { return s.meshKey }
+
+// PrivateKey returns the server's private key.
+func (s *Server) PrivateKey() key.Private { return s.privateKey }
+
+// PublicKey returns the server's public key.
+func (s *Server) PublicKey() key.Public { return s.publicKey }
+
 // Close closes the server and waits for the connections to disconnect.
 func (s *Server) Close() error {
 	s.mu.Lock()
@@ -187,7 +251,23 @@ func (s *Server) registerClient(c *sclient) {
 	}
 	s.clients[c.key] = c
 	s.clientsEver[c.key] = true
+	if _, ok := s.clientsMesh[c.key]; !ok {
+		s.clientsMesh[c.key] = nil // just for varz of total users in cluster
+	}
 	s.curClients.Add(1)
+	s.broadcastPeerStateChangeLocked(c.key, true)
+}
+
+// broadcastPeerStateChangeLocked enqueues a message to all watchers
+// (other DERP nodes in the region, or trusted clients) that peer's
+// presence changed.
+//
+// s.mu must be held.
+func (s *Server) broadcastPeerStateChangeLocked(peer key.Public, present bool) {
+	for w := range s.watchers {
+		w.peerStateChange = append(w.peerStateChange, peerConnState{peer: peer, present: present})
+		go w.requestMeshUpdate()
+	}
 }
 
 // unregisterClient removes a client from the server.
@@ -198,30 +278,66 @@ func (s *Server) unregisterClient(c *sclient) {
 	if cur == c {
 		c.logf("removing connection")
 		delete(s.clients, c.key)
+		if v, ok := s.clientsMesh[c.key]; ok && v == nil {
+			delete(s.clientsMesh, c.key)
+			s.notePeerGoneFromRegionLocked(c.key)
+		}
+		s.broadcastPeerStateChangeLocked(c.key, false)
+	}
+	if c.canMesh {
+		delete(s.watchers, c)
 	}
 
 	s.curClients.Add(-1)
 	if c.preferred {
 		s.curHomeClients.Add(-1)
 	}
+}
+
+// notePeerGoneFromRegionLocked sends peerGone frames to parties that
+// key has sent to previously (whether those sends were from a local
+// client or forwarded).  It must only be called after the key has
+// been removed from clientsMesh.
+func (s *Server) notePeerGoneFromRegionLocked(key key.Public) {
+	if _, ok := s.clientsMesh[key]; ok {
+		panic("usage")
+	}
 
 	// Find still-connected peers and either notify that we've gone away
 	// so they can drop their route entries to us (issue 150)
 	// or move them over to the active client (in case a replaced client
 	// connection is being unregistered).
-	for pubKey, connNum := range c.sentTo {
+	for pubKey, connNum := range s.sentTo[key] {
 		if peer, ok := s.clients[pubKey]; ok && peer.connNum == connNum {
-			if cur == c {
-				go peer.requestPeerGoneWrite(c.key)
-			} else {
-				// Only if the current client has not already accepted a newer
-				// connection from the peer.
-				if _, ok := cur.sentTo[pubKey]; !ok {
-					cur.sentTo[pubKey] = connNum
-				}
-			}
+			go peer.requestPeerGoneWrite(key)
 		}
 	}
+	delete(s.sentTo, key)
+}
+
+func (s *Server) addWatcher(c *sclient) {
+	if !c.canMesh {
+		panic("invariant: addWatcher called without permissions")
+	}
+
+	if c.key == s.publicKey {
+		// We're connecting to ourself. Do nothing.
+		return
+	}
+
+	s.mu.Lock()
+	defer s.mu.Unlock()
+
+	// Queue messages for each already-connected client.
+	for peer := range s.clients {
+		c.peerStateChange = append(c.peerStateChange, peerConnState{peer: peer, present: true})
+	}
+
+	// And enroll the watcher in future updates (of both
+	// connections & disconnections).
+	s.watchers[c] = true
+
+	go c.requestMeshUpdate()
 }
 
 func (s *Server) accept(nc Conn, brw *bufio.ReadWriter, remoteAddr string, connNum int64) error {
@@ -258,7 +374,10 @@ func (s *Server) accept(nc Conn, brw *bufio.ReadWriter, remoteAddr string, connN
 		connectedAt: time.Now(),
 		sendQueue:   make(chan pkt, perClientSendQueueDepth),
 		peerGone:    make(chan key.Public),
-		sentTo:      make(map[key.Public]int64),
+		canMesh:     clientInfo.MeshKey != "" && clientInfo.MeshKey == s.meshKey,
+	}
+	if c.canMesh {
+		c.meshUpdate = make(chan struct{})
 	}
 	if clientInfo != nil {
 		c.info = *clientInfo
@@ -307,6 +426,12 @@ func (c *sclient) run(ctx context.Context) error {
 			err = c.handleFrameNotePreferred(ft, fl)
 		case frameSendPacket:
 			err = c.handleFrameSendPacket(ft, fl)
+		case frameForwardPacket:
+			err = c.handleFrameForwardPacket(ft, fl)
+		case frameWatchConns:
+			err = c.handleFrameWatchConns(ft, fl)
+		case frameClosePeer:
+			err = c.handleFrameClosePeer(ft, fl)
 		default:
 			err = c.handleUnknownFrame(ft, fl)
 		}
@@ -333,6 +458,92 @@ func (c *sclient) handleFrameNotePreferred(ft frameType, fl uint32) error {
 	return nil
 }
 
+func (c *sclient) handleFrameWatchConns(ft frameType, fl uint32) error {
+	if fl != 0 {
+		return fmt.Errorf("handleFrameWatchConns wrong size")
+	}
+	if !c.canMesh {
+		return fmt.Errorf("insufficient permissions")
+	}
+	c.s.addWatcher(c)
+	return nil
+}
+
+func (c *sclient) handleFrameClosePeer(ft frameType, fl uint32) error {
+	if fl != keyLen {
+		return fmt.Errorf("handleFrameClosePeer wrong size")
+	}
+	if !c.canMesh {
+		return fmt.Errorf("insufficient permissions")
+	}
+	var targetKey key.Public
+	if _, err := io.ReadFull(c.br, targetKey[:]); err != nil {
+		return err
+	}
+	s := c.s
+
+	s.mu.Lock()
+	defer s.mu.Unlock()
+
+	if target, ok := s.clients[targetKey]; ok {
+		c.logf("frameClosePeer closing peer %x", targetKey)
+		go target.nc.Close()
+	} else {
+		c.logf("frameClosePeer failed to find peer %x", targetKey)
+	}
+
+	return nil
+}
+
+// handleFrameForwardPacket reads a "forward packet" frame from the client
+// (which must be a trusted client, a peer in our mesh).
+func (c *sclient) handleFrameForwardPacket(ft frameType, fl uint32) error {
+	if !c.canMesh {
+		return fmt.Errorf("insufficient permissions")
+	}
+	s := c.s
+
+	srcKey, dstKey, contents, err := s.recvForwardPacket(c.br, fl)
+	if err != nil {
+		return fmt.Errorf("client %x: recvForwardPacket: %v", c.key, err)
+	}
+	s.packetsForwardedIn.Add(1)
+
+	s.mu.Lock()
+	dst := s.clients[dstKey]
+	if dst != nil {
+		s.notePeerSendLocked(srcKey, dst)
+	}
+	s.mu.Unlock()
+
+	if dst == nil {
+		s.packetsDropped.Add(1)
+		s.packetsDroppedFwdUnknown.Add(1)
+		if debug {
+			c.logf("dropping forwarded packet for unknown %x", dstKey)
+		}
+		return nil
+	}
+
+	return c.sendPkt(dst, pkt{
+		bs:  contents,
+		src: srcKey,
+	})
+}
+
+// notePeerSendLocked records that src sent to dst.  We keep track of
+// that so when src disconnects, we can tell dst (if it's still
+// around) that src is gone (a peerGone frame).
+func (s *Server) notePeerSendLocked(src key.Public, dst *sclient) {
+	m, ok := s.sentTo[src]
+	if !ok {
+		m = map[key.Public]int64{}
+		s.sentTo[src] = m
+	}
+	m[dst.key] = dst.connNum
+}
+
+// handleFrameSendPacket reads a "send packet" frame from the client.
 func (c *sclient) handleFrameSendPacket(ft frameType, fl uint32) error {
 	s := c.s
 
@@ -341,17 +552,25 @@ func (c *sclient) handleFrameSendPacket(ft frameType, fl uint32) error {
 		return fmt.Errorf("client %x: recvPacket: %v", c.key, err)
 	}
 
+	var fwd PacketForwarder
 	s.mu.Lock()
 	dst := s.clients[dstKey]
-	if dst != nil {
-		// Track that we've sent to this peer, so if/when we
-		// disconnect first, the server can inform all our old
-		// recipients that we're gone. (Issue 150 optimization)
-		c.sentTo[dstKey] = dst.connNum
+	if dst == nil {
+		fwd = s.clientsMesh[dstKey]
+	} else {
+		s.notePeerSendLocked(c.key, dst)
 	}
 	s.mu.Unlock()
 
 	if dst == nil {
+		if fwd != nil {
+			s.packetsForwardedOut.Add(1)
+			if err := fwd.ForwardPacket(c.key, dstKey, contents); err != nil {
+				// TODO:
+				return nil
+			}
+			return nil
+		}
 		s.packetsDropped.Add(1)
 		s.packetsDroppedUnknown.Add(1)
 		if debug {
@@ -366,6 +585,13 @@ func (c *sclient) handleFrameSendPacket(ft frameType, fl uint32) error {
 	if dst.info.Version >= protocolSrcAddrs {
 		p.src = c.key
 	}
+	return c.sendPkt(dst, p)
+}
+
+func (c *sclient) sendPkt(dst *sclient, p pkt) error {
+	s := c.s
+	dstKey := dst.key
+
 	// Attempt to queue for sending up to 3 times. On each attempt, if
 	// the queue is full, try to drop from queue head to prioritize
 	// fresher packets.
@@ -418,6 +644,16 @@ func (c *sclient) requestPeerGoneWrite(peer key.Public) {
 	}
 }
 
+func (c *sclient) requestMeshUpdate() {
+	if !c.canMesh {
+		panic("unexpected requestMeshUpdate")
+	}
+	select {
+	case c.meshUpdate <- struct{}{}:
+	case <-c.done:
+	}
+}
+
 func (s *Server) verifyClient(clientKey key.Public, info *clientInfo) error {
 	// TODO(crawshaw): implement policy constraints on who can use the DERP server
 	// TODO(bradfitz): ... and at what rate.
@@ -464,60 +700,86 @@ func (s *Server) sendServerInfo(bw *bufio.Writer, clientKey key.Public) error {
 func (s *Server) recvClientKey(br *bufio.Reader) (clientKey key.Public, info *clientInfo, err error) {
 	fl, err := readFrameTypeHeader(br, frameClientInfo)
 	if err != nil {
-		return key.Public{}, nil, err
+		return zpub, nil, err
 	}
 	const minLen = keyLen + nonceLen
 	if fl < minLen {
-		return key.Public{}, nil, errors.New("short client info")
+		return zpub, nil, errors.New("short client info")
 	}
 	// We don't trust the client at all yet, so limit its input size to limit
 	// things like JSON resource exhausting (http://github.com/golang/go/issues/31789).
 	if fl > 256<<10 {
-		return key.Public{}, nil, errors.New("long client info")
+		return zpub, nil, errors.New("long client info")
 	}
 	if _, err := io.ReadFull(br, clientKey[:]); err != nil {
-		return key.Public{}, nil, err
+		return zpub, nil, err
 	}
 	var nonce [24]byte
 	if _, err := io.ReadFull(br, nonce[:]); err != nil {
-		return key.Public{}, nil, fmt.Errorf("nonce: %v", err)
+		return zpub, nil, fmt.Errorf("nonce: %v", err)
 	}
 	msgLen := int(fl - minLen)
 	msgbox := make([]byte, msgLen)
 	if _, err := io.ReadFull(br, msgbox); err != nil {
-		return key.Public{}, nil, fmt.Errorf("msgbox: %v", err)
+		return zpub, nil, fmt.Errorf("msgbox: %v", err)
 	}
 	msg, ok := box.Open(nil, msgbox, &nonce, (*[32]byte)(&clientKey), s.privateKey.B32())
 	if !ok {
-		return key.Public{}, nil, fmt.Errorf("msgbox: cannot open len=%d with client key %x", msgLen, clientKey[:])
+		return zpub, nil, fmt.Errorf("msgbox: cannot open len=%d with client key %x", msgLen, clientKey[:])
 	}
 	info = new(clientInfo)
 	if err := json.Unmarshal(msg, info); err != nil {
-		return key.Public{}, nil, fmt.Errorf("msg: %v", err)
+		return zpub, nil, fmt.Errorf("msg: %v", err)
 	}
 	return clientKey, info, nil
 }
 
 func (s *Server) recvPacket(br *bufio.Reader, frameLen uint32) (dstKey key.Public, contents []byte, err error) {
 	if frameLen < keyLen {
-		return key.Public{}, nil, errors.New("short send packet frame")
+		return zpub, nil, errors.New("short send packet frame")
 	}
 	if _, err := io.ReadFull(br, dstKey[:]); err != nil {
-		return key.Public{}, nil, err
+		return zpub, nil, err
 	}
 	packetLen := frameLen - keyLen
 	if packetLen > MaxPacketSize {
-		return key.Public{}, nil, fmt.Errorf("data packet longer (%d) than max of %v", packetLen, MaxPacketSize)
+		return zpub, nil, fmt.Errorf("data packet longer (%d) than max of %v", packetLen, MaxPacketSize)
 	}
 	contents = make([]byte, packetLen)
 	if _, err := io.ReadFull(br, contents); err != nil {
-		return key.Public{}, nil, err
+		return zpub, nil, err
 	}
 	s.packetsRecv.Add(1)
 	s.bytesRecv.Add(int64(len(contents)))
 	return dstKey, contents, nil
 }
 
+// zpub is the key.Public zero value.
+var zpub key.Public
+
+func (s *Server) recvForwardPacket(br *bufio.Reader, frameLen uint32) (srcKey, dstKey key.Public, contents []byte, err error) {
+	if frameLen < keyLen*2 {
+		return zpub, zpub, nil, errors.New("short send packet frame")
+	}
+	if _, err := io.ReadFull(br, srcKey[:]); err != nil {
+		return zpub, zpub, nil, err
+	}
+	if _, err := io.ReadFull(br, dstKey[:]); err != nil {
+		return zpub, zpub, nil, err
+	}
+	packetLen := frameLen - keyLen*2
+	if packetLen > MaxPacketSize {
+		return zpub, zpub, nil, fmt.Errorf("data packet longer (%d) than max of %v", packetLen, MaxPacketSize)
+	}
+	contents = make([]byte, packetLen)
+	if _, err := io.ReadFull(br, contents); err != nil {
+		return zpub, zpub, nil, err
+	}
+	// TODO: was s.packetsRecv.Add(1)
+	// TODO: was s.bytesRecv.Add(int64(len(contents)))
+	return srcKey, dstKey, contents, nil
+}
+
 // sclient is a client connection to the server.
 //
 // (The "s" prefix is to more explicitly distinguish it from Client in derp_client.go)
@@ -532,7 +794,9 @@ type sclient struct {
 	done       <-chan struct{} // closed when connection closes
 	remoteAddr string          // usually ip:port from net.Conn.RemoteAddr().String()
 	sendQueue  chan pkt        // packets queued to this client; never closed
-	peerGone   chan key.Public // write request that a previous sender has disconnected
+	peerGone   chan key.Public // write request that a previous sender has disconnected (not used by mesh peers)
+	meshUpdate chan struct{}   // write request to write peerStateChange
+	canMesh    bool            // clientInfo had correct mesh token for inter-region routing
 
 	// Owned by run, not thread-safe.
 	br          *bufio.Reader
@@ -542,11 +806,20 @@ type sclient struct {
 	// Owned by sender, not thread-safe.
 	bw *bufio.Writer
 
-	// Guarded by s.mu.
+	// Guarded by s.mu
 	//
-	// sentTo tracks all the peers this client has ever sent a packet to, and at which
-	// connection number.
-	sentTo map[key.Public]int64 // recipient => rcpt's latest sclient.connNum
+	// peerStateChange is used by mesh peers (a set of regional
+	// DERP servers) and contains records that need to be sent to
+	// the client for them to update their map of who's connected
+	// to this node.
+	peerStateChange []peerConnState
+}
+
+// peerConnState represents whether a peer is connected to the server
+// or not.
+type peerConnState struct {
+	peer    key.Public
+	present bool
 }
 
 // pkt is a request to write a data frame to an sclient.
@@ -628,6 +901,9 @@ func (c *sclient) sendLoop(ctx context.Context) error {
 		case peer := <-c.peerGone:
 			werr = c.sendPeerGone(peer)
 			continue
+		case <-c.meshUpdate:
+			werr = c.sendMeshUpdates()
+			continue
 		case msg := <-c.sendQueue:
 			werr = c.sendPacket(msg.src, msg.bs)
 			continue
@@ -648,6 +924,9 @@ func (c *sclient) sendLoop(ctx context.Context) error {
 			return nil
 		case peer := <-c.peerGone:
 			werr = c.sendPeerGone(peer)
+		case <-c.meshUpdate:
+			werr = c.sendMeshUpdates()
+			continue
 		case msg := <-c.sendQueue:
 			werr = c.sendPacket(msg.src, msg.bs)
 		case <-keepAliveTick.C:
@@ -677,6 +956,59 @@ func (c *sclient) sendPeerGone(peer key.Public) error {
 	return err
 }
 
+// sendPeerPresent sends a peerPresent frame, without flushing.
+func (c *sclient) sendPeerPresent(peer key.Public) error {
+	c.setWriteDeadline()
+	if err := writeFrameHeader(c.bw, framePeerPresent, keyLen); err != nil {
+		return err
+	}
+	_, err := c.bw.Write(peer[:])
+	return err
+}
+
+// sendMeshUpdates drains as many mesh peerStateChange entries as
+// possible into the write buffer WITHOUT flushing or otherwise
+// blocking (as it holds c.s.mu while working). If it can't drain them
+// all, it schedules itself to be called again in the future.
+func (c *sclient) sendMeshUpdates() error {
+	c.s.mu.Lock()
+	defer c.s.mu.Unlock()
+
+	writes := 0
+	for _, pcs := range c.peerStateChange {
+		if c.bw.Available() <= frameHeaderLen+keyLen {
+			break
+		}
+		var err error
+		if pcs.present {
+			err = c.sendPeerPresent(pcs.peer)
+		} else {
+			err = c.sendPeerGone(pcs.peer)
+		}
+		if err != nil {
+			// Shouldn't happen, though, as we're writing
+			// into available buffer space, not the
+			// network.
+			return err
+		}
+		writes++
+	}
+
+	remain := copy(c.peerStateChange, c.peerStateChange[writes:])
+	c.peerStateChange = c.peerStateChange[:remain]
+
+	// Did we manage to write them all into the bufio buffer without flushing?
+	if len(c.peerStateChange) == 0 {
+		if cap(c.peerStateChange) > 16 {
+			c.peerStateChange = nil
+		}
+	} else {
+		// Didn't finish in the buffer space provided; schedule a future run.
+		go c.requestMeshUpdate()
+	}
+	return nil
+}
+
 // sendPacket writes contents to the client in a RecvPacket frame. If
 // srcKey.IsZero, uses the old DERPv1 framing format, otherwise uses
 // DERPv2. The bytes of contents are only valid until this function
@@ -716,6 +1048,114 @@ func (c *sclient) sendPacket(srcKey key.Public, contents []byte) (err error) {
 	return err
 }
 
+// AddPacketForwarder registers fwd as a packet forwarder for dst.
+// fwd must be comparable.
+func (s *Server) AddPacketForwarder(dst key.Public, fwd PacketForwarder) {
+	s.mu.Lock()
+	defer s.mu.Unlock()
+	if prev, ok := s.clientsMesh[dst]; ok {
+		if prev == fwd {
+			// Duplicate registration of same forwarder. Ignore.
+			return
+		}
+		if m, ok := prev.(multiForwarder); ok {
+			if _, ok := m[fwd]; !ok {
+				// Duplicate registration of same forwarder in set; ignore.
+				return
+			}
+			m[fwd] = m.maxVal() + 1
+			return
+		}
+		if prev != nil {
+			// Otherwise, the existing value is not a set,
+			// not a dup, and not local-only (nil) so make
+			// it a set.
+			fwd = multiForwarder{
+				prev: 1, // existed 1st, higher priority
+				fwd:  2, // the passed in fwd is in 2nd place
+			}
+			s.multiForwarderCreated.Add(1)
+		}
+	}
+	s.clientsMesh[dst] = fwd
+}
+
+// RemovePacketForwarder removes fwd as a packet forwarder for dst.
+// fwd must be comparable.
+func (s *Server) RemovePacketForwarder(dst key.Public, fwd PacketForwarder) {
+	s.mu.Lock()
+	defer s.mu.Unlock()
+	v, ok := s.clientsMesh[dst]
+	if !ok {
+		return
+	}
+	if m, ok := v.(multiForwarder); ok {
+		if len(m) < 2 {
+			panic("unexpected")
+		}
+		delete(m, fwd)
+		// If fwd was in m and we no longer need to be a
+		// multiForwarder, replace the entry with the
+		// remaining PacketForwarder.
+		if len(m) == 1 {
+			var remain PacketForwarder
+			for k := range m {
+				remain = k
+			}
+			s.clientsMesh[dst] = remain
+			s.multiForwarderDeleted.Add(1)
+		}
+		return
+	}
+	if v != fwd {
+		s.removePktForwardOther.Add(1)
+		// Delete of an entry that wasn't in the
+		// map. Harmless, so ignore.
+		// (This might happen if a user is moving around
+		// between nodes and/or the server sent duplicate
+		// connection change broadcasts.)
+		return
+	}
+
+	if _, isLocal := s.clients[dst]; isLocal {
+		s.clientsMesh[dst] = nil
+	} else {
+		delete(s.clientsMesh, dst)
+		s.notePeerGoneFromRegionLocked(dst)
+	}
+}
+
+// multiForwarder is a PacketForwarder that represents a set of
+// forwarding options. It's used in the rare cases that a client is
+// connected to multiple DERP nodes in a region. That shouldn't really
+// happen except for perhaps during brief moments while the client is
+// reconfiguring, in which case we don't want to forget where the
+// client is. The map value is unique connection number; the lowest
+// one has been seen the longest. It's used to make sure we forward
+// packets consistently to the same node and don't pick randomly.
+type multiForwarder map[PacketForwarder]uint8
+
+func (m multiForwarder) maxVal() (max uint8) {
+	for _, v := range m {
+		if v > max {
+			max = v
+		}
+	}
+	return
+}
+
+func (m multiForwarder) ForwardPacket(src, dst key.Public, payload []byte) error {
+	var fwd PacketForwarder
+	var lowest uint8
+	for k, v := range m {
+		if fwd == nil || v < lowest {
+			fwd = k
+			lowest = v
+		}
+	}
+	return fwd.ForwardPacket(src, dst, payload)
+}
+
 func (s *Server) expVarFunc(f func() interface{}) expvar.Func {
 	return expvar.Func(func() interface{} {
 		s.mu.Lock()
@@ -729,8 +1169,12 @@ func (s *Server) ExpVar() expvar.Var {
 	m := new(metrics.Set)
 	m.Set("counter_unique_clients_ever", s.expVarFunc(func() interface{} { return len(s.clientsEver) }))
 	m.Set("gauge_memstats_sys0", expvar.Func(func() interface{} { return int64(s.memSys0) }))
-	m.Set("gauge_current_connnections", &s.curClients)
-	m.Set("gauge_current_home_connnections", &s.curHomeClients)
+	m.Set("gauge_watchers", s.expVarFunc(func() interface{} { return len(s.watchers) }))
+	m.Set("gauge_current_connections", &s.curClients)
+	m.Set("gauge_current_home_connections", &s.curHomeClients)
+	m.Set("gauge_clients_total", expvar.Func(func() interface{} { return len(s.clientsMesh) }))
+	m.Set("gauge_clients_local", expvar.Func(func() interface{} { return len(s.clients) }))
+	m.Set("gauge_clients_remote", expvar.Func(func() interface{} { return len(s.clientsMesh) - len(s.clients) }))
 	m.Set("accepts", &s.accepts)
 	m.Set("clients_replaced", &s.clientsReplaced)
 	m.Set("bytes_received", &s.bytesRecv)
@@ -743,5 +1187,52 @@ func (s *Server) ExpVar() expvar.Var {
 	m.Set("home_moves_in", &s.homeMovesIn)
 	m.Set("home_moves_out", &s.homeMovesOut)
 	m.Set("peer_gone_frames", &s.peerGoneFrames)
+	m.Set("packets_forwarded_out", &s.packetsForwardedOut)
+	m.Set("packets_forwarded_in", &s.packetsForwardedIn)
+	m.Set("multiforwarder_created", &s.multiForwarderCreated)
+	m.Set("multiforwarder_deleted", &s.multiForwarderDeleted)
+	m.Set("packet_forwarder_delete_other_value", &s.removePktForwardOther)
+	var expvarVersion expvar.String
+	expvarVersion.Set(version.LONG)
+	m.Set("version", &expvarVersion)
 	return m
 }
+
+func (s *Server) ConsistencyCheck() error {
+	s.mu.Lock()
+	defer s.mu.Unlock()
+
+	var errs []string
+
+	var nilMeshNotInClient int
+	for k, f := range s.clientsMesh {
+		if f == nil {
+			if _, ok := s.clients[k]; !ok {
+				nilMeshNotInClient++
+			}
+		}
+	}
+	if nilMeshNotInClient != 0 {
+		errs = append(errs, fmt.Sprintf("%d s.clientsMesh keys not in s.clients", nilMeshNotInClient))
+	}
+
+	var clientNotInMesh int
+	for k := range s.clients {
+		if _, ok := s.clientsMesh[k]; !ok {
+			clientNotInMesh++
+		}
+	}
+	if clientNotInMesh != 0 {
+		errs = append(errs, fmt.Sprintf("%d s.clients keys not in s.clientsMesh", clientNotInMesh))
+	}
+
+	if s.curClients.Value() != int64(len(s.clients)) {
+		errs = append(errs, fmt.Sprintf("expvar connections = %d != clients map says of %d",
+			s.curClients.Value(),
+			len(s.clients)))
+	}
+	if len(errs) == 0 {
+		return nil
+	}
+	return errors.New(strings.Join(errs, ", "))
+}

derp/derp_test.go

@@ -13,17 +13,20 @@ import (
 	"fmt"
 	"io"
 	"net"
+	"reflect"
+	"sync"
 	"testing"
 	"time"
 
 	"tailscale.com/net/nettest"
 	"tailscale.com/types/key"
+	"tailscale.com/types/logger"
 )
 
-func newPrivateKey(t *testing.T) (k key.Private) {
-	t.Helper()
+func newPrivateKey(tb testing.TB) (k key.Private) {
+	tb.Helper()
 	if _, err := crand.Read(k[:]); err != nil {
-		t.Fatal(err)
+		tb.Fatal(err)
 	}
 	return
 }
@@ -87,8 +90,7 @@ func TestSendRecv(t *testing.T) {
 	for i := 0; i < numClients; i++ {
 		go func(i int) {
 			for {
-				b := make([]byte, 1<<16)
-				m, err := clients[i].Recv(b)
+				m, err := clients[i].Recv()
 				if err != nil {
 					errCh <- err
 					return
@@ -103,7 +105,7 @@ func TestSendRecv(t *testing.T) {
 					if m.Source.IsZero() {
 						t.Errorf("zero Source address in ReceivedPacket")
 					}
-					recvChs[i] <- m.Data
+					recvChs[i] <- append([]byte(nil), m.Data...)
 				}
 			}
 		}(i)
@@ -256,8 +258,7 @@ func TestSendFreeze(t *testing.T) {
 	recv := func(name string, client *Client) {
 		ch := chs(name)
 		for {
-			b := make([]byte, 1<<9)
-			m, err := client.Recv(b)
+			m, err := client.Recv()
 			if err != nil {
 				errCh <- fmt.Errorf("%s: %w", name, err)
 				return
@@ -391,3 +392,414 @@ func TestSendFreeze(t *testing.T) {
 		}
 	}
 }
+
+type testServer struct {
+	s    *Server
+	ln   net.Listener
+	logf logger.Logf
+
+	mu      sync.Mutex
+	pubName map[key.Public]string
+	clients map[*testClient]bool
+}
+
+func (ts *testServer) addTestClient(c *testClient) {
+	ts.mu.Lock()
+	defer ts.mu.Unlock()
+	ts.clients[c] = true
+}
+
+func (ts *testServer) addKeyName(k key.Public, name string) {
+	ts.mu.Lock()
+	defer ts.mu.Unlock()
+	ts.pubName[k] = name
+	ts.logf("test adding named key %q for %x", name, k)
+}
+
+func (ts *testServer) keyName(k key.Public) string {
+	ts.mu.Lock()
+	defer ts.mu.Unlock()
+	if name, ok := ts.pubName[k]; ok {
+		return name
+	}
+	return k.ShortString()
+}
+
+func (ts *testServer) close(t *testing.T) error {
+	ts.ln.Close()
+	ts.s.Close()
+	for c := range ts.clients {
+		c.close(t)
+	}
+	return nil
+}
+
+func newTestServer(t *testing.T) *testServer {
+	t.Helper()
+	logf := logger.WithPrefix(t.Logf, "derp-server: ")
+	s := NewServer(newPrivateKey(t), logf)
+	s.SetMeshKey("mesh-key")
+	ln, err := net.Listen("tcp", "127.0.0.1:0")
+	if err != nil {
+		t.Fatal(err)
+	}
+	go func() {
+		i := 0
+		for {
+			i++
+			c, err := ln.Accept()
+			if err != nil {
+				return
+			}
+			// TODO: register c in ts so Close also closes it?
+			go func(i int) {
+				brwServer := bufio.NewReadWriter(bufio.NewReader(c), bufio.NewWriter(c))
+				go s.Accept(c, brwServer, fmt.Sprintf("test-client-%d", i))
+			}(i)
+		}
+	}()
+	return &testServer{
+		s:       s,
+		ln:      ln,
+		logf:    logf,
+		clients: map[*testClient]bool{},
+		pubName: map[key.Public]string{},
+	}
+}
+
+type testClient struct {
+	name   string
+	c      *Client
+	nc     net.Conn
+	pub    key.Public
+	ts     *testServer
+	closed bool
+}
+
+func newTestClient(t *testing.T, ts *testServer, name string, newClient func(net.Conn, key.Private, logger.Logf) (*Client, error)) *testClient {
+	t.Helper()
+	nc, err := net.Dial("tcp", ts.ln.Addr().String())
+	if err != nil {
+		t.Fatal(err)
+	}
+	key := newPrivateKey(t)
+	ts.addKeyName(key.Public(), name)
+	c, err := newClient(nc, key, logger.WithPrefix(t.Logf, "client-"+name+": "))
+	if err != nil {
+		t.Fatal(err)
+	}
+	tc := &testClient{
+		name: name,
+		nc:   nc,
+		c:    c,
+		ts:   ts,
+		pub:  key.Public(),
+	}
+	ts.addTestClient(tc)
+	return tc
+}
+
+func newRegularClient(t *testing.T, ts *testServer, name string) *testClient {
+	return newTestClient(t, ts, name, func(nc net.Conn, priv key.Private, logf logger.Logf) (*Client, error) {
+		brw := bufio.NewReadWriter(bufio.NewReader(nc), bufio.NewWriter(nc))
+		return NewClient(priv, nc, brw, logf)
+	})
+}
+
+func newTestWatcher(t *testing.T, ts *testServer, name string) *testClient {
+	return newTestClient(t, ts, name, func(nc net.Conn, priv key.Private, logf logger.Logf) (*Client, error) {
+		brw := bufio.NewReadWriter(bufio.NewReader(nc), bufio.NewWriter(nc))
+		c, err := NewClient(priv, nc, brw, logf, MeshKey("mesh-key"))
+		if err != nil {
+			return nil, err
+		}
+		if err := c.WatchConnectionChanges(); err != nil {
+			return nil, err
+		}
+		return c, nil
+	})
+}
+
+func (tc *testClient) wantPresent(t *testing.T, peers ...key.Public) {
+	t.Helper()
+	want := map[key.Public]bool{}
+	for _, k := range peers {
+		want[k] = true
+	}
+
+	for {
+		m, err := tc.c.recvTimeout(time.Second)
+		if err != nil {
+			t.Fatal(err)
+		}
+		switch m := m.(type) {
+		case PeerPresentMessage:
+			got := key.Public(m)
+			if !want[got] {
+				t.Fatalf("got peer present for %v; want present for %v", tc.ts.keyName(got), logger.ArgWriter(func(bw *bufio.Writer) {
+					for _, pub := range peers {
+						fmt.Fprintf(bw, "%s ", tc.ts.keyName(pub))
+					}
+				}))
+			}
+			delete(want, got)
+			if len(want) == 0 {
+				return
+			}
+		default:
+			t.Fatalf("unexpected message type %T", m)
+		}
+	}
+}
+
+func (tc *testClient) wantGone(t *testing.T, peer key.Public) {
+	t.Helper()
+	m, err := tc.c.recvTimeout(time.Second)
+	if err != nil {
+		t.Fatal(err)
+	}
+	switch m := m.(type) {
+	case PeerGoneMessage:
+		got := key.Public(m)
+		if peer != got {
+			t.Errorf("got gone message for %v; want gone for %v", tc.ts.keyName(got), tc.ts.keyName(peer))
+		}
+	default:
+		t.Fatalf("unexpected message type %T", m)
+	}
+}
+
+func (c *testClient) close(t *testing.T) {
+	t.Helper()
+	if c.closed {
+		return
+	}
+	c.closed = true
+	t.Logf("closing client %q (%x)", c.name, c.pub)
+	c.nc.Close()
+}
+
+// TestWatch tests the connection watcher mechanism used by regional
+// DERP nodes to mesh up with each other.
+func TestWatch(t *testing.T) {
+	ts := newTestServer(t)
+	defer ts.close(t)
+
+	w1 := newTestWatcher(t, ts, "w1")
+	w1.wantPresent(t, w1.pub)
+
+	c1 := newRegularClient(t, ts, "c1")
+	w1.wantPresent(t, c1.pub)
+
+	c2 := newRegularClient(t, ts, "c2")
+	w1.wantPresent(t, c2.pub)
+
+	w2 := newTestWatcher(t, ts, "w2")
+	w1.wantPresent(t, w2.pub)
+	w2.wantPresent(t, w1.pub, w2.pub, c1.pub, c2.pub)
+
+	c3 := newRegularClient(t, ts, "c3")
+	w1.wantPresent(t, c3.pub)
+	w2.wantPresent(t, c3.pub)
+
+	c2.close(t)
+	w1.wantGone(t, c2.pub)
+	w2.wantGone(t, c2.pub)
+
+	w3 := newTestWatcher(t, ts, "w3")
+	w1.wantPresent(t, w3.pub)
+	w2.wantPresent(t, w3.pub)
+	w3.wantPresent(t, c1.pub, c3.pub, w1.pub, w2.pub, w3.pub)
+
+	c1.close(t)
+	w1.wantGone(t, c1.pub)
+	w2.wantGone(t, c1.pub)
+	w3.wantGone(t, c1.pub)
+}
+
+type testFwd int
+
+func (testFwd) ForwardPacket(key.Public, key.Public, []byte) error { panic("not called in tests") }
+
+func pubAll(b byte) (ret key.Public) {
+	for i := range ret {
+		ret[i] = b
+	}
+	return
+}
+
+func TestForwarderRegistration(t *testing.T) {
+	s := &Server{
+		clients:     make(map[key.Public]*sclient),
+		clientsMesh: map[key.Public]PacketForwarder{},
+	}
+	want := func(want map[key.Public]PacketForwarder) {
+		t.Helper()
+		if got := s.clientsMesh; !reflect.DeepEqual(got, want) {
+			t.Fatalf("mismatch\n got: %v\nwant: %v\n", got, want)
+		}
+	}
+	wantCounter := func(c *expvar.Int, want int) {
+		t.Helper()
+		if got := c.Value(); got != int64(want) {
+			t.Errorf("counter = %v; want %v", got, want)
+		}
+	}
+
+	u1 := pubAll(1)
+	u2 := pubAll(2)
+	u3 := pubAll(3)
+
+	s.AddPacketForwarder(u1, testFwd(1))
+	s.AddPacketForwarder(u2, testFwd(2))
+	want(map[key.Public]PacketForwarder{
+		u1: testFwd(1),
+		u2: testFwd(2),
+	})
+
+	// Verify a remove of non-registered forwarder is no-op.
+	s.RemovePacketForwarder(u2, testFwd(999))
+	want(map[key.Public]PacketForwarder{
+		u1: testFwd(1),
+		u2: testFwd(2),
+	})
+
+	// Verify a remove of non-registered user is no-op.
+	s.RemovePacketForwarder(u3, testFwd(1))
+	want(map[key.Public]PacketForwarder{
+		u1: testFwd(1),
+		u2: testFwd(2),
+	})
+
+	// Actual removal.
+	s.RemovePacketForwarder(u2, testFwd(2))
+	want(map[key.Public]PacketForwarder{
+		u1: testFwd(1),
+	})
+
+	// Adding a dup for a user.
+	wantCounter(&s.multiForwarderCreated, 0)
+	s.AddPacketForwarder(u1, testFwd(100))
+	want(map[key.Public]PacketForwarder{
+		u1: multiForwarder{
+			testFwd(1):   1,
+			testFwd(100): 2,
+		},
+	})
+	wantCounter(&s.multiForwarderCreated, 1)
+
+	// Removing a forwarder in a multi set that doesn't exist; does nothing.
+	s.RemovePacketForwarder(u1, testFwd(55))
+	want(map[key.Public]PacketForwarder{
+		u1: multiForwarder{
+			testFwd(1):   1,
+			testFwd(100): 2,
+		},
+	})
+
+	// Removing a forwarder in a multi set that does exist should collapse it away
+	// from being a multiForwarder.
+	wantCounter(&s.multiForwarderDeleted, 0)
+	s.RemovePacketForwarder(u1, testFwd(1))
+	want(map[key.Public]PacketForwarder{
+		u1: testFwd(100),
+	})
+	wantCounter(&s.multiForwarderDeleted, 1)
+
+	// Removing an entry for a client that's still connected locally should result
+	// in a nil forwarder.
+	u1c := &sclient{
+		key:  u1,
+		logf: logger.Discard,
+	}
+	s.clients[u1] = u1c
+	s.RemovePacketForwarder(u1, testFwd(100))
+	want(map[key.Public]PacketForwarder{
+		u1: nil,
+	})
+
+	// But once that client disconnects, it should go away.
+	s.unregisterClient(u1c)
+	want(map[key.Public]PacketForwarder{})
+
+	// But if it already has a forwarder, it's not removed.
+	s.AddPacketForwarder(u1, testFwd(2))
+	s.unregisterClient(u1c)
+	want(map[key.Public]PacketForwarder{
+		u1: testFwd(2),
+	})
+
+	// Now pretend u1 was already connected locally (so clientsMesh[u1] is nil), and then we heard
+	// that they're also connected to a peer of ours. That sholdn't transition the forwarder
+	// from nil to the new one, not a multiForwarder.
+	s.clients[u1] = u1c
+	s.clientsMesh[u1] = nil
+	want(map[key.Public]PacketForwarder{
+		u1: nil,
+	})
+	s.AddPacketForwarder(u1, testFwd(3))
+	want(map[key.Public]PacketForwarder{
+		u1: testFwd(3),
+	})
+}
+
+func BenchmarkSendRecv(b *testing.B) {
+	for _, size := range []int{10, 100, 1000, 10000} {
+		b.Run(fmt.Sprintf("msgsize=%d", size), func(b *testing.B) { benchmarkSendRecvSize(b, size) })
+	}
+}
+
+func benchmarkSendRecvSize(b *testing.B, packetSize int) {
+	serverPrivateKey := newPrivateKey(b)
+	s := NewServer(serverPrivateKey, logger.Discard)
+	defer s.Close()
+
+	key := newPrivateKey(b)
+	clientKey := key.Public()
+
+	ln, err := net.Listen("tcp", "127.0.0.1:0")
+	if err != nil {
+		b.Fatal(err)
+	}
+	defer ln.Close()
+
+	connOut, err := net.Dial("tcp", ln.Addr().String())
+	if err != nil {
+		b.Fatal(err)
+	}
+	defer connOut.Close()
+
+	connIn, err := ln.Accept()
+	if err != nil {
+		b.Fatal(err)
+	}
+	defer connIn.Close()
+
+	brwServer := bufio.NewReadWriter(bufio.NewReader(connIn), bufio.NewWriter(connIn))
+	go s.Accept(connIn, brwServer, "test-client")
+
+	brw := bufio.NewReadWriter(bufio.NewReader(connOut), bufio.NewWriter(connOut))
+	client, err := NewClient(key, connOut, brw, logger.Discard)
+	if err != nil {
+		b.Fatalf("client: %v", err)
+	}
+
+	go func() {
+		for {
+			_, err := client.Recv()
+			if err != nil {
+				return
+			}
+		}
+	}()
+
+	msg := make([]byte, packetSize)
+	b.SetBytes(int64(len(msg)))
+	b.ReportAllocs()
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		if err := client.Send(clientKey, msg); err != nil {
+			b.Fatal(err)
+		}
+	}
+}

derp/derphttp/derphttp_client.go

@@ -43,6 +43,7 @@ import (
 type Client struct {
 	TLSConfig *tls.Config        // optional; nil means default
 	DNSCache  *dnscache.Resolver // optional; nil means no caching
+	MeshKey   string             // optional; for trusted clients
 
 	privateKey key.Private
 	logf       logger.Logf
@@ -54,11 +55,13 @@ type Client struct {
 	ctx       context.Context // closed via cancelCtx in Client.Close
 	cancelCtx context.CancelFunc
 
-	mu        sync.Mutex
-	preferred bool
-	closed    bool
-	netConn   io.Closer
-	client    *derp.Client
+	mu           sync.Mutex
+	preferred    bool
+	closed       bool
+	netConn      io.Closer
+	client       *derp.Client
+	connGen      int // incremented once per new connection; valid values are >0
+	serverPubKey key.Public
 }
 
 // NewRegionClient returns a new DERP-over-HTTP client. It connects lazily.
@@ -106,10 +109,20 @@ func NewClient(privateKey key.Private, serverURL string, logf logger.Logf) (*Cli
 // Connect connects or reconnects to the server, unless already connected.
 // It returns nil if there was already a good connection, or if one was made.
 func (c *Client) Connect(ctx context.Context) error {
-	_, err := c.connect(ctx, "derphttp.Client.Connect")
+	_, _, err := c.connect(ctx, "derphttp.Client.Connect")
 	return err
 }
 
+// ServerPublicKey returns the server's public key.
+//
+// It only returns a non-zero value once a connection has succeeded
+// from an earlier call.
+func (c *Client) ServerPublicKey() key.Public {
+	c.mu.Lock()
+	defer c.mu.Unlock()
+	return c.serverPubKey
+}
+
 func urlPort(u *url.URL) string {
 	if p := u.Port(); p != "" {
 		return p
@@ -152,14 +165,14 @@ func (c *Client) urlString(node *tailcfg.DERPNode) string {
 	return fmt.Sprintf("https://%s/derp", node.HostName)
 }
 
-func (c *Client) connect(ctx context.Context, caller string) (client *derp.Client, err error) {
+func (c *Client) connect(ctx context.Context, caller string) (client *derp.Client, connGen int, err error) {
 	c.mu.Lock()
 	defer c.mu.Unlock()
 	if c.closed {
-		return nil, ErrClientClosed
+		return nil, 0, ErrClientClosed
 	}
 	if c.client != nil {
-		return c.client, nil
+		return c.client, c.connGen, nil
 	}
 
 	// timeout is the fallback maximum time (if ctx doesn't limit
@@ -185,7 +198,7 @@ func (c *Client) connect(ctx context.Context, caller string) (client *derp.Clien
 	if c.getRegion != nil {
 		reg = c.getRegion()
 		if reg == nil {
-			return nil, errors.New("DERP region not available")
+			return nil, 0, errors.New("DERP region not available")
 		}
 	}
 
@@ -212,7 +225,7 @@ func (c *Client) connect(ctx context.Context, caller string) (client *derp.Clien
 		tcpConn, node, err = c.dialRegion(ctx, reg)
 	}
 	if err != nil {
-		return nil, err
+		return nil, 0, err
 	}
 
 	// Now that we have a TCP connection, force close it if the
@@ -250,42 +263,44 @@ func (c *Client) connect(ctx context.Context, caller string) (client *derp.Clien
 
 	req, err := http.NewRequest("GET", c.urlString(node), nil)
 	if err != nil {
-		return nil, err
+		return nil, 0, err
 	}
 	req.Header.Set("Upgrade", "DERP")
 	req.Header.Set("Connection", "Upgrade")
 
 	if err := req.Write(brw); err != nil {
-		return nil, err
+		return nil, 0, err
 	}
 	if err := brw.Flush(); err != nil {
-		return nil, err
+		return nil, 0, err
 	}
 
 	resp, err := http.ReadResponse(brw.Reader, req)
 	if err != nil {
-		return nil, err
+		return nil, 0, err
 	}
 	if resp.StatusCode != http.StatusSwitchingProtocols {
 		b, _ := ioutil.ReadAll(resp.Body)
 		resp.Body.Close()
-		return nil, fmt.Errorf("GET failed: %v: %s", err, b)
+		return nil, 0, fmt.Errorf("GET failed: %v: %s", err, b)
 	}
 
-	derpClient, err := derp.NewClient(c.privateKey, httpConn, brw, c.logf)
+	derpClient, err := derp.NewClient(c.privateKey, httpConn, brw, c.logf, derp.MeshKey(c.MeshKey))
 	if err != nil {
-		return nil, err
+		return nil, 0, err
 	}
 	if c.preferred {
 		if err := derpClient.NotePreferred(true); err != nil {
 			go httpConn.Close()
-			return nil, err
+			return nil, 0, err
 		}
 	}
 
+	c.serverPubKey = derpClient.ServerPublicKey()
 	c.client = derpClient
 	c.netConn = tcpConn
-	return c.client, nil
+	c.connGen++
+	return c.client, c.connGen, nil
 }
 
 func (c *Client) dialURL(ctx context.Context) (net.Conn, error) {
@@ -323,6 +338,9 @@ func (c *Client) dialRegion(ctx context.Context, reg *tailcfg.DERPRegion) (net.C
 	var firstErr error
 	for _, n := range reg.Nodes {
 		if n.STUNOnly {
+			if firstErr == nil {
+				firstErr = fmt.Errorf("no non-STUNOnly nodes for %s", c.targetString(reg))
+			}
 			continue
 		}
 		c, err := c.dialNode(ctx, n)
@@ -463,7 +481,7 @@ func (c *Client) dialNode(ctx context.Context, n *tailcfg.DERPNode) (net.Conn, e
 }
 
 func (c *Client) Send(dstKey key.Public, b []byte) error {
-	client, err := c.connect(context.TODO(), "derphttp.Client.Send")
+	client, _, err := c.connect(context.TODO(), "derphttp.Client.Send")
 	if err != nil {
 		return err
 	}
@@ -473,6 +491,17 @@ func (c *Client) Send(dstKey key.Public, b []byte) error {
 	return err
 }
 
+func (c *Client) ForwardPacket(from, to key.Public, b []byte) error {
+	client, _, err := c.connect(context.TODO(), "derphttp.Client.ForwardPacket")
+	if err != nil {
+		return err
+	}
+	if err := client.ForwardPacket(from, to, b); err != nil {
+		c.closeForReconnect(client)
+	}
+	return err
+}
+
 // NotePreferred notes whether this Client is the caller's preferred
 // (home) DERP node. It's only used for stats.
 func (c *Client) NotePreferred(v bool) {
@@ -492,18 +521,58 @@ func (c *Client) NotePreferred(v bool) {
 	}
 }
 
-func (c *Client) Recv(b []byte) (derp.ReceivedMessage, error) {
-	client, err := c.connect(context.TODO(), "derphttp.Client.Recv")
+// WatchConnectionChanges sends a request to subscribe to
+// notifications about clients connecting & disconnecting.
+//
+// Only trusted connections (using MeshKey) are allowed to use this.
+func (c *Client) WatchConnectionChanges() error {
+	client, _, err := c.connect(context.TODO(), "derphttp.Client.WatchConnectionChanges")
 	if err != nil {
-		return nil, err
+		return err
 	}
-	m, err := client.Recv(b)
+	err = client.WatchConnectionChanges()
 	if err != nil {
 		c.closeForReconnect(client)
 	}
+	return err
+}
+
+// ClosePeer asks the server to close target's TCP connection.
+//
+// Only trusted connections (using MeshKey) are allowed to use this.
+func (c *Client) ClosePeer(target key.Public) error {
+	client, _, err := c.connect(context.TODO(), "derphttp.Client.ClosePeer")
+	if err != nil {
+		return err
+	}
+	err = client.ClosePeer(target)
+	if err != nil {
+		c.closeForReconnect(client)
+	}
+	return err
+}
+
+// Recv reads a message from c. The returned message may alias memory from Client.
+// The message should only be used until the next Client call.
+func (c *Client) Recv() (derp.ReceivedMessage, error) {
+	m, _, err := c.RecvDetail()
 	return m, err
 }
 
+// RecvDetail is like Recv, but additional returns the connection generation on each message.
+// The connGen value is incremented every time the derphttp.Client reconnects to the server.
+func (c *Client) RecvDetail() (m derp.ReceivedMessage, connGen int, err error) {
+	client, connGen, err := c.connect(context.TODO(), "derphttp.Client.Recv")
+	if err != nil {
+		return nil, 0, err
+	}
+	m, err = client.Recv()
+	if err != nil {
+		c.closeForReconnect(client)
+	}
+	return m, connGen, err
+}
+
 // Close closes the client. It will not automatically reconnect after
 // being closed.
 func (c *Client) Close() error {

derp/derphttp/derphttp_test.go

@@ -93,8 +93,7 @@ func TestSendRecv(t *testing.T) {
 					return
 				default:
 				}
-				b := make([]byte, 1<<16)
-				m, err := c.Recv(b)
+				m, err := c.Recv()
 				if err != nil {
 					t.Logf("client%d: %v", i, err)
 					break
@@ -106,7 +105,7 @@ func TestSendRecv(t *testing.T) {
 				case derp.PeerGoneMessage:
 					// Ignore.
 				case derp.ReceivedPacket:
-					recvChs[i] <- m.Data
+					recvChs[i] <- append([]byte(nil), m.Data...)
 				}
 			}
 		}(i)

derp/derphttp/mesh_client.go

@@ -0,0 +1,122 @@
+// Copyright (c) 2020 Tailscale Inc & AUTHORS All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package derphttp
+
+import (
+	"sync"
+	"time"
+
+	"tailscale.com/derp"
+	"tailscale.com/types/key"
+)
+
+// RunWatchConnectionLoop loops forever, sending WatchConnectionChanges and subscribing to
+// connection changes.
+//
+// If the server's public key is ignoreServerKey, RunWatchConnectionLoop returns.
+//
+// Otherwise, the add and remove funcs are called as clients come & go.
+func (c *Client) RunWatchConnectionLoop(ignoreServerKey key.Public, add, remove func(key.Public)) {
+	logf := c.logf
+	const retryInterval = 5 * time.Second
+	const statusInterval = 10 * time.Second
+	var (
+		mu              sync.Mutex
+		present         = map[key.Public]bool{}
+		loggedConnected = false
+	)
+	clear := func() {
+		mu.Lock()
+		defer mu.Unlock()
+		if len(present) == 0 {
+			return
+		}
+		logf("reconnected; clearing %d forwarding mappings", len(present))
+		for k := range present {
+			remove(k)
+		}
+		present = map[key.Public]bool{}
+	}
+	lastConnGen := 0
+	lastStatus := time.Now()
+	logConnectedLocked := func() {
+		if loggedConnected {
+			return
+		}
+		logf("connected; %d peers", len(present))
+		loggedConnected = true
+	}
+
+	const logConnectedDelay = 200 * time.Millisecond
+	timer := time.AfterFunc(2*time.Second, func() {
+		mu.Lock()
+		defer mu.Unlock()
+		logConnectedLocked()
+	})
+	defer timer.Stop()
+
+	updatePeer := func(k key.Public, isPresent bool) {
+		if isPresent {
+			add(k)
+		} else {
+			remove(k)
+		}
+
+		mu.Lock()
+		defer mu.Unlock()
+		if isPresent {
+			present[k] = true
+			if !loggedConnected {
+				timer.Reset(logConnectedDelay)
+			}
+		} else {
+			// If we got a peerGone message, that means the initial connection's
+			// flood of peerPresent messages is done, so we can log already:
+			logConnectedLocked()
+			delete(present, k)
+		}
+	}
+
+	for {
+		err := c.WatchConnectionChanges()
+		if err != nil {
+			clear()
+			logf("WatchConnectionChanges: %v", err)
+			time.Sleep(retryInterval)
+			continue
+		}
+
+		if c.ServerPublicKey() == ignoreServerKey {
+			logf("detected self-connect; ignoring host")
+			return
+		}
+		for {
+			m, connGen, err := c.RecvDetail()
+			if err != nil {
+				clear()
+				logf("Recv: %v", err)
+				time.Sleep(retryInterval)
+				break
+			}
+			if connGen != lastConnGen {
+				lastConnGen = connGen
+				clear()
+			}
+			switch m := m.(type) {
+			case derp.PeerPresentMessage:
+				updatePeer(key.Public(m), true)
+			case derp.PeerGoneMessage:
+				updatePeer(key.Public(m), false)
+			default:
+				continue
+			}
+			if now := time.Now(); now.Sub(lastStatus) > statusInterval {
+				lastStatus = now
+				logf("%d peers", len(present))
+			}
+		}
+	}
+
+}

disco/disco.go

@@ -0,0 +1,179 @@
+// Copyright (c) 2020 Tailscale Inc & AUTHORS All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+// Package disco contains the discovery message types.
+//
+// A discovery message is:
+//
+// Header:
+//     magic          [6]byte  // “TS💬” (0x54 53 f0 9f 92 ac)
+//     senderDiscoPub [32]byte // nacl public key
+//     nonce          [24]byte
+//
+// The recipient then decrypts the bytes following (the nacl secretbox)
+// and then the inner payload structure is:
+//
+//     messageType    byte  (the MessageType constants below)
+//     messageVersion byte  (0 for now; but always ignore bytes at the end)
+//     message-paylod [...]byte
+package disco
+
+import (
+	"encoding/binary"
+	"errors"
+	"fmt"
+	"net"
+
+	"inet.af/netaddr"
+)
+
+// Magic is the 6 byte header of all discovery messages.
+const Magic = "TS💬" // 6 bytes: 0x54 53 f0 9f 92 ac
+
+const keyLen = 32
+
+// NonceLen is the length of the nonces used by nacl secretboxes.
+const NonceLen = 24
+
+type MessageType byte
+
+const (
+	TypePing        = MessageType(0x01)
+	TypePong        = MessageType(0x02)
+	TypeCallMeMaybe = MessageType(0x03)
+)
+
+const v0 = byte(0)
+
+var errShort = errors.New("short message")
+
+// LooksLikeDiscoWrapper reports whether p looks like it's a packet
+// containing an encrypted disco message.
+func LooksLikeDiscoWrapper(p []byte) bool {
+	if len(p) < len(Magic)+keyLen+NonceLen {
+		return false
+	}
+	return string(p[:len(Magic)]) == Magic
+}
+
+// Parse parses the encrypted part of the message from inside the
+// nacl secretbox.
+func Parse(p []byte) (Message, error) {
+	if len(p) < 2 {
+		return nil, errShort
+	}
+	t, ver, p := MessageType(p[0]), p[1], p[2:]
+	switch t {
+	case TypePing:
+		return parsePing(ver, p)
+	case TypePong:
+		return parsePong(ver, p)
+	case TypeCallMeMaybe:
+		return CallMeMaybe{}, nil
+	default:
+		return nil, fmt.Errorf("unknown message type 0x%02x", byte(t))
+	}
+}
+
+// Message a discovery message.
+type Message interface {
+	// AppendMarshal appends the message's marshaled representation.
+	AppendMarshal([]byte) []byte
+}
+
+// appendMsgHeader appends two bytes (for t and ver) and then also
+// dataLen bytes to b, returning the appended slice in all. The
+// returned data slice is a subslice of all with just dataLen bytes of
+// where the caller will fill in the data.
+func appendMsgHeader(b []byte, t MessageType, ver uint8, dataLen int) (all, data []byte) {
+	// TODO: optimize this?
+	all = append(b, make([]byte, dataLen+2)...)
+	all[len(b)] = byte(t)
+	all[len(b)+1] = ver
+	data = all[len(b)+2:]
+	return
+}
+
+type Ping struct {
+	TxID [12]byte
+}
+
+func (m *Ping) AppendMarshal(b []byte) []byte {
+	ret, d := appendMsgHeader(b, TypePing, v0, 12)
+	copy(d, m.TxID[:])
+	return ret
+}
+
+func parsePing(ver uint8, p []byte) (m *Ping, err error) {
+	if len(p) < 12 {
+		return nil, errShort
+	}
+	m = new(Ping)
+	copy(m.TxID[:], p)
+	return m, nil
+}
+
+// CallMeMaybe is a message sent only over DERP to request that the recipient try
+// to open up a magicsock path back to the sender.
+//
+// The sender should've already sent UDP packets to the peer to open
+// up the stateful firewall mappings inbound.
+//
+// The recipient may choose to not open a path back, if it's already
+// happy with its path. But usually it will.
+type CallMeMaybe struct{}
+
+func (CallMeMaybe) AppendMarshal(b []byte) []byte {
+	ret, _ := appendMsgHeader(b, TypeCallMeMaybe, v0, 0)
+	return ret
+}
+
+// Pong is a response a Ping.
+//
+// It includes the sender's source IP + port, so it's effectively a
+// STUN response.
+type Pong struct {
+	TxID [12]byte
+	Src  netaddr.IPPort // 18 bytes (16+2) on the wire; v4-mapped ipv6 for IPv4
+}
+
+const pongLen = 12 + 16 + 2
+
+func (m *Pong) AppendMarshal(b []byte) []byte {
+	ret, d := appendMsgHeader(b, TypePong, v0, pongLen)
+	d = d[copy(d, m.TxID[:]):]
+	ip16 := m.Src.IP.As16()
+	d = d[copy(d, ip16[:]):]
+	binary.BigEndian.PutUint16(d, m.Src.Port)
+	return ret
+}
+
+func parsePong(ver uint8, p []byte) (m *Pong, err error) {
+	if len(p) < pongLen {
+		return nil, errShort
+	}
+	m = new(Pong)
+	copy(m.TxID[:], p)
+	p = p[12:]
+
+	m.Src.IP, _ = netaddr.FromStdIP(net.IP(p[:16]))
+	p = p[16:]
+
+	m.Src.Port = binary.BigEndian.Uint16(p)
+	return m, nil
+}
+
+// MessageSummary returns a short summary of m for logging purposes.
+func MessageSummary(m Message) string {
+	switch m := m.(type) {
+	case *Ping:
+		return fmt.Sprintf("ping tx=%x", m.TxID[:6])
+	case *Pong:
+		return fmt.Sprintf("pong tx=%x", m.TxID[:6])
+	case CallMeMaybe:
+		return "call-me-maybe"
+	default:
+		return fmt.Sprintf("%#v", m)
+	}
+}

disco/disco_test.go

@@ -0,0 +1,82 @@
+// Copyright (c) 2020 Tailscale Inc & AUTHORS All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package disco
+
+import (
+	"fmt"
+	"reflect"
+	"strings"
+	"testing"
+
+	"inet.af/netaddr"
+)
+
+func TestMarshalAndParse(t *testing.T) {
+	tests := []struct {
+		name string
+		want string
+		m    Message
+	}{
+		{
+			name: "ping",
+			m: &Ping{
+				TxID: [12]byte{1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12},
+			},
+			want: "01 00 01 02 03 04 05 06 07 08 09 0a 0b 0c",
+		},
+		{
+			name: "pong",
+			m: &Pong{
+				TxID: [12]byte{1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12},
+				Src:  mustIPPort("2.3.4.5:1234"),
+			},
+			want: "02 00 01 02 03 04 05 06 07 08 09 0a 0b 0c 00 00 00 00 00 00 00 00 00 00 ff ff 02 03 04 05 04 d2",
+		},
+		{
+			name: "pongv6",
+			m: &Pong{
+				TxID: [12]byte{1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12},
+				Src:  mustIPPort("[fed0::12]:6666"),
+			},
+			want: "02 00 01 02 03 04 05 06 07 08 09 0a 0b 0c fe d0 00 00 00 00 00 00 00 00 00 00 00 00 00 12 1a 0a",
+		},
+		{
+			name: "call_me_maybe",
+			m:    CallMeMaybe{},
+			want: "03 00",
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			foo := []byte("foo")
+			got := string(tt.m.AppendMarshal(foo))
+			if !strings.HasPrefix(got, "foo") {
+				t.Fatalf("didn't start with foo: got %q", got)
+			}
+			got = strings.TrimPrefix(got, "foo")
+
+			gotHex := fmt.Sprintf("% x", got)
+			if gotHex != tt.want {
+				t.Fatalf("wrong marshal\n got: %s\nwant: %s\n", gotHex, tt.want)
+			}
+
+			back, err := Parse([]byte(got))
+			if err != nil {
+				t.Fatalf("parse back: %v", err)
+			}
+			if !reflect.DeepEqual(back, tt.m) {
+				t.Errorf("message in %+v doesn't match Parse back result %+v", tt.m, back)
+			}
+		})
+	}
+}
+
+func mustIPPort(s string) netaddr.IPPort {
+	ipp, err := netaddr.ParseIPPort(s)
+	if err != nil {
+		panic(err)
+	}
+	return ipp
+}

go.mod

@@ -1,6 +1,6 @@
 module tailscale.com
 
-go 1.13
+go 1.14
 
 require (
 	github.com/anmitsu/go-shlex v0.0.0-20161002113705-648efa622239 // indirect
@@ -9,25 +9,30 @@ require (
 	github.com/flynn/go-shlex v0.0.0-20150515145356-3f9db97f8568 // indirect
 	github.com/gliderlabs/ssh v0.2.2
 	github.com/go-ole/go-ole v1.2.4
+	github.com/godbus/dbus/v5 v5.0.3
 	github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e
 	github.com/google/go-cmp v0.4.0
 	github.com/goreleaser/nfpm v1.1.10
-	github.com/klauspost/compress v1.9.8
+	github.com/jsimonetti/rtnetlink v0.0.0-20200117123717-f846d4f6c1f4
+	github.com/klauspost/compress v1.10.10
 	github.com/kr/pty v1.1.1
 	github.com/mdlayher/netlink v1.1.0
+	github.com/miekg/dns v1.1.30
 	github.com/pborman/getopt v0.0.0-20190409184431-ee0cd42419d3
 	github.com/peterbourgon/ff/v2 v2.0.0
 	github.com/tailscale/winipcfg-go v0.0.0-20200413171540-609dcf2df55f
-	github.com/tailscale/wireguard-go v0.0.0-20200515231107-62868271d710
+	github.com/tailscale/wireguard-go v0.0.0-20200806235025-91988cfbaa3a
 	github.com/tcnksm/go-httpstat v0.2.0
 	github.com/toqueteos/webbrowser v1.2.0
-	go4.org/mem v0.0.0-20200601023850-d8ee1dfa5518
-	golang.org/x/crypto v0.0.0-20200317142112-1b76d66859c6
-	golang.org/x/net v0.0.0-20200301022130-244492dfa37a
+	go4.org/mem v0.0.0-20200706164138-185c595c3ecc
+	golang.org/x/crypto v0.0.0-20200429183012-4b2356b1ed79
+	golang.org/x/net v0.0.0-20200501053045-e0ff5e5a1de5
 	golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d
 	golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e
-	golang.org/x/sys v0.0.0-20200501052902-10377860bb8e
+	golang.org/x/sys v0.0.0-20200501145240-bc7a7d42d5c3
 	golang.org/x/time v0.0.0-20191024005414-555d28b269f0
-	inet.af/netaddr v0.0.0-20200513162223-787f13e36cbe
+	golang.org/x/tools v0.0.0-20191216052735-49a3e744a425
+	honnef.co/go/tools v0.0.1-2020.1.4
+	inet.af/netaddr v0.0.0-20200810144936-56928fe48a98
 	rsc.io/goversion v1.2.0
 )

go.sum

@@ -30,6 +30,8 @@ github.com/gliderlabs/ssh v0.2.2 h1:6zsha5zo/TWhRhwqCD3+EarCAgZ2yN28ipRnGPnwkI0=
 github.com/gliderlabs/ssh v0.2.2/go.mod h1:U7qILu1NlMHj9FlMhZLlkCdDnU1DBEAqr0aevW3Awn0=
 github.com/go-ole/go-ole v1.2.4 h1:nNBDSCOigTSiarFpYE9J/KtEA1IOW4CNeqT9TQDqCxI=
 github.com/go-ole/go-ole v1.2.4/go.mod h1:XCwSNxSkXRo4vlyPy93sltvi/qJq0jqQhjqQNIwKuxM=
+github.com/godbus/dbus/v5 v5.0.3 h1:ZqHaoEF7TBzh4jzPmqVhE/5A1z9of6orkAe5uHoAeME=
+github.com/godbus/dbus/v5 v5.0.3/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA=
 github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e h1:1r7pUrabqp18hOBcwBwiTsbnFeTZHV9eER/QT5JVZxY=
 github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
 github.com/golang/protobuf v1.2.0 h1:P3YflyNX/ehuJFLhxviNdFxQPkGK5cDcApsge1SqnvM=
@@ -38,6 +40,7 @@ github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5a
 github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
 github.com/google/go-cmp v0.4.0 h1:xsAVV57WRhGj6kEIi8ReJzQlHHqcBYCElAvkovg3B/4=
 github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
+github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI=
 github.com/google/rpmpack v0.0.0-20191226140753-aa36bfddb3a0 h1:BW6OvS3kpT5UEPbCZ+KyX/OB4Ks9/MNMhWjqPPkZxsE=
 github.com/google/rpmpack v0.0.0-20191226140753-aa36bfddb3a0/go.mod h1:RaTPr0KUf2K7fnZYLNDrr8rxAamWs3iNywJLtQ2AzBg=
 github.com/goreleaser/nfpm v1.1.10 h1:0nwzKUJTcygNxTzVKq2Dh9wpVP1W2biUH6SNKmoxR3w=
@@ -47,8 +50,9 @@ github.com/imdario/mergo v0.3.8/go.mod h1:2EnlNZ0deacrJVfApfmtdGgDfMuh/nq6Ok1EcJ
 github.com/jsimonetti/rtnetlink v0.0.0-20190606172950-9527aa82566a/go.mod h1:Oz+70psSo5OFh8DBl0Zv2ACw7Esh6pPUphlvZG9x7uw=
 github.com/jsimonetti/rtnetlink v0.0.0-20200117123717-f846d4f6c1f4 h1:nwOc1YaOrYJ37sEBrtWZrdqzK22hiJs3GpDmP3sR2Yw=
 github.com/jsimonetti/rtnetlink v0.0.0-20200117123717-f846d4f6c1f4/go.mod h1:WGuG/smIU4J/54PblvSbh+xvCZmpJnFgr3ds6Z55XMQ=
-github.com/klauspost/compress v1.9.8 h1:VMAMUUOh+gaxKTMk+zqbjsSjsIcUcL/LF4o63i82QyA=
-github.com/klauspost/compress v1.9.8/go.mod h1:RyIbtBH6LamlWaDj8nUwkbUhJ87Yi3uG0guNDohfE1A=
+github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck=
+github.com/klauspost/compress v1.10.10 h1:a/y8CglcM7gLGYmlbP/stPE5sR3hbhFRUjCBfd/0B3I=
+github.com/klauspost/compress v1.10.10/go.mod h1:aoV0uJVorq1K+umq18yTdKaF57EivdYsUV+/s2qKfXs=
 github.com/kr/pretty v0.1.0 h1:L/CwN0zerZDmRFUapSPitk6f+Q3+0za1rQkzVuMiMFI=
 github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo=
 github.com/kr/pty v1.1.1 h1:VkoXIwSboBpnk99O/KFauAEILuNHv5DVFKZMBN/gUgw=
@@ -61,6 +65,8 @@ github.com/mdlayher/netlink v0.0.0-20190409211403-11939a169225/go.mod h1:eQB3mZE
 github.com/mdlayher/netlink v1.0.0/go.mod h1:KxeJAFOFLG6AjpyDkQ/iIhxygIUKD+vcwqcnu43w/+M=
 github.com/mdlayher/netlink v1.1.0 h1:mpdLgm+brq10nI9zM1BpX1kpDbh3NLl3RSnVq6ZSkfg=
 github.com/mdlayher/netlink v1.1.0/go.mod h1:H4WCitaheIsdF9yOYu8CFmCgQthAPIWZmcKp9uZHgmY=
+github.com/miekg/dns v1.1.30 h1:Qww6FseFn8PRfw07jueqIXqodm0JKiiKuK0DeXSqfyo=
+github.com/miekg/dns v1.1.30/go.mod h1:KNUDUusw/aVsxyTYZM1oqvCicbwhgbNgztCETuNZ7xM=
 github.com/op/go-logging v0.0.0-20160315200505-970db520ece7 h1:lDH9UUVJtmYCjyT0CI4q8xvlXPxeZ0gYCVvWbmPlp88=
 github.com/op/go-logging v0.0.0-20160315200505-970db520ece7/go.mod h1:HzydrMdWErDVzsI23lYNej1Htcns9BCg93Dk0bBINWk=
 github.com/pborman/getopt v0.0.0-20190409184431-ee0cd42419d3 h1:YtFkrqsMEj7YqpIhRteVxJxCeC3jJBieuLr0d4C4rSA=
@@ -72,6 +78,7 @@ github.com/pkg/errors v0.8.1 h1:iURUrRGxPUNPdy5/HRSm+Yj6okJ6UtLINN0Q9M4+h3I=
 github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
 github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
 github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
+github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4=
 github.com/sassoftware/go-rpmutils v0.0.0-20190420191620-a8f1baeba37b h1:+gCnWOZV8Z/8jehJ2CdqB47Z3S+SREmQcuXkRFLNsiI=
 github.com/sassoftware/go-rpmutils v0.0.0-20190420191620-a8f1baeba37b/go.mod h1:am+Fp8Bt506lA3Rk3QCmSqmYmLMnPDhdDUcosQCAx+I=
 github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
@@ -79,8 +86,8 @@ github.com/stretchr/testify v1.4.0 h1:2E4SXV/wtOkTonXsotYi4li6zVWxYlZuYNCXe9XRJy
 github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4=
 github.com/tailscale/winipcfg-go v0.0.0-20200413171540-609dcf2df55f h1:uFj5bslHsMzxIM8UTjAhq4VXeo6GfNW91rpoh/WMJaY=
 github.com/tailscale/winipcfg-go v0.0.0-20200413171540-609dcf2df55f/go.mod h1:x880GWw5fvrl2DVTQ04ttXQD4DuppTt1Yz6wLibbjNE=
-github.com/tailscale/wireguard-go v0.0.0-20200515231107-62868271d710 h1:I6aq3tOYbZob9uwhGpr7R266qTeU9PFqS6NnpfCqEzo=
-github.com/tailscale/wireguard-go v0.0.0-20200515231107-62868271d710/go.mod h1:JPm5cTfu1K+qDFRbiHy0sOlHUylYQbpl356sdYFD8V4=
+github.com/tailscale/wireguard-go v0.0.0-20200806235025-91988cfbaa3a h1:dQEgNpoOJf+8MswlvXJicb8ZDQqZAGe8f/WfzbDMvtE=
+github.com/tailscale/wireguard-go v0.0.0-20200806235025-91988cfbaa3a/go.mod h1:WXq+IkSOJGIgfF1XW+4z4oW+LX/TXzU9DcKlT5EZLi4=
 github.com/tcnksm/go-httpstat v0.2.0 h1:rP7T5e5U2HfmOBmZzGgGZjBQ5/GluWUylujl0tJ04I0=
 github.com/tcnksm/go-httpstat v0.2.0/go.mod h1:s3JVJFtQxtBEBC9dwcdTTXS9xFnM3SXAZwPG41aurT8=
 github.com/toqueteos/webbrowser v1.2.0 h1:tVP/gpK69Fx+qMJKsLE7TD8LuGWPnEV71wBN9rrstGQ=
@@ -89,27 +96,33 @@ github.com/ulikunitz/xz v0.5.6 h1:jGHAfXawEGZQ3blwU5wnWKQJvAraT7Ftq9EXjnXYgt8=
 github.com/ulikunitz/xz v0.5.6/go.mod h1:2bypXElzHzzJZwzH67Y6wb67pO62Rzfn7BSiF4ABRW8=
 github.com/xi2/xz v0.0.0-20171230120015-48954b6210f8 h1:nIPpBwaJSVYIxUFsDv3M8ofmx9yWTog9BfvIu0q41lo=
 github.com/xi2/xz v0.0.0-20171230120015-48954b6210f8/go.mod h1:HUYIGzjTL3rfEspMxjDjgmT5uz5wzYJKVo23qUhYTos=
-go4.org/mem v0.0.0-20200601023850-d8ee1dfa5518 h1:AA3bSGklCgkrqIGnvL4894oa/2K9ltE0RejXh8CgyvA=
-go4.org/mem v0.0.0-20200601023850-d8ee1dfa5518/go.mod h1:NEYvpHWemiG/E5UWfaN5QAIGZeT1sa0Z2UNk6oeMb/k=
+go4.org/mem v0.0.0-20200706164138-185c595c3ecc h1:paujszgN6SpsO/UsXC7xax3gQAKz/XQKCYZLQdU34Tw=
+go4.org/mem v0.0.0-20200706164138-185c595c3ecc/go.mod h1:NEYvpHWemiG/E5UWfaN5QAIGZeT1sa0Z2UNk6oeMb/k=
 golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
+golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
 golang.org/x/crypto v0.0.0-20191002192127-34f69633bfdc/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
-golang.org/x/crypto v0.0.0-20200317142112-1b76d66859c6 h1:TjszyFsQsyZNHwdVdZ5m7bjmreu0znc2kRYsEml9/Ww=
-golang.org/x/crypto v0.0.0-20200317142112-1b76d66859c6/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
+golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
+golang.org/x/crypto v0.0.0-20200429183012-4b2356b1ed79 h1:IaQbIIB2X/Mp/DKctl6ROxz1KyMlKp4uyvL6+kQ7C88=
+golang.org/x/crypto v0.0.0-20200429183012-4b2356b1ed79/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
+golang.org/x/mod v0.0.0-20190513183733-4bf6d317e70e/go.mod h1:mXi4GBBbnImb6dmsKGUJ2LatrhH/nqhxcFungHvyanc=
+golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg=
 golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/net v0.0.0-20190108225652-1e06a53dbb7e/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
 golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
+golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
 golang.org/x/net v0.0.0-20190827160401-ba9fcec4b297/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
-golang.org/x/net v0.0.0-20191003171128-d98b1b443823/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
+golang.org/x/net v0.0.0-20190923162816-aa69164e4478/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
 golang.org/x/net v0.0.0-20191007182048-72f939374954/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
 golang.org/x/net v0.0.0-20200202094626-16171245cfb2 h1:CCH4IOTTfewWjGOlSp+zGcjutRKlBEZQ6wTn8ozI/nI=
 golang.org/x/net v0.0.0-20200202094626-16171245cfb2/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
-golang.org/x/net v0.0.0-20200301022130-244492dfa37a h1:GuSPYbZzB5/dcLNCwLQLsg3obCJtX9IJhpXkvY7kzk0=
-golang.org/x/net v0.0.0-20200301022130-244492dfa37a/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
+golang.org/x/net v0.0.0-20200501053045-e0ff5e5a1de5 h1:WQ8q63x+f/zpC8Ac1s9wLElVoHhm32p6tudrU72n1QA=
+golang.org/x/net v0.0.0-20200501053045-e0ff5e5a1de5/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A=
 golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d h1:TzXSXBo42m9gQenoE3b9BGiEpg5IG2JkU5FkPIawgtw=
 golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
 golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4 h1:YUO/7uOKsKeq9UokNS62b8FYywz3ker1l1vDZRCRefw=
 golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e h1:vcxGaoTs7kV8m5Np9uUNQin4BrLOthgV7252N8V+FwY=
 golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
@@ -119,30 +132,43 @@ golang.org/x/sys v0.0.0-20190405154228-4b34438f7a67/go.mod h1:h1NjWce9XRLGQEsW7w
 golang.org/x/sys v0.0.0-20190411185658-b44545bcd369/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20190826190057-c7b8b68b1456/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20191003212358-c178f38b412c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20190924154521-2837fb4f24fe/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20191008105621-543471e840be/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20200202164722-d101bd2416d5 h1:LfCXLvNmTYH9kEmVgqbnsWfruoXZIrh4YBgqVHtDvw0=
 golang.org/x/sys v0.0.0-20200202164722-d101bd2416d5/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20200501052902-10377860bb8e h1:hq86ru83GdWTlfQFZGO4nZJTU4Bs2wfHl8oFHRaXsfc=
-golang.org/x/sys v0.0.0-20200501052902-10377860bb8e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200501145240-bc7a7d42d5c3 h1:5B6i6EAiSYyejWfvc5Rc9BbI3rzIsrrXfAQBWnYfn+w=
+golang.org/x/sys v0.0.0-20200501145240-bc7a7d42d5c3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
 golang.org/x/text v0.3.2 h1:tW2bmiBqwgJj/UpqtC8EpXEZVYOwU0yG4iWbprSVAcs=
 golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk=
 golang.org/x/time v0.0.0-20191024005414-555d28b269f0 h1:/5xXl8Y5W96D+TtHSlonuFqGHIWVuyCkGJLwGh9JJFs=
 golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
 golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
+golang.org/x/tools v0.0.0-20191130070609-6e064ea0cf2d h1:/iIZNFGxc/a7C3yWjGcnboV+Tkc7mxr+p6fDztwoxuM=
+golang.org/x/tools v0.0.0-20191130070609-6e064ea0cf2d/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
+golang.org/x/tools v0.0.0-20191216052735-49a3e744a425 h1:VvQyQJN0tSuecqgcIxMWnnfG5kSmgy9KZR9sW3W5QeA=
+golang.org/x/tools v0.0.0-20191216052735-49a3e744a425/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=
+golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
+golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
 golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543 h1:E7g+9GITq07hpfrRu66IVDexMakfv52eLZ2CXBWiKr4=
 golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
 google.golang.org/appengine v1.4.0 h1:/wp5JvzpHIxhs/dumFmF7BXTf3Z+dd4uXta4kVyO508=
 google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4=
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
+gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
 gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15 h1:YR8cESwS4TdDjEe65xsg0ogRM/Nc3DYOhEAlW+xobZo=
 gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
+gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI=
 gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
 gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
 gopkg.in/yaml.v2 v2.2.7 h1:VUgggvou5XRW9mHwD/yXxIYSMtY0zoKQf/v226p2nyo=
 gopkg.in/yaml.v2 v2.2.7/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
-inet.af/netaddr v0.0.0-20200513162223-787f13e36cbe h1:WjJ6wZhXEWQA3FFSwOjG8tO2q1NDFSqrUwNcTvxwMEQ=
-inet.af/netaddr v0.0.0-20200513162223-787f13e36cbe/go.mod h1:qqYzz/2whtrbWJvt+DNWQyvekNN4ePQZcg2xc2/Yjww=
+honnef.co/go/tools v0.0.1-2020.1.4 h1:UoveltGrhghAA7ePc+e+QYDHXrBps2PqFZiHkGR/xK8=
+honnef.co/go/tools v0.0.1-2020.1.4/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k=
+inet.af/netaddr v0.0.0-20200718043157-99321d6ad24c h1:si3Owrfem175Ry6gKqnh59eOXxDojyBTIHxUKuvK/Eo=
+inet.af/netaddr v0.0.0-20200718043157-99321d6ad24c/go.mod h1:qqYzz/2whtrbWJvt+DNWQyvekNN4ePQZcg2xc2/Yjww=
+inet.af/netaddr v0.0.0-20200810144936-56928fe48a98 h1:bWyWDZP0l6VnQ1TDKf6yNwuiEDV6Q3q1Mv34m+lzT1I=
+inet.af/netaddr v0.0.0-20200810144936-56928fe48a98/go.mod h1:qqYzz/2whtrbWJvt+DNWQyvekNN4ePQZcg2xc2/Yjww=
 rsc.io/goversion v1.2.0 h1:SPn+NLTiAG7w30IRK/DKp1BjvpWabYgxlLp/+kx5J8w=
 rsc.io/goversion v1.2.0/go.mod h1:Eih9y/uIBS3ulggl7KNJ09xGSLcuNaLgmvvqa07sgfo=

internal/deepprint/deepprint.go

@@ -0,0 +1,103 @@
+// Copyright (c) 2020 Tailscale Inc & AUTHORS All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+// Package deepprint walks a Go value recursively, in a predictable
+// order, without looping, and prints each value out to a given
+// Writer, which is assumed to be a hash.Hash, as this package doesn't
+// format things nicely.
+//
+// This is intended as a lighter version of go-spew, etc. We don't need its
+// features when our writer is just a hash.
+package deepprint
+
+import (
+	"crypto/sha256"
+	"fmt"
+	"io"
+	"reflect"
+)
+
+func Hash(v ...interface{}) string {
+	h := sha256.New()
+	Print(h, v)
+	return fmt.Sprintf("%x", h.Sum(nil))
+}
+
+// UpdateHash sets last to the hash of v and reports whether its value changed.
+func UpdateHash(last *string, v ...interface{}) (changed bool) {
+	sig := Hash(v)
+	if *last != sig {
+		*last = sig
+		return true
+	}
+	return false
+}
+
+func Print(w io.Writer, v ...interface{}) {
+	print(w, reflect.ValueOf(v), make(map[uintptr]bool))
+}
+
+func print(w io.Writer, v reflect.Value, visited map[uintptr]bool) {
+	if !v.IsValid() {
+		return
+	}
+	switch v.Kind() {
+	default:
+		panic(fmt.Sprintf("unhandled kind %v for type %v", v.Kind(), v.Type()))
+	case reflect.Ptr:
+		ptr := v.Pointer()
+		if visited[ptr] {
+			return
+		}
+		visited[ptr] = true
+		print(w, v.Elem(), visited)
+		return
+	case reflect.Struct:
+		fmt.Fprintf(w, "struct{\n")
+		t := v.Type()
+		for i, n := 0, v.NumField(); i < n; i++ {
+			sf := t.Field(i)
+			fmt.Fprintf(w, "%s: ", sf.Name)
+			print(w, v.Field(i), visited)
+			fmt.Fprintf(w, "\n")
+		}
+	case reflect.Slice, reflect.Array:
+		if v.Type().Elem().Kind() == reflect.Uint8 && v.CanInterface() {
+			fmt.Fprintf(w, "%q", v.Interface())
+			return
+		}
+		fmt.Fprintf(w, "[%d]{\n", v.Len())
+		for i, ln := 0, v.Len(); i < ln; i++ {
+			fmt.Fprintf(w, " [%d]: ", i)
+			print(w, v.Index(i), visited)
+			fmt.Fprintf(w, "\n")
+		}
+		fmt.Fprintf(w, "}\n")
+	case reflect.Interface:
+		print(w, v.Elem(), visited)
+	case reflect.Map:
+		sm := newSortedMap(v)
+		fmt.Fprintf(w, "map[%d]{\n", len(sm.Key))
+		for i, k := range sm.Key {
+			print(w, k, visited)
+			fmt.Fprintf(w, ": ")
+			print(w, sm.Value[i], visited)
+			fmt.Fprintf(w, "\n")
+		}
+		fmt.Fprintf(w, "}\n")
+
+	case reflect.String:
+		fmt.Fprintf(w, "%s", v.String())
+	case reflect.Bool:
+		fmt.Fprintf(w, "%v", v.Bool())
+	case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64:
+		fmt.Fprintf(w, "%v", v.Int())
+	case reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64, reflect.Uintptr:
+		fmt.Fprintf(w, "%v", v.Uint())
+	case reflect.Float32, reflect.Float64:
+		fmt.Fprintf(w, "%v", v.Float())
+	case reflect.Complex64, reflect.Complex128:
+		fmt.Fprintf(w, "%v", v.Complex())
+	}
+}

internal/deepprint/deepprint_test.go

@@ -0,0 +1,71 @@
+// Copyright (c) 2020 Tailscale Inc & AUTHORS All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package deepprint
+
+import (
+	"bytes"
+	"testing"
+
+	"github.com/tailscale/wireguard-go/wgcfg"
+	"inet.af/netaddr"
+	"tailscale.com/wgengine/router"
+	"tailscale.com/wgengine/router/dns"
+)
+
+func TestDeepPrint(t *testing.T) {
+	// v contains the types of values we care about for our current callers.
+	// Mostly we're just testing that we don't panic on handled types.
+	v := getVal()
+
+	var buf bytes.Buffer
+	Print(&buf, v)
+	t.Logf("Got: %s", buf.Bytes())
+
+	hash1 := Hash(v)
+	t.Logf("hash: %v", hash1)
+	for i := 0; i < 20; i++ {
+		hash2 := Hash(getVal())
+		if hash1 != hash2 {
+			t.Error("second hash didn't match")
+		}
+	}
+}
+
+func getVal() []interface{} {
+	return []interface{}{
+		&wgcfg.Config{
+			Name:       "foo",
+			Addresses:  []wgcfg.CIDR{{Mask: 5, IP: wgcfg.IP{Addr: [16]byte{3: 3}}}},
+			ListenPort: 5,
+			Peers: []wgcfg.Peer{
+				{
+					Endpoints: []wgcfg.Endpoint{
+						{
+							Host: "foo",
+							Port: 5,
+						},
+					},
+				},
+			},
+		},
+		&router.Config{
+			DNS: dns.Config{
+				Nameservers: []netaddr.IP{netaddr.IPv4(8, 8, 8, 8)},
+				Domains:     []string{"tailscale.net"},
+			},
+		},
+		map[string]string{
+			"key1": "val1",
+			"key2": "val2",
+			"key3": "val3",
+			"key4": "val4",
+			"key5": "val5",
+			"key6": "val6",
+			"key7": "val7",
+			"key8": "val8",
+			"key9": "val9",
+		},
+	}
+}

internal/deepprint/fmtsort.go

@@ -0,0 +1,224 @@
+// Copyright (c) 2020 Tailscale Inc & AUTHORS All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+// and
+
+// Copyright 2018 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+// This is a slightly modified fork of Go's src/internal/fmtsort/sort.go
+
+package deepprint
+
+import (
+	"reflect"
+	"sort"
+)
+
+// Note: Throughout this package we avoid calling reflect.Value.Interface as
+// it is not always legal to do so and it's easier to avoid the issue than to face it.
+
+// sortedMap represents a map's keys and values. The keys and values are
+// aligned in index order: Value[i] is the value in the map corresponding to Key[i].
+type sortedMap struct {
+	Key   []reflect.Value
+	Value []reflect.Value
+}
+
+func (o *sortedMap) Len() int           { return len(o.Key) }
+func (o *sortedMap) Less(i, j int) bool { return compare(o.Key[i], o.Key[j]) < 0 }
+func (o *sortedMap) Swap(i, j int) {
+	o.Key[i], o.Key[j] = o.Key[j], o.Key[i]
+	o.Value[i], o.Value[j] = o.Value[j], o.Value[i]
+}
+
+// Sort accepts a map and returns a sortedMap that has the same keys and
+// values but in a stable sorted order according to the keys, modulo issues
+// raised by unorderable key values such as NaNs.
+//
+// The ordering rules are more general than with Go's < operator:
+//
+//  - when applicable, nil compares low
+//  - ints, floats, and strings order by <
+//  - NaN compares less than non-NaN floats
+//  - bool compares false before true
+//  - complex compares real, then imag
+//  - pointers compare by machine address
+//  - channel values compare by machine address
+//  - structs compare each field in turn
+//  - arrays compare each element in turn.
+//    Otherwise identical arrays compare by length.
+//  - interface values compare first by reflect.Type describing the concrete type
+//    and then by concrete value as described in the previous rules.
+//
+func newSortedMap(mapValue reflect.Value) *sortedMap {
+	if mapValue.Type().Kind() != reflect.Map {
+		return nil
+	}
+	// Note: this code is arranged to not panic even in the presence
+	// of a concurrent map update. The runtime is responsible for
+	// yelling loudly if that happens. See issue 33275.
+	n := mapValue.Len()
+	key := make([]reflect.Value, 0, n)
+	value := make([]reflect.Value, 0, n)
+	iter := mapValue.MapRange()
+	for iter.Next() {
+		key = append(key, iter.Key())
+		value = append(value, iter.Value())
+	}
+	sorted := &sortedMap{
+		Key:   key,
+		Value: value,
+	}
+	sort.Stable(sorted)
+	return sorted
+}
+
+// compare compares two values of the same type. It returns -1, 0, 1
+// according to whether a > b (1), a == b (0), or a < b (-1).
+// If the types differ, it returns -1.
+// See the comment on Sort for the comparison rules.
+func compare(aVal, bVal reflect.Value) int {
+	aType, bType := aVal.Type(), bVal.Type()
+	if aType != bType {
+		return -1 // No good answer possible, but don't return 0: they're not equal.
+	}
+	switch aVal.Kind() {
+	case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64:
+		a, b := aVal.Int(), bVal.Int()
+		switch {
+		case a < b:
+			return -1
+		case a > b:
+			return 1
+		default:
+			return 0
+		}
+	case reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64, reflect.Uintptr:
+		a, b := aVal.Uint(), bVal.Uint()
+		switch {
+		case a < b:
+			return -1
+		case a > b:
+			return 1
+		default:
+			return 0
+		}
+	case reflect.String:
+		a, b := aVal.String(), bVal.String()
+		switch {
+		case a < b:
+			return -1
+		case a > b:
+			return 1
+		default:
+			return 0
+		}
+	case reflect.Float32, reflect.Float64:
+		return floatCompare(aVal.Float(), bVal.Float())
+	case reflect.Complex64, reflect.Complex128:
+		a, b := aVal.Complex(), bVal.Complex()
+		if c := floatCompare(real(a), real(b)); c != 0 {
+			return c
+		}
+		return floatCompare(imag(a), imag(b))
+	case reflect.Bool:
+		a, b := aVal.Bool(), bVal.Bool()
+		switch {
+		case a == b:
+			return 0
+		case a:
+			return 1
+		default:
+			return -1
+		}
+	case reflect.Ptr:
+		a, b := aVal.Pointer(), bVal.Pointer()
+		switch {
+		case a < b:
+			return -1
+		case a > b:
+			return 1
+		default:
+			return 0
+		}
+	case reflect.Chan:
+		if c, ok := nilCompare(aVal, bVal); ok {
+			return c
+		}
+		ap, bp := aVal.Pointer(), bVal.Pointer()
+		switch {
+		case ap < bp:
+			return -1
+		case ap > bp:
+			return 1
+		default:
+			return 0
+		}
+	case reflect.Struct:
+		for i := 0; i < aVal.NumField(); i++ {
+			if c := compare(aVal.Field(i), bVal.Field(i)); c != 0 {
+				return c
+			}
+		}
+		return 0
+	case reflect.Array:
+		for i := 0; i < aVal.Len(); i++ {
+			if c := compare(aVal.Index(i), bVal.Index(i)); c != 0 {
+				return c
+			}
+		}
+		return 0
+	case reflect.Interface:
+		if c, ok := nilCompare(aVal, bVal); ok {
+			return c
+		}
+		c := compare(reflect.ValueOf(aVal.Elem().Type()), reflect.ValueOf(bVal.Elem().Type()))
+		if c != 0 {
+			return c
+		}
+		return compare(aVal.Elem(), bVal.Elem())
+	default:
+		// Certain types cannot appear as keys (maps, funcs, slices), but be explicit.
+		panic("bad type in compare: " + aType.String())
+	}
+}
+
+// nilCompare checks whether either value is nil. If not, the boolean is false.
+// If either value is nil, the boolean is true and the integer is the comparison
+// value. The comparison is defined to be 0 if both are nil, otherwise the one
+// nil value compares low. Both arguments must represent a chan, func,
+// interface, map, pointer, or slice.
+func nilCompare(aVal, bVal reflect.Value) (int, bool) {
+	if aVal.IsNil() {
+		if bVal.IsNil() {
+			return 0, true
+		}
+		return -1, true
+	}
+	if bVal.IsNil() {
+		return 1, true
+	}
+	return 0, false
+}
+
+// floatCompare compares two floating-point values. NaNs compare low.
+func floatCompare(a, b float64) int {
+	switch {
+	case isNaN(a):
+		return -1 // No good answer if b is a NaN so don't bother checking.
+	case isNaN(b):
+		return 1
+	case a < b:
+		return -1
+	case a > b:
+		return 1
+	}
+	return 0
+}
+
+func isNaN(a float64) bool {
+	return a != a
+}

ipn/backend.go

@@ -8,6 +8,7 @@ import (
 	"net/http"
 	"time"
 
+	"golang.org/x/oauth2"
 	"tailscale.com/control/controlclient"
 	"tailscale.com/ipn/ipnstate"
 	"tailscale.com/tailcfg"
@@ -27,6 +28,10 @@ const (
 	Running
 )
 
+// GoogleIDToken Type is the oauth2.Token.TokenType for the Google
+// ID tokens used by the Android client.
+const GoogleIDTokenType = "ts_android_google_login"
+
 func (s State) String() string {
 	return [...]string{"NoState", "NeedsLogin", "NeedsMachineAuth",
 		"Stopped", "Starting", "Running"}[s]
@@ -58,6 +63,12 @@ type Notify struct {
 	BrowseToURL   *string                   // UI should open a browser right now
 	BackendLogID  *string                   // public logtail id used by backend
 
+	// LocalTCPPort, if non-nil, informs the UI frontend which
+	// (non-zero) localhost TCP port it's listening on.
+	// This is currently only used by Tailscale when run in the
+	// macOS Network Extension.
+	LocalTCPPort *uint16 `json:",omitempty"`
+
 	// type is mirrored in xcode/Shared/IPN.swift
 }
 
@@ -123,6 +134,8 @@ type Backend interface {
 	// flow. This should trigger a new BrowseToURL notification
 	// eventually.
 	StartLoginInteractive()
+	// Login logs in with an OAuth2 token.
+	Login(token *oauth2.Token)
 	// Logout terminates the current login session and stops the
 	// wireguard engine.
 	Logout()

ipn/fake_test.go

@@ -8,6 +8,7 @@ import (
 	"log"
 	"time"
 
+	"golang.org/x/oauth2"
 	"tailscale.com/control/controlclient"
 	"tailscale.com/ipn/ipnstate"
 )
@@ -42,6 +43,14 @@ func (b *FakeBackend) newState(s State) {
 func (b *FakeBackend) StartLoginInteractive() {
 	u := b.serverURL + "/this/is/fake"
 	b.notify(Notify{BrowseToURL: &u})
+	b.login()
+}
+
+func (b *FakeBackend) Login(token *oauth2.Token) {
+	b.login()
+}
+
+func (b *FakeBackend) login() {
 	b.newState(NeedsMachineAuth)
 	b.newState(Stopped)
 	// TODO(apenwarr): Fill in a more interesting netmap here.

ipn/handle.go

@@ -9,6 +9,7 @@ import (
 	"time"
 
 	"github.com/tailscale/wireguard-go/wgcfg"
+	"golang.org/x/oauth2"
 	"tailscale.com/control/controlclient"
 	"tailscale.com/types/logger"
 )
@@ -154,6 +155,10 @@ func (h *Handle) StartLoginInteractive() {
 	h.b.StartLoginInteractive()
 }
 
+func (h *Handle) Login(token *oauth2.Token) {
+	h.b.Login(token)
+}
+
 func (h *Handle) Logout() {
 	h.b.Logout()
 }

ipn/ipnserver/server.go

@@ -18,11 +18,11 @@ import (
 	"syscall"
 	"time"
 
-	"github.com/klauspost/compress/zstd"
 	"tailscale.com/control/controlclient"
 	"tailscale.com/ipn"
 	"tailscale.com/logtail/backoff"
 	"tailscale.com/safesocket"
+	"tailscale.com/smallzstd"
 	"tailscale.com/types/logger"
 	"tailscale.com/version"
 	"tailscale.com/wgengine"
@@ -33,15 +33,19 @@ type Options struct {
 	// SocketPath, on unix systems, is the unix socket path to listen
 	// on for frontend connections.
 	SocketPath string
+
 	// Port, on windows, is the localhost TCP port to listen on for
 	// frontend connections.
 	Port int
+
 	// StatePath is the path to the stored agent state.
 	StatePath string
+
 	// AutostartStateKey, if non-empty, immediately starts the agent
 	// using the given StateKey. If empty, the agent stays idle and
 	// waits for a frontend to start it.
 	AutostartStateKey ipn.StateKey
+
 	// LegacyConfigPath optionally specifies the old-style relaynode
 	// relay.conf location. If both LegacyConfigPath and
 	// AutostartStateKey are specified and the requested state doesn't
@@ -51,10 +55,15 @@ type Options struct {
 	// TODO(danderson): remove some time after the transition to
 	// tailscaled is done.
 	LegacyConfigPath string
+
 	// SurviveDisconnects specifies how the server reacts to its
 	// frontend disconnecting. If true, the server keeps running on
 	// its existing state, and accepts new frontend connections. If
 	// false, the server dumps its state and becomes idle.
+	//
+	// To support CLI connections (notably, "tailscale status"),
+	// the actual definition of "disconnect" is when the
+	// connection count transitions from 1 to 0.
 	SurviveDisconnects bool
 
 	// DebugMux, if non-nil, specifies an HTTP ServeMux in which
@@ -62,42 +71,146 @@ type Options struct {
 	DebugMux *http.ServeMux
 }
 
-func pump(logf logger.Logf, ctx context.Context, bs *ipn.BackendServer, s net.Conn) {
-	defer logf("Control connection done.")
+// server is an IPN backend and its set of 0 or more active connections
+// talking to an IPN backend.
+type server struct {
+	resetOnZero bool // call bs.Reset on transition from 1->0 connections
 
-	for ctx.Err() == nil && !bs.GotQuit {
-		msg, err := ipn.ReadMsg(s)
+	bsMu sync.Mutex // lock order: bsMu, then mu
+	bs   *ipn.BackendServer
+
+	mu      sync.Mutex
+	clients map[net.Conn]bool
+}
+
+func (s *server) serveConn(ctx context.Context, c net.Conn, logf logger.Logf) {
+	s.addConn(c)
+	logf("incoming control connection")
+	defer s.removeAndCloseConn(c)
+	for ctx.Err() == nil {
+		msg, err := ipn.ReadMsg(c)
 		if err != nil {
-			logf("ReadMsg: %v", err)
-			break
+			if ctx.Err() == nil {
+				logf("ReadMsg: %v", err)
+			}
+			return
 		}
-		err = bs.GotCommandMsg(msg)
-		if err != nil {
+		s.bsMu.Lock()
+		if err := s.bs.GotCommandMsg(msg); err != nil {
 			logf("GotCommandMsg: %v", err)
-			break
+		}
+		gotQuit := s.bs.GotQuit
+		s.bsMu.Unlock()
+		if gotQuit {
+			return
 		}
 	}
 }
 
-func Run(rctx context.Context, logf logger.Logf, logid string, opts Options, e wgengine.Engine) (err error) {
-	runDone := make(chan error, 1)
-	defer func() { runDone <- err }()
+func (s *server) addConn(c net.Conn) {
+	s.mu.Lock()
+	defer s.mu.Unlock()
+	if s.clients == nil {
+		s.clients = map[net.Conn]bool{}
+	}
+	s.clients[c] = true
+}
+
+func (s *server) removeAndCloseConn(c net.Conn) {
+	s.mu.Lock()
+	delete(s.clients, c)
+	remain := len(s.clients)
+	s.mu.Unlock()
+
+	if remain == 0 && s.resetOnZero {
+		s.bsMu.Lock()
+		s.bs.Reset()
+		s.bsMu.Unlock()
+	}
+	c.Close()
+}
+
+func (s *server) stopAll() {
+	s.mu.Lock()
+	defer s.mu.Unlock()
+	for c := range s.clients {
+		safesocket.ConnCloseRead(c)
+		safesocket.ConnCloseWrite(c)
+	}
+	s.clients = nil
+}
+
+func (s *server) writeToClients(b []byte) {
+	s.mu.Lock()
+	defer s.mu.Unlock()
+	for c := range s.clients {
+		ipn.WriteMsg(c, b)
+	}
+}
+
+// Run runs a Tailscale backend service.
+// The getEngine func is called repeatedly, once per connection, until it returns an engine successfully.
+func Run(ctx context.Context, logf logger.Logf, logid string, getEngine func() (wgengine.Engine, error), opts Options) error {
+	runDone := make(chan struct{})
+	defer close(runDone)
 
 	listen, _, err := safesocket.Listen(opts.SocketPath, uint16(opts.Port))
 	if err != nil {
 		return fmt.Errorf("safesocket.Listen: %v", err)
 	}
 
-	// Go listeners can't take a context, close it instead.
+	server := &server{
+		resetOnZero: !opts.SurviveDisconnects,
+	}
+
+	// When the context is closed or when we return, whichever is first, close our listner
+	// and all open connections.
 	go func() {
 		select {
-		case <-rctx.Done():
+		case <-ctx.Done():
 		case <-runDone:
 		}
+		server.stopAll()
 		listen.Close()
 	}()
 	logf("Listening on %v", listen.Addr())
 
+	bo := backoff.NewBackoff("ipnserver", logf, 30*time.Second)
+
+	var unservedConn net.Conn // if non-nil, accepted, but hasn't served yet
+
+	eng, err := getEngine()
+	if err != nil {
+		logf("Initial getEngine call: %v", err)
+		for i := 1; ctx.Err() == nil; i++ {
+			c, err := listen.Accept()
+			if err != nil {
+				logf("%d: Accept: %v", i, err)
+				bo.BackOff(ctx, err)
+				continue
+			}
+			logf("%d: trying getEngine again...", i)
+			eng, err = getEngine()
+			if err == nil {
+				logf("%d: GetEngine worked; exiting failure loop", i)
+				unservedConn = c
+				break
+			}
+			logf("%d: getEngine failed again: %v", i, err)
+			errMsg := err.Error()
+			go func() {
+				defer c.Close()
+				serverToClient := func(b []byte) { ipn.WriteMsg(c, b) }
+				bs := ipn.NewBackendServer(logf, nil, serverToClient)
+				bs.SendErrorMessage(errMsg)
+				time.Sleep(time.Second)
+			}()
+		}
+		if err := ctx.Err(); err != nil {
+			return err
+		}
+	}
+
 	var store ipn.StateStore
 	if opts.StatePath != "" {
 		store, err = ipn.NewFileStore(opts.StatePath)
@@ -108,15 +221,13 @@ func Run(rctx context.Context, logf logger.Logf, logid string, opts Options, e w
 		store = &ipn.MemoryStore{}
 	}
 
-	b, err := ipn.NewLocalBackend(logf, logid, store, e)
+	b, err := ipn.NewLocalBackend(logf, logid, store, eng)
 	if err != nil {
 		return fmt.Errorf("NewLocalBackend: %v", err)
 	}
+	defer b.Shutdown()
 	b.SetDecompressor(func() (controlclient.Decompressor, error) {
-		return zstd.NewReader(nil,
-			zstd.WithDecoderLowmem(true),
-			zstd.WithDecoderConcurrency(1),
-		)
+		return smallzstd.NewDecoder(nil)
 	})
 
 	if opts.DebugMux != nil {
@@ -128,17 +239,10 @@ func Run(rctx context.Context, logf logger.Logf, logid string, opts Options, e w
 		})
 	}
 
-	var s net.Conn
-	serverToClient := func(b []byte) {
-		if s != nil { // TODO: racy access to s?
-			ipn.WriteMsg(s, b)
-		}
-	}
-
-	bs := ipn.NewBackendServer(logf, b, serverToClient)
+	server.bs = ipn.NewBackendServer(logf, b, server.writeToClients)
 
 	if opts.AutostartStateKey != "" {
-		bs.GotCommand(&ipn.Command{
+		server.bs.GotCommand(&ipn.Command{
 			Version: version.LONG,
 			Start: &ipn.StartArgs{
 				Opts: ipn.Options{
@@ -149,55 +253,25 @@ func Run(rctx context.Context, logf logger.Logf, logid string, opts Options, e w
 		})
 	}
 
-	var (
-		oldS   net.Conn
-		ctx    context.Context
-		cancel context.CancelFunc
-	)
-	stopAll := func() {
-		// Currently we only support one client connection at a time.
-		// Theoretically we could allow multiple clients, by passing
-		// notifications to all of them and accepting commands from
-		// any of them, but there doesn't seem to be much need for
-		// that right now.
-		if oldS != nil {
-			cancel()
-			safesocket.ConnCloseRead(oldS)
-			safesocket.ConnCloseWrite(oldS)
+	for i := 1; ctx.Err() == nil; i++ {
+		var c net.Conn
+		var err error
+		if unservedConn != nil {
+			c = unservedConn
+			unservedConn = nil
+		} else {
+			c, err = listen.Accept()
 		}
-	}
-
-	bo := backoff.NewBackoff("ipnserver", logf)
-
-	for i := 1; rctx.Err() == nil; i++ {
-		s, err = listen.Accept()
 		if err != nil {
-			logf("%d: Accept: %v", i, err)
-			bo.BackOff(rctx, err)
+			if ctx.Err() == nil {
+				logf("ipnserver: Accept: %v", err)
+				bo.BackOff(ctx, err)
+			}
 			continue
 		}
-		logf("%d: Incoming control connection.", i)
-		stopAll()
-
-		ctx, cancel = context.WithCancel(rctx)
-		oldS = s
-
-		go func(ctx context.Context, s net.Conn, i int) {
-			logf := logger.WithPrefix(logf, fmt.Sprintf("%d: ", i))
-			pump(logf, ctx, bs, s)
-			if !opts.SurviveDisconnects || bs.GotQuit {
-				bs.Reset()
-				s.Close()
-			}
-			// Quitting not allowed, just keep going.
-			bs.GotQuit = false
-		}(ctx, s, i)
-
-		bo.BackOff(ctx, nil)
+		go server.serveConn(ctx, c, logger.WithPrefix(logf, fmt.Sprintf("ipnserver: conn%d: ", i)))
 	}
-	stopAll()
-
-	return rctx.Err()
+	return ctx.Err()
 }
 
 func BabysitProc(ctx context.Context, args []string, logf logger.Logf) {
@@ -232,7 +306,7 @@ func BabysitProc(ctx context.Context, args []string, logf logger.Logf) {
 		proc.mu.Unlock()
 	}()
 
-	bo := backoff.NewBackoff("BabysitProc", logf)
+	bo := backoff.NewBackoff("BabysitProc", logf, 30*time.Second)
 
 	for {
 		startTime := time.Now()
@@ -315,3 +389,8 @@ func BabysitProc(ctx context.Context, args []string, logf logger.Logf) {
 		}
 	}
 }
+
+// FixedEngine returns a func that returns eng and a nil error.
+func FixedEngine(eng wgengine.Engine) func() (wgengine.Engine, error) {
+	return func() (wgengine.Engine, error) { return eng, nil }
+}

ipn/ipnserver/server_test.go

@@ -72,6 +72,6 @@ func TestRunMultipleAccepts(t *testing.T) {
 		SocketPath: socketPath,
 	}
 	t.Logf("pre-Run")
-	err = ipnserver.Run(ctx, logTriggerTestf, "dummy_logid", opts, eng)
+	err = ipnserver.Run(ctx, logTriggerTestf, "dummy_logid", ipnserver.FixedEngine(eng), opts)
 	t.Logf("ipnserver.Run = %v", err)
 }

ipn/ipnstate/ipnstate.go

@@ -18,6 +18,7 @@ import (
 	"sync"
 	"time"
 
+	"inet.af/netaddr"
 	"tailscale.com/tailcfg"
 	"tailscale.com/types/key"
 )
@@ -25,6 +26,7 @@ import (
 // Status represents the entire state of the IPN network.
 type Status struct {
 	BackendState string
+	TailscaleIPs []netaddr.IP // Tailscale IP(s) assigned to this node
 	Peer         map[key.Public]*PeerStatus
 	User         map[tailcfg.UserID]tailcfg.UserProfile
 }
@@ -49,10 +51,12 @@ type PeerStatus struct {
 	// Endpoints:
 	Addrs   []string
 	CurAddr string // one of Addrs, or unique if roaming
+	Relay   string // DERP region
 
 	RxBytes       int64
 	TxBytes       int64
 	Created       time.Time // time registered with tailcontrol
+	LastWrite     time.Time // time last packet sent
 	LastSeen      time.Time // last seen to tailcontrol
 	LastHandshake time.Time // with local wireguard
 	KeepAlive     bool
@@ -107,6 +111,18 @@ func (sb *StatusBuilder) AddUser(id tailcfg.UserID, up tailcfg.UserProfile) {
 	sb.st.User[id] = up
 }
 
+// AddIP adds a Tailscale IP address to the status.
+func (sb *StatusBuilder) AddTailscaleIP(ip netaddr.IP) {
+	sb.mu.Lock()
+	defer sb.mu.Unlock()
+	if sb.locked {
+		log.Printf("[unexpected] ipnstate: AddIP after Locked")
+		return
+	}
+
+	sb.st.TailscaleIPs = append(sb.st.TailscaleIPs, ip)
+}
+
 // AddPeer adds a peer node to the status.
 //
 // Its PeerStatus is mixed with any previous status already added.
@@ -135,6 +151,9 @@ func (sb *StatusBuilder) AddPeer(peer key.Public, st *PeerStatus) {
 	if v := st.HostName; v != "" {
 		e.HostName = v
 	}
+	if v := st.Relay; v != "" {
+		e.Relay = v
+	}
 	if v := st.UserID; v != 0 {
 		e.UserID = v
 	}
@@ -165,6 +184,9 @@ func (sb *StatusBuilder) AddPeer(peer key.Public, st *PeerStatus) {
 	if v := st.LastSeen; !v.IsZero() {
 		e.LastSeen = v
 	}
+	if v := st.LastWrite; !v.IsZero() {
+		e.LastWrite = v
+	}
 	if st.InNetworkMap {
 		e.InNetworkMap = true
 	}
@@ -210,29 +232,26 @@ table tbody tr:nth-child(even) td { background-color: #f5f5f5; }
 	//f("<p><b>logid:</b> %s</p>\n", logid)
 	//f("<p><b>opts:</b> <code>%s</code></p>\n", html.EscapeString(fmt.Sprintf("%+v", opts)))
 
+	ips := make([]string, 0, len(st.TailscaleIPs))
+	for _, ip := range st.TailscaleIPs {
+		ips = append(ips, ip.String())
+	}
+	f("<p>Tailscale IP: %s", strings.Join(ips, ", "))
+
 	f("<table>\n<thead>\n")
-	f("<tr><th>Peer</th><th>Node</th><th>Owner</th><th>Rx</th><th>Tx</th><th>Handshake</th><th>Endpoints</th></tr>\n")
+	f("<tr><th>Peer</th><th>Node</th><th>Owner</th><th>Rx</th><th>Tx</th><th>Activity</th><th>Endpoints</th></tr>\n")
 	f("</thead>\n<tbody>\n")
 
 	now := time.Now()
 
-	// The tailcontrol server rounds LastSeen to 10 minutes. So we
-	// declare that a longAgo seen time of 15 minutes means
-	// they're not connected.
-	longAgo := now.Add(-15 * time.Minute)
-
 	for _, peer := range st.Peers() {
 		ps := st.Peer[peer]
-		var hsAgo string
-		if !ps.LastHandshake.IsZero() {
-			hsAgo = now.Sub(ps.LastHandshake).Round(time.Second).String() + " ago"
-		} else {
-			if ps.LastSeen.Before(longAgo) {
-				hsAgo = "<i>offline</i>"
-			} else if !ps.KeepAlive {
-				hsAgo = "on demand"
-			} else {
-				hsAgo = "<b>pending</b>"
+		var actAgo string
+		if !ps.LastWrite.IsZero() {
+			ago := now.Sub(ps.LastWrite)
+			actAgo = ago.Round(time.Second).String() + " ago"
+			if ago < 5*time.Minute {
+				actAgo = "<b>" + actAgo + "</b>"
 			}
 		}
 		var owner string
@@ -250,9 +269,20 @@ table tbody tr:nth-child(even) td { background-color: #f5f5f5; }
 			html.EscapeString(owner),
 			ps.RxBytes,
 			ps.TxBytes,
-			hsAgo,
+			actAgo,
 		)
 		f("<td class=\"aright\">")
+		// TODO: let server report this active bool instead
+		active := !ps.LastWrite.IsZero() && time.Since(ps.LastWrite) < 2*time.Minute
+		relay := ps.Relay
+		if relay != "" {
+			if active && ps.CurAddr == "" {
+				f("🔗 <b>derp-%v</b><br>", html.EscapeString(relay))
+			} else {
+				f("derp-%v<br>", html.EscapeString(relay))
+			}
+		}
+
 		match := false
 		for _, addr := range ps.Addrs {
 			if addr == ps.CurAddr {

ipn/local.go

@@ -13,10 +13,13 @@ import (
 	"time"
 
 	"github.com/tailscale/wireguard-go/wgcfg"
+	"golang.org/x/oauth2"
 	"inet.af/netaddr"
 	"tailscale.com/control/controlclient"
+	"tailscale.com/internal/deepprint"
 	"tailscale.com/ipn/ipnstate"
 	"tailscale.com/ipn/policy"
+	"tailscale.com/net/tsaddr"
 	"tailscale.com/portlist"
 	"tailscale.com/tailcfg"
 	"tailscale.com/types/empty"
@@ -26,6 +29,8 @@ import (
 	"tailscale.com/wgengine"
 	"tailscale.com/wgengine/filter"
 	"tailscale.com/wgengine/router"
+	"tailscale.com/wgengine/router/dns"
+	"tailscale.com/wgengine/tsdns"
 )
 
 // LocalBackend is the glue between the major pieces of the Tailscale
@@ -48,22 +53,23 @@ type LocalBackend struct {
 	store           StateStore
 	backendLogID    string
 	portpoll        *portlist.Poller // may be nil
+	portpollOnce    sync.Once
+	serverURL       string // tailcontrol URL
 	newDecompressor func() (controlclient.Decompressor, error)
 
-	// TODO: these fields are accessed unsafely by concurrent
-	// goroutines. They need to be protected.
-	serverURL       string // tailcontrol URL
-	lastFilterPrint time.Time
+	filterHash string
 
 	// The mutex protects the following elements.
-	mu           sync.Mutex
-	notify       func(Notify)
-	c            *controlclient.Client
-	stateKey     StateKey
-	prefs        *Prefs
-	state        State
-	hiCache      *tailcfg.Hostinfo
-	netMapCache  *controlclient.NetworkMap
+	mu       sync.Mutex
+	notify   func(Notify)
+	c        *controlclient.Client
+	stateKey StateKey
+	prefs    *Prefs
+	state    State
+	// hostinfo is mutated in-place while mu is held.
+	hostinfo *tailcfg.Hostinfo
+	// netMap is not mutated in-place once set.
+	netMap       *controlclient.NetworkMap
 	engineStatus EngineStatus
 	endpoints    []string
 	blocked      bool
@@ -105,11 +111,6 @@ func NewLocalBackend(logf logger.Logf, logid string, store StateStore, e wgengin
 	}
 	b.statusChanged = sync.NewCond(&b.statusLock)
 
-	if b.portpoll != nil {
-		go b.portpoll.Run(ctx)
-		go b.readPoller()
-	}
-
 	return b, nil
 }
 
@@ -145,11 +146,11 @@ func (b *LocalBackend) UpdateStatus(sb *ipnstate.StatusBuilder) {
 
 	// TODO: hostinfo, and its networkinfo
 	// TODO: EngineStatus copy (and deprecate it?)
-	if b.netMapCache != nil {
-		for id, up := range b.netMapCache.UserProfiles {
+	if b.netMap != nil {
+		for id, up := range b.netMap.UserProfiles {
 			sb.AddUser(id, up)
 		}
-		for _, p := range b.netMapCache.Peers {
+		for _, p := range b.netMap.Peers {
 			var lastSeen time.Time
 			if p.LastSeen != nil {
 				lastSeen = *p.LastSeen
@@ -183,6 +184,136 @@ func (b *LocalBackend) SetDecompressor(fn func() (controlclient.Decompressor, er
 	b.newDecompressor = fn
 }
 
+// setClientStatus is the callback invoked by the control client whenever it posts a new status.
+// Among other things, this is where we update the netmap, packet filters, DNS and DERP maps.
+func (b *LocalBackend) setClientStatus(st controlclient.Status) {
+	// The following do not depend on any data for which we need to lock b.
+	if st.Err != "" {
+		// TODO(crawshaw): display in the UI.
+		b.logf("Received error: %v", st.Err)
+		return
+	}
+	if st.LoginFinished != nil {
+		// Auth completed, unblock the engine
+		b.blockEngineUpdates(false)
+		b.authReconfig()
+		b.send(Notify{LoginFinished: &empty.Message{}})
+	}
+
+	prefsChanged := false
+
+	// Lock b once and do only the things that require locking.
+	b.mu.Lock()
+
+	prefs := b.prefs
+	stateKey := b.stateKey
+	netMap := b.netMap
+	interact := b.interact
+
+	if st.Persist != nil {
+		if !b.prefs.Persist.Equals(st.Persist) {
+			prefsChanged = true
+			b.prefs.Persist = st.Persist.Clone()
+		}
+	}
+	if st.NetMap != nil {
+		b.netMap = st.NetMap
+	}
+	if st.URL != "" {
+		b.authURL = st.URL
+	}
+	if b.state == NeedsLogin {
+		if !b.prefs.WantRunning {
+			prefsChanged = true
+		}
+		b.prefs.WantRunning = true
+	}
+	// Prefs will be written out; this is not safe unless locked or cloned.
+	if prefsChanged {
+		prefs = b.prefs.Clone()
+	}
+
+	b.mu.Unlock()
+
+	// Now complete the lock-free parts of what we started while locked.
+	if prefsChanged {
+		if stateKey != "" {
+			if err := b.store.WriteState(stateKey, prefs.ToBytes()); err != nil {
+				b.logf("Failed to save new controlclient state: %v", err)
+			}
+		}
+		b.send(Notify{Prefs: prefs})
+	}
+	if st.NetMap != nil {
+		if netMap != nil {
+			diff := st.NetMap.ConciseDiffFrom(netMap)
+			if strings.TrimSpace(diff) == "" {
+				b.logf("netmap diff: (none)")
+			} else {
+				b.logf("netmap diff:\n%v", diff)
+			}
+		}
+
+		b.updateFilter(st.NetMap, prefs)
+		b.e.SetNetworkMap(st.NetMap)
+
+		if !dnsMapsEqual(st.NetMap, netMap) {
+			b.updateDNSMap(st.NetMap)
+		}
+
+		disableDERP := prefs != nil && prefs.DisableDERP
+		if disableDERP {
+			b.e.SetDERPMap(nil)
+		} else {
+			b.e.SetDERPMap(st.NetMap.DERPMap)
+		}
+
+		b.send(Notify{NetMap: st.NetMap})
+	}
+	if st.URL != "" {
+		b.logf("Received auth URL: %.20v...", st.URL)
+		if interact > 0 {
+			b.popBrowserAuthNow()
+		}
+	}
+	b.stateMachine()
+	// This is currently (2020-07-28) necessary; conditionally disabling it is fragile!
+	// This is where netmap information gets propagated to router and magicsock.
+	b.authReconfig()
+}
+
+// setWgengineStatus is the callback by the wireguard engine whenever it posts a new status.
+// This updates the endpoints both in the backend and in the control client.
+func (b *LocalBackend) setWgengineStatus(s *wgengine.Status, err error) {
+	if err != nil {
+		b.logf("wgengine status error: %#v", err)
+		return
+	}
+	if s == nil {
+		b.logf("[unexpected] non-error wgengine update with status=nil: %v", s)
+		return
+	}
+
+	es := b.parseWgStatus(s)
+
+	b.mu.Lock()
+	c := b.c
+	b.engineStatus = es
+	b.endpoints = append([]string{}, s.LocalAddrs...)
+	b.mu.Unlock()
+
+	if c != nil {
+		c.UpdateEndpoints(0, s.LocalAddrs)
+	}
+	b.stateMachine()
+
+	b.statusLock.Lock()
+	b.statusChanged.Broadcast()
+	b.statusLock.Unlock()
+
+	b.send(Notify{Engine: &es})
+}
+
 // Start applies the configuration specified in opts, and starts the
 // state machine.
 //
@@ -204,9 +335,9 @@ func (b *LocalBackend) Start(opts Options) error {
 		b.logf("Start")
 	}
 
-	hi := controlclient.NewHostinfo()
-	hi.BackendLogID = b.backendLogID
-	hi.FrontendLogID = opts.FrontendLogID
+	hostinfo := controlclient.NewHostinfo()
+	hostinfo.BackendLogID = b.backendLogID
+	hostinfo.FrontendLogID = opts.FrontendLogID
 
 	b.mu.Lock()
 
@@ -221,11 +352,11 @@ func (b *LocalBackend) Start(opts Options) error {
 		b.c.Shutdown()
 	}
 
-	if b.hiCache != nil {
-		hi.Services = b.hiCache.Services // keep any previous session and netinfo
-		hi.NetInfo = b.hiCache.NetInfo
+	if b.hostinfo != nil {
+		hostinfo.Services = b.hostinfo.Services // keep any previous session and netinfo
+		hostinfo.NetInfo = b.hostinfo.NetInfo
 	}
-	b.hiCache = hi
+	b.hostinfo = hostinfo
 	b.state = NoState
 
 	if err := b.loadStateLocked(opts.StateKey, opts.Prefs, opts.LegacyConfigPath); err != nil {
@@ -234,15 +365,21 @@ func (b *LocalBackend) Start(opts Options) error {
 	}
 
 	b.serverURL = b.prefs.ControlURL
-	hi.RoutableIPs = append(hi.RoutableIPs, b.prefs.AdvertiseRoutes...)
-	hi.RequestTags = append(hi.RequestTags, b.prefs.AdvertiseTags...)
+	hostinfo.RoutableIPs = append(hostinfo.RoutableIPs, b.prefs.AdvertiseRoutes...)
+	hostinfo.RequestTags = append(hostinfo.RequestTags, b.prefs.AdvertiseTags...)
+	applyPrefsToHostinfo(hostinfo, b.prefs)
 
 	b.notify = opts.Notify
-	b.netMapCache = nil
+	b.netMap = nil
 	persist := b.prefs.Persist
 	b.mu.Unlock()
 
-	b.updateFilter(nil)
+	b.updateFilter(nil, nil)
+
+	var discoPublic tailcfg.DiscoKey
+	if controlclient.Debug.Disco {
+		discoPublic = b.e.DiscoPublicKey()
+	}
 
 	var err error
 	if persist == nil {
@@ -254,15 +391,25 @@ func (b *LocalBackend) Start(opts Options) error {
 		Persist:         *persist,
 		ServerURL:       b.serverURL,
 		AuthKey:         opts.AuthKey,
-		Hostinfo:        hi,
+		Hostinfo:        hostinfo,
 		KeepAlive:       true,
 		NewDecompressor: b.newDecompressor,
 		HTTPTestClient:  opts.HTTPTestClient,
+		DiscoPublicKey:  discoPublic,
 	})
 	if err != nil {
 		return err
 	}
 
+	// At this point, we have finished using hostinfo without synchronization,
+	// so it is safe to start readPoller which concurrently writes to it.
+	if b.portpoll != nil {
+		b.portpollOnce.Do(func() {
+			go b.portpoll.Run(b.ctx)
+			go b.readPoller()
+		})
+	}
+
 	b.mu.Lock()
 	b.c = cli
 	endpoints := b.endpoints
@@ -272,111 +419,8 @@ func (b *LocalBackend) Start(opts Options) error {
 		cli.UpdateEndpoints(0, endpoints)
 	}
 
-	cli.SetStatusFunc(func(newSt controlclient.Status) {
-		if newSt.LoginFinished != nil {
-			// Auth completed, unblock the engine
-			b.blockEngineUpdates(false)
-			b.authReconfig()
-			b.send(Notify{LoginFinished: &empty.Message{}})
-		}
-		if newSt.Persist != nil {
-			persist := *newSt.Persist // copy
-
-			b.mu.Lock()
-			b.prefs.Persist = &persist
-			prefs := b.prefs.Clone()
-			stateKey := b.stateKey
-			b.mu.Unlock()
-
-			if stateKey != "" {
-				if err := b.store.WriteState(stateKey, prefs.ToBytes()); err != nil {
-					b.logf("Failed to save new controlclient state: %v", err)
-				}
-			}
-			b.send(Notify{Prefs: prefs})
-		}
-		if newSt.NetMap != nil {
-			b.mu.Lock()
-			if b.netMapCache != nil {
-				diff := newSt.NetMap.ConciseDiffFrom(b.netMapCache)
-				if strings.TrimSpace(diff) == "" {
-					b.logf("netmap diff: (none)")
-				} else {
-					b.logf("netmap diff:\n%v", diff)
-				}
-			}
-			disableDERP := b.prefs != nil && b.prefs.DisableDERP
-			b.netMapCache = newSt.NetMap
-			b.mu.Unlock()
-
-			b.send(Notify{NetMap: newSt.NetMap})
-			b.updateFilter(newSt.NetMap)
-			if disableDERP {
-				b.e.SetDERPMap(nil)
-			} else {
-				b.e.SetDERPMap(newSt.NetMap.DERPMap)
-			}
-		}
-		if newSt.URL != "" {
-			b.logf("Received auth URL: %.20v...", newSt.URL)
-
-			b.mu.Lock()
-			interact := b.interact
-			b.authURL = newSt.URL
-			b.mu.Unlock()
-
-			if interact > 0 {
-				b.popBrowserAuthNow()
-			}
-		}
-		if newSt.Err != "" {
-			// TODO(crawshaw): display in the UI.
-			b.logf("Received error: %v", newSt.Err)
-			return
-		}
-		if newSt.NetMap != nil {
-			b.mu.Lock()
-			if b.state == NeedsLogin {
-				b.prefs.WantRunning = true
-			}
-			prefs := b.prefs
-			b.mu.Unlock()
-
-			b.SetPrefs(prefs)
-		}
-		b.stateMachine()
-	})
-
-	b.e.SetStatusCallback(func(s *wgengine.Status, err error) {
-		if err != nil {
-			b.logf("wgengine status error: %#v", err)
-			return
-		}
-		if s == nil {
-			b.logf("weird: non-error wgengine update with status=nil: %v", s)
-			return
-		}
-
-		es := b.parseWgStatus(s)
-
-		b.mu.Lock()
-		c := b.c
-		b.engineStatus = es
-		b.endpoints = append([]string{}, s.LocalAddrs...)
-		b.mu.Unlock()
-
-		if c != nil {
-			c.UpdateEndpoints(0, s.LocalAddrs)
-		}
-		b.stateMachine()
-
-		b.statusLock.Lock()
-		b.statusChanged.Broadcast()
-		b.statusLock.Unlock()
-
-		b.send(Notify{Engine: &es})
-	})
-
+	cli.SetStatusFunc(b.setClientStatus)
+	b.e.SetStatusCallback(b.setWgengineStatus)
 	b.e.SetNetInfoCallback(b.setNetInfo)
 
 	b.mu.Lock()
@@ -394,37 +438,121 @@ func (b *LocalBackend) Start(opts Options) error {
 
 // updateFilter updates the packet filter in wgengine based on the
 // given netMap and user preferences.
-func (b *LocalBackend) updateFilter(netMap *controlclient.NetworkMap) {
-	if netMap == nil {
-		// Not configured yet, block everything
+func (b *LocalBackend) updateFilter(netMap *controlclient.NetworkMap, prefs *Prefs) {
+	// NOTE(danderson): keep change detection as the first thing in
+	// this function. Don't try to optimize by returning early, more
+	// likely than not you'll just end up breaking the change
+	// detection and end up with the wrong filter installed. This is
+	// quite hard to debug, so save yourself the trouble.
+	var (
+		haveNetmap   = netMap != nil
+		addrs        []wgcfg.CIDR
+		packetFilter filter.Matches
+		advRoutes    []wgcfg.CIDR
+		shieldsUp    = prefs == nil || prefs.ShieldsUp // Be conservative when not ready
+	)
+	if haveNetmap {
+		addrs = netMap.Addresses
+		packetFilter = netMap.PacketFilter
+	}
+	if prefs != nil {
+		advRoutes = prefs.AdvertiseRoutes
+	}
+
+	changed := deepprint.UpdateHash(&b.filterHash, haveNetmap, addrs, packetFilter, advRoutes, shieldsUp)
+	if !changed {
+		return
+	}
+
+	if !haveNetmap {
 		b.logf("netmap packet filter: (not ready yet)")
 		b.e.SetFilter(filter.NewAllowNone(b.logf))
 		return
 	}
 
-	b.mu.Lock()
-	advRoutes := b.prefs.AdvertiseRoutes
-	b.mu.Unlock()
 	localNets := wgCIDRsToFilter(netMap.Addresses, advRoutes)
 
-	if b.shieldsAreUp() {
-		// Shields up, block everything
+	if shieldsUp {
 		b.logf("netmap packet filter: (shields up)")
 		var prevFilter *filter.Filter // don't reuse old filter state
 		b.e.SetFilter(filter.New(filter.Matches{}, localNets, prevFilter, b.logf))
+	} else {
+		b.logf("netmap packet filter: %v", packetFilter)
+		b.e.SetFilter(filter.New(packetFilter, localNets, b.e.GetFilter(), b.logf))
+	}
+}
+
+// dnsCIDRsEqual determines whether two CIDR lists are equal
+// for DNS map construction purposes (that is, only the first entry counts).
+func dnsCIDRsEqual(newAddr, oldAddr []wgcfg.CIDR) bool {
+	if len(newAddr) != len(oldAddr) {
+		return false
+	}
+	if len(newAddr) == 0 || newAddr[0] == oldAddr[0] {
+		return true
+	}
+	return false
+}
+
+// dnsMapsEqual determines whether the new and the old network map
+// induce the same DNS map. It does so without allocating memory,
+// at the expense of giving false negatives if peers are reordered.
+func dnsMapsEqual(new, old *controlclient.NetworkMap) bool {
+	if (old == nil) != (new == nil) {
+		return false
+	}
+	if old == nil && new == nil {
+		return true
+	}
+
+	if len(new.Peers) != len(old.Peers) {
+		return false
+	}
+
+	if new.Name != old.Name {
+		return false
+	}
+	if !dnsCIDRsEqual(new.Addresses, old.Addresses) {
+		return false
+	}
+
+	for i, newPeer := range new.Peers {
+		oldPeer := old.Peers[i]
+		if newPeer.Name != oldPeer.Name {
+			return false
+		}
+		if !dnsCIDRsEqual(newPeer.Addresses, oldPeer.Addresses) {
+			return false
+		}
+	}
+
+	return true
+}
+
+// updateDNSMap updates the domain map in the DNS resolver in wgengine
+// based on the given netMap and user preferences.
+func (b *LocalBackend) updateDNSMap(netMap *controlclient.NetworkMap) {
+	if netMap == nil {
+		b.logf("dns map: (not ready)")
 		return
 	}
 
-	// TODO(apenwarr): don't replace filter at all if unchanged.
-	// TODO(apenwarr): print a diff instead of full filter.
-	now := time.Now()
-	if now.Sub(b.lastFilterPrint) > 1*time.Minute {
-		b.logf("netmap packet filter: %v", netMap.PacketFilter)
-		b.lastFilterPrint = now
-	} else {
-		b.logf("netmap packet filter: (length %d)", len(netMap.PacketFilter))
+	nameToIP := make(map[string]netaddr.IP)
+	set := func(name string, addrs []wgcfg.CIDR) {
+		if len(addrs) == 0 {
+			return
+		}
+		nameToIP[name] = netaddr.IPFrom16(addrs[0].IP.Addr)
 	}
-	b.e.SetFilter(filter.New(netMap.PacketFilter, localNets, b.e.GetFilter(), b.logf))
+
+	for _, peer := range netMap.Peers {
+		set(peer.Name, peer.Addresses)
+	}
+	set(netMap.Name, netMap.Addresses)
+
+	dnsMap := tsdns.NewMap(nameToIP)
+	// map diff will be logged in tsdns.Resolver.SetMap.
+	b.e.SetDNSMap(dnsMap)
 }
 
 // readPoller is a goroutine that receives service lists from
@@ -448,13 +576,11 @@ func (b *LocalBackend) readPoller() {
 		}
 
 		b.mu.Lock()
-		if b.hiCache == nil {
-			// TODO(bradfitz): it's a little weird that this port poller
-			// is started (by NewLocalBackend) before the Start call.
-			b.hiCache = new(tailcfg.Hostinfo)
+		if b.hostinfo == nil {
+			b.hostinfo = new(tailcfg.Hostinfo)
 		}
-		b.hiCache.Services = sl
-		hi := b.hiCache
+		b.hostinfo.Services = sl
+		hi := b.hostinfo
 		b.mu.Unlock()
 
 		b.doSetHostinfoFilterServices(hi)
@@ -471,6 +597,8 @@ func (b *LocalBackend) send(n Notify) {
 	if notify != nil {
 		n.Version = version.LONG
 		notify(n)
+	} else {
+		b.logf("nil notify callback; dropping %+v", n)
 	}
 }
 
@@ -523,7 +651,7 @@ func (b *LocalBackend) loadStateLocked(key StateKey, prefs *Prefs, legacyPath st
 	if err != nil {
 		if errors.Is(err, ErrStateNotExist) {
 			if legacyPath != "" {
-				b.prefs, err = LoadPrefs(legacyPath, true)
+				b.prefs, err = LoadPrefs(legacyPath)
 				if err != nil {
 					b.logf("Failed to load legacy prefs: %v", err)
 					b.prefs = NewPrefs()
@@ -565,6 +693,16 @@ func (b *LocalBackend) getEngineStatus() EngineStatus {
 	return b.engineStatus
 }
 
+// Login implements Backend.
+func (b *LocalBackend) Login(token *oauth2.Token) {
+	b.mu.Lock()
+	b.assertClientLocked()
+	c := b.c
+	b.mu.Unlock()
+
+	c.Login(token, controlclient.LoginInteractive)
+}
+
 // StartLoginInteractive implements Backend. It requests a new
 // interactive login from controlclient, unless such a flow is already
 // in progress, in which case StartLoginInteractive attempts to pick
@@ -588,13 +726,23 @@ func (b *LocalBackend) StartLoginInteractive() {
 // FakeExpireAfter implements Backend.
 func (b *LocalBackend) FakeExpireAfter(x time.Duration) {
 	b.logf("FakeExpireAfter: %v", x)
-	if b.netMapCache != nil {
-		e := b.netMapCache.Expiry
-		if e.IsZero() || time.Until(e) > x {
-			b.netMapCache.Expiry = time.Now().Add(x)
-		}
-		b.send(Notify{NetMap: b.netMapCache})
+
+	b.mu.Lock()
+	defer b.mu.Unlock()
+
+	if b.netMap == nil {
+		return
 	}
+
+	// This function is called very rarely,
+	// so we prefer to fully copy the netmap over introducing in-place modification here.
+	mapCopy := *b.netMap
+	e := mapCopy.Expiry
+	if e.IsZero() || time.Until(e) > x {
+		mapCopy.Expiry = time.Now().Add(x)
+	}
+	b.netMap = &mapCopy
+	b.send(Notify{NetMap: b.netMap})
 }
 
 func (b *LocalBackend) parseWgStatus(s *wgengine.Status) (ret EngineStatus) {
@@ -643,30 +791,46 @@ func (b *LocalBackend) SetPrefs(new *Prefs) {
 	}
 
 	b.mu.Lock()
+
+	netMap := b.netMap
+	stateKey := b.stateKey
+
 	old := b.prefs
 	new.Persist = old.Persist // caller isn't allowed to override this
 	b.prefs = new
-	if b.stateKey != "" {
-		if err := b.store.WriteState(b.stateKey, b.prefs.ToBytes()); err != nil {
+	// We do this to avoid holding the lock while doing everything else.
+	new = b.prefs.Clone()
+
+	oldHi := b.hostinfo
+	newHi := oldHi.Clone()
+	newHi.RoutableIPs = append([]wgcfg.CIDR(nil), b.prefs.AdvertiseRoutes...)
+	applyPrefsToHostinfo(newHi, new)
+	b.hostinfo = newHi
+	hostInfoChanged := !oldHi.Equal(newHi)
+
+	b.mu.Unlock()
+
+	if stateKey != "" {
+		if err := b.store.WriteState(stateKey, new.ToBytes()); err != nil {
 			b.logf("Failed to save new controlclient state: %v", err)
 		}
 	}
-	oldHi := b.hiCache
-	newHi := oldHi.Clone()
-	newHi.RoutableIPs = append([]wgcfg.CIDR(nil), b.prefs.AdvertiseRoutes...)
-	if h := new.Hostname; h != "" {
-		newHi.Hostname = h
-	}
-	b.hiCache = newHi
-	b.mu.Unlock()
 
 	b.logf("SetPrefs: %v", new.Pretty())
 
-	if old.ShieldsUp != new.ShieldsUp || !oldHi.Equal(newHi) {
+	if old.ShieldsUp != new.ShieldsUp || hostInfoChanged {
 		b.doSetHostinfoFilterServices(newHi)
 	}
 
-	b.updateFilter(b.netMapCache)
+	b.updateFilter(netMap, new)
+
+	turnDERPOff := new.DisableDERP && !old.DisableDERP
+	turnDERPOn := !new.DisableDERP && old.DisableDERP
+	if turnDERPOff {
+		b.e.SetDERPMap(nil)
+	} else if turnDERPOn && netMap != nil {
+		b.e.SetDERPMap(netMap.DERPMap)
+	}
 
 	if old.WantRunning != new.WantRunning {
 		b.stateMachine()
@@ -703,7 +867,9 @@ func (b *LocalBackend) doSetHostinfoFilterServices(hi *tailcfg.Hostinfo) {
 // NetMap returns the latest cached network map received from
 // controlclient, or nil if no network map was received yet.
 func (b *LocalBackend) NetMap() *controlclient.NetworkMap {
-	return b.netMapCache
+	b.mu.Lock()
+	defer b.mu.Unlock()
+	return b.netMap
 }
 
 // blockEngineUpdate sets b.blocked to block, while holding b.mu. Its
@@ -724,7 +890,7 @@ func (b *LocalBackend) authReconfig() {
 	b.mu.Lock()
 	blocked := b.blocked
 	uc := b.prefs
-	nm := b.netMapCache
+	nm := b.netMap
 	b.mu.Unlock()
 
 	if blocked {
@@ -740,44 +906,88 @@ func (b *LocalBackend) authReconfig() {
 		return
 	}
 
-	uflags := controlclient.UDefault
+	var flags controlclient.WGConfigFlags
 	if uc.RouteAll {
-		uflags |= controlclient.UAllowDefaultRoute
+		flags |= controlclient.AllowDefaultRoute
 		// TODO(apenwarr): Make subnet routes a different pref?
-		uflags |= controlclient.UAllowSubnetRoutes
+		flags |= controlclient.AllowSubnetRoutes
 		// TODO(apenwarr): Remove this once we sort out subnet routes.
 		//  Right now default routes are broken in Windows, but
 		//  controlclient doesn't properly send subnet routes. So
 		//  let's convert a default route into a subnet route in order
 		//  to allow experimentation.
-		uflags |= controlclient.UHackDefaultRoute
+		flags |= controlclient.HackDefaultRoute
 	}
 	if uc.AllowSingleHosts {
-		uflags |= controlclient.UAllowSingleHosts
+		flags |= controlclient.AllowSingleHosts
 	}
 
-	dns := nm.DNS
-	dom := nm.DNSDomains
-	if !uc.CorpDNS {
-		dns = []wgcfg.IP{}
-		dom = []string{}
-	}
-	cfg, err := nm.WGCfg(uflags, dns)
+	cfg, err := nm.WGCfg(b.logf, flags)
 	if err != nil {
 		b.logf("wgcfg: %v", err)
 		return
 	}
 
-	err = b.e.Reconfig(cfg, routerConfig(cfg, uc, dom))
+	rcfg := routerConfig(cfg, uc)
+
+	// If CorpDNS is false, rcfg.DNS remains the zero value.
+	if uc.CorpDNS {
+		domains := nm.DNS.Domains
+		proxied := nm.DNS.Proxied
+		if proxied {
+			if len(nm.DNS.Nameservers) == 0 {
+				b.logf("[unexpected] dns proxied but no nameservers")
+				proxied = false
+			} else {
+				// Domains for proxying should come first to avoid leaking queries.
+				domains = append(domainsForProxying(nm), domains...)
+			}
+		}
+		rcfg.DNS = dns.Config{
+			Nameservers: nm.DNS.Nameservers,
+			Domains:     domains,
+			PerDomain:   nm.DNS.PerDomain,
+			Proxied:     proxied,
+		}
+	}
+
+	err = b.e.Reconfig(cfg, rcfg)
 	if err == wgengine.ErrNoChanges {
 		return
 	}
-	b.logf("authReconfig: ra=%v dns=%v 0x%02x: %v", uc.RouteAll, uc.CorpDNS, uflags, err)
+	b.logf("authReconfig: ra=%v dns=%v 0x%02x: %v", uc.RouteAll, uc.CorpDNS, flags, err)
 }
 
-// routerConfig produces a router.Config from a wireguard config,
-// IPN prefs, and the dnsDomains pulled from control's network map.
-func routerConfig(cfg *wgcfg.Config, prefs *Prefs, dnsDomains []string) *router.Config {
+// domainsForProxying produces a list of search domains for proxied DNS.
+func domainsForProxying(nm *controlclient.NetworkMap) []string {
+	var domains []string
+	if idx := strings.IndexByte(nm.Name, '.'); idx != -1 {
+		domains = append(domains, nm.Name[idx+1:])
+	}
+	for _, peer := range nm.Peers {
+		idx := strings.IndexByte(peer.Name, '.')
+		if idx == -1 {
+			continue
+		}
+		domain := peer.Name[idx+1:]
+		seen := false
+		// In theory this makes the function O(n^2) worst case,
+		// but in practice we expect domains to contain very few elements
+		// (only one until invitations are introduced).
+		for _, seenDomain := range domains {
+			if domain == seenDomain {
+				seen = true
+			}
+		}
+		if !seen {
+			domains = append(domains, domain)
+		}
+	}
+	return domains
+}
+
+// routerConfig produces a router.Config from a wireguard config and IPN prefs.
+func routerConfig(cfg *wgcfg.Config, prefs *Prefs) *router.Config {
 	var addrs []wgcfg.CIDR
 	for _, addr := range cfg.Addresses {
 		addrs = append(addrs, wgcfg.CIDR{
@@ -788,8 +998,6 @@ func routerConfig(cfg *wgcfg.Config, prefs *Prefs, dnsDomains []string) *router.
 
 	rs := &router.Config{
 		LocalAddrs:       wgCIDRToNetaddr(addrs),
-		DNS:              wgIPToNetaddr(cfg.DNS),
-		DNSDomains:       dnsDomains,
 		SubnetRoutes:     wgCIDRToNetaddr(prefs.AdvertiseRoutes),
 		SNATSubnetRoutes: !prefs.NoSNAT,
 		NetfilterMode:    prefs.NetfilterMode,
@@ -799,6 +1007,11 @@ func routerConfig(cfg *wgcfg.Config, prefs *Prefs, dnsDomains []string) *router.
 		rs.Routes = append(rs.Routes, wgCIDRToNetaddr(peer.AllowedIPs)...)
 	}
 
+	rs.Routes = append(rs.Routes, netaddr.IPPrefix{
+		IP:   tsaddr.TailscaleServiceIP(),
+		Bits: 32,
+	})
+
 	return rs
 }
 
@@ -819,17 +1032,6 @@ func wgCIDRsToFilter(cidrLists ...[]wgcfg.CIDR) (ret []filter.Net) {
 	return ret
 }
 
-func wgIPToNetaddr(ips []wgcfg.IP) (ret []netaddr.IP) {
-	for _, ip := range ips {
-		nip, ok := netaddr.FromStdIP(ip.IP())
-		if !ok {
-			panic(fmt.Sprintf("conversion of %s from wgcfg to netaddr IP failed", ip))
-		}
-		ret = append(ret, nip.Unmap())
-	}
-	return ret
-}
-
 func wgCIDRToNetaddr(cidrs []wgcfg.CIDR) (ret []netaddr.IPPrefix) {
 	for _, cidr := range cidrs {
 		ncidr, ok := netaddr.FromStdIPNet(cidr.IPNet())
@@ -842,6 +1044,18 @@ func wgCIDRToNetaddr(cidrs []wgcfg.CIDR) (ret []netaddr.IPPrefix) {
 	return ret
 }
 
+func applyPrefsToHostinfo(hi *tailcfg.Hostinfo, prefs *Prefs) {
+	if h := prefs.Hostname; h != "" {
+		hi.Hostname = h
+	}
+	if v := prefs.OSVersion; v != "" {
+		hi.OSVersion = v
+	}
+	if m := prefs.DeviceModel; m != "" {
+		hi.DeviceModel = m
+	}
+}
+
 // enterState transitions the backend into newState, updating internal
 // state and propagating events out as needed.
 //
@@ -852,8 +1066,10 @@ func wgCIDRToNetaddr(cidrs []wgcfg.CIDR) (ret []netaddr.IPPrefix) {
 func (b *LocalBackend) enterState(newState State) {
 	b.mu.Lock()
 	state := b.state
+	b.state = newState
 	prefs := b.prefs
 	notify := b.notify
+	bc := b.c
 	b.mu.Unlock()
 
 	if state == newState {
@@ -865,7 +1081,10 @@ func (b *LocalBackend) enterState(newState State) {
 		b.send(Notify{State: &newState})
 	}
 
-	b.state = newState
+	if bc != nil {
+		bc.SetPaused(newState == Stopped)
+	}
+
 	switch newState {
 	case NeedsLogin:
 		b.blockEngineUpdates(true)
@@ -894,7 +1113,7 @@ func (b *LocalBackend) nextState() State {
 	b.assertClientLocked()
 	var (
 		c           = b.c
-		netMap      = b.netMapCache
+		netMap      = b.netMap
 		state       = b.state
 		wantRunning = b.prefs.WantRunning
 	)
@@ -941,7 +1160,7 @@ func (b *LocalBackend) RequestEngineStatus() {
 // RequestStatus implements Backend.
 func (b *LocalBackend) RequestStatus() {
 	st := b.Status()
-	b.notify(Notify{Status: st})
+	b.send(Notify{Status: st})
 }
 
 // stateMachine updates the state machine state based on other things
@@ -992,13 +1211,13 @@ func (b *LocalBackend) Logout() {
 	b.mu.Lock()
 	b.assertClientLocked()
 	c := b.c
-	b.netMapCache = nil
+	b.netMap = nil
 	b.mu.Unlock()
 
 	c.Logout()
 
 	b.mu.Lock()
-	b.netMapCache = nil
+	b.netMap = nil
 	b.mu.Unlock()
 
 	b.stateMachine()
@@ -1011,13 +1230,13 @@ func (b *LocalBackend) assertClientLocked() {
 	}
 }
 
-// setNetInfo sets b.hiCache.NetInfo to ni, and passes ni along to the
+// setNetInfo sets b.hostinfo.NetInfo to ni, and passes ni along to the
 // controlclient, if one exists.
 func (b *LocalBackend) setNetInfo(ni *tailcfg.NetInfo) {
 	b.mu.Lock()
 	c := b.c
-	if b.hiCache != nil {
-		b.hiCache.NetInfo = ni.Clone()
+	if b.hostinfo != nil {
+		b.hostinfo.NetInfo = ni.Clone()
 	}
 	b.mu.Unlock()
 

ipn/message.go

@@ -13,6 +13,7 @@ import (
 	"log"
 	"time"
 
+	"golang.org/x/oauth2"
 	"tailscale.com/types/logger"
 	"tailscale.com/types/structs"
 	"tailscale.com/version"
@@ -49,6 +50,7 @@ type Command struct {
 	Quit                  *NoArgs
 	Start                 *StartArgs
 	StartLoginInteractive *NoArgs
+	Login                 *oauth2.Token
 	Logout                *NoArgs
 	SetPrefs              *SetPrefsArgs
 	RequestEngineStatus   *NoArgs
@@ -80,6 +82,10 @@ func (bs *BackendServer) send(n Notify) {
 	bs.sendNotifyMsg(b)
 }
 
+func (bs *BackendServer) SendErrorMessage(msg string) {
+	bs.send(Notify{ErrMessage: &msg})
+}
+
 // GotCommandMsg parses the incoming message b as a JSON Command and
 // calls GotCommand with it.
 func (bs *BackendServer) GotCommandMsg(b []byte) error {
@@ -124,6 +130,9 @@ func (bs *BackendServer) GotCommand(cmd *Command) error {
 	} else if c := cmd.StartLoginInteractive; c != nil {
 		bs.b.StartLoginInteractive()
 		return nil
+	} else if c := cmd.Login; c != nil {
+		bs.b.Login(c)
+		return nil
 	} else if c := cmd.Logout; c != nil {
 		bs.b.Logout()
 		return nil
@@ -221,6 +230,10 @@ func (bc *BackendClient) StartLoginInteractive() {
 	bc.send(Command{StartLoginInteractive: &NoArgs{}})
 }
 
+func (bc *BackendClient) Login(token *oauth2.Token) {
+	bc.send(Command{Login: token})
+}
+
 func (bc *BackendClient) Logout() {
 	bc.send(Command{Logout: &NoArgs{}})
 }
@@ -242,7 +255,7 @@ func (bc *BackendClient) FakeExpireAfter(x time.Duration) {
 }
 
 // MaxMessageSize is the maximum message size, in bytes.
-const MaxMessageSize = 1 << 20
+const MaxMessageSize = 10 << 20
 
 // TODO(apenwarr): incremental json decode?
 //  That would let us avoid storing the whole byte array uselessly in RAM.

ipn/message_test.go

@@ -9,6 +9,7 @@ import (
 	"testing"
 	"time"
 
+	"golang.org/x/oauth2"
 	"tailscale.com/tstest"
 )
 
@@ -177,4 +178,10 @@ func TestClientServer(t *testing.T) {
 
 	h.Logout()
 	flushUntil(NeedsLogin)
+
+	h.Login(&oauth2.Token{
+		AccessToken: "google_id_token",
+		TokenType:   GoogleIDTokenType,
+	})
+	flushUntil(Running)
 }

ipn/prefs.go

@@ -54,6 +54,10 @@ type Prefs struct {
 	// Hostname is the hostname to use for identifying the node. If
 	// not set, os.Hostname is used.
 	Hostname string
+	// OSVersion overrides tailcfg.Hostinfo's OSVersion.
+	OSVersion string
+	// DeviceModel overrides tailcfg.Hostinfo's DeviceModel.
+	DeviceModel string
 
 	// NotepadURLs is a debugging setting that opens OAuth URLs in
 	// notepad.exe on Windows, rather than loading them in a browser.
@@ -138,6 +142,8 @@ func (p *Prefs) Equals(p2 *Prefs) bool {
 		p.NoSNAT == p2.NoSNAT &&
 		p.NetfilterMode == p2.NetfilterMode &&
 		p.Hostname == p2.Hostname &&
+		p.OSVersion == p2.OSVersion &&
+		p.DeviceModel == p2.DeviceModel &&
 		compareIPNets(p.AdvertiseRoutes, p2.AdvertiseRoutes) &&
 		compareStrings(p.AdvertiseTags, p2.AdvertiseTags) &&
 		p.Persist.Equals(p2.Persist)
@@ -217,10 +223,9 @@ func (p *Prefs) Clone() *Prefs {
 	return p2
 }
 
-// LoadLegacyPrefs loads a legacy relaynode config file into Prefs
-// with sensible migration defaults set. If enforceDefaults is true,
-// Prefs.RouteAll and Prefs.AllowSingleHosts are forced on.
-func LoadPrefs(filename string, enforceDefaults bool) (*Prefs, error) {
+// LoadPrefs loads a legacy relaynode config file into Prefs
+// with sensible migration defaults set.
+func LoadPrefs(filename string) (*Prefs, error) {
 	data, err := ioutil.ReadFile(filename)
 	if err != nil {
 		return nil, fmt.Errorf("loading prefs from %q: %v", filename, err)

ipn/prefs_test.go

@@ -24,7 +24,7 @@ func fieldsOf(t reflect.Type) (fields []string) {
 func TestPrefsEqual(t *testing.T) {
 	tstest.PanicOnLog()
 
-	prefsHandles := []string{"ControlURL", "RouteAll", "AllowSingleHosts", "CorpDNS", "WantRunning", "ShieldsUp", "AdvertiseTags", "Hostname", "NotepadURLs", "DisableDERP", "AdvertiseRoutes", "NoSNAT", "NetfilterMode", "Persist"}
+	prefsHandles := []string{"ControlURL", "RouteAll", "AllowSingleHosts", "CorpDNS", "WantRunning", "ShieldsUp", "AdvertiseTags", "Hostname", "OSVersion", "DeviceModel", "NotepadURLs", "DisableDERP", "AdvertiseRoutes", "NoSNAT", "NetfilterMode", "Persist"}
 	if have := fieldsOf(reflect.TypeOf(Prefs{})); !reflect.DeepEqual(have, prefsHandles) {
 		t.Errorf("Prefs.Equal check might be out of sync\nfields: %q\nhandled: %q\n",
 			have, prefsHandles)

log/logheap/logheap.go

@@ -7,9 +7,9 @@ package logheap
 
 import (
 	"bytes"
-	"encoding/json"
-	"io"
-	"os"
+	"context"
+	"log"
+	"net/http"
 	"runtime"
 	"runtime/pprof"
 	"time"
@@ -17,29 +17,25 @@ import (
 
 // LogHeap writes a JSON logtail record with the base64 heap pprof to
 // os.Stderr.
-func LogHeap() {
-	logHeap(os.Stderr)
-}
-
-type logTail struct {
-	ClientTime string `json:"client_time"`
-}
-
-type pprofRec struct {
-	Heap []byte `json:"heap,omitempty"`
-}
-
-type logLine struct {
-	LogTail logTail  `json:"logtail"`
-	Pprof   pprofRec `json:"pprof"`
-}
-
-func logHeap(w io.Writer) error {
+func LogHeap(postURL string) {
+	if postURL == "" {
+		return
+	}
 	runtime.GC()
 	buf := new(bytes.Buffer)
 	pprof.WriteHeapProfile(buf)
-	return json.NewEncoder(w).Encode(logLine{
-		LogTail: logTail{ClientTime: time.Now().Format(time.RFC3339Nano)},
-		Pprof:   pprofRec{Heap: buf.Bytes()},
-	})
+
+	ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
+	defer cancel()
+	req, err := http.NewRequestWithContext(ctx, "POST", postURL, buf)
+	if err != nil {
+		log.Printf("LogHeap: %v", err)
+		return
+	}
+	res, err := http.DefaultClient.Do(req)
+	if err != nil {
+		log.Printf("LogHeap: %v", err)
+		return
+	}
+	defer res.Body.Close()
 }

log/logheap/logheap_test.go

@@ -1,40 +0,0 @@
-// Copyright (c) 2020 Tailscale Inc & AUTHORS All rights reserved.
-// Use of this source code is governed by a BSD-style
-// license that can be found in the LICENSE file.
-
-package logheap
-
-import (
-	"bytes"
-	"compress/gzip"
-	"encoding/json"
-	"io/ioutil"
-	"testing"
-)
-
-func TestLogHeap(t *testing.T) {
-	var buf bytes.Buffer
-	if err := logHeap(&buf); err != nil {
-		t.Fatal(err)
-	}
-	t.Logf("Got line: %s", buf.Bytes())
-
-	var ll logLine
-	if err := json.Unmarshal(buf.Bytes(), &ll); err != nil {
-		t.Fatal(err)
-	}
-
-	zr, err := gzip.NewReader(bytes.NewReader(ll.Pprof.Heap))
-	if err != nil {
-		t.Fatal(err)
-	}
-	rawProto, err := ioutil.ReadAll(zr)
-	if err != nil {
-		t.Fatal(err)
-	}
-	// Just sanity check it. Too lazy to properly decode the protobuf. But see that
-	// it contains an expected sample name.
-	if !bytes.Contains(rawProto, []byte("alloc_objects")) {
-		t.Errorf("raw proto didn't contain `alloc_objects`: %q", rawProto)
-	}
-}

logpolicy/logpolicy.go

@@ -25,13 +25,14 @@ import (
 	"strings"
 	"time"
 
-	"github.com/klauspost/compress/zstd"
 	"golang.org/x/crypto/ssh/terminal"
 	"tailscale.com/atomicfile"
 	"tailscale.com/logtail"
 	"tailscale.com/logtail/filch"
 	"tailscale.com/net/netns"
 	"tailscale.com/net/tlsdial"
+	"tailscale.com/smallzstd"
+	"tailscale.com/types/logger"
 	"tailscale.com/version"
 )
 
@@ -54,7 +55,7 @@ type Policy struct {
 func (c *Config) ToBytes() []byte {
 	data, err := json.MarshalIndent(c, "", "\t")
 	if err != nil {
-		log.Fatalf("logpolicy.Config marshal: %v\n", err)
+		log.Fatalf("logpolicy.Config marshal: %v", err)
 	}
 	return data
 }
@@ -101,21 +102,25 @@ func (l logWriter) Write(buf []byte) (int, error) {
 
 // logsDir returns the directory to use for log configuration and
 // buffer storage.
-func logsDir() string {
+func logsDir(logf logger.Logf) string {
 	systemdStateDir := os.Getenv("STATE_DIRECTORY")
 	if systemdStateDir != "" {
+		logf("logpolicy: using $STATE_DIRECTORY, %q", systemdStateDir)
 		return systemdStateDir
 	}
 
 	cacheDir, err := os.UserCacheDir()
 	if err == nil {
-		return filepath.Join(cacheDir, "Tailscale")
+		d := filepath.Join(cacheDir, "Tailscale")
+		logf("logpolicy: using UserCacheDir, %q", d)
+		return d
 	}
 
 	// Use the current working directory, unless we're being run by a
 	// service manager that sets it to /.
 	wd, err := os.Getwd()
 	if err == nil && wd != "/" {
+		logf("logpolicy: using current directory, %q", wd)
 		return wd
 	}
 
@@ -126,6 +131,7 @@ func logsDir() string {
 	if err != nil {
 		panic("no safe place found to store log state")
 	}
+	logf("logpolicy: using temp directory, %q", tmp)
 	return tmp
 }
 
@@ -149,6 +155,14 @@ func runningUnderSystemd() bool {
 // moved from whereever it does exist, into dir. Leftover logs state
 // in / and $CACHE_DIRECTORY is deleted.
 func tryFixLogStateLocation(dir, cmdname string) {
+	switch runtime.GOOS {
+	case "linux", "freebsd", "openbsd":
+		// These are the OSes where we might have written stuff into
+		// root. Others use different logic to find the logs storage
+		// dir.
+	default:
+		return
+	}
 	if cmdname == "" {
 		log.Printf("[unexpected] no cmdname given to tryFixLogStateLocation, please file a bug at https://github.com/tailscale/tailscale")
 		return
@@ -163,14 +177,6 @@ func tryFixLogStateLocation(dir, cmdname string) {
 		// Only root could have written log configs to weird places.
 		return
 	}
-	switch runtime.GOOS {
-	case "linux", "freebsd", "openbsd":
-		// These are the OSes where we might have written stuff into
-		// root. Others use different logic to find the logs storage
-		// dir.
-	default:
-		return
-	}
 
 	// We stored logs in 2 incorrect places: either /, or CACHE_DIR
 	// (aka /var/cache/tailscale). We want to move files into the
@@ -303,21 +309,28 @@ func New(collection string) *Policy {
 	}
 	console := log.New(stderrWriter{}, "", lflags)
 
-	dir := logsDir()
+	var earlyErrBuf bytes.Buffer
+	earlyLogf := func(format string, a ...interface{}) {
+		fmt.Fprintf(&earlyErrBuf, format, a...)
+		earlyErrBuf.WriteByte('\n')
+	}
 
-	tryFixLogStateLocation(dir, version.CmdName())
+	dir := logsDir(earlyLogf)
 
-	cfgPath := filepath.Join(dir, fmt.Sprintf("%s.log.conf", version.CmdName()))
+	cmdName := version.CmdName()
+	tryFixLogStateLocation(dir, cmdName)
+
+	cfgPath := filepath.Join(dir, fmt.Sprintf("%s.log.conf", cmdName))
 	var oldc *Config
 	data, err := ioutil.ReadFile(cfgPath)
 	if err != nil {
-		log.Printf("logpolicy.Read %v: %v\n", cfgPath, err)
+		earlyLogf("logpolicy.Read %v: %v", cfgPath, err)
 		oldc = &Config{}
 		oldc.Collection = collection
 	} else {
 		oldc, err = ConfigFromBytes(data)
 		if err != nil {
-			log.Printf("logpolicy.Config unmarshal: %v\n", err)
+			earlyLogf("logpolicy.Config unmarshal: %v", err)
 			oldc = &Config{}
 		}
 	}
@@ -339,7 +352,7 @@ func New(collection string) *Policy {
 	newc.PublicID = newc.PrivateID.Public()
 	if newc != *oldc {
 		if err := newc.save(cfgPath); err != nil {
-			log.Printf("logpolicy.Config.Save: %v\n", err)
+			earlyLogf("logpolicy.Config.Save: %v", err)
 		}
 	}
 
@@ -348,11 +361,7 @@ func New(collection string) *Policy {
 		PrivateID:  newc.PrivateID,
 		Stderr:     logWriter{console},
 		NewZstdEncoder: func() logtail.Encoder {
-			w, err := zstd.NewWriter(nil,
-				zstd.WithEncoderLevel(zstd.SpeedFastest),
-				zstd.WithEncoderConcurrency(1),
-				zstd.WithWindowSize(8192),
-			)
+			w, err := smallzstd.NewEncoder(nil)
 			if err != nil {
 				panic(err)
 			}
@@ -361,7 +370,7 @@ func New(collection string) *Policy {
 		HTTPC: &http.Client{Transport: newLogtailTransport(logtail.DefaultHost)},
 	}
 
-	filchBuf, filchErr := filch.New(filepath.Join(dir, version.CmdName()), filch.Options{})
+	filchBuf, filchErr := filch.New(filepath.Join(dir, cmdName), filch.Options{})
 	if filchBuf != nil {
 		c.Buffer = filchBuf
 	}
@@ -369,14 +378,17 @@ func New(collection string) *Policy {
 	log.SetFlags(0) // other logflags are set on console, not here
 	log.SetOutput(lw)
 
-	log.Printf("Program starting: v%v, Go %v: %#v\n",
+	log.Printf("Program starting: v%v, Go %v: %#v",
 		version.LONG,
 		strings.TrimPrefix(runtime.Version(), "go"),
 		os.Args)
-	log.Printf("LogID: %v\n", newc.PublicID)
+	log.Printf("LogID: %v", newc.PublicID)
 	if filchErr != nil {
 		log.Printf("filch failed: %v", filchErr)
 	}
+	if earlyErrBuf.Len() != 0 {
+		log.Printf("%s", earlyErrBuf.Bytes())
+	}
 
 	return &Policy{
 		Logtail:  lw,
@@ -395,7 +407,7 @@ func (p *Policy) Close() {
 // log upload if it can be done before ctx is canceled.
 func (p *Policy) Shutdown(ctx context.Context) error {
 	if p.Logtail != nil {
-		log.Printf("flushing log.\n")
+		log.Printf("flushing log.")
 		return p.Logtail.Shutdown(ctx)
 	}
 	return nil

logtail/backoff/backoff.go

@@ -2,6 +2,7 @@
 // Use of this source code is governed by a BSD-style
 // license that can be found in the LICENSE file.
 
+// Package backoff provides a back-off timer type.
 package backoff
 
 import (
@@ -12,54 +13,70 @@ import (
 	"tailscale.com/types/logger"
 )
 
-const MAX_BACKOFF_MSEC = 30000
-
+// Backoff tracks state the history of consecutive failures and sleeps
+// an increasing amount of time, up to a provided limit.
 type Backoff struct {
-	n int
+	n          int // number of consecutive failures
+	maxBackoff time.Duration
+
 	// Name is the name of this backoff timer, for logging purposes.
 	name string
 	// logf is the function used for log messages when backing off.
 	logf logger.Logf
-	// NewTimer is the function that acts like time.NewTimer().
-	// You can override this in unit tests.
-	NewTimer func(d time.Duration) *time.Timer
+
+	// NewTimer is the function that acts like time.NewTimer.
+	// It's for use in unit tests.
+	NewTimer func(time.Duration) *time.Timer
+
 	// LogLongerThan sets the minimum time of a single backoff interval
 	// before we mention it in the log.
 	LogLongerThan time.Duration
 }
 
-func NewBackoff(name string, logf logger.Logf) Backoff {
-	return Backoff{
-		name:     name,
-		logf:     logf,
-		NewTimer: time.NewTimer,
+// NewBackoff returns a new Backoff timer with the provided name (for logging), logger,
+// and max backoff time. By default, all failures (calls to BackOff with a non-nil err)
+// are logged unless the returned Backoff.LogLongerThan is adjusted.
+func NewBackoff(name string, logf logger.Logf, maxBackoff time.Duration) *Backoff {
+	return &Backoff{
+		name:       name,
+		logf:       logf,
+		maxBackoff: maxBackoff,
+		NewTimer:   time.NewTimer,
 	}
 }
 
+// Backoff sleeps an increasing amount of time if err is non-nil.
+// and the context is not a
+// It resets the backoff schedule once err is nil.
 func (b *Backoff) BackOff(ctx context.Context, err error) {
-	if ctx.Err() == nil && err != nil {
-		b.n++
-		// n^2 backoff timer is a little smoother than the
-		// common choice of 2^n.
-		msec := b.n * b.n * 10
-		if msec > MAX_BACKOFF_MSEC {
-			msec = MAX_BACKOFF_MSEC
-		}
-		// Randomize the delay between 0.5-1.5 x msec, in order
-		// to prevent accidental "thundering herd" problems.
-		msec = rand.Intn(msec) + msec/2
-		dur := time.Duration(msec) * time.Millisecond
-		if dur >= b.LogLongerThan {
-			b.logf("%s: backoff: %d msec\n", b.name, msec)
-		}
-		t := b.NewTimer(dur)
-		select {
-		case <-ctx.Done():
-			t.Stop()
-		case <-t.C:
-		}
-	} else {
-		// not a regular error
+	if err == nil {
+		// No error. Reset number of consecutive failures.
 		b.n = 0
+		return
+	}
+	if ctx.Err() != nil {
+		// Fast path.
+		return
+	}
+
+	b.n++
+	// n^2 backoff timer is a little smoother than the
+	// common choice of 2^n.
+	d := time.Duration(b.n*b.n) * 10 * time.Millisecond
+	if d > b.maxBackoff {
+		d = b.maxBackoff
+	}
+	// Randomize the delay between 0.5-1.5 x msec, in order
+	// to prevent accidental "thundering herd" problems.
+	d = time.Duration(float64(d) * (rand.Float64() + 0.5))
+
+	if d >= b.LogLongerThan {
+		b.logf("%s: backoff: %d msec", b.name, d.Milliseconds())
+	}
+	t := b.NewTimer(d)
+	select {
+	case <-ctx.Done():
+		t.Stop()
+	case <-t.C:
 	}
 }

logtail/logtail.go

@@ -105,7 +105,7 @@ func Log(cfg Config, logf tslogger.Logf) Logger {
 		sentinel:       make(chan int32, 16),
 		drainLogs:      cfg.DrainLogs,
 		timeNow:        cfg.TimeNow,
-		bo:             backoff.NewBackoff("logtail", logf),
+		bo:             backoff.NewBackoff("logtail", logf, 30*time.Second),
 
 		shutdownStart: make(chan struct{}),
 		shutdownDone:  make(chan struct{}),
@@ -133,7 +133,7 @@ type logger struct {
 	drainLogs      <-chan struct{} // if non-nil, external signal to attempt a drain
 	sentinel       chan int32
 	timeNow        func() time.Time
-	bo             backoff.Backoff
+	bo             *backoff.Backoff
 	zstdEncoder    Encoder
 	uploadCancel   func()
 
@@ -273,10 +273,10 @@ func (l *logger) uploading(ctx context.Context) {
 			if err != nil {
 				fmt.Fprintf(l.stderr, "logtail: upload: %v\n", err)
 			}
+			l.bo.BackOff(ctx, err)
 			if uploaded {
 				break
 			}
-			l.bo.BackOff(ctx, err)
 		}
 
 		select {
@@ -462,5 +462,6 @@ func (l *logger) Write(buf []byte) (int, error) {
 		}
 	}
 	b := l.encode(buf)
-	return l.send(b)
+	_, err := l.send(b)
+	return len(buf), err
 }

logtail/logtail_test.go

@@ -32,3 +32,18 @@ func TestLoggerEncodeTextAllocs(t *testing.T) {
 		t.Logf("allocs = %d; want 1", int(n))
 	}
 }
+
+func TestLoggerWriteLength(t *testing.T) {
+	lg := &logger{
+		timeNow: time.Now,
+		buffer:  NewMemoryBuffer(1024),
+	}
+	inBuf := []byte("some text to encode")
+	n, err := lg.Write(inBuf)
+	if err != nil {
+		t.Error(err)
+	}
+	if n != len(inBuf) {
+		t.Errorf("logger.Write wrote %d bytes, expected %d", n, len(inBuf))
+	}
+}

metrics/metrics.go

@@ -40,3 +40,10 @@ func (m *LabelMap) Get(key string) *expvar.Int {
 	m.Add(key, 0)
 	return m.Map.Get(key).(*expvar.Int)
 }
+
+// GetFloat returns a direct pointer to the expvar.Float for key, creating it
+// if necessary.
+func (m *LabelMap) GetFloat(key string) *expvar.Float {
+	m.AddFloat(key, 0.0)
+	return m.Map.Get(key).(*expvar.Float)
+}

net/interfaces/interfaces.go

@@ -12,6 +12,7 @@ import (
 	"strings"
 
 	"inet.af/netaddr"
+	"tailscale.com/net/tsaddr"
 )
 
 // Tailscale returns the current machine's Tailscale interface, if any.
@@ -52,7 +53,7 @@ func maybeTailscaleInterfaceName(s string) bool {
 // Tailscale virtual network interfaces.
 func IsTailscaleIP(ip net.IP) bool {
 	nip, _ := netaddr.FromStdIP(ip) // TODO: push this up to caller, change func signature
-	return cgNAT.Contains(nip)
+	return tsaddr.IsTailscaleIP(nip)
 }
 
 func isUp(nif *net.Interface) bool       { return nif.Flags&net.FlagUp != 0 }
@@ -95,7 +96,7 @@ func LocalAddresses() (regular, loopback []string, err error) {
 				// very well be something we can route to
 				// directly, because both nodes are
 				// behind the same CGNAT router.
-				if cgNAT.Contains(ip) {
+				if tsaddr.IsTailscaleIP(ip) {
 					continue
 				}
 				if linkLocalIPv4.Contains(ip) {
@@ -230,6 +231,38 @@ func HTTPOfListener(ln net.Listener) string {
 
 }
 
+var likelyHomeRouterIP func() (netaddr.IP, bool)
+
+// LikelyHomeRouterIP returns the likely IP of the residential router,
+// which will always be an IPv4 private address, if found.
+// In addition, it returns the IP address of the current machine on
+// the LAN using that gateway.
+// This is used as the destination for UPnP, NAT-PMP, PCP, etc queries.
+func LikelyHomeRouterIP() (gateway, myIP netaddr.IP, ok bool) {
+	if likelyHomeRouterIP != nil {
+		gateway, ok = likelyHomeRouterIP()
+		if !ok {
+			return
+		}
+	}
+	if !ok {
+		return
+	}
+	ForeachInterfaceAddress(func(i Interface, ip netaddr.IP) {
+		if !i.IsUp() || ip.IsZero() || !myIP.IsZero() {
+			return
+		}
+		for _, prefix := range privatev4s {
+			if prefix.Contains(gateway) && prefix.Contains(ip) {
+				myIP = ip
+				ok = true
+				return
+			}
+		}
+	})
+	return gateway, myIP, !myIP.IsZero()
+}
+
 func isPrivateIP(ip netaddr.IP) bool {
 	return private1.Contains(ip) || private2.Contains(ip) || private3.Contains(ip)
 }
@@ -250,7 +283,7 @@ var (
 	private1      = mustCIDR("10.0.0.0/8")
 	private2      = mustCIDR("172.16.0.0/12")
 	private3      = mustCIDR("192.168.0.0/16")
-	cgNAT         = mustCIDR("100.64.0.0/10")
+	privatev4s    = []netaddr.IPPrefix{private1, private2, private3}
 	linkLocalIPv4 = mustCIDR("169.254.0.0/16")
 	v6Global1     = mustCIDR("2000::/3")
 )

net/interfaces/interfaces_darwin.go

@@ -0,0 +1,69 @@
+// Copyright (c) 2020 Tailscale Inc & AUTHORS All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package interfaces
+
+import (
+	"os/exec"
+
+	"go4.org/mem"
+	"inet.af/netaddr"
+	"tailscale.com/util/lineread"
+	"tailscale.com/version"
+)
+
+/*
+Parse out 10.0.0.1 from:
+
+$ netstat -r -n -f inet
+Routing tables
+
+Internet:
+Destination        Gateway            Flags        Netif Expire
+default            10.0.0.1           UGSc           en0
+default            link#14            UCSI         utun2
+10/16              link#4             UCS            en0      !
+10.0.0.1/32        link#4             UCS            en0      !
+...
+
+*/
+func likelyHomeRouterIPDarwinExec() (ret netaddr.IP, ok bool) {
+	if version.IsMobile() {
+		// Don't try to do subprocesses on iOS. Ends up with log spam like:
+		// kernel: "Sandbox: IPNExtension(86580) deny(1) process-fork"
+		// This is why we have likelyHomeRouterIPDarwinSyscall.
+		return ret, false
+	}
+	cmd := exec.Command("/usr/sbin/netstat", "-r", "-n", "-f", "inet")
+	stdout, err := cmd.StdoutPipe()
+	if err != nil {
+		return
+	}
+	if err := cmd.Start(); err != nil {
+		return
+	}
+	defer cmd.Wait()
+
+	var f []mem.RO
+	lineread.Reader(stdout, func(lineb []byte) error {
+		line := mem.B(lineb)
+		if !mem.Contains(line, mem.S("default")) {
+			return nil
+		}
+		f = mem.AppendFields(f[:0], line)
+		if len(f) < 3 || !f[0].EqualString("default") {
+			return nil
+		}
+		ipm, flagsm := f[1], f[2]
+		if !mem.Contains(flagsm, mem.S("G")) {
+			return nil
+		}
+		ip, err := netaddr.ParseIP(string(mem.Append(nil, ipm)))
+		if err == nil && isPrivateIP(ip) {
+			ret = ip
+		}
+		return nil
+	})
+	return ret, !ret.IsZero()
+}

net/interfaces/interfaces_darwin_cgo.go

@@ -0,0 +1,127 @@
+// Copyright (c) 2020 Tailscale Inc & AUTHORS All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+// +build darwin,cgo
+
+package interfaces
+
+/*
+#import "route.h"
+#import <netinet/in.h>
+#import <sys/sysctl.h>
+#import <stdlib.h>
+#import <stdio.h>
+
+// privateGatewayIPFromRoute returns the private gateway ip address from rtm, if it exists.
+// Otherwise, it returns 0.
+int privateGatewayIPFromRoute(struct rt_msghdr2 *rtm)
+{
+    // sockaddrs are after the message header
+    struct sockaddr* dst_sa = (struct sockaddr *)(rtm + 1);
+
+    if((rtm->rtm_addrs & (RTA_DST|RTA_GATEWAY)) != (RTA_DST|RTA_GATEWAY))
+        return 0; // missing dst or gateway addr
+    if (dst_sa->sa_family != AF_INET)
+        return 0; // dst not IPv4
+    if ((rtm->rtm_flags & RTF_GATEWAY) == 0)
+        return 0; // gateway flag not set
+
+    struct sockaddr_in* dst_si = (struct sockaddr_in *)dst_sa;
+    if (dst_si->sin_addr.s_addr != INADDR_ANY)
+        return 0; // not default route
+
+    #define ROUNDUP(a) ((a) > 0 ? (1 + (((a) - 1) | (sizeof(long) - 1))) : sizeof(long))
+
+    struct sockaddr* gateway_sa = (struct sockaddr *)((char *)dst_sa + ROUNDUP(dst_sa->sa_len));
+    if (gateway_sa->sa_family != AF_INET)
+        return 0; // gateway not IPv4
+
+    struct sockaddr_in* gateway_si= (struct sockaddr_in *)gateway_sa;
+    int ip;
+    ip = gateway_si->sin_addr.s_addr;
+
+    unsigned char a, b;
+    a = (ip >> 0) & 0xff;
+    b = (ip >> 8) & 0xff;
+
+    // Check whether ip is private, that is, whether it is
+    // in one of 10.0.0.0/8, 172.16.0.0/12, or 192.168.0.0/16.
+    if (a == 10)
+        return ip; // matches 10.0.0.0/8
+    if (a == 172 && (b >> 4) == 1)
+        return ip; // matches 172.16.0.0/12
+    if (a == 192 && b == 168)
+        return ip; // matches 192.168.0.0/16
+
+    // Not a private IP.
+    return 0;
+}
+
+// privateGatewayIP returns the private gateway IP address, if it exists.
+// If no private gateway IP address was found, it returns 0.
+// On an error, it returns an error code in (0, 255].
+// Any private gateway IP address is > 255.
+int privateGatewayIP()
+{
+    size_t needed;
+    int mib[6];
+    char *buf;
+
+    mib[0] = CTL_NET;
+    mib[1] = PF_ROUTE;
+    mib[2] = 0;
+    mib[3] = 0;
+    mib[4] = NET_RT_DUMP2;
+    mib[5] = 0;
+
+    if (sysctl(mib, 6, NULL, &needed, NULL, 0) < 0)
+        return 1; // route dump size estimation failed
+    if ((buf = malloc(needed)) == 0)
+        return 2; // malloc failed
+    if (sysctl(mib, 6, buf, &needed, NULL, 0) < 0) {
+        free(buf);
+        return 3; // route dump failed
+    }
+
+    // Loop over all routes.
+    char *next, *lim;
+    lim = buf + needed;
+	struct rt_msghdr2 *rtm;
+    for (next = buf; next < lim; next += rtm->rtm_msglen) {
+        rtm = (struct rt_msghdr2 *)next;
+		int ip;
+        ip = privateGatewayIPFromRoute(rtm);
+        if (ip) {
+            free(buf);
+            return ip;
+        }
+    }
+    free(buf);
+    return 0; // no gateway found
+}
+*/
+import "C"
+
+import (
+	"encoding/binary"
+	"fmt"
+	"os"
+
+	"inet.af/netaddr"
+)
+
+func init() {
+	likelyHomeRouterIP = likelyHomeRouterIPDarwinSyscall
+}
+
+func likelyHomeRouterIPDarwinSyscall() (ret netaddr.IP, ok bool) {
+	ip := C.privateGatewayIP()
+	fmt.Fprintln(os.Stderr, "likelyHomeRouterIPDarwinSyscall", ip)
+	if ip < 255 {
+		return netaddr.IP{}, false
+	}
+	var q [4]byte
+	binary.LittleEndian.PutUint32(q[:], uint32(ip))
+	return netaddr.IPv4(q[0], q[1], q[2], q[3]), true
+}

net/interfaces/interfaces_darwin_cgo_test.go

@@ -0,0 +1,20 @@
+// Copyright (c) 2020 Tailscale Inc & AUTHORS All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+// +build cgo,darwin
+
+package interfaces
+
+import "testing"
+
+func TestLikelyHomeRouterIPSyscallExec(t *testing.T) {
+	syscallIP, syscallOK := likelyHomeRouterIPDarwinSyscall()
+	netstatIP, netstatOK := likelyHomeRouterIPDarwinExec()
+	if syscallOK != netstatOK || syscallIP != netstatIP {
+		t.Errorf("syscall() = %v, %v, netstat = %v, %v",
+			syscallIP, syscallOK,
+			netstatIP, netstatOK,
+		)
+	}
+}

net/interfaces/interfaces_darwin_nocgo.go

@@ -0,0 +1,11 @@
+// Copyright (c) 2020 Tailscale Inc & AUTHORS All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+// +build darwin,!cgo
+
+package interfaces
+
+func init() {
+	likelyHomeRouterIP = likelyHomeRouterIPDarwinExec
+}

net/interfaces/interfaces_linux.go

@@ -0,0 +1,210 @@
+// Copyright (c) 2020 Tailscale Inc & AUTHORS All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package interfaces
+
+import (
+	"bufio"
+	"bytes"
+	"errors"
+	"io"
+	"log"
+	"os"
+	"os/exec"
+	"runtime"
+	"strings"
+
+	"go4.org/mem"
+	"inet.af/netaddr"
+	"tailscale.com/syncs"
+	"tailscale.com/util/lineread"
+)
+
+func init() {
+	likelyHomeRouterIP = likelyHomeRouterIPLinux
+}
+
+var procNetRouteErr syncs.AtomicBool
+
+/*
+Parse 10.0.0.1 out of:
+
+$ cat /proc/net/route
+Iface   Destination     Gateway         Flags   RefCnt  Use     Metric  Mask            MTU     Window  IRTT
+ens18   00000000        0100000A        0003    0       0       0       00000000        0       0       0
+ens18   0000000A        00000000        0001    0       0       0       0000FFFF        0       0       0
+*/
+func likelyHomeRouterIPLinux() (ret netaddr.IP, ok bool) {
+	if procNetRouteErr.Get() {
+		// If we failed to read /proc/net/route previously, don't keep trying.
+		// But if we're on Android, go into the Android path.
+		if runtime.GOOS == "android" {
+			return likelyHomeRouterIPAndroid()
+		}
+		return ret, false
+	}
+	lineNum := 0
+	var f []mem.RO
+	err := lineread.File("/proc/net/route", func(line []byte) error {
+		lineNum++
+		if lineNum == 1 {
+			// Skip header line.
+			return nil
+		}
+		f = mem.AppendFields(f[:0], mem.B(line))
+		if len(f) < 4 {
+			return nil
+		}
+		gwHex, flagsHex := f[2], f[3]
+		flags, err := mem.ParseUint(flagsHex, 16, 16)
+		if err != nil {
+			return nil // ignore error, skip line and keep going
+		}
+		const RTF_UP = 0x0001
+		const RTF_GATEWAY = 0x0002
+		if flags&(RTF_UP|RTF_GATEWAY) != RTF_UP|RTF_GATEWAY {
+			return nil
+		}
+		ipu32, err := mem.ParseUint(gwHex, 16, 32)
+		if err != nil {
+			return nil // ignore error, skip line and keep going
+		}
+		ip := netaddr.IPv4(byte(ipu32), byte(ipu32>>8), byte(ipu32>>16), byte(ipu32>>24))
+		if isPrivateIP(ip) {
+			ret = ip
+		}
+		return nil
+	})
+	if err != nil {
+		procNetRouteErr.Set(true)
+		if runtime.GOOS == "android" {
+			return likelyHomeRouterIPAndroid()
+		}
+		log.Printf("interfaces: failed to read /proc/net/route: %v", err)
+	}
+	return ret, !ret.IsZero()
+}
+
+// Android apps don't have permission to read /proc/net/route, at
+// least on Google devices and the Android emulator.
+func likelyHomeRouterIPAndroid() (ret netaddr.IP, ok bool) {
+	cmd := exec.Command("/system/bin/ip", "route", "show", "table", "0")
+	out, err := cmd.StdoutPipe()
+	if err != nil {
+		return
+	}
+	if err := cmd.Start(); err != nil {
+		log.Printf("interfaces: running /system/bin/ip: %v", err)
+		return
+	}
+	// Search for line like "default via 10.0.2.2 dev radio0 table 1016 proto static mtu 1500 "
+	lineread.Reader(out, func(line []byte) error {
+		const pfx = "default via "
+		if !mem.HasPrefix(mem.B(line), mem.S(pfx)) {
+			return nil
+		}
+		line = line[len(pfx):]
+		sp := bytes.IndexByte(line, ' ')
+		if sp == -1 {
+			return nil
+		}
+		ipb := line[:sp]
+		if ip, err := netaddr.ParseIP(string(ipb)); err == nil && ip.Is4() {
+			ret = ip
+			log.Printf("interfaces: found Android default route %v", ip)
+		}
+		return nil
+	})
+	cmd.Process.Kill()
+	cmd.Wait()
+	return ret, !ret.IsZero()
+}
+
+// DefaultRouteInterface returns the name of the network interface that owns
+// the default route, not including any tailscale interfaces.
+func DefaultRouteInterface() (string, error) {
+	v, err := defaultRouteInterfaceProcNet()
+	if err == nil {
+		return v, nil
+	}
+	if runtime.GOOS == "android" {
+		return defaultRouteInterfaceAndroidIPRoute()
+	}
+	return v, err
+}
+
+var zeroRouteBytes = []byte("00000000")
+
+func defaultRouteInterfaceProcNet() (string, error) {
+	f, err := os.Open("/proc/net/route")
+	if err != nil {
+		return "", err
+	}
+	defer f.Close()
+	br := bufio.NewReaderSize(f, 128)
+	for {
+		line, err := br.ReadSlice('\n')
+		if err == io.EOF {
+			break
+		}
+		if err != nil {
+			return "", err
+		}
+		if !bytes.Contains(line, zeroRouteBytes) {
+			continue
+		}
+		fields := strings.Fields(string(line))
+		ifc := fields[0]
+		ip := fields[1]
+		netmask := fields[7]
+
+		if strings.HasPrefix(ifc, "tailscale") ||
+			strings.HasPrefix(ifc, "wg") {
+			continue
+		}
+		if ip == "00000000" && netmask == "00000000" {
+			// default route
+			return ifc, nil // interface name
+		}
+	}
+
+	return "", errors.New("no default routes found")
+
+}
+
+// defaultRouteInterfaceAndroidIPRoute tries to find the machine's default route interface name
+// by parsing the "ip route" command output. We use this on Android where /proc/net/route
+// can be missing entries or have locked-down permissions.
+// See also comments in https://github.com/tailscale/tailscale/pull/666.
+func defaultRouteInterfaceAndroidIPRoute() (ifname string, err error) {
+	cmd := exec.Command("/system/bin/ip", "route", "show", "table", "0")
+	out, err := cmd.StdoutPipe()
+	if err != nil {
+		return "", err
+	}
+	if err := cmd.Start(); err != nil {
+		log.Printf("interfaces: running /system/bin/ip: %v", err)
+		return "", err
+	}
+	// Search for line like "default via 10.0.2.2 dev radio0 table 1016 proto static mtu 1500 "
+	lineread.Reader(out, func(line []byte) error {
+		const pfx = "default via "
+		if !mem.HasPrefix(mem.B(line), mem.S(pfx)) {
+			return nil
+		}
+		ff := strings.Fields(string(line))
+		for i, v := range ff {
+			if i > 0 && ff[i-1] == "dev" && ifname == "" {
+				ifname = v
+			}
+		}
+		return nil
+	})
+	cmd.Process.Kill()
+	cmd.Wait()
+	if ifname == "" {
+		return "", errors.New("no default routes found")
+	}
+	return ifname, nil
+}

net/interfaces/interfaces_linux_test.go

@@ -0,0 +1,16 @@
+// Copyright (c) 2020 Tailscale Inc & AUTHORS All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package interfaces
+
+import "testing"
+
+func BenchmarkDefaultRouteInterface(b *testing.B) {
+	b.ReportAllocs()
+	for i := 0; i < b.N; i++ {
+		if _, err := DefaultRouteInterface(); err != nil {
+			b.Fatal(err)
+		}
+	}
+}

net/interfaces/interfaces_test.go

@@ -47,3 +47,12 @@ func TestGetState(t *testing.T) {
 		t.Fatal("two States back-to-back were not equal")
 	}
 }
+
+func TestLikelyHomeRouterIP(t *testing.T) {
+	gw, my, ok := LikelyHomeRouterIP()
+	if !ok {
+		t.Logf("no result")
+		return
+	}
+	t.Logf("myIP = %v; gw = %v", my, gw)
+}

net/interfaces/interfaces_windows.go

@@ -0,0 +1,73 @@
+// Copyright (c) 2020 Tailscale Inc & AUTHORS All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package interfaces
+
+import (
+	"os/exec"
+	"syscall"
+
+	"go4.org/mem"
+	"inet.af/netaddr"
+	"tailscale.com/util/lineread"
+)
+
+func init() {
+	likelyHomeRouterIP = likelyHomeRouterIPWindows
+}
+
+/*
+Parse out 10.0.0.1 from:
+
+Z:\>route print -4
+===========================================================================
+Interface List
+ 15...aa 15 48 ff 1c 72 ......Red Hat VirtIO Ethernet Adapter
+  5...........................Tailscale Tunnel
+  1...........................Software Loopback Interface 1
+===========================================================================
+
+IPv4 Route Table
+===========================================================================
+Active Routes:
+Network Destination        Netmask          Gateway       Interface  Metric
+          0.0.0.0          0.0.0.0         10.0.0.1       10.0.28.63      5
+         10.0.0.0      255.255.0.0         On-link        10.0.28.63    261
+       10.0.28.63  255.255.255.255         On-link        10.0.28.63    261
+        10.0.42.0    255.255.255.0   100.103.42.106   100.103.42.106      5
+     10.0.255.255  255.255.255.255         On-link        10.0.28.63    261
+   34.193.248.174  255.255.255.255   100.103.42.106   100.103.42.106      5
+
+*/
+func likelyHomeRouterIPWindows() (ret netaddr.IP, ok bool) {
+	cmd := exec.Command("route", "print", "-4")
+	cmd.SysProcAttr = &syscall.SysProcAttr{HideWindow: true}
+	stdout, err := cmd.StdoutPipe()
+	if err != nil {
+		return
+	}
+	if err := cmd.Start(); err != nil {
+		return
+	}
+	defer cmd.Wait()
+
+	var f []mem.RO
+	lineread.Reader(stdout, func(lineb []byte) error {
+		line := mem.B(lineb)
+		if !mem.Contains(line, mem.S("0.0.0.0")) {
+			return nil
+		}
+		f = mem.AppendFields(f[:0], line)
+		if len(f) < 3 || !f[0].EqualString("0.0.0.0") || !f[1].EqualString("0.0.0.0") {
+			return nil
+		}
+		ipm := f[2]
+		ip, err := netaddr.ParseIP(string(mem.Append(nil, ipm)))
+		if err == nil && isPrivateIP(ip) {
+			ret = ip
+		}
+		return nil
+	})
+	return ret, !ret.IsZero()
+}

net/interfaces/route.h

@@ -0,0 +1,257 @@
+/*
+ * Copyright (c) 2000-2017 Apple Inc. All rights reserved.
+ *
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
+ *
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
+ *
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ *
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
+ * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
+ *
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
+ */
+/*
+ * Copyright (c) 1980, 1986, 1993
+ *	The Regents of the University of California.  All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ * 3. All advertising materials mentioning features or use of this software
+ *    must display the following acknowledgement:
+ *	This product includes software developed by the University of
+ *	California, Berkeley and its contributors.
+ * 4. Neither the name of the University nor the names of its contributors
+ *    may be used to endorse or promote products derived from this software
+ *    without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED.  IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ *
+ *	@(#)route.h	8.3 (Berkeley) 4/19/94
+ * $FreeBSD: src/sys/net/route.h,v 1.36.2.1 2000/08/16 06:14:23 jayanth Exp $
+ */
+
+#ifndef _NET_ROUTE_H_
+#define _NET_ROUTE_H_
+#include <sys/appleapiopts.h>
+#include <stdint.h>
+#include <sys/types.h>
+#include <sys/socket.h>
+
+/*
+ * These numbers are used by reliable protocols for determining
+ * retransmission behavior and are included in the routing structure.
+ */
+struct rt_metrics {
+	u_int32_t       rmx_locks;      /* Kernel leaves these values alone */
+	u_int32_t       rmx_mtu;        /* MTU for this path */
+	u_int32_t       rmx_hopcount;   /* max hops expected */
+	int32_t         rmx_expire;     /* lifetime for route, e.g. redirect */
+	u_int32_t       rmx_recvpipe;   /* inbound delay-bandwidth product */
+	u_int32_t       rmx_sendpipe;   /* outbound delay-bandwidth product */
+	u_int32_t       rmx_ssthresh;   /* outbound gateway buffer limit */
+	u_int32_t       rmx_rtt;        /* estimated round trip time */
+	u_int32_t       rmx_rttvar;     /* estimated rtt variance */
+	u_int32_t       rmx_pksent;     /* packets sent using this route */
+	u_int32_t       rmx_state;      /* route state */
+	u_int32_t       rmx_filler[3];  /* will be used for T/TCP later */
+};
+
+/*
+ * rmx_rtt and rmx_rttvar are stored as microseconds;
+ */
+#define RTM_RTTUNIT     1000000 /* units for rtt, rttvar, as units per sec */
+
+
+
+#define RTF_UP          0x1             /* route usable */
+#define RTF_GATEWAY     0x2             /* destination is a gateway */
+#define RTF_HOST        0x4             /* host entry (net otherwise) */
+#define RTF_REJECT      0x8             /* host or net unreachable */
+#define RTF_DYNAMIC     0x10            /* created dynamically (by redirect) */
+#define RTF_MODIFIED    0x20            /* modified dynamically (by redirect) */
+#define RTF_DONE        0x40            /* message confirmed */
+#define RTF_DELCLONE    0x80            /* delete cloned route */
+#define RTF_CLONING     0x100           /* generate new routes on use */
+#define RTF_XRESOLVE    0x200           /* external daemon resolves name */
+#define RTF_LLINFO      0x400           /* DEPRECATED - exists ONLY for backward
+	                                 *  compatibility */
+#define RTF_LLDATA      0x400           /* used by apps to add/del L2 entries */
+#define RTF_STATIC      0x800           /* manually added */
+#define RTF_BLACKHOLE   0x1000          /* just discard pkts (during updates) */
+#define RTF_NOIFREF     0x2000          /* not eligible for RTF_IFREF */
+#define RTF_PROTO2      0x4000          /* protocol specific routing flag */
+#define RTF_PROTO1      0x8000          /* protocol specific routing flag */
+
+#define RTF_PRCLONING   0x10000         /* protocol requires cloning */
+#define RTF_WASCLONED   0x20000         /* route generated through cloning */
+#define RTF_PROTO3      0x40000         /* protocol specific routing flag */
+                                        /* 0x80000 unused */
+#define RTF_PINNED      0x100000        /* future use */
+#define RTF_LOCAL       0x200000        /* route represents a local address */
+#define RTF_BROADCAST   0x400000        /* route represents a bcast address */
+#define RTF_MULTICAST   0x800000        /* route represents a mcast address */
+#define RTF_IFSCOPE     0x1000000       /* has valid interface scope */
+#define RTF_CONDEMNED   0x2000000       /* defunct; no longer modifiable */
+#define RTF_IFREF       0x4000000       /* route holds a ref to interface */
+#define RTF_PROXY       0x8000000       /* proxying, no interface scope */
+#define RTF_ROUTER      0x10000000      /* host is a router */
+#define RTF_DEAD        0x20000000      /* Route entry is being freed */
+                                        /* 0x40000000 and up unassigned */
+
+#define RTPRF_OURS      RTF_PROTO3      /* set on routes we manage */
+#define RTF_BITS \
+	"\020\1UP\2GATEWAY\3HOST\4REJECT\5DYNAMIC\6MODIFIED\7DONE" \
+	"\10DELCLONE\11CLONING\12XRESOLVE\13LLINFO\14STATIC\15BLACKHOLE" \
+	"\16NOIFREF\17PROTO2\20PROTO1\21PRCLONING\22WASCLONED\23PROTO3" \
+	"\25PINNED\26LOCAL\27BROADCAST\30MULTICAST\31IFSCOPE\32CONDEMNED" \
+	"\33IFREF\34PROXY\35ROUTER"
+
+#define IS_DIRECT_HOSTROUTE(rt) \
+	(((rt)->rt_flags & (RTF_HOST | RTF_GATEWAY)) == RTF_HOST)
+/*
+ * Routing statistics.
+ */
+struct  rtstat {
+	short   rts_badredirect;        /* bogus redirect calls */
+	short   rts_dynamic;            /* routes created by redirects */
+	short   rts_newgateway;         /* routes modified by redirects */
+	short   rts_unreach;            /* lookups which failed */
+	short   rts_wildcard;           /* lookups satisfied by a wildcard */
+	short   rts_badrtgwroute;       /* route to gateway is not direct */
+};
+
+/*
+ * Structures for routing messages.
+ */
+struct rt_msghdr {
+	u_short rtm_msglen;     /* to skip over non-understood messages */
+	u_char  rtm_version;    /* future binary compatibility */
+	u_char  rtm_type;       /* message type */
+	u_short rtm_index;      /* index for associated ifp */
+	int     rtm_flags;      /* flags, incl. kern & message, e.g. DONE */
+	int     rtm_addrs;      /* bitmask identifying sockaddrs in msg */
+	pid_t   rtm_pid;        /* identify sender */
+	int     rtm_seq;        /* for sender to identify action */
+	int     rtm_errno;      /* why failed */
+	int     rtm_use;        /* from rtentry */
+	u_int32_t rtm_inits;    /* which metrics we are initializing */
+	struct rt_metrics rtm_rmx; /* metrics themselves */
+};
+
+struct rt_msghdr2 {
+	u_short rtm_msglen;     /* to skip over non-understood messages */
+	u_char  rtm_version;    /* future binary compatibility */
+	u_char  rtm_type;       /* message type */
+	u_short rtm_index;      /* index for associated ifp */
+	int     rtm_flags;      /* flags, incl. kern & message, e.g. DONE */
+	int     rtm_addrs;      /* bitmask identifying sockaddrs in msg */
+	int32_t rtm_refcnt;     /* reference count */
+	int     rtm_parentflags; /* flags of the parent route */
+	int     rtm_reserved;   /* reserved field set to 0 */
+	int     rtm_use;        /* from rtentry */
+	u_int32_t rtm_inits;    /* which metrics we are initializing */
+	struct rt_metrics rtm_rmx; /* metrics themselves */
+};
+
+
+#define RTM_VERSION     5       /* Up the ante and ignore older versions */
+
+/*
+ * Message types.
+ */
+#define RTM_ADD         0x1     /* Add Route */
+#define RTM_DELETE      0x2     /* Delete Route */
+#define RTM_CHANGE      0x3     /* Change Metrics or flags */
+#define RTM_GET         0x4     /* Report Metrics */
+#define RTM_LOSING      0x5     /* RTM_LOSING is no longer generated by xnu
+	                         *  and is deprecated */
+#define RTM_REDIRECT    0x6     /* Told to use different route */
+#define RTM_MISS        0x7     /* Lookup failed on this address */
+#define RTM_LOCK        0x8     /* fix specified metrics */
+#define RTM_OLDADD      0x9     /* caused by SIOCADDRT */
+#define RTM_OLDDEL      0xa     /* caused by SIOCDELRT */
+#define RTM_RESOLVE     0xb     /* req to resolve dst to LL addr */
+#define RTM_NEWADDR     0xc     /* address being added to iface */
+#define RTM_DELADDR     0xd     /* address being removed from iface */
+#define RTM_IFINFO      0xe     /* iface going up/down etc. */
+#define RTM_NEWMADDR    0xf     /* mcast group membership being added to if */
+#define RTM_DELMADDR    0x10    /* mcast group membership being deleted */
+#define RTM_IFINFO2     0x12    /* */
+#define RTM_NEWMADDR2   0x13    /* */
+#define RTM_GET2        0x14    /* */
+
+/*
+ * Bitmask values for rtm_inits and rmx_locks.
+ */
+#define RTV_MTU         0x1     /* init or lock _mtu */
+#define RTV_HOPCOUNT    0x2     /* init or lock _hopcount */
+#define RTV_EXPIRE      0x4     /* init or lock _expire */
+#define RTV_RPIPE       0x8     /* init or lock _recvpipe */
+#define RTV_SPIPE       0x10    /* init or lock _sendpipe */
+#define RTV_SSTHRESH    0x20    /* init or lock _ssthresh */
+#define RTV_RTT         0x40    /* init or lock _rtt */
+#define RTV_RTTVAR      0x80    /* init or lock _rttvar */
+
+/*
+ * Bitmask values for rtm_addrs.
+ */
+#define RTA_DST         0x1     /* destination sockaddr present */
+#define RTA_GATEWAY     0x2     /* gateway sockaddr present */
+#define RTA_NETMASK     0x4     /* netmask sockaddr present */
+#define RTA_GENMASK     0x8     /* cloning mask sockaddr present */
+#define RTA_IFP         0x10    /* interface name sockaddr present */
+#define RTA_IFA         0x20    /* interface addr sockaddr present */
+#define RTA_AUTHOR      0x40    /* sockaddr for author of redirect */
+#define RTA_BRD         0x80    /* for NEWADDR, broadcast or p-p dest addr */
+
+/*
+ * Index offsets for sockaddr array for alternate internal encoding.
+ */
+#define RTAX_DST        0       /* destination sockaddr present */
+#define RTAX_GATEWAY    1       /* gateway sockaddr present */
+#define RTAX_NETMASK    2       /* netmask sockaddr present */
+#define RTAX_GENMASK    3       /* cloning mask sockaddr present */
+#define RTAX_IFP        4       /* interface name sockaddr present */
+#define RTAX_IFA        5       /* interface addr sockaddr present */
+#define RTAX_AUTHOR     6       /* sockaddr for author of redirect */
+#define RTAX_BRD        7       /* for NEWADDR, broadcast or p-p dest addr */
+#define RTAX_MAX        8       /* size of array to allocate */
+
+struct rt_addrinfo {
+	int     rti_addrs;
+	struct  sockaddr *rti_info[RTAX_MAX];
+};
+
+
+#endif /* _NET_ROUTE_H_ */

net/netcheck/netcheck.go

@@ -6,9 +6,11 @@
 package netcheck
 
 import (
-	"bytes"
+	"bufio"
 	"context"
+	"crypto/rand"
 	"crypto/tls"
+	"encoding/binary"
 	"errors"
 	"fmt"
 	"io"
@@ -16,11 +18,14 @@ import (
 	"log"
 	"net"
 	"net/http"
+	"os"
 	"sort"
+	"strconv"
 	"sync"
 	"time"
 
 	"github.com/tcnksm/go-httpstat"
+	"go4.org/mem"
 	"inet.af/netaddr"
 	"tailscale.com/derp/derphttp"
 	"tailscale.com/net/dnscache"
@@ -33,16 +38,66 @@ import (
 	"tailscale.com/types/opt"
 )
 
+// Debugging and experimentation tweakables.
+var (
+	debugNetcheck, _ = strconv.ParseBool(os.Getenv("TS_DEBUG_NETCHECK"))
+)
+
+// The various default timeouts for things.
+const (
+	// overallProbeTimeout is the maximum amount of time netcheck will
+	// spend gathering a single report.
+	overallProbeTimeout = 5 * time.Second
+	// stunTimeout is the maximum amount of time netcheck will spend
+	// probing with STUN packets without getting a reply before
+	// switching to HTTP probing, on the assumption that outbound UDP
+	// is blocked.
+	stunProbeTimeout = 3 * time.Second
+	// hairpinCheckTimeout is the amount of time we wait for a
+	// hairpinned packet to come back.
+	hairpinCheckTimeout = 100 * time.Millisecond
+	// defaultActiveRetransmitTime is the retransmit interval we use
+	// for STUN probes when we're in steady state (not in start-up),
+	// but don't have previous latency information for a DERP
+	// node. This is a somewhat conservative guess because if we have
+	// no data, likely the DERP node is very far away and we have no
+	// data because we timed out the last time we probed it.
+	defaultActiveRetransmitTime = 200 * time.Millisecond
+	// defaultInitialRetransmitTime is the retransmit interval used
+	// when netcheck first runs. We have no past context to work with,
+	// and we want answers relatively quickly, so it's biased slightly
+	// more aggressive than defaultActiveRetransmitTime. A few extra
+	// packets at startup is fine.
+	defaultInitialRetransmitTime = 100 * time.Millisecond
+	// portMapServiceProbeTimeout is the time we wait for port mapping
+	// services (UPnP, NAT-PMP, PCP) to respond before we give up and
+	// decide that they're not there. Since these services are on the
+	// same LAN as this machine and a single L3 hop away, we don't
+	// give them much time to respond.
+	portMapServiceProbeTimeout = 100 * time.Millisecond
+)
+
 type Report struct {
-	UDP                   bool                  // UDP works
-	IPv6                  bool                  // IPv6 works
-	IPv4                  bool                  // IPv4 works
-	MappingVariesByDestIP opt.Bool              // for IPv4
-	HairPinning           opt.Bool              // for IPv4
-	PreferredDERP         int                   // or 0 for unknown
-	RegionLatency         map[int]time.Duration // keyed by DERP Region ID
-	RegionV4Latency       map[int]time.Duration // keyed by DERP Region ID
-	RegionV6Latency       map[int]time.Duration // keyed by DERP Region ID
+	UDP                   bool     // UDP works
+	IPv6                  bool     // IPv6 works
+	IPv4                  bool     // IPv4 works
+	MappingVariesByDestIP opt.Bool // for IPv4
+	HairPinning           opt.Bool // for IPv4
+
+	// UPnP is whether UPnP appears present on the LAN.
+	// Empty means not checked.
+	UPnP opt.Bool
+	// PMP is whether NAT-PMP appears present on the LAN.
+	// Empty means not checked.
+	PMP opt.Bool
+	// PCP is whether PCP appears present on the LAN.
+	// Empty means not checked.
+	PCP opt.Bool
+
+	PreferredDERP   int                   // or 0 for unknown
+	RegionLatency   map[int]time.Duration // keyed by DERP Region ID
+	RegionV4Latency map[int]time.Duration // keyed by DERP Region ID
+	RegionV6Latency map[int]time.Duration // keyed by DERP Region ID
 
 	GlobalV4 string // ip:port of global IPv4
 	GlobalV6 string // [ip]:port of global IPv6
@@ -50,6 +105,11 @@ type Report struct {
 	// TODO: update Clone when adding new fields
 }
 
+// AnyPortMappingChecked reports whether any of UPnP, PMP, or PCP are non-empty.
+func (r *Report) AnyPortMappingChecked() bool {
+	return r.UPnP != "" || r.PMP != "" || r.PCP != ""
+}
+
 func (r *Report) Clone() *Report {
 	if r == nil {
 		return nil
@@ -120,14 +180,14 @@ func (c *Client) logf(format string, a ...interface{}) {
 }
 
 func (c *Client) vlogf(format string, a ...interface{}) {
-	if c.Verbose {
+	if c.Verbose || debugNetcheck {
 		c.logf(format, a...)
 	}
 }
 
 // handleHairSTUN reports whether pkt (from src) was our magic hairpin
 // probe packet that we sent to ourselves.
-func (c *Client) handleHairSTUNLocked(pkt []byte, src *net.UDPAddr) bool {
+func (c *Client) handleHairSTUNLocked(pkt []byte, src netaddr.IPPort) bool {
 	rs := c.curState
 	if rs == nil {
 		return false
@@ -150,10 +210,8 @@ func (c *Client) MakeNextReportFull() {
 	c.mu.Unlock()
 }
 
-func (c *Client) ReceiveSTUNPacket(pkt []byte, src *net.UDPAddr) {
-	if src == nil || src.IP == nil {
-		panic("bogus src")
-	}
+func (c *Client) ReceiveSTUNPacket(pkt []byte, src netaddr.IPPort) {
+	c.vlogf("received STUN packet from %s", src)
 
 	c.mu.Lock()
 	if c.handleHairSTUNLocked(pkt, src) {
@@ -315,7 +373,7 @@ func makeProbePlan(dm *tailcfg.DERPMap, ifState *interfaces.State, last *Report)
 			n := reg.Nodes[try%len(reg.Nodes)]
 			prevLatency := last.RegionLatency[reg.RegionID] * 120 / 100
 			if prevLatency == 0 {
-				prevLatency = 200 * time.Millisecond
+				prevLatency = defaultActiveRetransmitTime
 			}
 			delay := time.Duration(try) * prevLatency
 			if do4 {
@@ -338,16 +396,12 @@ func makeProbePlan(dm *tailcfg.DERPMap, ifState *interfaces.State, last *Report)
 func makeProbePlanInitial(dm *tailcfg.DERPMap, ifState *interfaces.State) (plan probePlan) {
 	plan = make(probePlan)
 
-	// initialSTUNTimeout is only 100ms because some extra retransmits
-	// when starting up is tolerable.
-	const initialSTUNTimeout = 100 * time.Millisecond
-
 	for _, reg := range dm.Regions {
 		var p4 []probe
 		var p6 []probe
 		for try := 0; try < 3; try++ {
 			n := reg.Nodes[try%len(reg.Nodes)]
-			delay := time.Duration(try) * initialSTUNTimeout
+			delay := time.Duration(try) * defaultInitialRetransmitTime
 			if ifState.HaveV4 && nodeMight4(n) {
 				p4 = append(p4, probe{delay: delay, node: n.Name, proto: probeIPv4})
 			}
@@ -421,7 +475,9 @@ func (c *Client) readPackets(ctx context.Context, pc net.PacketConn) {
 		if !stun.Is(pkt) {
 			continue
 		}
-		c.ReceiveSTUNPacket(pkt, ua)
+		if ipp, ok := netaddr.FromStdAddr(ua.IP, ua.Port, ua.Zone); ok {
+			c.ReceiveSTUNPacket(pkt, ipp)
+		}
 	}
 }
 
@@ -429,13 +485,14 @@ func (c *Client) readPackets(ctx context.Context, pc net.PacketConn) {
 type reportState struct {
 	c           *Client
 	hairTX      stun.TxID
-	gotHairSTUN chan *net.UDPAddr
+	gotHairSTUN chan netaddr.IPPort
 	hairTimeout chan struct{} // closed on timeout
 	pc4         STUNConn
 	pc6         STUNConn
 	pc4Hair     net.PacketConn
 	incremental bool // doing a lite, follow-up netcheck
 	stopProbeCh chan struct{}
+	waitPortMap sync.WaitGroup
 
 	mu            sync.Mutex
 	sentHairCheck bool
@@ -500,7 +557,7 @@ func (rs *reportState) startHairCheckLocked(dst netaddr.IPPort) {
 	ua := dst.UDPAddr()
 	rs.pc4Hair.WriteTo(stun.Request(rs.hairTX), ua)
 	rs.c.vlogf("sent haircheck to %v", ua)
-	time.AfterFunc(500*time.Millisecond, func() { close(rs.hairTimeout) })
+	time.AfterFunc(hairpinCheckTimeout, func() { close(rs.hairTimeout) })
 }
 
 func (rs *reportState) waitHairCheck(ctx context.Context) {
@@ -521,6 +578,7 @@ func (rs *reportState) waitHairCheck(ctx context.Context) {
 	case <-rs.gotHairSTUN:
 		ret.HairPinning.Set(true)
 	case <-rs.hairTimeout:
+		rs.c.vlogf("hairCheck timeout")
 		ret.HairPinning.Set(false)
 	default:
 		select {
@@ -601,6 +659,102 @@ func (rs *reportState) stopProbes() {
 	}
 }
 
+func (rs *reportState) setOptBool(b *opt.Bool, v bool) {
+	rs.mu.Lock()
+	defer rs.mu.Unlock()
+	b.Set(v)
+}
+
+func (rs *reportState) probePortMapServices() {
+	defer rs.waitPortMap.Done()
+	gw, myIP, ok := interfaces.LikelyHomeRouterIP()
+	if !ok {
+		return
+	}
+
+	rs.setOptBool(&rs.report.UPnP, false)
+	rs.setOptBool(&rs.report.PMP, false)
+	rs.setOptBool(&rs.report.PCP, false)
+
+	port1900 := netaddr.IPPort{IP: gw, Port: 1900}.UDPAddr()
+	port5351 := netaddr.IPPort{IP: gw, Port: 5351}.UDPAddr()
+
+	rs.c.logf("probePortMapServices: me %v -> gw %v", myIP, gw)
+
+	// Create a UDP4 socket used just for querying for UPnP, NAT-PMP, and PCP.
+	uc, err := netns.Listener().ListenPacket(context.Background(), "udp4", ":0")
+	if err != nil {
+		rs.c.logf("probePortMapServices: %v", err)
+		return
+	}
+	defer uc.Close()
+	tempPort := uc.LocalAddr().(*net.UDPAddr).Port
+	uc.SetReadDeadline(time.Now().Add(portMapServiceProbeTimeout))
+
+	// Send request packets for all three protocols.
+	uc.WriteTo(uPnPPacket, port1900)
+	uc.WriteTo(pmpPacket, port5351)
+	uc.WriteTo(pcpPacket(myIP, tempPort, false), port5351)
+
+	res := make([]byte, 1500)
+	for {
+		n, addr, err := uc.ReadFrom(res)
+		if err != nil {
+			return
+		}
+		switch addr.(*net.UDPAddr).Port {
+		case 1900:
+			if mem.Contains(mem.B(res[:n]), mem.S(":InternetGatewayDevice:")) {
+				rs.setOptBool(&rs.report.UPnP, true)
+			}
+		case 5351:
+			if n == 12 && res[0] == 0x00 { // right length and version 0
+				rs.setOptBool(&rs.report.PMP, true)
+			}
+			if n == 60 && res[0] == 0x02 { // right length and version 2
+				rs.setOptBool(&rs.report.PCP, true)
+
+				// And now delete the mapping.
+				// (PCP is the only protocol of the three that requires
+				// we cause a side effect to detect whether it's present,
+				// so we need to redo that side effect now.)
+				uc.WriteTo(pcpPacket(myIP, tempPort, true), port5351)
+			}
+		}
+	}
+}
+
+var pmpPacket = []byte{0, 0} // version 0, opcode 0 = "Public address request"
+
+var uPnPPacket = []byte("M-SEARCH * HTTP/1.1\r\n" +
+	"HOST: 239.255.255.250:1900\r\n" +
+	"ST: ssdp:all\r\n" +
+	"MAN: \"ssdp:discover\"\r\n" +
+	"MX: 2\r\n\r\n")
+
+var v4unspec, _ = netaddr.ParseIP("0.0.0.0")
+
+func pcpPacket(myIP netaddr.IP, mapToLocalPort int, delete bool) []byte {
+	const udpProtoNumber = 17
+	lifetimeSeconds := uint32(1)
+	if delete {
+		lifetimeSeconds = 0
+	}
+	const opMap = 1
+	pkt := make([]byte, (32+32+128)/8+(96+8+24+16+16+128)/8)
+	pkt[0] = 2 // version
+	pkt[1] = opMap
+	binary.BigEndian.PutUint32(pkt[4:8], lifetimeSeconds)
+	myIP16 := myIP.As16()
+	copy(pkt[8:], myIP16[:])
+	rand.Read(pkt[24 : 24+12])
+	pkt[36] = udpProtoNumber
+	binary.BigEndian.PutUint16(pkt[40:], uint16(mapToLocalPort))
+	v4unspec16 := v4unspec.As16()
+	copy(pkt[40:], v4unspec16[:])
+	return pkt
+}
+
 func newReport() *Report {
 	return &Report{
 		RegionLatency:   make(map[int]time.Duration),
@@ -613,15 +767,10 @@ func newReport() *Report {
 //
 // It may not be called concurrently with itself.
 func (c *Client) GetReport(ctx context.Context, dm *tailcfg.DERPMap) (*Report, error) {
-	// Wait for STUN for 3 seconds, but then give HTTP probing
-	// another 2 seconds if all UDP failed.
-	const overallTimeout = 5 * time.Second
-	const stunTimeout = 3 * time.Second
-
 	// Mask user context with ours that we guarantee to cancel so
 	// we can depend on it being closed in goroutines later.
 	// (User ctx might be context.Background, etc)
-	ctx, cancel := context.WithTimeout(ctx, overallTimeout)
+	ctx, cancel := context.WithTimeout(ctx, overallProbeTimeout)
 	defer cancel()
 
 	if dm == nil {
@@ -638,7 +787,7 @@ func (c *Client) GetReport(ctx context.Context, dm *tailcfg.DERPMap) (*Report, e
 		report:      newReport(),
 		inFlight:    map[stun.TxID]func(netaddr.IPPort){},
 		hairTX:      stun.NewTxID(), // random payload
-		gotHairSTUN: make(chan *net.UDPAddr, 1),
+		gotHairSTUN: make(chan netaddr.IPPort, 1),
 		hairTimeout: make(chan struct{}),
 		stopProbeCh: make(chan struct{}, 1),
 	}
@@ -673,6 +822,22 @@ func (c *Client) GetReport(ctx context.Context, dm *tailcfg.DERPMap) (*Report, e
 	}
 	defer rs.pc4Hair.Close()
 
+	rs.waitPortMap.Add(1)
+	go rs.probePortMapServices()
+
+	// At least the Apple Airport Extreme doesn't allow hairpin
+	// sends from a private socket until it's seen traffic from
+	// that src IP:port to something else out on the internet.
+	//
+	// See https://github.com/tailscale/tailscale/issues/188#issuecomment-600728643
+	//
+	// And it seems that even sending to a likely-filtered RFC 5737
+	// documentation-only IPv4 range is enough to set up the mapping.
+	// So do that for now. In the future we might want to classify networks
+	// that do and don't require this separately. But for now help it.
+	const documentationIP = "203.0.113.1"
+	rs.pc4Hair.WriteTo([]byte("tailscale netcheck; see https://github.com/tailscale/tailscale/issues/188"), &net.UDPAddr{IP: net.ParseIP(documentationIP), Port: 12345})
+
 	if f := c.GetSTUNConn4; f != nil {
 		rs.pc4 = f()
 	} else {
@@ -714,7 +879,7 @@ func (c *Client) GetReport(ctx context.Context, dm *tailcfg.DERPMap) (*Report, e
 		}(probeSet)
 	}
 
-	stunTimer := time.NewTimer(stunTimeout)
+	stunTimer := time.NewTimer(stunProbeTimeout)
 	defer stunTimer.Stop()
 
 	select {
@@ -727,6 +892,9 @@ func (c *Client) GetReport(ctx context.Context, dm *tailcfg.DERPMap) (*Report, e
 	}
 
 	rs.waitHairCheck(ctx)
+	c.vlogf("hairCheck done")
+	rs.waitPortMap.Wait()
+	c.vlogf("portMap done")
 	rs.stopTimers()
 
 	// Try HTTPS latency check if all STUN probes failed due to UDP presumably being blocked.
@@ -781,7 +949,7 @@ func (c *Client) GetReport(ctx context.Context, dm *tailcfg.DERPMap) (*Report, e
 
 func (c *Client) measureHTTPSLatency(ctx context.Context, reg *tailcfg.DERPRegion) (time.Duration, netaddr.IP, error) {
 	var result httpstat.Result
-	ctx, cancel := context.WithTimeout(httpstat.WithHTTPStat(ctx, &result), 5*time.Second)
+	ctx, cancel := context.WithTimeout(httpstat.WithHTTPStat(ctx, &result), overallProbeTimeout)
 	defer cancel()
 
 	var ip netaddr.IP
@@ -841,42 +1009,48 @@ func (c *Client) measureHTTPSLatency(ctx context.Context, reg *tailcfg.DERPRegio
 }
 
 func (c *Client) logConciseReport(r *Report, dm *tailcfg.DERPMap) {
-	buf := bytes.NewBuffer(make([]byte, 0, 256)) // empirically: 5 DERPs + IPv6 == ~233 bytes
-	fmt.Fprintf(buf, "udp=%v", r.UDP)
-	if !r.IPv4 {
-		fmt.Fprintf(buf, " v4=%v", r.IPv4)
-	}
-	fmt.Fprintf(buf, " v6=%v", r.IPv6)
-	fmt.Fprintf(buf, " mapvarydest=%v", r.MappingVariesByDestIP)
-	fmt.Fprintf(buf, " hair=%v", r.HairPinning)
-	if r.GlobalV4 != "" {
-		fmt.Fprintf(buf, " v4a=%v", r.GlobalV4)
-	}
-	if r.GlobalV6 != "" {
-		fmt.Fprintf(buf, " v6a=%v", r.GlobalV6)
-	}
-	fmt.Fprintf(buf, " derp=%v", r.PreferredDERP)
-	if r.PreferredDERP != 0 {
-		fmt.Fprintf(buf, " derpdist=")
-		for i, rid := range dm.RegionIDs() {
-			if i != 0 {
-				buf.WriteByte(',')
-			}
+	c.logf("%v", logger.ArgWriter(func(w *bufio.Writer) {
+		fmt.Fprintf(w, "udp=%v", r.UDP)
+		if !r.IPv4 {
+			fmt.Fprintf(w, " v4=%v", r.IPv4)
+		}
+
+		fmt.Fprintf(w, " v6=%v", r.IPv6)
+		fmt.Fprintf(w, " mapvarydest=%v", r.MappingVariesByDestIP)
+		fmt.Fprintf(w, " hair=%v", r.HairPinning)
+		if r.AnyPortMappingChecked() {
+			fmt.Fprintf(w, " portmap=%v%v%v", conciseOptBool(r.UPnP, "U"), conciseOptBool(r.PMP, "M"), conciseOptBool(r.PCP, "C"))
+		} else {
+			fmt.Fprintf(w, " portmap=?")
+		}
+		if r.GlobalV4 != "" {
+			fmt.Fprintf(w, " v4a=%v", r.GlobalV4)
+		}
+		if r.GlobalV6 != "" {
+			fmt.Fprintf(w, " v6a=%v", r.GlobalV6)
+		}
+		fmt.Fprintf(w, " derp=%v", r.PreferredDERP)
+		if r.PreferredDERP != 0 {
+			fmt.Fprintf(w, " derpdist=")
 			needComma := false
-			if d := r.RegionV4Latency[rid]; d != 0 {
-				fmt.Fprintf(buf, "%dv4:%v", rid, d.Round(time.Millisecond))
-				needComma = true
-			}
-			if d := r.RegionV6Latency[rid]; d != 0 {
-				if needComma {
-					buf.WriteByte(',')
+			for _, rid := range dm.RegionIDs() {
+				if d := r.RegionV4Latency[rid]; d != 0 {
+					if needComma {
+						w.WriteByte(',')
+					}
+					fmt.Fprintf(w, "%dv4:%v", rid, d.Round(time.Millisecond))
+					needComma = true
+				}
+				if d := r.RegionV6Latency[rid]; d != 0 {
+					if needComma {
+						w.WriteByte(',')
+					}
+					fmt.Fprintf(w, "%dv6:%v", rid, d.Round(time.Millisecond))
+					needComma = true
 				}
-				fmt.Fprintf(buf, "%dv6:%v", rid, d.Round(time.Millisecond))
 			}
 		}
-	}
-
-	c.logf("%s", buf.Bytes())
+	}))
 }
 
 func (c *Client) timeNow() time.Time {
@@ -1009,6 +1183,20 @@ func (c *Client) nodeAddr(ctx context.Context, n *tailcfg.DERPNode, proto probeP
 	if port < 0 || port > 1<<16-1 {
 		return nil
 	}
+	if n.STUNTestIP != "" {
+		ip, err := netaddr.ParseIP(n.STUNTestIP)
+		if err != nil {
+			return nil
+		}
+		if proto == probeIPv4 && ip.Is6() {
+			return nil
+		}
+		if proto == probeIPv6 && ip.Is4() {
+			return nil
+		}
+		return netaddr.IPPort{IP: ip, Port: uint16(port)}.UDPAddr()
+	}
+
 	switch proto {
 	case probeIPv4:
 		if n.IPv4 != "" {
@@ -1016,7 +1204,7 @@ func (c *Client) nodeAddr(ctx context.Context, n *tailcfg.DERPNode, proto probeP
 			if !ip.Is4() {
 				return nil
 			}
-			return netaddr.IPPort{ip, uint16(port)}.UDPAddr()
+			return netaddr.IPPort{IP: ip, Port: uint16(port)}.UDPAddr()
 		}
 	case probeIPv6:
 		if n.IPv6 != "" {
@@ -1024,7 +1212,7 @@ func (c *Client) nodeAddr(ctx context.Context, n *tailcfg.DERPNode, proto probeP
 			if !ip.Is6() {
 				return nil
 			}
-			return netaddr.IPPort{ip, uint16(port)}.UDPAddr()
+			return netaddr.IPPort{IP: ip, Port: uint16(port)}.UDPAddr()
 		}
 	default:
 		return nil
@@ -1057,3 +1245,17 @@ func maxDurationValue(m map[int]time.Duration) (max time.Duration) {
 	}
 	return max
 }
+
+func conciseOptBool(b opt.Bool, trueVal string) string {
+	if b == "" {
+		return "_"
+	}
+	v, ok := b.Get()
+	if !ok {
+		return "x"
+	}
+	if v {
+		return trueVal
+	}
+	return ""
+}

net/netcheck/netcheck_test.go

@@ -5,6 +5,7 @@
 package netcheck
 
 import (
+	"bytes"
 	"context"
 	"fmt"
 	"net"
@@ -15,6 +16,7 @@ import (
 	"testing"
 	"time"
 
+	"inet.af/netaddr"
 	"tailscale.com/net/interfaces"
 	"tailscale.com/net/stun"
 	"tailscale.com/net/stun/stuntest"
@@ -26,14 +28,14 @@ func TestHairpinSTUN(t *testing.T) {
 	c := &Client{
 		curState: &reportState{
 			hairTX:      tx,
-			gotHairSTUN: make(chan *net.UDPAddr, 1),
+			gotHairSTUN: make(chan netaddr.IPPort, 1),
 		},
 	}
 	req := stun.Request(tx)
 	if !stun.Is(req) {
 		t.Fatal("expected STUN message")
 	}
-	if !c.handleHairSTUNLocked(req, nil) {
+	if !c.handleHairSTUNLocked(req, netaddr.IPPort{}) {
 		t.Fatal("expected true")
 	}
 	select {
@@ -98,6 +100,9 @@ func TestWorksWhenUDPBlocked(t *testing.T) {
 		t.Fatal(err)
 	}
 	want := newReport()
+	r.UPnP = ""
+	r.PMP = ""
+	r.PCP = ""
 
 	if !reflect.DeepEqual(r, want) {
 		t.Errorf("mismatch\n got: %+v\nwant: %+v\n", r, want)
@@ -443,3 +448,114 @@ func (p probeProto) String() string {
 	}
 	return "?"
 }
+
+func TestLogConciseReport(t *testing.T) {
+	dm := &tailcfg.DERPMap{
+		Regions: map[int]*tailcfg.DERPRegion{
+			1: nil,
+			2: nil,
+			3: nil,
+		},
+	}
+	const ms = time.Millisecond
+	tests := []struct {
+		name string
+		r    *Report
+		want string
+	}{
+		{
+			name: "no_udp",
+			r:    &Report{},
+			want: "udp=false v4=false v6=false mapvarydest= hair= portmap=? derp=0",
+		},
+		{
+			name: "ipv4_one_region",
+			r: &Report{
+				UDP:           true,
+				IPv4:          true,
+				PreferredDERP: 1,
+				RegionLatency: map[int]time.Duration{
+					1: 10 * ms,
+				},
+				RegionV4Latency: map[int]time.Duration{
+					1: 10 * ms,
+				},
+			},
+			want: "udp=true v6=false mapvarydest= hair= portmap=? derp=1 derpdist=1v4:10ms",
+		},
+		{
+			name: "ipv4_all_region",
+			r: &Report{
+				UDP:           true,
+				IPv4:          true,
+				PreferredDERP: 1,
+				RegionLatency: map[int]time.Duration{
+					1: 10 * ms,
+					2: 20 * ms,
+					3: 30 * ms,
+				},
+				RegionV4Latency: map[int]time.Duration{
+					1: 10 * ms,
+					2: 20 * ms,
+					3: 30 * ms,
+				},
+			},
+			want: "udp=true v6=false mapvarydest= hair= portmap=? derp=1 derpdist=1v4:10ms,2v4:20ms,3v4:30ms",
+		},
+		{
+			name: "ipboth_all_region",
+			r: &Report{
+				UDP:           true,
+				IPv4:          true,
+				IPv6:          true,
+				PreferredDERP: 1,
+				RegionLatency: map[int]time.Duration{
+					1: 10 * ms,
+					2: 20 * ms,
+					3: 30 * ms,
+				},
+				RegionV4Latency: map[int]time.Duration{
+					1: 10 * ms,
+					2: 20 * ms,
+					3: 30 * ms,
+				},
+				RegionV6Latency: map[int]time.Duration{
+					1: 10 * ms,
+					2: 20 * ms,
+					3: 30 * ms,
+				},
+			},
+			want: "udp=true v6=true mapvarydest= hair= portmap=? derp=1 derpdist=1v4:10ms,1v6:10ms,2v4:20ms,2v6:20ms,3v4:30ms,3v6:30ms",
+		},
+		{
+			name: "portmap_all",
+			r: &Report{
+				UDP:  true,
+				UPnP: "true",
+				PMP:  "true",
+				PCP:  "true",
+			},
+			want: "udp=true v4=false v6=false mapvarydest= hair= portmap=UMC derp=0",
+		},
+		{
+			name: "portmap_some",
+			r: &Report{
+				UDP:  true,
+				UPnP: "true",
+				PMP:  "false",
+				PCP:  "true",
+			},
+			want: "udp=true v4=false v6=false mapvarydest= hair= portmap=UC derp=0",
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			var buf bytes.Buffer
+			c := &Client{Logf: func(f string, a ...interface{}) { fmt.Fprintf(&buf, f, a...) }}
+			c.logConciseReport(tt.r, dm)
+			if got := buf.String(); got != tt.want {
+				t.Errorf("unexpected result.\n got: %#q\nwant: %#q\n", got, tt.want)
+			}
+		})
+	}
+}

net/netns/netns_linux.go

@@ -5,19 +5,15 @@
 package netns
 
 import (
-	"bufio"
-	"bytes"
-	"errors"
 	"flag"
 	"fmt"
-	"io"
 	"os"
 	"os/exec"
-	"strings"
 	"sync"
 	"syscall"
 
 	"golang.org/x/sys/unix"
+	"tailscale.com/net/interfaces"
 )
 
 // tailscaleBypassMark is the mark indicating that packets originating
@@ -26,7 +22,7 @@ import (
 //
 // Keep this in sync with tailscaleBypassMark in
 // wgengine/router/router_linux.go.
-const tailscaleBypassMark = 0x20000
+const tailscaleBypassMark = 0x80000
 
 // ipRuleOnce is the sync.Once & cached value for ipRuleAvailable.
 var ipRuleOnce struct {
@@ -43,47 +39,6 @@ func ipRuleAvailable() bool {
 	return ipRuleOnce.v
 }
 
-var zeroRouteBytes = []byte("00000000")
-
-// defaultRouteInterface returns the name of the network interface that owns
-// the default route, not including any tailscale interfaces. We only use
-// this in SO_BINDTODEVICE mode.
-func defaultRouteInterface() (string, error) {
-	f, err := os.Open("/proc/net/route")
-	if err != nil {
-		return "", err
-	}
-	defer f.Close()
-	br := bufio.NewReaderSize(f, 128)
-	for {
-		line, err := br.ReadSlice('\n')
-		if err == io.EOF {
-			break
-		}
-		if err != nil {
-			return "", err
-		}
-		if !bytes.Contains(line, zeroRouteBytes) {
-			continue
-		}
-		fields := strings.Fields(string(line))
-		ifc := fields[0]
-		ip := fields[1]
-		netmask := fields[7]
-
-		if strings.HasPrefix(ifc, "tailscale") ||
-			strings.HasPrefix(ifc, "wg") {
-			continue
-		}
-		if ip == "00000000" && netmask == "00000000" {
-			// default route
-			return ifc, nil // interface name
-		}
-	}
-
-	return "", errors.New("no default routes found")
-}
-
 // ignoreErrors returns true if we should ignore setsocketopt errors in
 // this instance.
 func ignoreErrors() bool {
@@ -133,7 +88,7 @@ func setBypassMark(fd uintptr) error {
 }
 
 func bindToDevice(fd uintptr) error {
-	ifc, err := defaultRouteInterface()
+	ifc, err := interfaces.DefaultRouteInterface()
 	if err != nil {
 		// Make sure we bind to *some* interface,
 		// or we could get a routing loop.

net/netns/netns_linux_test.go

@@ -49,12 +49,3 @@ func TestBypassMarkInSync(t *testing.T) {
 	}
 	t.Errorf("tailscaleBypassMark not found in router_linux.go")
 }
-
-func BenchmarkDefaultRouteInterface(b *testing.B) {
-	b.ReportAllocs()
-	for i := 0; i < b.N; i++ {
-		if _, err := defaultRouteInterface(); err != nil {
-			b.Fatal(err)
-		}
-	}
-}

net/stun/stun.go

@@ -29,8 +29,6 @@ const (
 	bindingRequest = "\x00\x01"
 	magicCookie    = "\x21\x12\xa4\x42"
 	lenFingerprint = 8 // 2+byte header + 2-byte length + 4-byte crc32
-	ipv4Len        = 4
-	ipv6Len        = 16
 	headerLen      = 20
 )
 
@@ -135,7 +133,6 @@ var (
 func foreachAttr(b []byte, fn func(attrType uint16, a []byte) error) error {
 	for len(b) > 0 {
 		if len(b) < 4 {
-			return errors.New("effed-f1")
 			return ErrMalformedAttrs
 		}
 		attrType := binary.BigEndian.Uint16(b[:2])
@@ -143,7 +140,6 @@ func foreachAttr(b []byte, fn func(attrType uint16, a []byte) error) error {
 		attrLenPad := attrLen % 4
 		b = b[4:]
 		if attrLen+attrLenPad > len(b) {
-			return errors.New("effed-f2")
 			return ErrMalformedAttrs
 		}
 		if err := fn(attrType, b[:attrLen]); err != nil {
@@ -161,9 +157,9 @@ func Response(txID TxID, ip net.IP, port uint16) []byte {
 	}
 	var fam byte
 	switch len(ip) {
-	case 4:
+	case net.IPv4len:
 		fam = 1
-	case 16:
+	case net.IPv6len:
 		fam = 2
 	default:
 		return nil
@@ -194,8 +190,6 @@ func Response(txID TxID, ip net.IP, port uint16) []byte {
 	return b
 }
 
-func beu16(b []byte) uint16 { return binary.BigEndian.Uint16(b) }
-
 // ParseResponse parses a successful binding response STUN packet.
 // The IP address is extracted from the XOR-MAPPED-ADDRESS attribute.
 // The returned addr slice is owned by the caller and does not alias b.
@@ -207,7 +201,7 @@ func ParseResponse(b []byte) (tID TxID, addr []byte, port uint16, err error) {
 	if b[0] != 0x01 || b[1] != 0x01 {
 		return tID, nil, 0, ErrNotSuccessResponse
 	}
-	attrsLen := int(beu16(b[2:4]))
+	attrsLen := int(binary.BigEndian.Uint16(b[2:4]))
 	b = b[headerLen:] // remove STUN header
 	if attrsLen > len(b) {
 		return tID, nil, 0, ErrMalformedAttrs
@@ -272,7 +266,7 @@ func xorMappedAddress(tID TxID, b []byte) (addr []byte, port uint16, err error)
 	if len(b) < 4 {
 		return nil, 0, ErrMalformedAttrs
 	}
-	xorPort := beu16(b[2:4])
+	xorPort := binary.BigEndian.Uint16(b[2:4])
 	addrField := b[4:]
 	port = xorPort ^ 0x2112 // first half of magicCookie
 
@@ -298,9 +292,9 @@ func xorMappedAddress(tID TxID, b []byte) (addr []byte, port uint16, err error)
 func familyAddrLen(fam byte) int {
 	switch fam {
 	case 0x01: // IPv4
-		return ipv4Len
+		return net.IPv4len
 	case 0x02: // IPv6
-		return ipv6Len
+		return net.IPv6len
 	default:
 		return 0
 	}

net/stun/stuntest/stuntest.go

@@ -6,6 +6,7 @@
 package stuntest
 
 import (
+	"context"
 	"fmt"
 	"net"
 	"strconv"
@@ -16,6 +17,7 @@ import (
 	"inet.af/netaddr"
 	"tailscale.com/net/stun"
 	"tailscale.com/tailcfg"
+	"tailscale.com/types/nettype"
 )
 
 type stunStats struct {
@@ -25,18 +27,22 @@ type stunStats struct {
 }
 
 func Serve(t *testing.T) (addr *net.UDPAddr, cleanupFn func()) {
+	return ServeWithPacketListener(t, nettype.Std{})
+}
+
+func ServeWithPacketListener(t *testing.T, ln nettype.PacketListener) (addr *net.UDPAddr, cleanupFn func()) {
 	t.Helper()
 
 	// TODO(crawshaw): use stats to test re-STUN logic
 	var stats stunStats
 
-	pc, err := net.ListenPacket("udp4", ":0")
+	pc, err := ln.ListenPacket(context.Background(), "udp4", ":0")
 	if err != nil {
 		t.Fatalf("failed to open STUN listener: %v", err)
 	}
-	addr = &net.UDPAddr{
-		IP:   net.ParseIP("127.0.0.1"),
-		Port: pc.LocalAddr().(*net.UDPAddr).Port,
+	addr = pc.LocalAddr().(*net.UDPAddr)
+	if len(addr.IP) == 0 || addr.IP.IsUnspecified() {
+		addr.IP = net.ParseIP("127.0.0.1")
 	}
 	doneCh := make(chan struct{})
 	go runSTUN(t, pc, &stats, doneCh)

net/tsaddr/tsaddr.go

@@ -0,0 +1,74 @@
+// Copyright (c) 2020 Tailscale Inc & AUTHORS All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+// Package tsaddr handles Tailscale-specific IPs and ranges.
+package tsaddr
+
+import (
+	"sync"
+
+	"inet.af/netaddr"
+)
+
+// ChromeOSVMRange returns the subset of the CGNAT IPv4 range used by
+// ChromeOS to interconnect the host OS to containers and VMs. We
+// avoid allocating Tailscale IPs from it, to avoid conflicts.
+func ChromeOSVMRange() netaddr.IPPrefix {
+	chromeOSRange.Do(func() { mustPrefix(&chromeOSRange.v, "100.115.92.0/23") })
+	return chromeOSRange.v
+}
+
+var chromeOSRange oncePrefix
+
+// CGNATRange returns the Carrier Grade NAT address range that
+// is the superset range that Tailscale assigns out of.
+// See https://tailscale.com/kb/1015/100.x-addresses.
+// Note that Tailscale does not assign out of the ChromeOSVMRange.
+func CGNATRange() netaddr.IPPrefix {
+	cgnatRange.Do(func() { mustPrefix(&cgnatRange.v, "100.64.0.0/10") })
+	return cgnatRange.v
+}
+
+var cgnatRange oncePrefix
+
+// TailscaleServiceIP returns the listen address of services
+// provided by Tailscale itself such as the Magic DNS proxy.
+func TailscaleServiceIP() netaddr.IP {
+	serviceIP.Do(func() { mustIP(&serviceIP.v, "100.100.100.100") })
+	return serviceIP.v
+}
+
+var serviceIP onceIP
+
+// IsTailscaleIP reports whether ip is an IP address in a range that
+// Tailscale assigns from.
+func IsTailscaleIP(ip netaddr.IP) bool {
+	return CGNATRange().Contains(ip) && !ChromeOSVMRange().Contains(ip)
+}
+
+func mustPrefix(v *netaddr.IPPrefix, prefix string) {
+	var err error
+	*v, err = netaddr.ParseIPPrefix(prefix)
+	if err != nil {
+		panic(err)
+	}
+}
+
+type oncePrefix struct {
+	sync.Once
+	v netaddr.IPPrefix
+}
+
+func mustIP(v *netaddr.IP, ip string) {
+	var err error
+	*v, err = netaddr.ParseIP(ip)
+	if err != nil {
+		panic(err)
+	}
+}
+
+type onceIP struct {
+	sync.Once
+	v netaddr.IP
+}

net/tsaddr/tsaddr_test.go

@@ -0,0 +1,19 @@
+// Copyright (c) 2020 Tailscale Inc & AUTHORS All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package tsaddr
+
+import "testing"
+
+func TestChromeOSVMRange(t *testing.T) {
+	if got, want := ChromeOSVMRange().String(), "100.115.92.0/23"; got != want {
+		t.Errorf("got %q; want %q", got, want)
+	}
+}
+
+func TestCGNATRange(t *testing.T) {
+	if got, want := CGNATRange().String(), "100.64.0.0/10"; got != want {
+		t.Errorf("got %q; want %q", got, want)
+	}
+}

paths/paths.go

@@ -11,9 +11,15 @@ import (
 	"runtime"
 )
 
-// LegacyConfigPath is the path used by the pre-tailscaled "relaynode"
-// daemon's config file.
-const LegacyConfigPath = "/var/lib/tailscale/relay.conf"
+// LegacyConfigPath returns the path used by the pre-tailscaled
+// "relaynode" daemon's config file. It returns the empty string for
+// platforms where relaynode never ran.
+func LegacyConfigPath() string {
+	if runtime.GOOS == "windows" {
+		return ""
+	}
+	return "/var/lib/tailscale/relay.conf"
+}
 
 // DefaultTailscaledSocket returns the path to the tailscaled Unix socket
 // or the empty string if there's no reasonable default.

portlist/netstat_exec.go

@@ -13,12 +13,22 @@ import (
 	exec "tailscale.com/tempfork/osexec"
 )
 
+var osHideWindow func(*exec.Cmd) // non-nil on Windows; see portlist_windows.go
+
+// hideWindow returns c. On Windows it first sets SysProcAttr.HideWindow.
+func hideWindow(c *exec.Cmd) *exec.Cmd {
+	if osHideWindow != nil {
+		osHideWindow(c)
+	}
+	return c
+}
+
 func listPortsNetstat(arg string) (List, error) {
 	exe, err := exec.LookPath("netstat")
 	if err != nil {
 		return nil, fmt.Errorf("netstat: lookup: %v", err)
 	}
-	output, err := exec.Command(exe, arg).Output()
+	output, err := hideWindow(exec.Command(exe, arg)).Output()
 	if err != nil {
 		xe, ok := err.(*exec.ExitError)
 		stderr := ""

portlist/portlist_linux.go

@@ -10,12 +10,15 @@ import (
 	"io"
 	"io/ioutil"
 	"os"
+	"runtime"
 	"sort"
 	"strconv"
 	"strings"
+	"syscall"
 	"time"
 
 	"golang.org/x/sys/unix"
+	"tailscale.com/syncs"
 )
 
 // Reading the sockfiles on Linux is very fast, so we can do it often.
@@ -26,13 +29,30 @@ const pollInterval = 1 * time.Second
 var sockfiles = []string{"/proc/net/tcp", "/proc/net/udp"}
 var protos = []string{"tcp", "udp"}
 
+var sawProcNetPermissionErr syncs.AtomicBool
+
 func listPorts() (List, error) {
+	if sawProcNetPermissionErr.Get() {
+		return nil, nil
+	}
 	l := []Port{}
 
 	for pi, fname := range sockfiles {
 		proto := protos[pi]
 
+		// Android 10+ doesn't allow access to this anymore.
+		// https://developer.android.com/about/versions/10/privacy/changes#proc-net-filesystem
+		// Ignore it rather than have the system log about our violation.
+		if runtime.GOOS == "android" && syscall.Access(fname, unix.R_OK) != nil {
+			sawProcNetPermissionErr.Set(true)
+			return nil, nil
+		}
+
 		f, err := os.Open(fname)
+		if os.IsPermission(err) {
+			sawProcNetPermissionErr.Set(true)
+			return nil, nil
+		}
 		if err != nil {
 			return nil, fmt.Errorf("%s: %s", fname, err)
 		}
@@ -96,7 +116,18 @@ func addProcesses(pl []Port) ([]Port, error) {
 	}
 
 	err := foreachPID(func(pid string) error {
-		fdDir, err := os.Open(fmt.Sprintf("/proc/%s/fd", pid))
+		fdPath := fmt.Sprintf("/proc/%s/fd", pid)
+
+		// Android logs a bunch of audit violations in logcat
+		// if we try to open things we don't have access
+		// to. So on Android only, ask if we have permission
+		// rather than just trying it to determine whether we
+		// have permission.
+		if runtime.GOOS == "android" && syscall.Access(fdPath, unix.R_OK) != nil {
+			return nil
+		}
+
+		fdDir, err := os.Open(fdPath)
 		if err != nil {
 			// Can't open fd list for this pid. Maybe
 			// don't have access. Ignore it.

portlist/portlist_windows.go

@@ -5,7 +5,10 @@
 package portlist
 
 import (
+	"syscall"
 	"time"
+
+	exec "tailscale.com/tempfork/osexec"
 )
 
 // Forking on Windows is insanely expensive, so don't do it too often.
@@ -18,3 +21,9 @@ func listPorts() (List, error) {
 func addProcesses(pl []Port) ([]Port, error) {
 	return listPortsNetstat("-nab")
 }
+
+func init() {
+	osHideWindow = func(c *exec.Cmd) {
+		c.SysProcAttr = &syscall.SysProcAttr{HideWindow: true}
+	}
+}

safesocket/unixsocket.go

@@ -11,6 +11,7 @@ import (
 	"bytes"
 	"fmt"
 	"io"
+	"io/ioutil"
 	"log"
 	"net"
 	"os"
@@ -77,6 +78,16 @@ func listen(path string, port uint16) (ln net.Listener, _ uint16, err error) {
 //   * it also picks a random hex string that acts as an auth token
 //   * it then creates a file named "sameuserproof-$PORT-$TOKEN" and leaves
 //     that file descriptor open forever.
+//
+// Then, we do different things depending on whether the user is
+// running cmd/tailscale that they built themselves (running as
+// themselves, outside the App Sandbox), or whether the user is
+// running the CLI via the GUI binary
+// (e.g. /Applications/Tailscale.app/Contents/MacOS/Tailscale <args>),
+// in which case we're running within the App Sandbox.
+//
+// If we're outside the App Sandbox:
+//
 //   * then we come along here, running as the same UID, but outside
 //     of the sandbox, and look for it. We can run lsof on our own processes,
 //     but other users on the system can't.
@@ -86,7 +97,38 @@ func listen(path string, port uint16) (ln net.Listener, _ uint16, err error) {
 //   * server verifies $TOKEN, sends "#IPN\n" if okay.
 //   * server is now protocol switched
 //   * we return the net.Conn and the caller speaks the normal protocol
+//
+// If we're inside the App Sandbox, then TS_MACOS_CLI_SHARED_DIR has
+// been set to our shared directory. We now have to find the most
+// recent "sameuserproof" file (there should only be 1, but previous
+// versions of the macOS app didn't clean them up).
 func connectMacOSAppSandbox() (net.Conn, error) {
+	// Are we running the Tailscale.app GUI binary as a CLI, running within the App Sandbox?
+	if d := os.Getenv("TS_MACOS_CLI_SHARED_DIR"); d != "" {
+		fis, err := ioutil.ReadDir(d)
+		if err != nil {
+			return nil, fmt.Errorf("reading TS_MACOS_CLI_SHARED_DIR: %w", err)
+		}
+		var best os.FileInfo
+		for _, fi := range fis {
+			if !strings.HasPrefix(fi.Name(), "sameuserproof-") || strings.Count(fi.Name(), "-") != 2 {
+				continue
+			}
+			if best == nil || fi.ModTime().After(best.ModTime()) {
+				best = fi
+			}
+		}
+		if best == nil {
+			return nil, fmt.Errorf("no sameuserproof token found in TS_MACOS_CLI_SHARED_DIR %q", d)
+		}
+		f := strings.SplitN(best.Name(), "-", 3)
+		portStr, token := f[1], f[2]
+		return connectMacTCP(portStr, token)
+	}
+
+	// Otherwise, assume we're running the cmd/tailscale binary from outside the
+	// App Sandbox.
+
 	out, err := exec.Command("lsof",
 		"-n",                             // numeric sockets; don't do DNS lookups, etc
 		"-a",                             // logical AND remaining options
@@ -110,22 +152,26 @@ func connectMacOSAppSandbox() (net.Conn, error) {
 			continue
 		}
 		portStr, token := f[0], f[1]
-		c, err := net.Dial("tcp", "localhost:"+portStr)
-		if err != nil {
-			return nil, fmt.Errorf("error dialing IPNExtension: %w", err)
-		}
-		if _, err := io.WriteString(c, token+"\n"); err != nil {
-			return nil, fmt.Errorf("error writing auth token: %w", err)
-		}
-		buf := make([]byte, 5)
-		const authOK = "#IPN\n"
-		if _, err := io.ReadFull(c, buf); err != nil {
-			return nil, fmt.Errorf("error reading from IPNExtension post-auth: %w", err)
-		}
-		if string(buf) != authOK {
-			return nil, fmt.Errorf("invalid response reading from IPNExtension post-auth")
-		}
-		return c, nil
+		return connectMacTCP(portStr, token)
 	}
 	return nil, fmt.Errorf("failed to find Tailscale's IPNExtension process")
 }
+
+func connectMacTCP(portStr, token string) (net.Conn, error) {
+	c, err := net.Dial("tcp", "localhost:"+portStr)
+	if err != nil {
+		return nil, fmt.Errorf("error dialing IPNExtension: %w", err)
+	}
+	if _, err := io.WriteString(c, token+"\n"); err != nil {
+		return nil, fmt.Errorf("error writing auth token: %w", err)
+	}
+	buf := make([]byte, 5)
+	const authOK = "#IPN\n"
+	if _, err := io.ReadFull(c, buf); err != nil {
+		return nil, fmt.Errorf("error reading from IPNExtension post-auth: %w", err)
+	}
+	if string(buf) != authOK {
+		return nil, fmt.Errorf("invalid response reading from IPNExtension post-auth")
+	}
+	return c, nil
+}

smallzstd/testdata

@@ -0,0 +1,14 @@
+{"logtail":{"client_time":"2020-07-01T14:49:40.196597018-07:00","server_time":"2020-07-01T21:49:40.198371511Z"},"text":"9.8M/25.6M magicsock: starting endpoint update (periodic)\n"}
+{"logtail":{"client_time":"2020-07-01T14:49:40.345925455-07:00","server_time":"2020-07-01T21:49:40.347904717Z"},"text":"9.9M/25.6M netcheck: udp=true v6=false mapvarydest=false hair=false v4a=202.188.7.1:41641 derp=2 derpdist=1v4:7ms,2v4:3ms,4v4:18ms\n"}
+{"logtail":{"client_time":"2020-07-01T14:49:43.347155742-07:00","server_time":"2020-07-01T21:49:43.34828658Z"},"text":"9.9M/25.6M control: map response long-poll timed out!\n"}
+{"logtail":{"client_time":"2020-07-01T14:49:43.347539333-07:00","server_time":"2020-07-01T21:49:43.358809354Z"},"text":"9.9M/25.6M control: PollNetMap: context canceled\n"}
+{"logtail":{"client_time":"2020-07-01T14:49:43.347767812-07:00","server_time":"2020-07-01T21:49:43.358809354Z"},"text":"10.0M/25.6M control: sendStatus: mapRoutine1: state:authenticated\n"}
+{"logtail":{"client_time":"2020-07-01T14:49:43.347817165-07:00","server_time":"2020-07-01T21:49:43.358809354Z"},"text":"10.0M/25.6M blockEngineUpdates(false)\n"}
+{"logtail":{"client_time":"2020-07-01T14:49:43.347989028-07:00","server_time":"2020-07-01T21:49:43.358809354Z"},"text":"10.0M/25.6M wgcfg: [SViTM] skipping subnet route\n"}
+{"logtail":{"client_time":"2020-07-01T14:49:43.349997554-07:00","server_time":"2020-07-01T21:49:43.358809354Z"},"text":"9.3M/25.6M Received error: PollNetMap: context canceled\n"}
+{"logtail":{"client_time":"2020-07-01T14:49:43.350072606-07:00","server_time":"2020-07-01T21:49:43.358809354Z"},"text":"9.3M/25.6M control: mapRoutine: backoff: 30136 msec\n"}
+{"logtail":{"client_time":"2020-07-01T14:49:47.998364646-07:00","server_time":"2020-07-01T21:49:47.999333754Z"},"text":"9.5M/25.6M [W1NbE] - [UcppE] Send handshake init [127.3.3.40:1, 6.1.1.6:37388*, 10.3.2.6:41641]\n"}
+{"logtail":{"client_time":"2020-07-01T14:49:47.99881914-07:00","server_time":"2020-07-01T21:49:48.009859543Z"},"text":"9.6M/25.6M magicsock: adding connection to derp-1 for [W1NbE]\n"}
+{"logtail":{"client_time":"2020-07-01T14:49:47.998904932-07:00","server_time":"2020-07-01T21:49:48.009859543Z"},"text":"9.6M/25.6M magicsock: 2 active derp conns: derp-1=cr0s,wr0s derp-2=cr16h0m0s,wr14h38m0s\n"}
+{"logtail":{"client_time":"2020-07-01T14:49:47.999045606-07:00","server_time":"2020-07-01T21:49:48.009859543Z"},"text":"9.6M/25.6M derphttp.Client.Recv: connecting to derp-1 (nyc)\n"}
+{"logtail":{"client_time":"2020-07-01T14:49:48.091104119-07:00","server_time":"2020-07-01T21:49:48.09280535Z"},"text":"9.6M/25.6M magicsock: rx [W1NbE] from 6.1.1.6:37388 (1/3), set as new priority\n"}

smallzstd/zstd.go

@@ -0,0 +1,79 @@
+// Copyright (c) 2020 Tailscale Inc & AUTHORS All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+// Package smallzstd produces zstd encoders and decoders optimized for
+// low memory usage, at the expense of compression efficiency.
+//
+// This package is optimized primarily for the memory cost of
+// compressing and decompressing data. We reduce this cost in two
+// major ways: disable parallelism within the library (i.e. don't use
+// multiple CPU cores to decompress), and drop the compression window
+// down from the defaults of 4-16MiB, to 8kiB.
+//
+// Decompressors cost 2x the window size in RAM to run, so by using an
+// 8kiB window, we can run ~1000 more decompressors per unit of memory
+// than with the defaults.
+//
+// Depending on context, the benefit is either being able to run more
+// decoders (e.g. in our logs processing system), or having a lower
+// memory footprint when using compression in network protocols
+// (e.g. in tailscaled, which should have a minimal RAM cost).
+package smallzstd
+
+import (
+	"io"
+
+	"github.com/klauspost/compress/zstd"
+)
+
+// WindowSize is the window size used for zstd compression. Decoder
+// memory usage scales linearly with WindowSize.
+const WindowSize = 8 << 10 // 8kiB
+
+// NewDecoder returns a zstd.Decoder configured for low memory usage,
+// at the expense of decompression performance.
+func NewDecoder(r io.Reader, options ...zstd.DOption) (*zstd.Decoder, error) {
+	defaults := []zstd.DOption{
+		// Default is GOMAXPROCS, which costs many KiB in stacks.
+		zstd.WithDecoderConcurrency(1),
+		// Default is to allocate more upfront for performance. We
+		// prefer lower memory use and a bit of GC load.
+		zstd.WithDecoderLowmem(true),
+		// You might expect to see zstd.WithDecoderMaxMemory
+		// here. However, it's not terribly safe to use if you're
+		// doing stateless decoding, because it sets the maximum
+		// amount of memory the decompressed data can occupy, rather
+		// than the window size of the zstd stream. This means a very
+		// compressible piece of data might violate the max memory
+		// limit here, even if the window size (and thus total memory
+		// required to decompress the data) is small.
+		//
+		// As a result, we don't set a decoder limit here, and rely on
+		// the encoder below producing "cheap" streams. Callers are
+		// welcome to set their own max memory setting, if
+		// contextually there is a clearly correct value (e.g. it's
+		// known from the upper layer protocol that the decoded data
+		// can never be more than 1MiB).
+	}
+
+	return zstd.NewReader(r, append(defaults, options...)...)
+}
+
+// NewEncoder returns a zstd.Encoder configured for low memory usage,
+// both during compression and at decompression time, at the expense
+// of performance and compression efficiency.
+func NewEncoder(w io.Writer, options ...zstd.EOption) (*zstd.Encoder, error) {
+	defaults := []zstd.EOption{
+		// Default is GOMAXPROCS, which costs many KiB in stacks.
+		zstd.WithEncoderConcurrency(1),
+		// Default is several MiB, which bloats both encoders and
+		// their corresponding decoders.
+		zstd.WithWindowSize(WindowSize),
+		// Encode zero-length inputs in a way that the `zstd` utility
+		// can read, because interoperability is handy.
+		zstd.WithZeroFrames(true),
+	}
+
+	return zstd.NewWriter(w, append(defaults, options...)...)
+}

smallzstd/zstd_test.go

@@ -0,0 +1,131 @@
+// Copyright (c) 2020 Tailscale Inc & AUTHORS All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package smallzstd
+
+import (
+	"io/ioutil"
+	"testing"
+
+	"github.com/klauspost/compress/zstd"
+)
+
+func BenchmarkSmallEncoder(b *testing.B) {
+	benchEncoder(b, func() (*zstd.Encoder, error) { return NewEncoder(nil) })
+}
+
+func BenchmarkSmallEncoderWithBuild(b *testing.B) {
+	benchEncoderWithConstruction(b, func() (*zstd.Encoder, error) { return NewEncoder(nil) })
+}
+
+func BenchmarkStockEncoder(b *testing.B) {
+	benchEncoder(b, func() (*zstd.Encoder, error) { return zstd.NewWriter(nil) })
+}
+
+func BenchmarkStockEncoderWithBuild(b *testing.B) {
+	benchEncoderWithConstruction(b, func() (*zstd.Encoder, error) { return zstd.NewWriter(nil) })
+}
+
+func BenchmarkSmallDecoder(b *testing.B) {
+	benchDecoder(b, func() (*zstd.Decoder, error) { return NewDecoder(nil) })
+}
+
+func BenchmarkSmallDecoderWithBuild(b *testing.B) {
+	benchDecoderWithConstruction(b, func() (*zstd.Decoder, error) { return NewDecoder(nil) })
+}
+
+func BenchmarkStockDecoder(b *testing.B) {
+	benchDecoder(b, func() (*zstd.Decoder, error) { return zstd.NewReader(nil) })
+}
+
+func BenchmarkStockDecoderWithBuild(b *testing.B) {
+	benchDecoderWithConstruction(b, func() (*zstd.Decoder, error) { return zstd.NewReader(nil) })
+}
+
+func benchEncoder(b *testing.B, mk func() (*zstd.Encoder, error)) {
+	b.ReportAllocs()
+
+	in := testdata(b)
+	out := make([]byte, 0, 10<<10) // 10kiB
+
+	e, err := mk()
+	if err != nil {
+		b.Fatalf("making encoder: %v", err)
+	}
+
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		e.EncodeAll(in, out)
+	}
+}
+
+func benchEncoderWithConstruction(b *testing.B, mk func() (*zstd.Encoder, error)) {
+	b.ReportAllocs()
+
+	in := testdata(b)
+	out := make([]byte, 0, 10<<10) // 10kiB
+
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		e, err := mk()
+		if err != nil {
+			b.Fatalf("making encoder: %v", err)
+		}
+
+		e.EncodeAll(in, out)
+	}
+}
+
+func benchDecoder(b *testing.B, mk func() (*zstd.Decoder, error)) {
+	b.ReportAllocs()
+
+	in := compressedTestdata(b)
+	out := make([]byte, 0, 10<<10)
+
+	d, err := mk()
+	if err != nil {
+		b.Fatalf("creating decoder: %v", err)
+	}
+
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		d.DecodeAll(in, out)
+	}
+}
+
+func benchDecoderWithConstruction(b *testing.B, mk func() (*zstd.Decoder, error)) {
+	b.ReportAllocs()
+
+	in := compressedTestdata(b)
+	out := make([]byte, 0, 10<<10)
+
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		d, err := mk()
+		if err != nil {
+			b.Fatalf("creating decoder: %v", err)
+		}
+
+		d.DecodeAll(in, out)
+	}
+}
+
+func testdata(b *testing.B) []byte {
+	b.Helper()
+	in, err := ioutil.ReadFile("testdata")
+	if err != nil {
+		b.Fatalf("reading testdata: %v", err)
+	}
+	return in
+}
+
+func compressedTestdata(b *testing.B) []byte {
+	b.Helper()
+	uncomp := testdata(b)
+	e, err := NewEncoder(nil)
+	if err != nil {
+		b.Fatalf("creating encoder: %v", err)
+	}
+	return e.EncodeAll(uncomp, nil)
+}

tailcfg/derpmap.go

@@ -117,4 +117,8 @@ type DERPNode struct {
 	// of using the default port of 443. If non-zero, TLS
 	// verification is skipped.
 	DERPTestPort int `json:",omitempty"`
+
+	// STUNTestIP is used in tests to override the STUN server's IP.
+	// If empty, it's assumed to be the same as the DERP server.
+	STUNTestIP string `json:",omitempty"`
 }

tailcfg/tailcfg.go

@@ -4,6 +4,8 @@
 
 package tailcfg
 
+//go:generate go run tailscale.com/cmd/cloner -type=User,Node,Hostinfo,NetInfo -output=tailcfg_clone.go
+
 import (
 	"bytes"
 	"errors"
@@ -13,7 +15,10 @@ import (
 	"time"
 
 	"github.com/tailscale/wireguard-go/wgcfg"
+	"go4.org/mem"
 	"golang.org/x/oauth2"
+	"inet.af/netaddr"
+	"tailscale.com/types/key"
 	"tailscale.com/types/opt"
 	"tailscale.com/types/structs"
 )
@@ -38,6 +43,10 @@ type MachineKey [32]byte
 // NodeKey is the curve25519 public key for a node.
 type NodeKey [32]byte
 
+// DiscoKey is the curve25519 public key for path discovery key.
+// It's never written to disk or reused between network start-ups.
+type DiscoKey [32]byte
+
 type Group struct {
 	ID      GroupID
 	Name    string
@@ -87,18 +96,6 @@ type User struct {
 	// Note: be sure to update Clone when adding new fields
 }
 
-// Clone returns a copy of u that aliases no memory with the original.
-func (u *User) Clone() *User {
-	if u == nil {
-		return nil
-	}
-	u2 := new(User)
-	*u2 = *u
-	u2.Logins = append([]LoginID(nil), u.Logins...)
-	u2.Roles = append([]RoleID(nil), u.Roles...)
-	return u2
-}
-
 type Login struct {
 	_             structs.Incomparable
 	ID            LoginID
@@ -114,8 +111,8 @@ type Login struct {
 // It also includes derived data from one of the user's logins.
 type UserProfile struct {
 	ID            UserID
-	LoginName     string // for display purposes only (provider is not listed)
-	DisplayName   string
+	LoginName     string // "alice@smith.com"; for display purposes only (provider is not listed)
+	DisplayName   string // "Alice Smith"
 	ProfilePicURL string
 	Roles         []RoleID
 }
@@ -127,6 +124,7 @@ type Node struct {
 	Key        NodeKey
 	KeyExpiry  time.Time
 	Machine    MachineKey
+	DiscoKey   DiscoKey
 	Addresses  []wgcfg.CIDR // IP addresses of this Node directly
 	AllowedIPs []wgcfg.CIDR // range of IP addresses to route to this node
 	Endpoints  []string     `json:",omitempty"` // IP+port (public via STUN, and local LANs)
@@ -143,23 +141,6 @@ type Node struct {
 	//       require changes to Node.Clone.
 }
 
-// Clone makes a deep copy of Node.
-// The result aliases no memory with the original.
-func (n *Node) Clone() (res *Node) {
-	res = new(Node)
-	*res = *n
-
-	res.Addresses = append([]wgcfg.CIDR{}, res.Addresses...)
-	res.AllowedIPs = append([]wgcfg.CIDR{}, res.AllowedIPs...)
-	res.Endpoints = append([]string{}, res.Endpoints...)
-	if res.LastSeen != nil {
-		lastSeen := *res.LastSeen
-		res.LastSeen = &lastSeen
-	}
-	res.Hostinfo = *res.Hostinfo.Clone()
-	return res
-}
-
 type MachineStatus int
 
 const (
@@ -277,7 +258,10 @@ type Hostinfo struct {
 	FrontendLogID string       // logtail ID of frontend instance
 	BackendLogID  string       // logtail ID of backend instance
 	OS            string       // operating system the client runs on (a version.OS value)
+	OSVersion     string       // operating system version, with optional distro prefix ("Debian 10.4", "Windows 10 Pro 10.0.19041")
+	DeviceModel   string       // mobile phone model ("Pixel 3a", "iPhone 11 Pro")
 	Hostname      string       // name of the host the client runs on
+	GoArch        string       // the host's GOARCH value (of the running binary)
 	RoutableIPs   []wgcfg.CIDR `json:",omitempty"` // set of IP ranges this client can route
 	RequestTags   []string     `json:",omitempty"` // set of ACL tags this node wants to claim
 	Services      []Service    `json:",omitempty"` // services advertised by this machine
@@ -303,6 +287,18 @@ type NetInfo struct {
 	// WorkingUDP is whether UDP works.
 	WorkingUDP opt.Bool
 
+	// UPnP is whether UPnP appears present on the LAN.
+	// Empty means not checked.
+	UPnP opt.Bool
+
+	// PMP is whether NAT-PMP appears present on the LAN.
+	// Empty means not checked.
+	PMP opt.Bool
+
+	// PCP is whether PCP appears present on the LAN.
+	// Empty means not checked.
+	PCP opt.Bool
+
 	// PreferredDERP is this node's preferred DERP server
 	// for incoming traffic. The node might be be temporarily
 	// connected to multiple DERP servers (to send to other nodes)
@@ -331,9 +327,32 @@ func (ni *NetInfo) String() string {
 	if ni == nil {
 		return "NetInfo(nil)"
 	}
-	return fmt.Sprintf("NetInfo{varies=%v hairpin=%v ipv6=%v udp=%v derp=#%v link=%q}",
+	return fmt.Sprintf("NetInfo{varies=%v hairpin=%v ipv6=%v udp=%v derp=#%v portmap=%v link=%q}",
 		ni.MappingVariesByDestIP, ni.HairPinning, ni.WorkingIPv6,
-		ni.WorkingUDP, ni.PreferredDERP, ni.LinkType)
+		ni.WorkingUDP, ni.PreferredDERP,
+		ni.portMapSummary(),
+		ni.LinkType)
+}
+
+func (ni *NetInfo) portMapSummary() string {
+	if ni.UPnP == "" && ni.PMP == "" && ni.PCP == "" {
+		return "?"
+	}
+	return conciseOptBool(ni.UPnP, "U") + conciseOptBool(ni.PMP, "M") + conciseOptBool(ni.PCP, "C")
+}
+
+func conciseOptBool(b opt.Bool, trueVal string) string {
+	if b == "" {
+		return "_"
+	}
+	v, ok := b.Get()
+	if !ok {
+		return "x"
+	}
+	if v {
+		return trueVal
+	}
+	return ""
 }
 
 // BasicallyEqual reports whether ni and ni2 are basically equal, ignoring
@@ -349,39 +368,21 @@ func (ni *NetInfo) BasicallyEqual(ni2 *NetInfo) bool {
 		ni.HairPinning == ni2.HairPinning &&
 		ni.WorkingIPv6 == ni2.WorkingIPv6 &&
 		ni.WorkingUDP == ni2.WorkingUDP &&
+		ni.UPnP == ni2.UPnP &&
+		ni.PMP == ni2.PMP &&
+		ni.PCP == ni2.PCP &&
 		ni.PreferredDERP == ni2.PreferredDERP &&
 		ni.LinkType == ni2.LinkType
 }
 
-func (ni *NetInfo) Clone() (res *NetInfo) {
-	if ni == nil {
-		return nil
-	}
-	res = new(NetInfo)
-	*res = *ni
-	if ni.DERPLatency != nil {
-		res.DERPLatency = map[string]float64{}
-		for k, v := range ni.DERPLatency {
-			res.DERPLatency[k] = v
-		}
-	}
-	return res
-}
-
-// Clone makes a deep copy of Hostinfo.
-// The result aliases no memory with the original.
-func (h *Hostinfo) Clone() (res *Hostinfo) {
-	res = new(Hostinfo)
-	*res = *h
-
-	res.RoutableIPs = append([]wgcfg.CIDR{}, h.RoutableIPs...)
-	res.Services = append([]Service{}, h.Services...)
-	res.NetInfo = h.NetInfo.Clone()
-	return res
-}
-
 // Equal reports whether h and h2 are equal.
 func (h *Hostinfo) Equal(h2 *Hostinfo) bool {
+	if h == nil && h2 == nil {
+		return true
+	}
+	if (h == nil) != (h2 == nil) {
+		return false
+	}
 	return reflect.DeepEqual(h, h2)
 }
 
@@ -408,6 +409,8 @@ type RegisterRequest struct {
 
 // Clone makes a deep copy of RegisterRequest.
 // The result aliases no memory with the original.
+//
+// TODO: extend cmd/cloner to generate this method.
 func (req *RegisterRequest) Clone() *RegisterRequest {
 	res := new(RegisterRequest)
 	*res = *req
@@ -440,12 +443,20 @@ type RegisterResponse struct {
 type MapRequest struct {
 	Version     int    // current version is 4
 	Compress    string // "zstd" or "" (no compression)
-	KeepAlive   bool   // server sends keep-alives
+	KeepAlive   bool   // whether server should send keep-alives back to us
 	NodeKey     NodeKey
+	DiscoKey    DiscoKey
 	Endpoints   []string // caller's endpoints (IPv4 or IPv6)
 	IncludeIPv6 bool     // include IPv6 endpoints in returned Node Endpoints
+	DeltaPeers  bool     // whether the 2nd+ network map in response should be deltas, using PeersChanged, PeersRemoved
 	Stream      bool     // if true, multiple MapResponse objects are returned
 	Hostinfo    *Hostinfo
+
+	// DebugForceDisco is a temporary flag during the deployment
+	// of magicsock active discovery. It says that that the client
+	// has environment variables explicitly turning discovery on,
+	// so control should not disable it.
+	DebugForceDisco bool `json:"debugForceDisco,omitempty"`
 }
 
 // PortRange represents a range of UDP or TCP port numbers.
@@ -483,15 +494,45 @@ var FilterAllowAll = []FilterRule{
 	},
 }
 
+// DNSConfig is the DNS configuration.
+type DNSConfig struct {
+	Nameservers []netaddr.IP `json:",omitempty"`
+	Domains     []string     `json:",omitempty"`
+	PerDomain   bool
+	Proxied     bool
+}
+
 type MapResponse struct {
-	KeepAlive bool // if set, all other fields are ignored
+	KeepAlive bool `json:",omitempty"` // if set, all other fields are ignored
 
 	// Networking
-	Node        *Node
-	Peers       []*Node
-	DNS         []wgcfg.IP
-	SearchPaths []string
-	DERPMap     *DERPMap
+	Node    *Node
+	DERPMap *DERPMap `json:",omitempty"` // if non-empty, a change in the DERP map.
+
+	// Peers, if non-empty, is the complete list of peers.
+	// It will be set in the first MapResponse for a long-polled request/response.
+	// Subsequent responses will be delta-encoded if DeltaPeers was set in the request.
+	// If Peers is non-empty, PeersChanged and PeersRemoved should
+	// be ignored (and should be empty).
+	// Peers is always returned sorted by Node.ID.
+	Peers []*Node `json:",omitempty"`
+	// PeersChanged are the Nodes (identified by their ID) that
+	// have changed or been added since the past update on the
+	// HTTP response. It's only set if MapRequest.DeltaPeers was true.
+	// PeersChanged is always returned sorted by Node.ID.
+	PeersChanged []*Node `json:",omitempty"`
+	// PeersRemoved are the NodeIDs that are no longer in the peer list.
+	PeersRemoved []NodeID `json:",omitempty"`
+
+	// DNS is the same as DNSConfig.Nameservers.
+	//
+	// TODO(dmytro): should be sent in DNSConfig.Nameservers once clients have updated.
+	DNS []wgcfg.IP `json:",omitempty"`
+	// SearchPaths are the same as DNSConfig.Domains.
+	//
+	// TODO(dmytro): should be sent in DNSConfig.Domains once clients have updated.
+	SearchPaths []string  `json:",omitempty"`
+	DNSConfig   DNSConfig `json:",omitempty"`
 
 	// ACLs
 	Domain       string
@@ -509,65 +550,59 @@ type MapResponse struct {
 // Debug are instructions from the control server to the client
 // to adjust debug settings.
 type Debug struct {
-	// LogHeapPprof controls whether the client should logs
+	// LogHeapPprof controls whether the client should log
 	// its heap pprof data. Each true value sent from the server
 	// means that client should do one more log.
 	LogHeapPprof bool `json:",omitempty"`
+
+	// LogHeapURL is the URL to POST its heap pprof to.
+	// Empty means to not log.
+	LogHeapURL string `json:",omitempty"`
+
+	// ForceBackgroundSTUN controls whether magicsock should
+	// always do its background STUN queries (see magicsock's
+	// periodicReSTUN), regardless of inactivity.
+	ForceBackgroundSTUN bool `json:",omitempty"`
 }
 
-func (k MachineKey) String() string { return fmt.Sprintf("mkey:%x", k[:]) }
+func (k MachineKey) String() string                   { return fmt.Sprintf("mkey:%x", k[:]) }
+func (k MachineKey) MarshalText() ([]byte, error)     { return keyMarshalText("mkey:", k), nil }
+func (k *MachineKey) UnmarshalText(text []byte) error { return keyUnmarshalText(k[:], "mkey:", text) }
 
-func (k MachineKey) MarshalText() ([]byte, error) {
-	buf := new(bytes.Buffer)
-	fmt.Fprintf(buf, "mkey:%x", k[:])
-	return buf.Bytes(), nil
+func keyMarshalText(prefix string, k [32]byte) []byte {
+	buf := bytes.NewBuffer(make([]byte, 0, len(prefix)+64))
+	fmt.Fprintf(buf, "%s%x", prefix, k[:])
+	return buf.Bytes()
 }
 
-func (k *MachineKey) UnmarshalText(text []byte) error {
-	s := string(text)
-	if !strings.HasPrefix(s, "mkey:") {
-		return errors.New(`MachineKey.UnmarshalText: missing prefix`)
+func keyUnmarshalText(dst []byte, prefix string, text []byte) error {
+	if len(text) < len(prefix) || string(text[:len(prefix)]) != prefix {
+		return fmt.Errorf("UnmarshalText: missing %q prefix", prefix)
 	}
-	s = strings.TrimPrefix(s, `mkey:`)
-	key, err := wgcfg.ParseHexKey(s)
+	pub, err := key.NewPublicFromHexMem(mem.B(text[len(prefix):]))
 	if err != nil {
-		return fmt.Errorf("MachineKey.UnmarhsalText: %v", err)
+		return fmt.Errorf("UnmarshalText: after %q: %v", prefix, err)
 	}
-	copy(k[:], key[:])
+	copy(dst[:], pub[:])
 	return nil
 }
 
-func (k NodeKey) String() string { return fmt.Sprintf("nodekey:%x", k[:]) }
+func (k NodeKey) ShortString() string { return (key.Public(k)).ShortString() }
 
-func (k NodeKey) ShortString() string {
-	pk := wgcfg.Key(k)
-	return pk.ShortString()
-}
+func (k NodeKey) String() string                   { return fmt.Sprintf("nodekey:%x", k[:]) }
+func (k NodeKey) MarshalText() ([]byte, error)     { return keyMarshalText("nodekey:", k), nil }
+func (k *NodeKey) UnmarshalText(text []byte) error { return keyUnmarshalText(k[:], "nodekey:", text) }
 
-func (k NodeKey) MarshalText() ([]byte, error) {
-	buf := new(bytes.Buffer)
-	fmt.Fprintf(buf, "nodekey:%x", k[:])
-	return buf.Bytes(), nil
-}
+// IsZero reports whether k is the zero value.
+func (k NodeKey) IsZero() bool { return k == NodeKey{} }
 
-func (k *NodeKey) UnmarshalText(text []byte) error {
-	s := string(text)
-	if !strings.HasPrefix(s, "nodekey:") {
-		return errors.New(`Nodekey.UnmarshalText: missing prefix`)
-	}
-	s = strings.TrimPrefix(s, "nodekey:")
-	key, err := wgcfg.ParseHexKey(s)
-	if err != nil {
-		return fmt.Errorf("tailcfg.Ukey.UnmarhsalText: %v", err)
-	}
-	copy(k[:], key[:])
-	return nil
-}
+func (k DiscoKey) String() string                   { return fmt.Sprintf("discokey:%x", k[:]) }
+func (k DiscoKey) MarshalText() ([]byte, error)     { return keyMarshalText("discokey:", k), nil }
+func (k *DiscoKey) UnmarshalText(text []byte) error { return keyUnmarshalText(k[:], "discokey:", text) }
+func (k DiscoKey) ShortString() string              { return fmt.Sprintf("d:%x", k[:8]) }
 
-// IsZero reports whether k is the NodeKey zero value.
-func (k NodeKey) IsZero() bool {
-	return k == NodeKey{}
-}
+// IsZero reports whether k is the zero value.
+func (k DiscoKey) IsZero() bool { return k == DiscoKey{} }
 
 func (id ID) String() string           { return fmt.Sprintf("id:%x", int64(id)) }
 func (id UserID) String() string       { return fmt.Sprintf("userid:%x", int64(id)) }
@@ -589,11 +624,40 @@ func (n *Node) Equal(n2 *Node) bool {
 		n.Key == n2.Key &&
 		n.KeyExpiry.Equal(n2.KeyExpiry) &&
 		n.Machine == n2.Machine &&
-		reflect.DeepEqual(n.Addresses, n2.Addresses) &&
-		reflect.DeepEqual(n.AllowedIPs, n2.AllowedIPs) &&
-		reflect.DeepEqual(n.Endpoints, n2.Endpoints) &&
-		reflect.DeepEqual(n.Hostinfo, n2.Hostinfo) &&
+		n.DiscoKey == n2.DiscoKey &&
+		eqCIDRs(n.Addresses, n2.Addresses) &&
+		eqCIDRs(n.AllowedIPs, n2.AllowedIPs) &&
+		eqStrings(n.Endpoints, n2.Endpoints) &&
+		n.Hostinfo.Equal(&n2.Hostinfo) &&
 		n.Created.Equal(n2.Created) &&
-		reflect.DeepEqual(n.LastSeen, n2.LastSeen) &&
+		eqTimePtr(n.LastSeen, n2.LastSeen) &&
 		n.MachineAuthorized == n2.MachineAuthorized
 }
+
+func eqStrings(a, b []string) bool {
+	if len(a) != len(b) || ((a == nil) != (b == nil)) {
+		return false
+	}
+	for i, v := range a {
+		if v != b[i] {
+			return false
+		}
+	}
+	return true
+}
+
+func eqCIDRs(a, b []wgcfg.CIDR) bool {
+	if len(a) != len(b) || ((a == nil) != (b == nil)) {
+		return false
+	}
+	for i, v := range a {
+		if v != b[i] {
+			return false
+		}
+	}
+	return true
+}
+
+func eqTimePtr(a, b *time.Time) bool {
+	return ((a == nil) == (b == nil)) && (a == nil || a.Equal(*b))
+}

tailcfg/tailcfg_clone.go

@@ -0,0 +1,75 @@
+// Copyright (c) 2020 Tailscale Inc & AUTHORS All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+// Code generated by tailscale.com/cmd/cloner -type User,Node,Hostinfo,NetInfo; DO NOT EDIT.
+
+package tailcfg
+
+import (
+	"time"
+)
+
+// Clone makes a deep copy of User.
+// The result aliases no memory with the original.
+func (src *User) Clone() *User {
+	if src == nil {
+		return nil
+	}
+	dst := new(User)
+	*dst = *src
+	dst.Logins = append(src.Logins[:0:0], src.Logins...)
+	dst.Roles = append(src.Roles[:0:0], src.Roles...)
+	return dst
+}
+
+// Clone makes a deep copy of Node.
+// The result aliases no memory with the original.
+func (src *Node) Clone() *Node {
+	if src == nil {
+		return nil
+	}
+	dst := new(Node)
+	*dst = *src
+	dst.Addresses = append(src.Addresses[:0:0], src.Addresses...)
+	dst.AllowedIPs = append(src.AllowedIPs[:0:0], src.AllowedIPs...)
+	dst.Endpoints = append(src.Endpoints[:0:0], src.Endpoints...)
+	dst.Hostinfo = *src.Hostinfo.Clone()
+	if dst.LastSeen != nil {
+		dst.LastSeen = new(time.Time)
+		*dst.LastSeen = *src.LastSeen
+	}
+	return dst
+}
+
+// Clone makes a deep copy of Hostinfo.
+// The result aliases no memory with the original.
+func (src *Hostinfo) Clone() *Hostinfo {
+	if src == nil {
+		return nil
+	}
+	dst := new(Hostinfo)
+	*dst = *src
+	dst.RoutableIPs = append(src.RoutableIPs[:0:0], src.RoutableIPs...)
+	dst.RequestTags = append(src.RequestTags[:0:0], src.RequestTags...)
+	dst.Services = append(src.Services[:0:0], src.Services...)
+	dst.NetInfo = src.NetInfo.Clone()
+	return dst
+}
+
+// Clone makes a deep copy of NetInfo.
+// The result aliases no memory with the original.
+func (src *NetInfo) Clone() *NetInfo {
+	if src == nil {
+		return nil
+	}
+	dst := new(NetInfo)
+	*dst = *src
+	if dst.DERPLatency != nil {
+		dst.DERPLatency = map[string]float64{}
+		for k, v := range src.DERPLatency {
+			dst.DERPLatency[k] = v
+		}
+	}
+	return dst
+}

tailcfg/tailcfg_test.go

@@ -5,7 +5,9 @@
 package tailcfg
 
 import (
+	"encoding"
 	"reflect"
+	"strings"
 	"testing"
 	"time"
 
@@ -21,7 +23,8 @@ func fieldsOf(t reflect.Type) (fields []string) {
 
 func TestHostinfoEqual(t *testing.T) {
 	hiHandles := []string{
-		"IPNVersion", "FrontendLogID", "BackendLogID", "OS", "Hostname", "RoutableIPs", "RequestTags", "Services",
+		"IPNVersion", "FrontendLogID", "BackendLogID", "OS", "OSVersion",
+		"DeviceModel", "Hostname", "GoArch", "RoutableIPs", "RequestTags", "Services",
 		"NetInfo",
 	}
 	if have := fieldsOf(reflect.TypeOf(Hostinfo{})); !reflect.DeepEqual(have, hiHandles) {
@@ -176,7 +179,7 @@ func TestHostinfoEqual(t *testing.T) {
 }
 
 func TestNodeEqual(t *testing.T) {
-	nodeHandles := []string{"ID", "Name", "User", "Key", "KeyExpiry", "Machine", "Addresses", "AllowedIPs", "Endpoints", "DERP", "Hostinfo", "Created", "LastSeen", "KeepAlive", "MachineAuthorized"}
+	nodeHandles := []string{"ID", "Name", "User", "Key", "KeyExpiry", "Machine", "DiscoKey", "Addresses", "AllowedIPs", "Endpoints", "DERP", "Hostinfo", "Created", "LastSeen", "KeepAlive", "MachineAuthorized"}
 	if have := fieldsOf(reflect.TypeOf(Node{})); !reflect.DeepEqual(have, nodeHandles) {
 		t.Errorf("Node.Equal check might be out of sync\nfields: %q\nhandled: %q\n",
 			have, nodeHandles)
@@ -327,6 +330,9 @@ func TestNetInfoFields(t *testing.T) {
 		"HairPinning",
 		"WorkingIPv6",
 		"WorkingUDP",
+		"UPnP",
+		"PMP",
+		"PCP",
 		"PreferredDERP",
 		"LinkType",
 		"DERPLatency",
@@ -336,3 +342,91 @@ func TestNetInfoFields(t *testing.T) {
 			have, handled)
 	}
 }
+
+func TestMachineKeyMarshal(t *testing.T) {
+	var k1, k2 MachineKey
+	for i := range k1 {
+		k1[i] = byte(i)
+	}
+	testKey(t, "mkey:", k1, &k2)
+}
+
+func TestNodeKeyMarshal(t *testing.T) {
+	var k1, k2 NodeKey
+	for i := range k1 {
+		k1[i] = byte(i)
+	}
+	testKey(t, "nodekey:", k1, &k2)
+}
+
+func TestDiscoKeyMarshal(t *testing.T) {
+	var k1, k2 DiscoKey
+	for i := range k1 {
+		k1[i] = byte(i)
+	}
+	testKey(t, "discokey:", k1, &k2)
+}
+
+type keyIn interface {
+	String() string
+	MarshalText() ([]byte, error)
+}
+
+func testKey(t *testing.T, prefix string, in keyIn, out encoding.TextUnmarshaler) {
+	got, err := in.MarshalText()
+	if err != nil {
+		t.Fatal(err)
+	}
+	if err := out.UnmarshalText(got); err != nil {
+		t.Fatal(err)
+	}
+	if s := in.String(); string(got) != s {
+		t.Errorf("MarshalText = %q != String %q", got, s)
+	}
+	if !strings.HasPrefix(string(got), prefix) {
+		t.Errorf("%q didn't start with prefix %q", got, prefix)
+	}
+	if reflect.ValueOf(out).Elem().Interface() != in {
+		t.Errorf("mismatch after unmarshal")
+	}
+}
+
+func TestCloneUser(t *testing.T) {
+	tests := []struct {
+		name string
+		u    *User
+	}{
+		{"nil_logins", &User{}},
+		{"zero_logins", &User{Logins: make([]LoginID, 0)}},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			u2 := tt.u.Clone()
+			if !reflect.DeepEqual(tt.u, u2) {
+				t.Errorf("not equal")
+			}
+		})
+	}
+}
+
+func TestCloneNode(t *testing.T) {
+	tests := []struct {
+		name string
+		v    *Node
+	}{
+		{"nil_fields", &Node{}},
+		{"zero_fields", &Node{
+			Addresses:  make([]wgcfg.CIDR, 0),
+			AllowedIPs: make([]wgcfg.CIDR, 0),
+			Endpoints:  make([]string, 0),
+		}},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			v2 := tt.v.Clone()
+			if !reflect.DeepEqual(tt.v, v2) {
+				t.Errorf("not equal")
+			}
+		})
+	}
+}

tempfork/pprof/README.md

@@ -0,0 +1,4 @@
+This is a fork of net/http/pprof that doesn't use init side effects
+and doesn't use html/template (which ends up calling
+reflect.Value.MethodByName, which disables some linker deadcode
+optimizations).

tempfork/pprof/pprof.go

@@ -0,0 +1,301 @@
+// Copyright 2010 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+// Package pprof serves via its HTTP server runtime profiling data
+// in the format expected by the pprof visualization tool.
+//
+// See Go's net/http/pprof for docs.
+//
+// This is a fork of net/http/pprof that doesn't use init side effects
+// and doesn't use html/template (which ends up calling
+// reflect.Value.MethodByName, which disables some linker deadcode
+// optimizations).
+package pprof
+
+import (
+	"bufio"
+	"bytes"
+	"fmt"
+	"html"
+	"io"
+	"net/http"
+	"os"
+	"runtime"
+	"runtime/pprof"
+	"runtime/trace"
+	"sort"
+	"strconv"
+	"strings"
+	"time"
+)
+
+func AddHandlers(mux *http.ServeMux) {
+	mux.HandleFunc("/debug/pprof/", Index)
+	mux.HandleFunc("/debug/pprof/cmdline", Cmdline)
+	mux.HandleFunc("/debug/pprof/profile", Profile)
+	mux.HandleFunc("/debug/pprof/symbol", Symbol)
+	mux.HandleFunc("/debug/pprof/trace", Trace)
+}
+
+// Cmdline responds with the running program's
+// command line, with arguments separated by NUL bytes.
+// The package initialization registers it as /debug/pprof/cmdline.
+func Cmdline(w http.ResponseWriter, r *http.Request) {
+	w.Header().Set("X-Content-Type-Options", "nosniff")
+	w.Header().Set("Content-Type", "text/plain; charset=utf-8")
+	fmt.Fprintf(w, strings.Join(os.Args, "\x00"))
+}
+
+func sleep(w http.ResponseWriter, d time.Duration) {
+	var clientGone <-chan bool
+	if cn, ok := w.(http.CloseNotifier); ok {
+		clientGone = cn.CloseNotify()
+	}
+	select {
+	case <-time.After(d):
+	case <-clientGone:
+	}
+}
+
+func durationExceedsWriteTimeout(r *http.Request, seconds float64) bool {
+	srv, ok := r.Context().Value(http.ServerContextKey).(*http.Server)
+	return ok && srv.WriteTimeout != 0 && seconds >= srv.WriteTimeout.Seconds()
+}
+
+func serveError(w http.ResponseWriter, status int, txt string) {
+	w.Header().Set("Content-Type", "text/plain; charset=utf-8")
+	w.Header().Set("X-Go-Pprof", "1")
+	w.Header().Del("Content-Disposition")
+	w.WriteHeader(status)
+	fmt.Fprintln(w, txt)
+}
+
+// Profile responds with the pprof-formatted cpu profile.
+// Profiling lasts for duration specified in seconds GET parameter, or for 30 seconds if not specified.
+// The package initialization registers it as /debug/pprof/profile.
+func Profile(w http.ResponseWriter, r *http.Request) {
+	w.Header().Set("X-Content-Type-Options", "nosniff")
+	sec, err := strconv.ParseInt(r.FormValue("seconds"), 10, 64)
+	if sec <= 0 || err != nil {
+		sec = 30
+	}
+
+	if durationExceedsWriteTimeout(r, float64(sec)) {
+		serveError(w, http.StatusBadRequest, "profile duration exceeds server's WriteTimeout")
+		return
+	}
+
+	// Set Content Type assuming StartCPUProfile will work,
+	// because if it does it starts writing.
+	w.Header().Set("Content-Type", "application/octet-stream")
+	w.Header().Set("Content-Disposition", `attachment; filename="profile"`)
+	if err := pprof.StartCPUProfile(w); err != nil {
+		// StartCPUProfile failed, so no writes yet.
+		serveError(w, http.StatusInternalServerError,
+			fmt.Sprintf("Could not enable CPU profiling: %s", err))
+		return
+	}
+	sleep(w, time.Duration(sec)*time.Second)
+	pprof.StopCPUProfile()
+}
+
+// Trace responds with the execution trace in binary form.
+// Tracing lasts for duration specified in seconds GET parameter, or for 1 second if not specified.
+// The package initialization registers it as /debug/pprof/trace.
+func Trace(w http.ResponseWriter, r *http.Request) {
+	w.Header().Set("X-Content-Type-Options", "nosniff")
+	sec, err := strconv.ParseFloat(r.FormValue("seconds"), 64)
+	if sec <= 0 || err != nil {
+		sec = 1
+	}
+
+	if durationExceedsWriteTimeout(r, sec) {
+		serveError(w, http.StatusBadRequest, "profile duration exceeds server's WriteTimeout")
+		return
+	}
+
+	// Set Content Type assuming trace.Start will work,
+	// because if it does it starts writing.
+	w.Header().Set("Content-Type", "application/octet-stream")
+	w.Header().Set("Content-Disposition", `attachment; filename="trace"`)
+	if err := trace.Start(w); err != nil {
+		// trace.Start failed, so no writes yet.
+		serveError(w, http.StatusInternalServerError,
+			fmt.Sprintf("Could not enable tracing: %s", err))
+		return
+	}
+	sleep(w, time.Duration(sec*float64(time.Second)))
+	trace.Stop()
+}
+
+// Symbol looks up the program counters listed in the request,
+// responding with a table mapping program counters to function names.
+// The package initialization registers it as /debug/pprof/symbol.
+func Symbol(w http.ResponseWriter, r *http.Request) {
+	w.Header().Set("X-Content-Type-Options", "nosniff")
+	w.Header().Set("Content-Type", "text/plain; charset=utf-8")
+
+	// We have to read the whole POST body before
+	// writing any output. Buffer the output here.
+	var buf bytes.Buffer
+
+	// We don't know how many symbols we have, but we
+	// do have symbol information. Pprof only cares whether
+	// this number is 0 (no symbols available) or > 0.
+	fmt.Fprintf(&buf, "num_symbols: 1\n")
+
+	var b *bufio.Reader
+	if r.Method == "POST" {
+		b = bufio.NewReader(r.Body)
+	} else {
+		b = bufio.NewReader(strings.NewReader(r.URL.RawQuery))
+	}
+
+	for {
+		word, err := b.ReadSlice('+')
+		if err == nil {
+			word = word[0 : len(word)-1] // trim +
+		}
+		pc, _ := strconv.ParseUint(string(word), 0, 64)
+		if pc != 0 {
+			f := runtime.FuncForPC(uintptr(pc))
+			if f != nil {
+				fmt.Fprintf(&buf, "%#x %s\n", pc, f.Name())
+			}
+		}
+
+		// Wait until here to check for err; the last
+		// symbol will have an err because it doesn't end in +.
+		if err != nil {
+			if err != io.EOF {
+				fmt.Fprintf(&buf, "reading request: %v\n", err)
+			}
+			break
+		}
+	}
+
+	w.Write(buf.Bytes())
+}
+
+// Handler returns an HTTP handler that serves the named profile.
+func Handler(name string) http.Handler {
+	return handler(name)
+}
+
+type handler string
+
+func (name handler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
+	w.Header().Set("X-Content-Type-Options", "nosniff")
+	p := pprof.Lookup(string(name))
+	if p == nil {
+		serveError(w, http.StatusNotFound, "Unknown profile")
+		return
+	}
+	gc, _ := strconv.Atoi(r.FormValue("gc"))
+	if name == "heap" && gc > 0 {
+		runtime.GC()
+	}
+	debug, _ := strconv.Atoi(r.FormValue("debug"))
+	if debug != 0 {
+		w.Header().Set("Content-Type", "text/plain; charset=utf-8")
+	} else {
+		w.Header().Set("Content-Type", "application/octet-stream")
+		w.Header().Set("Content-Disposition", fmt.Sprintf(`attachment; filename="%s"`, name))
+	}
+	p.WriteTo(w, debug)
+}
+
+var profileDescriptions = map[string]string{
+	"allocs":       "A sampling of all past memory allocations",
+	"block":        "Stack traces that led to blocking on synchronization primitives",
+	"cmdline":      "The command line invocation of the current program",
+	"goroutine":    "Stack traces of all current goroutines",
+	"heap":         "A sampling of memory allocations of live objects. You can specify the gc GET parameter to run GC before taking the heap sample.",
+	"mutex":        "Stack traces of holders of contended mutexes",
+	"profile":      "CPU profile. You can specify the duration in the seconds GET parameter. After you get the profile file, use the go tool pprof command to investigate the profile.",
+	"threadcreate": "Stack traces that led to the creation of new OS threads",
+	"trace":        "A trace of execution of the current program. You can specify the duration in the seconds GET parameter. After you get the trace file, use the go tool trace command to investigate the trace.",
+}
+
+// Index responds with the pprof-formatted profile named by the request.
+// For example, "/debug/pprof/heap" serves the "heap" profile.
+// Index responds to a request for "/debug/pprof/" with an HTML page
+// listing the available profiles.
+func Index(w http.ResponseWriter, r *http.Request) {
+	if strings.HasPrefix(r.URL.Path, "/debug/pprof/") {
+		name := strings.TrimPrefix(r.URL.Path, "/debug/pprof/")
+		if name != "" {
+			handler(name).ServeHTTP(w, r)
+			return
+		}
+	}
+
+	type profile struct {
+		Name  string
+		Href  string
+		Desc  string
+		Count int
+	}
+	var profiles []profile
+	for _, p := range pprof.Profiles() {
+		profiles = append(profiles, profile{
+			Name:  p.Name(),
+			Href:  p.Name() + "?debug=1",
+			Desc:  profileDescriptions[p.Name()],
+			Count: p.Count(),
+		})
+	}
+
+	// Adding other profiles exposed from within this package
+	for _, p := range []string{"cmdline", "profile", "trace"} {
+		profiles = append(profiles, profile{
+			Name: p,
+			Href: p,
+			Desc: profileDescriptions[p],
+		})
+	}
+
+	sort.Slice(profiles, func(i, j int) bool {
+		return profiles[i].Name < profiles[j].Name
+	})
+
+	io.WriteString(w, `<html>
+<head>
+<title>/debug/pprof/</title>
+<style>
+.profile-name{
+	display:inline-block;
+	width:6rem;
+}
+</style>
+</head>
+<body>
+/debug/pprof/<br>
+<br>
+Types of profiles available:
+<table>
+<thead><td>Count</td><td>Profile</td></thead>
+`)
+	for _, p := range profiles {
+		fmt.Fprintf(w, "<tr><td>%d</td><td><a href=%v>%v</td></tr>\n",
+			p.Count, html.EscapeString(p.Href), html.EscapeString(p.Name))
+	}
+	io.WriteString(w, `</table>
+<a href="goroutine?debug=2">full goroutine stack dump</a>
+<br/>
+<p>
+Profile Descriptions:
+<ul>
+`)
+	for _, p := range profiles {
+		fmt.Fprintf(w, "<li><div class=profile-name>%s:</div> %s</li>\n",
+			html.EscapeString(p.Name), html.EscapeString(p.Desc))
+	}
+	io.WriteString(w, `
+</ul>
+</p>
+</body>
+</html>
+`)
+}

tempfork/pprof/pprof_test.go

@@ -0,0 +1,81 @@
+// Copyright 2018 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package pprof
+
+import (
+	"bytes"
+	"io/ioutil"
+	"net/http"
+	"net/http/httptest"
+	"runtime/pprof"
+	"testing"
+)
+
+// TestDescriptions checks that the profile names under runtime/pprof package
+// have a key in the description map.
+func TestDescriptions(t *testing.T) {
+	for _, p := range pprof.Profiles() {
+		_, ok := profileDescriptions[p.Name()]
+		if ok != true {
+			t.Errorf("%s does not exist in profileDescriptions map\n", p.Name())
+		}
+	}
+}
+
+func TestHandlers(t *testing.T) {
+	testCases := []struct {
+		path               string
+		handler            http.HandlerFunc
+		statusCode         int
+		contentType        string
+		contentDisposition string
+		resp               []byte
+	}{
+		{"/debug/pprof/<script>scripty<script>", Index, http.StatusNotFound, "text/plain; charset=utf-8", "", []byte("Unknown profile\n")},
+		{"/debug/pprof/heap", Index, http.StatusOK, "application/octet-stream", `attachment; filename="heap"`, nil},
+		{"/debug/pprof/heap?debug=1", Index, http.StatusOK, "text/plain; charset=utf-8", "", nil},
+		{"/debug/pprof/cmdline", Cmdline, http.StatusOK, "text/plain; charset=utf-8", "", nil},
+		{"/debug/pprof/profile?seconds=1", Profile, http.StatusOK, "application/octet-stream", `attachment; filename="profile"`, nil},
+		{"/debug/pprof/symbol", Symbol, http.StatusOK, "text/plain; charset=utf-8", "", nil},
+		{"/debug/pprof/trace", Trace, http.StatusOK, "application/octet-stream", `attachment; filename="trace"`, nil},
+	}
+	for _, tc := range testCases {
+		t.Run(tc.path, func(t *testing.T) {
+			req := httptest.NewRequest("GET", "http://example.com"+tc.path, nil)
+			w := httptest.NewRecorder()
+			tc.handler(w, req)
+
+			resp := w.Result()
+			if got, want := resp.StatusCode, tc.statusCode; got != want {
+				t.Errorf("status code: got %d; want %d", got, want)
+			}
+
+			body, err := ioutil.ReadAll(resp.Body)
+			if err != nil {
+				t.Errorf("when reading response body, expected non-nil err; got %v", err)
+			}
+			if got, want := resp.Header.Get("X-Content-Type-Options"), "nosniff"; got != want {
+				t.Errorf("X-Content-Type-Options: got %q; want %q", got, want)
+			}
+			if got, want := resp.Header.Get("Content-Type"), tc.contentType; got != want {
+				t.Errorf("Content-Type: got %q; want %q", got, want)
+			}
+			if got, want := resp.Header.Get("Content-Disposition"), tc.contentDisposition; got != want {
+				t.Errorf("Content-Disposition: got %q; want %q", got, want)
+			}
+
+			if resp.StatusCode == http.StatusOK {
+				return
+			}
+			if got, want := resp.Header.Get("X-Go-Pprof"), "1"; got != want {
+				t.Errorf("X-Go-Pprof: got %q; want %q", got, want)
+			}
+			if !bytes.Equal(body, tc.resp) {
+				t.Errorf("response: got %q; want %q", body, tc.resp)
+			}
+		})
+	}
+
+}

tempfork/registry/export_test.go

@@ -0,0 +1,11 @@
+// Copyright 2015 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+// +build windows
+
+package registry
+
+func (k Key) SetValue(name string, valtype uint32, data []byte) error {
+	return k.setValue(name, valtype, data)
+}

tempfork/registry/fix_corp_redo_build_test.go

@@ -0,0 +1,8 @@
+package registry_test
+
+// Tailscale's redo-based build system doesn't know how to skip running Go tests
+// in directories that don't contain files for the current OS.
+//
+// https://github.com/tailscale/corp/issues/293
+//
+// So this is a dummy file for now to make it happy.