net/netns, net/interfaces: move defaultRouteInterface, add Android fallback

Signed-off-by: Brad Fitzpatrick <bradfitz@tailscale.com>

.github/workflows/cross-darwin.yml

@@ -3,7 +3,7 @@ name: Darwin-Cross
 on:
   push:
     branches:
-      - master
+      - main
   pull_request:
     branches:
       - '*'

.github/workflows/cross-freebsd.yml

@@ -3,7 +3,7 @@ name: FreeBSD-Cross
 on:
   push:
     branches:
-      - master
+      - main
   pull_request:
     branches:
       - '*'

.github/workflows/cross-openbsd.yml

@@ -3,7 +3,7 @@ name: OpenBSD-Cross
 on:
   push:
     branches:
-      - master
+      - main
   pull_request:
     branches:
       - '*'

.github/workflows/cross-windows.yml

@@ -3,7 +3,7 @@ name: Windows-Cross
 on:
   push:
     branches:
-      - master
+      - main
   pull_request:
     branches:
       - '*'

.github/workflows/license.yml

@@ -3,7 +3,7 @@ name: license
 on:
   push:
     branches:
-      - master
+      - main
   pull_request:
     branches:
       - '*'

.github/workflows/linux.yml

@@ -3,7 +3,7 @@ name: Linux
 on:
   push:
     branches:
-      - master
+      - main
   pull_request:
     branches:
       - '*'

.github/workflows/linux32.yml

@@ -0,0 +1,48 @@
+name: Linux 32-bit
+
+on:
+  push:
+    branches:
+      - main
+  pull_request:
+    branches:
+      - '*'
+
+jobs:
+  build:
+    runs-on: ubuntu-latest
+
+    if: "!contains(github.event.head_commit.message, '[ci skip]')"
+
+    steps:
+
+    - name: Set up Go
+      uses: actions/setup-go@v1
+      with:
+        go-version: 1.14
+      id: go
+
+    - name: Check out code into the Go module directory
+      uses: actions/checkout@v1
+
+    - name: Basic build
+      run: GOARCH=386 go build ./cmd/...
+
+    - name: Run tests on linux
+      run: GOARCH=386 go test ./...
+
+    - uses: k0kubun/action-slack@v2.0.0
+      with:
+        payload: |
+          {
+            "attachments": [{
+              "text": "${{ job.status }}: ${{ github.workflow }} <https://github.com/${{ github.repository }}/commit/${{ github.sha }}/checks|${{ env.COMMIT_DATE }} #${{ env.COMMIT_NUMBER_OF_DAY }}> " +
+                      "(<https://github.com/${{ github.repository }}/commit/${{ github.sha }}|" + "${{ github.sha }}".substring(0, 10) + ">) " +
+                      "of ${{ github.repository }}@" + "${{ github.ref }}".split('/').reverse()[0] + " by ${{ github.event.head_commit.committer.name }}",
+              "color": "danger"
+            }]
+          }
+      env:
+        SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK_URL }}
+      if: failure() && github.event_name == 'push'
+

.github/workflows/staticcheck.yml

@@ -3,7 +3,7 @@ name: staticcheck
 on:
   push:
     branches:
-      - master
+      - main
   pull_request:
     branches:
       - '*'
@@ -21,6 +21,9 @@ jobs:
     - name: Check out code
       uses: actions/checkout@v1
 
+    - name: Run go vet
+      run: go vet ./...
+
     - name: Print staticcheck version
       run: go run honnef.co/go/tools/cmd/staticcheck -version
 

Dockerfile

@@ -34,5 +34,5 @@ COPY . .
 RUN go install -v ./cmd/...
 
 FROM alpine:3.11
-RUN apk add --no-cache ca-certificates iptables
+RUN apk add --no-cache ca-certificates iptables iproute2
 COPY --from=build-env /go/bin/* /usr/local/bin/

Makefile

@@ -0,0 +1,7 @@
+usage:
+	echo "See Makefile"
+
+check: staticcheck
+
+staticcheck:
+	go run honnef.co/go/tools/cmd/staticcheck -- $$(go list ./... | grep -v tempfork)

README.md

@@ -6,17 +6,24 @@ Private WireGuard® networks made easy
 
 ## Overview
 
-This repository contains all the open source Tailscale code.
-It currently includes the Linux client.
+This repository contains all the open source Tailscale client code and
+the `tailscaled` daemon and `tailscale` CLI tool. The `tailscaled`
+daemon runs primarily on Linux; it also works to varying degrees on
+FreeBSD, OpenBSD, Darwin, and Windows.
 
-The Linux client is currently `cmd/relaynode`, but will
-soon be replaced by `cmd/tailscaled`.
+The Android app is at https://github.com/tailscale/tailscale-android
 
 ## Using
 
 We serve packages for a variety of distros at
 https://pkgs.tailscale.com .
 
+## Other clients
+
+The [macOS, iOS, and Windows clients](https://tailscale.com/download)
+use the code in this repository but additionally include small GUI
+wrappers that are not open source.
+
 ## Building
 
 ```
@@ -35,10 +42,8 @@ Please file any issues about this code or the hosted service on
 
 ## Contributing
 
-`under_construction.gif`
-
-PRs welcome, but we are still working out our contribution process and
-tooling.
+PRs welcome! But please file bugs. Commit messages should [reference
+bugs](https://docs.github.com/en/github/writing-on-github/autolinked-references-and-urls).
 
 We require [Developer Certificate of
 Origin](https://en.wikipedia.org/wiki/Developer_Certificate_of_Origin)
@@ -46,7 +51,7 @@ Origin](https://en.wikipedia.org/wiki/Developer_Certificate_of_Origin)
 
 ## About Us
 
-We are apenwarr, bradfitz, crawshaw, danderson, dfcarney,
+We are apenwarr, bradfitz, crawshaw, danderson, dfcarney, josharian
 from Tailscale Inc.
 You can learn more about us from [our website](https://tailscale.com).
 

atomicfile/atomicfile.go

@@ -9,20 +9,39 @@
 package atomicfile // import "tailscale.com/atomicfile"
 
 import (
-	"fmt"
 	"io/ioutil"
 	"os"
+	"path/filepath"
+	"runtime"
 )
 
 // WriteFile writes data to filename+some suffix, then renames it
 // into filename.
-func WriteFile(filename string, data []byte, perm os.FileMode) error {
-	tmpname := filename + ".new.tmp"
-	if err := ioutil.WriteFile(tmpname, data, perm); err != nil {
-		return fmt.Errorf("%#v: %v", tmpname, err)
+func WriteFile(filename string, data []byte, perm os.FileMode) (err error) {
+	f, err := ioutil.TempFile(filepath.Dir(filename), filepath.Base(filename)+".tmp")
+	if err != nil {
+		return err
 	}
-	if err := os.Rename(tmpname, filename); err != nil {
-		return fmt.Errorf("%#v->%#v: %v", tmpname, filename, err)
+	tmpName := f.Name()
+	defer func() {
+		if err != nil {
+			f.Close()
+			os.Remove(tmpName)
+		}
+	}()
+	if _, err := f.Write(data); err != nil {
+		return err
 	}
-	return nil
+	if runtime.GOOS != "windows" {
+		if err := f.Chmod(perm); err != nil {
+			return err
+		}
+	}
+	if err := f.Sync(); err != nil {
+		return err
+	}
+	if err := f.Close(); err != nil {
+		return err
+	}
+	return os.Rename(tmpName, filename)
 }

cmd/cloner/cloner.go

@@ -0,0 +1,264 @@
+// Copyright (c) 2020 Tailscale Inc & AUTHORS All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+// Cloner is a tool to automate the creation of a Clone method.
+//
+// The result of the Clone method aliases no memory that can be edited
+// with the original.
+//
+// This tool makes lots of implicit assumptions about the types you feed it.
+// In particular, it can only write relatively "shallow" Clone methods.
+// That is, if a type contains another named struct type, cloner assumes that
+// named type will also have a Clone method.
+package main
+
+import (
+	"bytes"
+	"flag"
+	"fmt"
+	"go/ast"
+	"go/format"
+	"go/token"
+	"go/types"
+	"io/ioutil"
+	"log"
+	"os"
+	"strings"
+
+	"golang.org/x/tools/go/packages"
+)
+
+var (
+	flagTypes     = flag.String("type", "", "comma-separated list of types; required")
+	flagOutput    = flag.String("output", "", "output file; required")
+	flagBuildTags = flag.String("tags", "", "compiler build tags to apply")
+)
+
+func main() {
+	log.SetFlags(0)
+	log.SetPrefix("cloner: ")
+	flag.Parse()
+	if len(*flagTypes) == 0 {
+		flag.Usage()
+		os.Exit(2)
+	}
+	typeNames := strings.Split(*flagTypes, ",")
+
+	cfg := &packages.Config{
+		Mode:  packages.NeedTypes | packages.NeedTypesInfo | packages.NeedSyntax | packages.NeedName,
+		Tests: false,
+	}
+	if *flagBuildTags != "" {
+		cfg.BuildFlags = []string{"-tags=" + *flagBuildTags}
+	}
+	pkgs, err := packages.Load(cfg, ".")
+	if err != nil {
+		log.Fatal(err)
+	}
+	if len(pkgs) != 1 {
+		log.Fatalf("wrong number of packages: %d", len(pkgs))
+	}
+	pkg := pkgs[0]
+	buf := new(bytes.Buffer)
+	imports := make(map[string]struct{})
+	for _, typeName := range typeNames {
+		found := false
+		for _, file := range pkg.Syntax {
+			//var fbuf bytes.Buffer
+			//ast.Fprint(&fbuf, pkg.Fset, file, nil)
+			//fmt.Println(fbuf.String())
+
+			for _, d := range file.Decls {
+				decl, ok := d.(*ast.GenDecl)
+				if !ok || decl.Tok != token.TYPE {
+					continue
+				}
+				for _, s := range decl.Specs {
+					spec, ok := s.(*ast.TypeSpec)
+					if !ok || spec.Name.Name != typeName {
+						continue
+					}
+					typeNameObj := pkg.TypesInfo.Defs[spec.Name]
+					typ, ok := typeNameObj.Type().(*types.Named)
+					if !ok {
+						continue
+					}
+					pkg := typeNameObj.Pkg()
+					gen(buf, imports, typeName, typ, pkg)
+				}
+				found = true
+			}
+		}
+		if !found {
+			log.Fatalf("could not find type %s", typeName)
+		}
+	}
+
+	contents := new(bytes.Buffer)
+	fmt.Fprintf(contents, header, *flagTypes, pkg.Name)
+	fmt.Fprintf(contents, "import (\n")
+	for s := range imports {
+		fmt.Fprintf(contents, "\t%q\n", s)
+	}
+	fmt.Fprintf(contents, ")\n\n")
+	contents.Write(buf.Bytes())
+
+	out, err := format.Source(contents.Bytes())
+	if err != nil {
+		log.Fatalf("%s, in source:\n%s", err, contents.Bytes())
+	}
+
+	output := *flagOutput
+	if output == "" {
+		flag.Usage()
+		os.Exit(2)
+	}
+	if err := ioutil.WriteFile(output, out, 0666); err != nil {
+		log.Fatal(err)
+	}
+}
+
+const header = `// Copyright (c) 2020 Tailscale Inc & AUTHORS All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+// Code generated by tailscale.com/cmd/cloner -type %s; DO NOT EDIT.
+
+package %s
+
+`
+
+func gen(buf *bytes.Buffer, imports map[string]struct{}, name string, typ *types.Named, thisPkg *types.Package) {
+	pkgQual := func(pkg *types.Package) string {
+		if thisPkg == pkg {
+			return ""
+		}
+		imports[pkg.Path()] = struct{}{}
+		return pkg.Name()
+	}
+	importedName := func(t types.Type) string {
+		return types.TypeString(t, pkgQual)
+	}
+
+	switch t := typ.Underlying().(type) {
+	case *types.Struct:
+		_ = t
+		name := typ.Obj().Name()
+		fmt.Fprintf(buf, "// Clone makes a deep copy of %s.\n", name)
+		fmt.Fprintf(buf, "// The result aliases no memory with the original.\n")
+		fmt.Fprintf(buf, "func (src *%s) Clone() *%s {\n", name, name)
+		writef := func(format string, args ...interface{}) {
+			fmt.Fprintf(buf, "\t"+format+"\n", args...)
+		}
+		writef("if src == nil {")
+		writef("\treturn nil")
+		writef("}")
+		writef("dst := new(%s)", name)
+		writef("*dst = *src")
+		for i := 0; i < t.NumFields(); i++ {
+			fname := t.Field(i).Name()
+			ft := t.Field(i).Type()
+			if !containsPointers(ft) {
+				continue
+			}
+			if named, _ := ft.(*types.Named); named != nil && !hasBasicUnderlying(ft) {
+				writef("dst.%s = *src.%s.Clone()", fname, fname)
+				continue
+			}
+			switch ft := ft.Underlying().(type) {
+			case *types.Slice:
+				if containsPointers(ft.Elem()) {
+					n := importedName(ft.Elem())
+					writef("dst.%s = make([]%s, len(src.%s))", fname, n, fname)
+					writef("for i := range dst.%s {", fname)
+					if _, isPtr := ft.Elem().(*types.Pointer); isPtr {
+						writef("\tdst.%s[i] = src.%s[i].Clone()", fname, fname)
+					} else {
+						writef("\tdst.%s[i] = *src.%s[i].Clone()", fname, fname)
+					}
+					writef("}")
+				} else {
+					writef("dst.%s = append(src.%s[:0:0], src.%s...)", fname, fname, fname)
+				}
+			case *types.Pointer:
+				if named, _ := ft.Elem().(*types.Named); named != nil && containsPointers(ft.Elem()) {
+					writef("dst.%s = src.%s.Clone()", fname, fname)
+					continue
+				}
+				n := importedName(ft.Elem())
+				writef("if dst.%s != nil {", fname)
+				writef("\tdst.%s = new(%s)", fname, n)
+				writef("\t*dst.%s = *src.%s", fname, fname)
+				if containsPointers(ft.Elem()) {
+					writef("\t" + `panic("TODO pointers in pointers")`)
+				}
+				writef("}")
+			case *types.Map:
+				writef("if dst.%s != nil {", fname)
+				writef("\tdst.%s = map[%s]%s{}", fname, importedName(ft.Key()), importedName(ft.Elem()))
+				if sliceType, isSlice := ft.Elem().(*types.Slice); isSlice {
+					n := importedName(sliceType.Elem())
+					writef("\tfor k := range src.%s {", fname)
+					// use zero-length slice instead of nil to ensure
+					// the key is always copied.
+					writef("\t\tdst.%s[k] = append([]%s{}, src.%s[k]...)", fname, n, fname)
+					writef("\t}")
+				} else if containsPointers(ft.Elem()) {
+					writef("\t\t" + `panic("TODO map value pointers")`)
+				} else {
+					writef("\tfor k, v := range src.%s {", fname)
+					writef("\t\tdst.%s[k] = v", fname)
+					writef("\t}")
+				}
+				writef("}")
+			case *types.Struct:
+				writef(`panic("TODO struct %s")`, fname)
+			default:
+				writef(`panic(fmt.Sprintf("TODO: %T", ft))`)
+			}
+		}
+		writef("return dst")
+		fmt.Fprintf(buf, "}\n\n")
+	}
+}
+
+func hasBasicUnderlying(typ types.Type) bool {
+	switch typ.Underlying().(type) {
+	case *types.Slice, *types.Map:
+		return true
+	default:
+		return false
+	}
+}
+
+func containsPointers(typ types.Type) bool {
+	switch typ.String() {
+	case "time.Time":
+		// time.Time contains a pointer that does not need copying
+		return false
+	case "inet.af/netaddr.IP":
+		return false
+	}
+	switch ft := typ.Underlying().(type) {
+	case *types.Array:
+		return containsPointers(ft.Elem())
+	case *types.Chan:
+		return true
+	case *types.Interface:
+		return true // a little too broad
+	case *types.Map:
+		return true
+	case *types.Pointer:
+		return true
+	case *types.Slice:
+		return true
+	case *types.Struct:
+		for i := 0; i < ft.NumFields(); i++ {
+			if containsPointers(ft.Field(i).Type()) {
+				return true
+			}
+		}
+	}
+	return false
+}

cmd/derper/derper.go

@@ -20,6 +20,7 @@ import (
 	"os"
 	"path/filepath"
 	"regexp"
+	"strings"
 	"time"
 
 	"github.com/tailscale/wireguard-go/wgcfg"
@@ -29,9 +30,10 @@ import (
 	"tailscale.com/derp/derphttp"
 	"tailscale.com/logpolicy"
 	"tailscale.com/metrics"
-	"tailscale.com/stun"
+	"tailscale.com/net/stun"
 	"tailscale.com/tsweb"
 	"tailscale.com/types/key"
+	"tailscale.com/version"
 )
 
 var (
@@ -42,6 +44,8 @@ var (
 	hostname      = flag.String("hostname", "derp.tailscale.com", "LetsEncrypt host name, if addr's port is :443")
 	logCollection = flag.String("logcollection", "", "If non-empty, logtail collection to log to")
 	runSTUN       = flag.Bool("stun", false, "also run a STUN server")
+	meshPSKFile   = flag.String("mesh-psk-file", defaultMeshPSKFile(), "if non-empty, path to file containing the mesh pre-shared key file. It should contain some hex string; whitespace is trimmed.")
+	meshWith      = flag.String("mesh-with", "", "optional comma-separated list of hostnames to mesh with; the server's own hostname can be in the list")
 )
 
 type config struct {
@@ -118,6 +122,22 @@ func main() {
 	letsEncrypt := tsweb.IsProd443(*addr)
 
 	s := derp.NewServer(key.Private(cfg.PrivateKey), log.Printf)
+
+	if *meshPSKFile != "" {
+		b, err := ioutil.ReadFile(*meshPSKFile)
+		if err != nil {
+			log.Fatal(err)
+		}
+		key := strings.TrimSpace(string(b))
+		if matched, _ := regexp.MatchString(`(?i)^[0-9a-f]{64,}$`, key); !matched {
+			log.Fatalf("key in %s must contain 64+ hex digits", *meshPSKFile)
+		}
+		s.SetMeshKey(key)
+		log.Printf("DERP mesh key configured")
+	}
+	if err := startMesh(s); err != nil {
+		log.Fatalf("startMesh: %v", err)
+	}
 	expvar.Publish("derp", s.ExpVar())
 
 	// Create our own mux so we don't expose /debug/ stuff to the world.
@@ -166,7 +186,7 @@ func main() {
 		}
 		httpsrv.TLSConfig = certManager.TLSConfig()
 		go func() {
-			err := http.ListenAndServe(":80", certManager.HTTPHandler(tsweb.Port80Handler{mux}))
+			err := http.ListenAndServe(":80", certManager.HTTPHandler(tsweb.Port80Handler{Main: mux}))
 			if err != nil {
 				if err != http.ErrServerClosed {
 					log.Fatal(err)
@@ -185,6 +205,15 @@ func main() {
 
 func debugHandler(s *derp.Server) http.Handler {
 	return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		if r.RequestURI == "/debug/check" {
+			err := s.ConsistencyCheck()
+			if err != nil {
+				http.Error(w, err.Error(), 500)
+			} else {
+				io.WriteString(w, "derp.Server ConsistencyCheck okay")
+			}
+			return
+		}
 		f := func(format string, args ...interface{}) { fmt.Fprintf(w, format, args...) }
 		f(`<html><body>
 <h1>DERP debug</h1>
@@ -192,12 +221,15 @@ func debugHandler(s *derp.Server) http.Handler {
 `)
 		f("<li><b>Hostname:</b> %v</li>\n", *hostname)
 		f("<li><b>Uptime:</b> %v</li>\n", tsweb.Uptime())
+		f("<li><b>Mesh Key:</b> %v</li>\n", s.HasMeshKey())
+		f("<li><b>Version:</b> %v</li>\n", version.LONG)
 
 		f(`<li><a href="/debug/vars">/debug/vars</a> (Go)</li>
    <li><a href="/debug/varz">/debug/varz</a> (Prometheus)</li>
    <li><a href="/debug/pprof/">/debug/pprof/</a></li>
    <li><a href="/debug/pprof/goroutine?debug=1">/debug/pprof/goroutine</a> (collapsed)</li>
    <li><a href="/debug/pprof/goroutine?debug=2">/debug/pprof/goroutine</a> (full)</li>
+   <li><a href="/debug/check">/debug/check</a> internal consistency check</li>
 <ul>
 </html>
 `)
@@ -268,7 +300,7 @@ func serveSTUN() {
 	}
 }
 
-var validProdHostname = regexp.MustCompile(`^derp(\d+|\-\w+)?\.tailscale\.com\.?$`)
+var validProdHostname = regexp.MustCompile(`^derp([^.]*)\.tailscale\.com\.?$`)
 
 func prodAutocertHostPolicy(_ context.Context, host string) error {
 	if validProdHostname.MatchString(host) {
@@ -276,3 +308,16 @@ func prodAutocertHostPolicy(_ context.Context, host string) error {
 	}
 	return errors.New("invalid hostname")
 }
+
+func defaultMeshPSKFile() string {
+	try := []string{
+		"/home/derp/keys/derp-mesh.key",
+		filepath.Join(os.Getenv("HOME"), "keys", "derp-mesh.key"),
+	}
+	for _, p := range try {
+		if _, err := os.Stat(p); err == nil {
+			return p
+		}
+	}
+	return ""
+}

cmd/derper/derper_test.go

@@ -17,10 +17,11 @@ func TestProdAutocertHostPolicy(t *testing.T) {
 		{"derp.tailscale.com", true},
 		{"derp.tailscale.com.", true},
 		{"derp1.tailscale.com", true},
+		{"derp1b.tailscale.com", true},
 		{"derp2.tailscale.com", true},
 		{"derp02.tailscale.com", true},
 		{"derp-nyc.tailscale.com", true},
-		{"derpfoo.tailscale.com", false},
+		{"derpfoo.tailscale.com", true},
 		{"derp02.bar.tailscale.com", false},
 		{"example.net", false},
 	}

cmd/derper/mesh.go

@@ -0,0 +1,45 @@
+// Copyright (c) 2020 Tailscale Inc & AUTHORS All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package main
+
+import (
+	"errors"
+	"fmt"
+	"log"
+	"strings"
+
+	"tailscale.com/derp"
+	"tailscale.com/derp/derphttp"
+	"tailscale.com/types/key"
+	"tailscale.com/types/logger"
+)
+
+func startMesh(s *derp.Server) error {
+	if *meshWith == "" {
+		return nil
+	}
+	if !s.HasMeshKey() {
+		return errors.New("--mesh-with requires --mesh-psk-file")
+	}
+	for _, host := range strings.Split(*meshWith, ",") {
+		if err := startMeshWithHost(s, host); err != nil {
+			return err
+		}
+	}
+	return nil
+}
+
+func startMeshWithHost(s *derp.Server, host string) error {
+	logf := logger.WithPrefix(log.Printf, fmt.Sprintf("mesh(%q): ", host))
+	c, err := derphttp.NewClient(s.PrivateKey(), "https://"+host+"/derp", logf)
+	if err != nil {
+		return err
+	}
+	c.MeshKey = s.MeshKey()
+	add := func(k key.Public) { s.AddPacketForwarder(k, c) }
+	remove := func(k key.Public) { s.RemovePacketForwarder(k, c) }
+	go c.RunWatchConnectionLoop(s.PublicKey(), add, remove)
+	return nil
+}

cmd/relaynode/relaynode.od

@@ -1 +0,0 @@
-# placeholder to work around redo bug

cmd/tailscale/cli/cli.go

@@ -0,0 +1,128 @@
+// Copyright (c) 2020 Tailscale Inc & AUTHORS All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+// Package cli contains the cmd/tailscale CLI code in a package that can be included
+// in other wrapper binaries such as the Mac and Windows clients.
+package cli
+
+import (
+	"context"
+	"flag"
+	"log"
+	"net"
+	"os"
+	"os/signal"
+	"runtime"
+	"strings"
+	"syscall"
+
+	"github.com/peterbourgon/ff/v2/ffcli"
+	"tailscale.com/ipn"
+	"tailscale.com/paths"
+	"tailscale.com/safesocket"
+)
+
+// ActLikeCLI reports whether a GUI application should act like the
+// CLI based on os.Args, GOOS, the context the process is running in
+// (pty, parent PID), etc.
+func ActLikeCLI() bool {
+	if len(os.Args) < 2 {
+		return false
+	}
+	switch os.Args[1] {
+	case "up", "status", "netcheck", "version",
+		"-V", "--version", "-h", "--help":
+		return true
+	}
+	return false
+}
+
+// Run runs the CLI. The args do not include the binary name.
+func Run(args []string) error {
+	if len(args) == 1 && (args[0] == "-V" || args[0] == "--version") {
+		args = []string{"version"}
+	}
+
+	rootfs := flag.NewFlagSet("tailscale", flag.ExitOnError)
+	rootfs.StringVar(&rootArgs.socket, "socket", paths.DefaultTailscaledSocket(), "path to tailscaled's unix socket")
+
+	rootCmd := &ffcli.Command{
+		Name:       "tailscale",
+		ShortUsage: "tailscale subcommand [flags]",
+		ShortHelp:  "The easiest, most secure way to use WireGuard.",
+		LongHelp: strings.TrimSpace(`
+This CLI is still under active development. Commands and flags will
+change in the future.
+`),
+		Subcommands: []*ffcli.Command{
+			upCmd,
+			netcheckCmd,
+			statusCmd,
+			versionCmd,
+		},
+		FlagSet: rootfs,
+		Exec:    func(context.Context, []string) error { return flag.ErrHelp },
+	}
+
+	if err := rootCmd.Parse(args); err != nil {
+		return err
+	}
+
+	err := rootCmd.Run(context.Background())
+	if err == flag.ErrHelp {
+		return nil
+	}
+	return err
+}
+
+func fatalf(format string, a ...interface{}) {
+	log.SetFlags(0)
+	log.Fatalf(format, a...)
+}
+
+var rootArgs struct {
+	socket string
+}
+
+func connect(ctx context.Context) (net.Conn, *ipn.BackendClient, context.Context, context.CancelFunc) {
+	c, err := safesocket.Connect(rootArgs.socket, 41112)
+	if err != nil {
+		if runtime.GOOS != "windows" && rootArgs.socket == "" {
+			fatalf("--socket cannot be empty")
+		}
+		fatalf("Failed to connect to connect to tailscaled. (safesocket.Connect: %v)\n", err)
+	}
+	clientToServer := func(b []byte) {
+		ipn.WriteMsg(c, b)
+	}
+
+	ctx, cancel := context.WithCancel(ctx)
+
+	go func() {
+		interrupt := make(chan os.Signal, 1)
+		signal.Notify(interrupt, syscall.SIGINT, syscall.SIGTERM)
+		<-interrupt
+		c.Close()
+		cancel()
+	}()
+
+	bc := ipn.NewBackendClient(log.Printf, clientToServer)
+	return c, bc, ctx, cancel
+}
+
+// pump receives backend messages on conn and pushes them into bc.
+func pump(ctx context.Context, bc *ipn.BackendClient, conn net.Conn) {
+	defer conn.Close()
+	for ctx.Err() == nil {
+		msg, err := ipn.ReadMsg(conn)
+		if err != nil {
+			if ctx.Err() != nil {
+				return
+			}
+			log.Printf("ReadMsg: %v\n", err)
+			break
+		}
+		bc.GotNotifyMsg(msg)
+	}
+}

cmd/tailscale/cli/netcheck.go

@@ -0,0 +1,162 @@
+// Copyright (c) 2020 Tailscale Inc & AUTHORS All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package cli
+
+import (
+	"context"
+	"encoding/json"
+	"flag"
+	"fmt"
+	"log"
+	"os"
+	"sort"
+	"strings"
+	"time"
+
+	"github.com/peterbourgon/ff/v2/ffcli"
+	"tailscale.com/derp/derpmap"
+	"tailscale.com/net/dnscache"
+	"tailscale.com/net/netcheck"
+	"tailscale.com/tailcfg"
+	"tailscale.com/types/logger"
+)
+
+var netcheckCmd = &ffcli.Command{
+	Name:       "netcheck",
+	ShortUsage: "netcheck",
+	ShortHelp:  "Print an analysis of local network conditions",
+	Exec:       runNetcheck,
+	FlagSet: (func() *flag.FlagSet {
+		fs := flag.NewFlagSet("netcheck", flag.ExitOnError)
+		fs.StringVar(&netcheckArgs.format, "format", "", `output format; empty (for human-readable), "json" or "json-line"`)
+		fs.DurationVar(&netcheckArgs.every, "every", 0, "if non-zero, do an incremental report with the given frequency")
+		fs.BoolVar(&netcheckArgs.verbose, "verbose", false, "verbose logs")
+		return fs
+	})(),
+}
+
+var netcheckArgs struct {
+	format  string
+	every   time.Duration
+	verbose bool
+}
+
+func runNetcheck(ctx context.Context, args []string) error {
+	c := &netcheck.Client{
+		DNSCache: dnscache.Get(),
+	}
+	if netcheckArgs.verbose {
+		c.Logf = logger.WithPrefix(log.Printf, "netcheck: ")
+		c.Verbose = true
+	} else {
+		c.Logf = logger.Discard
+	}
+
+	if strings.HasPrefix(netcheckArgs.format, "json") {
+		fmt.Fprintln(os.Stderr, "# Warning: this JSON format is not yet considered a stable interface")
+	}
+
+	dm := derpmap.Prod()
+	for {
+		t0 := time.Now()
+		report, err := c.GetReport(ctx, dm)
+		d := time.Since(t0)
+		if netcheckArgs.verbose {
+			c.Logf("GetReport took %v; err=%v", d.Round(time.Millisecond), err)
+		}
+		if err != nil {
+			log.Fatalf("netcheck: %v", err)
+		}
+		if err := printReport(dm, report); err != nil {
+			return err
+		}
+		if netcheckArgs.every == 0 {
+			return nil
+		}
+		time.Sleep(netcheckArgs.every)
+	}
+}
+
+func printReport(dm *tailcfg.DERPMap, report *netcheck.Report) error {
+	var j []byte
+	var err error
+	switch netcheckArgs.format {
+	case "":
+		break
+	case "json":
+		j, err = json.MarshalIndent(report, "", "\t")
+	case "json-line":
+		j, err = json.Marshal(report)
+	default:
+		return fmt.Errorf("unknown output format %q", netcheckArgs.format)
+	}
+	if err != nil {
+		return err
+	}
+	if j != nil {
+		j = append(j, '\n')
+		os.Stdout.Write(j)
+		return nil
+	}
+
+	fmt.Printf("\nReport:\n")
+	fmt.Printf("\t* UDP: %v\n", report.UDP)
+	if report.GlobalV4 != "" {
+		fmt.Printf("\t* IPv4: yes, %v\n", report.GlobalV4)
+	} else {
+		fmt.Printf("\t* IPv4: (no addr found)\n")
+	}
+	if report.GlobalV6 != "" {
+		fmt.Printf("\t* IPv6: yes, %v\n", report.GlobalV6)
+	} else if report.IPv6 {
+		fmt.Printf("\t* IPv6: (no addr found)\n")
+	} else {
+		fmt.Printf("\t* IPv6: no\n")
+	}
+	fmt.Printf("\t* MappingVariesByDestIP: %v\n", report.MappingVariesByDestIP)
+	fmt.Printf("\t* HairPinning: %v\n", report.HairPinning)
+	fmt.Printf("\t* PortMapping: %v\n", portMapping(report))
+
+	// When DERP latency checking failed,
+	// magicsock will try to pick the DERP server that
+	// most of your other nodes are also using
+	if len(report.RegionLatency) == 0 {
+		fmt.Printf("\t* Nearest DERP: unknown (no response to latency probes)\n")
+	} else {
+		fmt.Printf("\t* Nearest DERP: %v (%v)\n", report.PreferredDERP, dm.Regions[report.PreferredDERP].RegionCode)
+		fmt.Printf("\t* DERP latency:\n")
+		var rids []int
+		for rid := range dm.Regions {
+			rids = append(rids, rid)
+		}
+		sort.Ints(rids)
+		for _, rid := range rids {
+			d, ok := report.RegionLatency[rid]
+			var latency string
+			if ok {
+				latency = d.Round(time.Millisecond / 10).String()
+			}
+			fmt.Printf("\t\t- %v, %3s = %s\n", rid, dm.Regions[rid].RegionCode, latency)
+		}
+	}
+	return nil
+}
+
+func portMapping(r *netcheck.Report) string {
+	if !r.AnyPortMappingChecked() {
+		return "not checked"
+	}
+	var got []string
+	if r.UPnP.EqualBool(true) {
+		got = append(got, "UPnP")
+	}
+	if r.PMP.EqualBool(true) {
+		got = append(got, "NAT-PMP")
+	}
+	if r.PCP.EqualBool(true) {
+		got = append(got, "PCP")
+	}
+	return strings.Join(got, ", ")
+}

cmd/tailscale/cli/status.go

@@ -2,7 +2,7 @@
 // Use of this source code is governed by a BSD-style
 // license that can be found in the LICENSE file.
 
-package main
+package cli
 
 import (
 	"bytes"
@@ -14,6 +14,7 @@ import (
 	"net"
 	"net/http"
 	"os"
+	"time"
 
 	"github.com/peterbourgon/ff/v2/ffcli"
 	"github.com/toqueteos/webbrowser"
@@ -24,13 +25,14 @@ import (
 
 var statusCmd = &ffcli.Command{
 	Name:       "status",
-	ShortUsage: "status [-web] [-json]",
+	ShortUsage: "status [-active] [-web] [-json]",
 	ShortHelp:  "Show state of tailscaled and its connections",
 	Exec:       runStatus,
 	FlagSet: (func() *flag.FlagSet {
 		fs := flag.NewFlagSet("status", flag.ExitOnError)
 		fs.BoolVar(&statusArgs.json, "json", false, "output in JSON format (WARNING: format subject to change)")
 		fs.BoolVar(&statusArgs.web, "web", false, "run webserver with HTML showing status")
+		fs.BoolVar(&statusArgs.active, "active", false, "filter output to only peers with active sessions (not applicable to web mode)")
 		fs.StringVar(&statusArgs.listen, "listen", "127.0.0.1:8384", "listen address; use port 0 for automatic")
 		fs.BoolVar(&statusArgs.browser, "browser", true, "Open a browser in web mode")
 		return fs
@@ -42,12 +44,15 @@ var statusArgs struct {
 	web     bool   // run webserver
 	listen  string // in web mode, webserver address to listen on, empty means auto
 	browser bool   // in web mode, whether to open browser
+	active  bool   // in CLI mode, filter output to only peers with active sessions
 }
 
 func runStatus(ctx context.Context, args []string) error {
 	c, bc, ctx, cancel := connect(ctx)
 	defer cancel()
 
+	bc.AllowVersionSkew = true
+
 	ch := make(chan *ipnstate.Status, 1)
 	bc.SetNotifyCallback(func(n ipn.Notify) {
 		if n.ErrMessage != nil {
@@ -73,6 +78,13 @@ func runStatus(ctx context.Context, args []string) error {
 		return err
 	}
 	if statusArgs.json {
+		if statusArgs.active {
+			for peer, ps := range st.Peer {
+				if !peerActive(ps) {
+					delete(st.Peer, peer)
+				}
+			}
+		}
 		j, err := json.MarshalIndent(st, "", "  ")
 		if err != nil {
 			return err
@@ -117,6 +129,10 @@ func runStatus(ctx context.Context, args []string) error {
 	f := func(format string, a ...interface{}) { fmt.Fprintf(&buf, format, a...) }
 	for _, peer := range st.Peers() {
 		ps := st.Peer[peer]
+		active := peerActive(ps)
+		if statusArgs.active && !active {
+			continue
+		}
 		f("%s %-7s %-15s %-18s tx=%8d rx=%8d ",
 			peer.ShortString(),
 			ps.OS,
@@ -125,6 +141,13 @@ func runStatus(ctx context.Context, args []string) error {
 			ps.TxBytes,
 			ps.RxBytes,
 		)
+		relay := ps.Relay
+		if active && relay != "" && ps.CurAddr == "" {
+			relay = "*" + relay + "*"
+		} else {
+			relay = " " + relay
+		}
+		f("%-6s", relay)
 		for i, addr := range ps.Addrs {
 			if i != 0 {
 				f(", ")
@@ -140,3 +163,10 @@ func runStatus(ctx context.Context, args []string) error {
 	os.Stdout.Write(buf.Bytes())
 	return nil
 }
+
+// peerActive reports whether ps has recent activity.
+//
+// TODO: have the server report this bool instead.
+func peerActive(ps *ipnstate.PeerStatus) bool {
+	return !ps.LastWrite.IsZero() && time.Since(ps.LastWrite) < 2*time.Minute
+}

cmd/tailscale/cli/up.go

@@ -0,0 +1,258 @@
+// Copyright (c) 2020 Tailscale Inc & AUTHORS All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package cli
+
+import (
+	"bytes"
+	"context"
+	"flag"
+	"fmt"
+	"log"
+	"os"
+	"os/exec"
+	"runtime"
+	"strconv"
+	"strings"
+
+	"github.com/peterbourgon/ff/v2/ffcli"
+	"github.com/tailscale/wireguard-go/wgcfg"
+	"inet.af/netaddr"
+	"tailscale.com/ipn"
+	"tailscale.com/tailcfg"
+	"tailscale.com/version"
+	"tailscale.com/wgengine/router"
+)
+
+// globalStateKey is the ipn.StateKey that tailscaled loads on
+// startup.
+//
+// We have to support multiple state keys for other OSes (Windows in
+// particular), but right now Unix daemons run with a single
+// node-global state. To keep open the option of having per-user state
+// later, the global state key doesn't look like a username.
+const globalStateKey = "_daemon"
+
+var upCmd = &ffcli.Command{
+	Name:       "up",
+	ShortUsage: "up [flags]",
+	ShortHelp:  "Connect to your Tailscale network",
+
+	LongHelp: strings.TrimSpace(`
+"tailscale up" connects this machine to your Tailscale network,
+triggering authentication if necessary.
+
+The flags passed to this command are specific to this machine. If you don't
+specify any flags, options are reset to their default.
+`),
+	FlagSet: (func() *flag.FlagSet {
+		upf := flag.NewFlagSet("up", flag.ExitOnError)
+		upf.StringVar(&upArgs.server, "login-server", "https://login.tailscale.com", "base URL of control server")
+		upf.BoolVar(&upArgs.acceptRoutes, "accept-routes", false, "accept routes advertised by other Tailscale nodes")
+		upf.BoolVar(&upArgs.acceptDNS, "accept-dns", true, "accept DNS configuration from the admin panel")
+		upf.BoolVar(&upArgs.singleRoutes, "host-routes", true, "install host routes to other Tailscale nodes")
+		upf.BoolVar(&upArgs.shieldsUp, "shields-up", false, "don't allow incoming connections")
+		upf.StringVar(&upArgs.advertiseTags, "advertise-tags", "", "ACL tags to request (comma-separated, e.g. eng,montreal,ssh)")
+		upf.StringVar(&upArgs.authKey, "authkey", "", "node authorization key")
+		upf.StringVar(&upArgs.hostname, "hostname", "", "hostname to use instead of the one provided by the OS")
+		upf.BoolVar(&upArgs.enableDERP, "enable-derp", true, "enable the use of DERP servers")
+		if runtime.GOOS == "linux" || isBSD(runtime.GOOS) || version.OS() == "macOS" {
+			upf.StringVar(&upArgs.advertiseRoutes, "advertise-routes", "", "routes to advertise to other nodes (comma-separated, e.g. 10.0.0.0/8,192.168.0.0/24)")
+		}
+		if runtime.GOOS == "linux" {
+			upf.BoolVar(&upArgs.snat, "snat-subnet-routes", true, "source NAT traffic to local routes advertised with -advertise-routes")
+			upf.StringVar(&upArgs.netfilterMode, "netfilter-mode", "on", "netfilter mode (one of on, nodivert, off)")
+		}
+		return upf
+	})(),
+	Exec: runUp,
+}
+
+var upArgs struct {
+	server          string
+	acceptRoutes    bool
+	acceptDNS       bool
+	singleRoutes    bool
+	shieldsUp       bool
+	advertiseRoutes string
+	advertiseTags   string
+	enableDERP      bool
+	snat            bool
+	netfilterMode   string
+	authKey         string
+	hostname        string
+}
+
+// parseIPOrCIDR parses an IP address or a CIDR prefix. If the input
+// is an IP address, it is returned in CIDR form with a /32 mask for
+// IPv4 or a /128 mask for IPv6.
+func parseIPOrCIDR(s string) (wgcfg.CIDR, bool) {
+	if strings.Contains(s, "/") {
+		ret, err := wgcfg.ParseCIDR(s)
+		if err != nil {
+			return wgcfg.CIDR{}, false
+		}
+		return ret, true
+	}
+
+	ip, ok := wgcfg.ParseIP(s)
+	if !ok {
+		return wgcfg.CIDR{}, false
+	}
+	if ip.Is4() {
+		return wgcfg.CIDR{IP: ip, Mask: 32}, true
+	} else {
+		return wgcfg.CIDR{IP: ip, Mask: 128}, true
+	}
+}
+
+func isBSD(s string) bool {
+	return s == "dragonfly" || s == "freebsd" || s == "netbsd" || s == "openbsd"
+}
+
+func warnf(format string, args ...interface{}) {
+	fmt.Printf("Warning: "+format+"\n", args...)
+}
+
+// checkIPForwarding prints warnings if IP forwarding is not
+// enabled, or if we were unable to verify the state of IP forwarding.
+func checkIPForwarding() {
+	var key string
+
+	if runtime.GOOS == "linux" {
+		key = "net.ipv4.ip_forward"
+	} else if isBSD(runtime.GOOS) || version.OS() == "macOS" {
+		key = "net.inet.ip.forwarding"
+	} else {
+		return
+	}
+
+	bs, err := exec.Command("sysctl", "-n", key).Output()
+	if err != nil {
+		warnf("couldn't check %s (%v).\nSubnet routes won't work without IP forwarding.", key, err)
+		return
+	}
+	on, err := strconv.ParseBool(string(bytes.TrimSpace(bs)))
+	if err != nil {
+		warnf("couldn't parse %s (%v).\nSubnet routes won't work without IP forwarding.", key, err)
+		return
+	}
+	if !on {
+		warnf("%s is disabled. Subnet routes won't work.", key)
+	}
+}
+
+func runUp(ctx context.Context, args []string) error {
+	if len(args) > 0 {
+		log.Fatalf("too many non-flag arguments: %q", args)
+	}
+
+	var routes []wgcfg.CIDR
+	if upArgs.advertiseRoutes != "" {
+		advroutes := strings.Split(upArgs.advertiseRoutes, ",")
+		for _, s := range advroutes {
+			cidr, ok := parseIPOrCIDR(s)
+			ipp, err := netaddr.ParseIPPrefix(s) // parse it with other pawith both packages
+			if !ok || err != nil {
+				fatalf("%q is not a valid IP address or CIDR prefix", s)
+			}
+			if ipp != ipp.Masked() {
+				fatalf("%s has non-address bits set; expected %s", ipp, ipp.Masked())
+			}
+			routes = append(routes, cidr)
+		}
+		checkIPForwarding()
+	}
+
+	var tags []string
+	if upArgs.advertiseTags != "" {
+		tags = strings.Split(upArgs.advertiseTags, ",")
+		for _, tag := range tags {
+			err := tailcfg.CheckTag(tag)
+			if err != nil {
+				fatalf("tag: %q: %s", tag, err)
+			}
+		}
+	}
+
+	if len(upArgs.hostname) > 256 {
+		fatalf("hostname too long: %d bytes (max 256)", len(upArgs.hostname))
+	}
+
+	// TODO(apenwarr): fix different semantics between prefs and uflags
+	// TODO(apenwarr): allow setting/using CorpDNS
+	prefs := ipn.NewPrefs()
+	prefs.ControlURL = upArgs.server
+	prefs.WantRunning = true
+	prefs.RouteAll = upArgs.acceptRoutes
+	prefs.CorpDNS = upArgs.acceptDNS
+	prefs.AllowSingleHosts = upArgs.singleRoutes
+	prefs.ShieldsUp = upArgs.shieldsUp
+	prefs.AdvertiseRoutes = routes
+	prefs.AdvertiseTags = tags
+	prefs.NoSNAT = !upArgs.snat
+	prefs.DisableDERP = !upArgs.enableDERP
+	prefs.Hostname = upArgs.hostname
+	if runtime.GOOS == "linux" {
+		switch upArgs.netfilterMode {
+		case "on":
+			prefs.NetfilterMode = router.NetfilterOn
+		case "nodivert":
+			prefs.NetfilterMode = router.NetfilterNoDivert
+			warnf("netfilter=nodivert; add iptables calls to ts-* chains manually.")
+		case "off":
+			prefs.NetfilterMode = router.NetfilterOff
+			warnf("netfilter=off; configure iptables yourself.")
+		default:
+			fatalf("invalid value --netfilter-mode: %q", upArgs.netfilterMode)
+		}
+	}
+
+	c, bc, ctx, cancel := connect(ctx)
+	defer cancel()
+
+	var printed bool
+
+	bc.SetPrefs(prefs)
+	opts := ipn.Options{
+		StateKey: globalStateKey,
+		AuthKey:  upArgs.authKey,
+		Notify: func(n ipn.Notify) {
+			if n.ErrMessage != nil {
+				fatalf("backend error: %v\n", *n.ErrMessage)
+			}
+			if s := n.State; s != nil {
+				switch *s {
+				case ipn.NeedsLogin:
+					printed = true
+					bc.StartLoginInteractive()
+				case ipn.NeedsMachineAuth:
+					printed = true
+					fmt.Fprintf(os.Stderr, "\nTo authorize your machine, visit (as admin):\n\n\t%s/admin/machines\n\n", upArgs.server)
+				case ipn.Starting, ipn.Running:
+					// Done full authentication process
+					if printed {
+						// Only need to print an update if we printed the "please click" message earlier.
+						fmt.Fprintf(os.Stderr, "Success.\n")
+					}
+					cancel()
+				}
+			}
+			if url := n.BrowseToURL; url != nil {
+				fmt.Fprintf(os.Stderr, "\nTo authenticate, visit:\n\n\t%s\n\n", *url)
+			}
+		},
+	}
+	// We still have to Start right now because it's the only way to
+	// set up notifications and whatnot. This causes a bunch of churn
+	// every time the CLI touches anything.
+	//
+	// TODO(danderson): redo the frontend/backend API to assume
+	// ephemeral frontends that read/modify/write state, once
+	// Windows/Mac state is moved into backend.
+	bc.Start(opts)
+	pump(ctx, bc, c)
+
+	return nil
+}

cmd/tailscale/cli/version.go

@@ -0,0 +1,69 @@
+// Copyright (c) 2020 Tailscale Inc & AUTHORS All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package cli
+
+import (
+	"context"
+	"flag"
+	"fmt"
+	"log"
+
+	"github.com/peterbourgon/ff/v2/ffcli"
+	"tailscale.com/ipn"
+	"tailscale.com/version"
+)
+
+var versionCmd = &ffcli.Command{
+	Name:       "version",
+	ShortUsage: "version [flags]",
+	ShortHelp:  "Print Tailscale version",
+	FlagSet: (func() *flag.FlagSet {
+		fs := flag.NewFlagSet("version", flag.ExitOnError)
+		fs.BoolVar(&versionArgs.daemon, "daemon", false, "also print local node's daemon version")
+		return fs
+	})(),
+	Exec: runVersion,
+}
+
+var versionArgs struct {
+	daemon bool // also check local node's daemon version
+}
+
+func runVersion(ctx context.Context, args []string) error {
+	if len(args) > 0 {
+		log.Fatalf("too many non-flag arguments: %q", args)
+	}
+	if !versionArgs.daemon {
+		fmt.Println(version.LONG)
+		return nil
+	}
+	fmt.Printf("Client: %s\n", version.LONG)
+
+	c, bc, ctx, cancel := connect(ctx)
+	defer cancel()
+
+	bc.AllowVersionSkew = true
+
+	done := make(chan struct{})
+
+	bc.SetNotifyCallback(func(n ipn.Notify) {
+		if n.ErrMessage != nil {
+			log.Fatal(*n.ErrMessage)
+		}
+		if n.Status != nil {
+			fmt.Printf("Daemon: %s\n", n.Version)
+			close(done)
+		}
+	})
+	go pump(ctx, bc, c)
+
+	bc.RequestStatus()
+	select {
+	case <-done:
+		return nil
+	case <-ctx.Done():
+		return ctx.Err()
+	}
+}

cmd/tailscale/netcheck.go

@@ -1,73 +0,0 @@
-// Copyright (c) 2020 Tailscale Inc & AUTHORS All rights reserved.
-// Use of this source code is governed by a BSD-style
-// license that can be found in the LICENSE file.
-
-package main
-
-import (
-	"context"
-	"fmt"
-	"log"
-	"sort"
-
-	"github.com/peterbourgon/ff/v2/ffcli"
-	"tailscale.com/derp/derpmap"
-	"tailscale.com/net/dnscache"
-	"tailscale.com/netcheck"
-	"tailscale.com/types/logger"
-)
-
-var netcheckCmd = &ffcli.Command{
-	Name:       "netcheck",
-	ShortUsage: "netcheck",
-	ShortHelp:  "Print an analysis of local network conditions",
-	Exec:       runNetcheck,
-}
-
-func runNetcheck(ctx context.Context, args []string) error {
-	c := &netcheck.Client{
-		DERP:     derpmap.Prod(),
-		Logf:     logger.WithPrefix(log.Printf, "netcheck: "),
-		DNSCache: dnscache.Get(),
-	}
-
-	report, err := c.GetReport(ctx)
-	if err != nil {
-		log.Fatalf("netcheck: %v", err)
-	}
-	fmt.Printf("\nReport:\n")
-	fmt.Printf("\t* UDP: %v\n", report.UDP)
-	if report.GlobalV4 != "" {
-		fmt.Printf("\t* IPv4: yes, %v\n", report.GlobalV4)
-	} else {
-		fmt.Printf("\t* IPv4: (no addr found)\n")
-	}
-	if report.GlobalV6 != "" {
-		fmt.Printf("\t* IPv6: yes, %v\n", report.GlobalV6)
-	} else if report.IPv6 {
-		fmt.Printf("\t* IPv6: (no addr found)\n")
-	} else {
-		fmt.Printf("\t* IPv6: no\n")
-	}
-	fmt.Printf("\t* MappingVariesByDestIP: %v\n", report.MappingVariesByDestIP)
-	fmt.Printf("\t* HairPinning: %v\n", report.HairPinning)
-
-	// When DERP latency checking failed,
-	// magicsock will try to pick the DERP server that
-	// most of your other nodes are also using
-	if len(report.DERPLatency) == 0 {
-		fmt.Printf("\t* Nearest DERP: unknown (no response to latency probes)\n")
-	} else {
-		fmt.Printf("\t* Nearest DERP: %v (%v)\n", report.PreferredDERP, c.DERP.LocationOfID(report.PreferredDERP))
-		fmt.Printf("\t* DERP latency:\n")
-		var ss []string
-		for s := range report.DERPLatency {
-			ss = append(ss, s)
-		}
-		sort.Strings(ss)
-		for _, s := range ss {
-			fmt.Printf("\t\t- %s = %v\n", s, report.DERPLatency[s])
-		}
-	}
-	return nil
-}

cmd/tailscale/tailscale.go

@@ -7,222 +7,22 @@
 package main // import "tailscale.com/cmd/tailscale"
 
 import (
-	"context"
-	"flag"
 	"fmt"
 	"log"
-	"net"
 	"os"
-	"os/signal"
-	"runtime"
-	"strings"
-	"syscall"
 
 	"github.com/apenwarr/fixconsole"
-	"github.com/peterbourgon/ff/v2/ffcli"
-	"github.com/tailscale/wireguard-go/wgcfg"
-	"tailscale.com/ipn"
-	"tailscale.com/paths"
-	"tailscale.com/safesocket"
-	"tailscale.com/tailcfg"
+	"tailscale.com/cmd/tailscale/cli"
 )
 
-// globalStateKey is the ipn.StateKey that tailscaled loads on
-// startup.
-//
-// We have to support multiple state keys for other OSes (Windows in
-// particular), but right now Unix daemons run with a single
-// node-global state. To keep open the option of having per-user state
-// later, the global state key doesn't look like a username.
-const globalStateKey = "_daemon"
-
-var rootArgs struct {
-	socket string
-}
-
 func main() {
 	err := fixconsole.FixConsoleIfNeeded()
 	if err != nil {
 		log.Printf("fixConsoleOutput: %v\n", err)
 	}
 
-	upf := flag.NewFlagSet("up", flag.ExitOnError)
-	upf.StringVar(&upArgs.server, "login-server", "https://login.tailscale.com", "base URL of control server")
-	upf.BoolVar(&upArgs.acceptRoutes, "accept-routes", false, "accept routes advertised by other Tailscale nodes")
-	upf.BoolVar(&upArgs.noSingleRoutes, "no-single-routes", false, "don't install routes to single nodes")
-	upf.BoolVar(&upArgs.shieldsUp, "shields-up", false, "don't allow incoming connections")
-	upf.StringVar(&upArgs.advertiseRoutes, "advertise-routes", "", "routes to advertise to other nodes (comma-separated, e.g. 10.0.0.0/8,192.168.0.0/24)")
-	upf.StringVar(&upArgs.advertiseTags, "advertise-tags", "", "ACL tags to request (comma-separated, e.g. eng,montreal,ssh)")
-	upf.StringVar(&upArgs.authKey, "authkey", "", "node authorization key")
-	upCmd := &ffcli.Command{
-		Name:       "up",
-		ShortUsage: "up [flags]",
-		ShortHelp:  "Connect to your Tailscale network",
-
-		LongHelp: strings.TrimSpace(`
-"tailscale up" connects this machine to your Tailscale network,
-triggering authentication if necessary.
-
-The flags passed to this command are specific to this machine. If you don't
-specify any flags, options are reset to their default.
-`),
-		FlagSet: upf,
-		Exec:    runUp,
-	}
-
-	rootfs := flag.NewFlagSet("tailscale", flag.ExitOnError)
-	rootfs.StringVar(&rootArgs.socket, "socket", paths.DefaultTailscaledSocket(), "path to tailscaled's unix socket")
-
-	rootCmd := &ffcli.Command{
-		Name:       "tailscale",
-		ShortUsage: "tailscale subcommand [flags]",
-		ShortHelp:  "The easiest, most secure way to use WireGuard.",
-		LongHelp: strings.TrimSpace(`
-This CLI is still under active development. Commands and flags will
-change in the future.
-`),
-		Subcommands: []*ffcli.Command{
-			upCmd,
-			netcheckCmd,
-			statusCmd,
-		},
-		FlagSet: rootfs,
-		Exec:    func(context.Context, []string) error { return flag.ErrHelp },
-	}
-
-	if err := rootCmd.ParseAndRun(context.Background(), os.Args[1:]); err != nil && err != flag.ErrHelp {
-		log.Fatal(err)
-	}
-}
-
-var upArgs struct {
-	server          string
-	acceptRoutes    bool
-	noSingleRoutes  bool
-	shieldsUp       bool
-	advertiseRoutes string
-	advertiseTags   string
-	authKey         string
-}
-
-func runUp(ctx context.Context, args []string) error {
-	if len(args) > 0 {
-		log.Fatalf("too many non-flag arguments: %q", args)
-	}
-
-	var routes []wgcfg.CIDR
-	if upArgs.advertiseRoutes != "" {
-		advroutes := strings.Split(upArgs.advertiseRoutes, ",")
-		for _, s := range advroutes {
-			cidr, err := wgcfg.ParseCIDR(s)
-			if err != nil {
-				log.Fatalf("%q is not a valid CIDR prefix: %v", s, err)
-			}
-			routes = append(routes, cidr)
-		}
-	}
-
-	var tags []string
-	if upArgs.advertiseTags != "" {
-		tags = strings.Split(upArgs.advertiseTags, ",")
-		for _, tag := range tags {
-			err := tailcfg.CheckTag(tag)
-			if err != nil {
-				log.Fatalf("tag: %q: %s", tag, err)
-			}
-		}
-	}
-
-	// TODO(apenwarr): fix different semantics between prefs and uflags
-	// TODO(apenwarr): allow setting/using CorpDNS
-	prefs := ipn.NewPrefs()
-	prefs.ControlURL = upArgs.server
-	prefs.WantRunning = true
-	prefs.RouteAll = upArgs.acceptRoutes
-	prefs.AllowSingleHosts = !upArgs.noSingleRoutes
-	prefs.ShieldsUp = upArgs.shieldsUp
-	prefs.AdvertiseRoutes = routes
-	prefs.AdvertiseTags = tags
-
-	c, bc, ctx, cancel := connect(ctx)
-	defer cancel()
-
-	bc.SetPrefs(prefs)
-	opts := ipn.Options{
-		StateKey: globalStateKey,
-		AuthKey:  upArgs.authKey,
-		Notify: func(n ipn.Notify) {
-			if n.ErrMessage != nil {
-				log.Fatalf("backend error: %v\n", *n.ErrMessage)
-			}
-			if s := n.State; s != nil {
-				switch *s {
-				case ipn.NeedsLogin:
-					bc.StartLoginInteractive()
-				case ipn.NeedsMachineAuth:
-					fmt.Fprintf(os.Stderr, "\nTo authorize your machine, visit (as admin):\n\n\t%s/admin/machines\n\n", upArgs.server)
-				case ipn.Starting, ipn.Running:
-					// Done full authentication process
-					fmt.Fprintf(os.Stderr, "tailscaled is authenticated, nothing more to do.\n")
-					cancel()
-				}
-			}
-			if url := n.BrowseToURL; url != nil {
-				fmt.Fprintf(os.Stderr, "\nTo authenticate, visit:\n\n\t%s\n\n", *url)
-			}
-		},
-	}
-	// We still have to Start right now because it's the only way to
-	// set up notifications and whatnot. This causes a bunch of churn
-	// every time the CLI touches anything.
-	//
-	// TODO(danderson): redo the frontend/backend API to assume
-	// ephemeral frontends that read/modify/write state, once
-	// Windows/Mac state is moved into backend.
-	bc.Start(opts)
-	pump(ctx, bc, c)
-
-	return nil
-}
-
-func connect(ctx context.Context) (net.Conn, *ipn.BackendClient, context.Context, context.CancelFunc) {
-	c, err := safesocket.Connect(rootArgs.socket, 41112)
-	if err != nil {
-		if runtime.GOOS != "windows" && rootArgs.socket == "" {
-			log.Fatalf("--socket cannot be empty")
-		}
-		log.Fatalf("Failed to connect to connect to tailscaled. (safesocket.Connect: %v)\n", err)
-	}
-	clientToServer := func(b []byte) {
-		ipn.WriteMsg(c, b)
-	}
-
-	ctx, cancel := context.WithCancel(ctx)
-
-	go func() {
-		interrupt := make(chan os.Signal, 1)
-		signal.Notify(interrupt, syscall.SIGINT, syscall.SIGTERM)
-		<-interrupt
-		c.Close()
-		cancel()
-	}()
-
-	bc := ipn.NewBackendClient(log.Printf, clientToServer)
-	return c, bc, ctx, cancel
-}
-
-// pump receives backend messages on conn and pushes them into bc.
-func pump(ctx context.Context, bc *ipn.BackendClient, conn net.Conn) {
-	defer conn.Close()
-	for ctx.Err() == nil {
-		msg, err := ipn.ReadMsg(conn)
-		if err != nil {
-			if ctx.Err() != nil {
-				return
-			}
-			log.Printf("ReadMsg: %v\n", err)
-			break
-		}
-		bc.GotNotifyMsg(msg)
+	if err := cli.Run(os.Args[1:]); err != nil {
+		fmt.Fprintln(os.Stderr, err)
+		os.Exit(1)
 	}
 }

cmd/tailscaled/tailscaled.go

@@ -14,15 +14,22 @@ import (
 	"log"
 	"net/http"
 	"net/http/pprof"
+	"os"
+	"os/signal"
 	"runtime"
+	"runtime/debug"
+	"syscall"
+	"time"
 
 	"github.com/apenwarr/fixconsole"
 	"github.com/pborman/getopt/v2"
 	"tailscale.com/ipn/ipnserver"
 	"tailscale.com/logpolicy"
 	"tailscale.com/paths"
+	"tailscale.com/types/logger"
 	"tailscale.com/wgengine"
 	"tailscale.com/wgengine/magicsock"
+	"tailscale.com/wgengine/router"
 )
 
 // globalStateKey is the ipn.StateKey that tailscaled loads on
@@ -34,80 +41,147 @@ import (
 // later, the global state key doesn't look like a username.
 const globalStateKey = "_daemon"
 
-var defaultTunName = "tailscale0"
-
-func init() {
-	if runtime.GOOS == "openbsd" {
-		defaultTunName = "tun"
+// defaultTunName returns the default tun device name for the platform.
+func defaultTunName() string {
+	switch runtime.GOOS {
+	case "openbsd":
+		return "tun"
+	case "windows":
+		return "Tailscale"
 	}
+	return "tailscale0"
+}
+
+var args struct {
+	cleanup    bool
+	fake       bool
+	debug      string
+	tunname    string
+	port       uint16
+	statepath  string
+	socketpath string
 }
 
 func main() {
-	fake := getopt.BoolLong("fake", 0, "fake tunnel+routing instead of tuntap")
-	debug := getopt.StringLong("debug", 0, "", "Address of debug server")
-	tunname := getopt.StringLong("tun", 0, defaultTunName, "tunnel interface name")
-	listenport := getopt.Uint16Long("port", 'p', magicsock.DefaultPort, "WireGuard port (0=autoselect)")
-	statepath := getopt.StringLong("state", 0, paths.DefaultTailscaledStateFile(), "Path of state file")
-	socketpath := getopt.StringLong("socket", 's', paths.DefaultTailscaledSocket(), "Path of the service unix socket")
+	// We aren't very performance sensitive, and the parts that are
+	// performance sensitive (wireguard) try hard not to do any memory
+	// allocations. So let's be aggressive about garbage collection,
+	// unless the user specifically overrides it in the usual way.
+	if _, ok := os.LookupEnv("GOGC"); !ok {
+		debug.SetGCPercent(10)
+	}
 
-	logf := wgengine.RusagePrefixLog(log.Printf)
+	// Set default values for getopt.
+	args.tunname = defaultTunName()
+	args.port = magicsock.DefaultPort
+	args.statepath = paths.DefaultTailscaledStateFile()
+	args.socketpath = paths.DefaultTailscaledSocket()
+
+	getopt.FlagLong(&args.cleanup, "cleanup", 0, "clean up system state and exit")
+	getopt.FlagLong(&args.fake, "fake", 0, "fake tunnel+routing instead of tuntap")
+	getopt.FlagLong(&args.debug, "debug", 0, "address of debug server")
+	getopt.FlagLong(&args.tunname, "tun", 0, "tunnel interface name")
+	getopt.FlagLong(&args.port, "port", 'p', "WireGuard port (0=autoselect)")
+	getopt.FlagLong(&args.statepath, "state", 0, "path of state file")
+	getopt.FlagLong(&args.socketpath, "socket", 's', "path of the service unix socket")
 
 	err := fixconsole.FixConsoleIfNeeded()
 	if err != nil {
-		logf("fixConsoleOutput: %v", err)
+		log.Fatalf("fixConsoleOutput: %v", err)
 	}
-	pol := logpolicy.New("tailnode.log.tailscale.io")
 
 	getopt.Parse()
 	if len(getopt.Args()) > 0 {
 		log.Fatalf("too many non-flag arguments: %#v", getopt.Args()[0])
 	}
 
-	if *statepath == "" {
+	if args.statepath == "" {
 		log.Fatalf("--state is required")
 	}
 
-	if *socketpath == "" {
+	if args.socketpath == "" && runtime.GOOS != "windows" {
 		log.Fatalf("--socket is required")
 	}
 
+	if err := run(); err != nil {
+		// No need to log; the func already did
+		os.Exit(1)
+	}
+}
+
+func run() error {
+	var err error
+
+	pol := logpolicy.New("tailnode.log.tailscale.io")
+	defer func() {
+		// Finish uploading logs after closing everything else.
+		ctx, cancel := context.WithTimeout(context.Background(), time.Second)
+		defer cancel()
+		pol.Shutdown(ctx)
+	}()
+
+	logf := wgengine.RusagePrefixLog(log.Printf)
+	logf = logger.RateLimitedFn(logf, 5*time.Second, 5, 100)
+
+	if args.cleanup {
+		router.Cleanup(logf, args.tunname)
+		return nil
+	}
+
 	var debugMux *http.ServeMux
-	if *debug != "" {
+	if args.debug != "" {
 		debugMux = newDebugMux()
-		go runDebugServer(debugMux, *debug)
+		go runDebugServer(debugMux, args.debug)
 	}
 
 	var e wgengine.Engine
-	if *fake {
+	if args.fake {
 		e, err = wgengine.NewFakeUserspaceEngine(logf, 0)
 	} else {
-		e, err = wgengine.NewUserspaceEngine(logf, *tunname, *listenport)
+		e, err = wgengine.NewUserspaceEngine(logf, args.tunname, args.port)
 	}
 	if err != nil {
-		log.Fatalf("wgengine.New: %v", err)
+		logf("wgengine.New: %v", err)
+		return err
 	}
 	e = wgengine.NewWatchdog(e)
 
+	ctx, cancel := context.WithCancel(context.Background())
+	// Exit gracefully by cancelling the ipnserver context in most common cases:
+	// interrupted from the TTY or killed by a service manager.
+	interrupt := make(chan os.Signal, 1)
+	signal.Notify(interrupt, syscall.SIGINT, syscall.SIGTERM)
+	// SIGPIPE sometimes gets generated when CLIs disconnect from
+	// tailscaled. The default action is to terminate the process, we
+	// want to keep running.
+	signal.Ignore(syscall.SIGPIPE)
+	go func() {
+		select {
+		case s := <-interrupt:
+			logf("tailscaled got signal %v; shutting down", s)
+			cancel()
+		case <-ctx.Done():
+			// continue
+		}
+	}()
+
 	opts := ipnserver.Options{
-		SocketPath:         *socketpath,
+		SocketPath:         args.socketpath,
 		Port:               41112,
-		StatePath:          *statepath,
+		StatePath:          args.statepath,
 		AutostartStateKey:  globalStateKey,
-		LegacyConfigPath:   paths.LegacyConfigPath,
+		LegacyConfigPath:   paths.LegacyConfigPath(),
 		SurviveDisconnects: true,
 		DebugMux:           debugMux,
 	}
-	err = ipnserver.Run(context.Background(), logf, pol.PublicID.String(), opts, e)
-	if err != nil {
-		log.Fatalf("tailscaled: %v", err)
+	err = ipnserver.Run(ctx, logf, pol.PublicID.String(), ipnserver.FixedEngine(e), opts)
+	// Cancelation is not an error: it is the only way to stop ipnserver.
+	if err != nil && err != context.Canceled {
+		logf("ipnserver.Run: %v", err)
+		return err
 	}
 
-	// TODO(crawshaw): It would be nice to start a timeout context the moment a signal
-	// is received and use that timeout to give us a moment to finish uploading logs
-	// here. But the signal is handled inside ipnserver.Run, so some plumbing is needed.
-	ctx, cancel := context.WithCancel(context.Background())
-	cancel()
-	pol.Shutdown(ctx)
+	return nil
 }
 
 func newDebugMux() *http.ServeMux {

cmd/tailscaled/tailscaled.service

@@ -3,12 +3,11 @@ Description=Tailscale node agent
 Documentation=https://tailscale.com/kb/
 Wants=network-pre.target
 After=network-pre.target
-StartLimitIntervalSec=0
-StartLimitBurst=0
 
 [Service]
 EnvironmentFile=/etc/default/tailscaled
 ExecStart=/usr/sbin/tailscaled --state=/var/lib/tailscale/tailscaled.state --socket=/run/tailscale/tailscaled.sock --port $PORT $FLAGS
+ExecStopPost=/usr/sbin/tailscaled --cleanup
 
 Restart=on-failure
 

control/controlclient/auto.go

@@ -17,6 +17,7 @@ import (
 	"sync"
 	"time"
 
+	"github.com/tailscale/wireguard-go/wgcfg"
 	"golang.org/x/oauth2"
 	"tailscale.com/logtail/backoff"
 	"tailscale.com/tailcfg"
@@ -25,36 +26,37 @@ import (
 	"tailscale.com/types/structs"
 )
 
-// TODO(apenwarr): eliminate the 'state' variable, as it's now obsolete.
-//  It's used only by the unit tests.
-type state int
+// State is the high-level state of the client. It is used only in
+// unit tests for proper sequencing, don't depend on it anywhere else.
+// TODO(apenwarr): eliminate 'state', as it's now obsolete.
+type State int
 
 const (
-	stateNew = state(iota)
-	stateNotAuthenticated
-	stateAuthenticating
-	stateURLVisitRequired
-	stateAuthenticated
-	stateSynchronized // connected and received map update
+	StateNew = State(iota)
+	StateNotAuthenticated
+	StateAuthenticating
+	StateURLVisitRequired
+	StateAuthenticated
+	StateSynchronized // connected and received map update
 )
 
-func (s state) MarshalText() ([]byte, error) {
+func (s State) MarshalText() ([]byte, error) {
 	return []byte(s.String()), nil
 }
 
-func (s state) String() string {
+func (s State) String() string {
 	switch s {
-	case stateNew:
+	case StateNew:
 		return "state:new"
-	case stateNotAuthenticated:
+	case StateNotAuthenticated:
 		return "state:not-authenticated"
-	case stateAuthenticating:
+	case StateAuthenticating:
 		return "state:authenticating"
-	case stateURLVisitRequired:
+	case StateURLVisitRequired:
 		return "state:url-visit-required"
-	case stateAuthenticated:
+	case StateAuthenticated:
 		return "state:authenticated"
-	case stateSynchronized:
+	case StateSynchronized:
 		return "state:synchronized"
 	default:
 		return fmt.Sprintf("state:unknown:%d", int(s))
@@ -69,7 +71,7 @@ type Status struct {
 	Persist       *Persist          // locally persisted configuration
 	NetMap        *NetworkMap       // server-pushed configuration
 	Hostinfo      *tailcfg.Hostinfo // current Hostinfo data
-	state         state
+	State         State
 }
 
 // Equal reports whether s and s2 are equal.
@@ -84,7 +86,7 @@ func (s *Status) Equal(s2 *Status) bool {
 		reflect.DeepEqual(s.Persist, s2.Persist) &&
 		reflect.DeepEqual(s.NetMap, s2.NetMap) &&
 		reflect.DeepEqual(s.Hostinfo, s2.Hostinfo) &&
-		s.state == s2.state
+		s.State == s2.State
 }
 
 func (s Status) String() string {
@@ -92,7 +94,7 @@ func (s Status) String() string {
 	if err != nil {
 		panic(err)
 	}
-	return s.state.String() + " " + string(b)
+	return s.State.String() + " " + string(b)
 }
 
 type LoginGoal struct {
@@ -115,13 +117,15 @@ type Client struct {
 	mu         sync.Mutex   // mutex guards the following fields
 	statusFunc func(Status) // called to update Client status
 
-	loggedIn     bool       // true if currently logged in
-	loginGoal    *LoginGoal // non-nil if some login activity is desired
-	synced       bool       // true if our netmap is up-to-date
-	hostinfo     *tailcfg.Hostinfo
-	inPollNetMap bool // true if currently running a PollNetMap
-	inSendStatus int  // number of sendStatus calls currently in progress
-	state        state
+	paused         bool // whether we should stop making HTTP requests
+	unpauseWaiters []chan struct{}
+	loggedIn       bool       // true if currently logged in
+	loginGoal      *LoginGoal // non-nil if some login activity is desired
+	synced         bool       // true if our netmap is up-to-date
+	hostinfo       *tailcfg.Hostinfo
+	inPollNetMap   bool // true if currently running a PollNetMap
+	inSendStatus   int  // number of sendStatus calls currently in progress
+	state          State
 
 	authCtx    context.Context // context used for auth requests
 	mapCtx     context.Context // context used for netmap requests
@@ -167,6 +171,27 @@ func NewNoStart(opts Options) (*Client, error) {
 	return c, nil
 }
 
+// SetPaused controls whether HTTP activity should be paused.
+//
+// The client can be paused and unpaused repeatedly, unlike Start and Shutdown, which can only be used once.
+func (c *Client) SetPaused(paused bool) {
+	c.mu.Lock()
+	defer c.mu.Unlock()
+	if paused == c.paused {
+		return
+	}
+	c.paused = paused
+	if paused {
+		// Just cancel the map routine. The auth routine isn't expensive.
+		c.cancelMapLocked()
+	} else {
+		for _, ch := range c.unpauseWaiters {
+			close(ch)
+		}
+		c.unpauseWaiters = nil
+	}
+}
+
 // Start starts the client's goroutines.
 //
 // It should only be called for clients created by NewNoStart.
@@ -239,7 +264,7 @@ func (c *Client) cancelMapSafely() {
 
 func (c *Client) authRoutine() {
 	defer close(c.authDone)
-	bo := backoff.Backoff{Name: "authRoutine"}
+	bo := backoff.NewBackoff("authRoutine", c.logf, 30*time.Second)
 
 	for {
 		c.mu.Lock()
@@ -270,6 +295,7 @@ func (c *Client) authRoutine() {
 		if goal == nil {
 			// Wait for something interesting to happen
 			var exp <-chan time.Time
+			var expTimer *time.Timer
 			if expiry != nil && !expiry.IsZero() {
 				// if expiry is in the future, don't delay
 				// past that time.
@@ -282,11 +308,15 @@ func (c *Client) authRoutine() {
 					if delay > 5*time.Second {
 						delay = time.Second
 					}
-					exp = time.After(delay)
+					expTimer = time.NewTimer(delay)
+					exp = expTimer.C
 				}
 			}
 			select {
 			case <-ctx.Done():
+				if expTimer != nil {
+					expTimer.Stop()
+				}
 				c.logf("authRoutine: context done.")
 			case <-exp:
 				// Unfortunately the key expiry isn't provided
@@ -308,7 +338,7 @@ func (c *Client) authRoutine() {
 				}
 			}
 		} else if !goal.wantLoggedIn {
-			err := c.direct.TryLogout(c.authCtx)
+			err := c.direct.TryLogout(ctx)
 			if err != nil {
 				report(err, "TryLogout")
 				bo.BackOff(ctx, err)
@@ -319,7 +349,7 @@ func (c *Client) authRoutine() {
 			c.mu.Lock()
 			c.loggedIn = false
 			c.loginGoal = nil
-			c.state = stateNotAuthenticated
+			c.state = StateNotAuthenticated
 			c.synced = false
 			c.mu.Unlock()
 
@@ -328,9 +358,9 @@ func (c *Client) authRoutine() {
 		} else { // ie. goal.wantLoggedIn
 			c.mu.Lock()
 			if goal.url != "" {
-				c.state = stateURLVisitRequired
+				c.state = StateURLVisitRequired
 			} else {
-				c.state = stateAuthenticating
+				c.state = StateAuthenticating
 			}
 			c.mu.Unlock()
 
@@ -353,13 +383,14 @@ func (c *Client) authRoutine() {
 					err = fmt.Errorf("weird: server required a new url?")
 					report(err, "WaitLoginURL")
 				}
-				goal.url = url
-				goal.token = nil
-				goal.flags = LoginDefault
 
 				c.mu.Lock()
-				c.loginGoal = goal
-				c.state = stateURLVisitRequired
+				c.loginGoal = &LoginGoal{
+					wantLoggedIn: true,
+					flags:        LoginDefault,
+					url:          url,
+				}
+				c.state = StateURLVisitRequired
 				c.synced = false
 				c.mu.Unlock()
 
@@ -372,7 +403,7 @@ func (c *Client) authRoutine() {
 			c.mu.Lock()
 			c.loggedIn = true
 			c.loginGoal = nil
-			c.state = stateAuthenticated
+			c.state = StateAuthenticated
 			c.mu.Unlock()
 
 			c.sendStatus("authRoutine4", nil, "", nil)
@@ -382,12 +413,49 @@ func (c *Client) authRoutine() {
 	}
 }
 
+// Expiry returns the credential expiration time, or the zero time if
+// the expiration time isn't known. Used in tests only.
+func (c *Client) Expiry() *time.Time {
+	c.mu.Lock()
+	defer c.mu.Unlock()
+	return c.expiry
+}
+
+// Direct returns the underlying direct client object. Used in tests
+// only.
+func (c *Client) Direct() *Direct {
+	return c.direct
+}
+
+// unpausedChanLocked returns a new channel that is closed when the
+// current Client pause is unpaused.
+//
+// c.mu must be held
+func (c *Client) unpausedChanLocked() <-chan struct{} {
+	unpaused := make(chan struct{})
+	c.unpauseWaiters = append(c.unpauseWaiters, unpaused)
+	return unpaused
+}
+
 func (c *Client) mapRoutine() {
 	defer close(c.mapDone)
-	bo := backoff.Backoff{Name: "mapRoutine"}
+	bo := backoff.NewBackoff("mapRoutine", c.logf, 30*time.Second)
 
 	for {
 		c.mu.Lock()
+		if c.paused {
+			unpaused := c.unpausedChanLocked()
+			c.mu.Unlock()
+			c.logf("mapRoutine: awaiting unpause")
+			select {
+			case <-unpaused:
+				c.logf("mapRoutine: unpaused")
+			case <-c.quit:
+				c.logf("mapRoutine: quit")
+				return
+			}
+			continue
+		}
 		c.logf("mapRoutine: %s", c.state)
 		loggedIn := c.loggedIn
 		ctx := c.mapCtx
@@ -449,7 +517,7 @@ func (c *Client) mapRoutine() {
 				c.synced = true
 				c.inPollNetMap = true
 				if c.loggedIn {
-					c.state = stateSynchronized
+					c.state = StateSynchronized
 				}
 				exp := nm.Expiry
 				c.expiry = &exp
@@ -467,11 +535,17 @@ func (c *Client) mapRoutine() {
 			c.mu.Lock()
 			c.synced = false
 			c.inPollNetMap = false
-			if c.state == stateSynchronized {
-				c.state = stateAuthenticated
+			if c.state == StateSynchronized {
+				c.state = StateAuthenticated
 			}
+			paused := c.paused
 			c.mu.Unlock()
 
+			if paused {
+				c.logf("mapRoutine: paused")
+				continue
+			}
+
 			if err != nil {
 				report(err, "PollNetMap")
 				bo.BackOff(ctx, err)
@@ -500,7 +574,7 @@ func (c *Client) SetHostinfo(hi *tailcfg.Hostinfo) {
 		panic("nil Hostinfo")
 	}
 	if !c.direct.SetHostinfo(hi) {
-		c.logf("[unexpected] duplicate Hostinfo: %v", hi)
+		// No changes. Don't log.
 		return
 	}
 	c.logf("Hostinfo: %v", hi)
@@ -537,7 +611,7 @@ func (c *Client) sendStatus(who string, err error, url string, nm *NetworkMap) {
 
 	var p *Persist
 	var fin *empty.Message
-	if state == stateAuthenticated {
+	if state == StateAuthenticated {
 		fin = new(empty.Message)
 	}
 	if nm != nil && loggedIn && synced {
@@ -554,7 +628,7 @@ func (c *Client) sendStatus(who string, err error, url string, nm *NetworkMap) {
 		Persist:       p,
 		NetMap:        nm,
 		Hostinfo:      hi,
-		state:         state,
+		State:         state,
 	}
 	if err != nil {
 		new.Err = err.Error()
@@ -623,3 +697,20 @@ func (c *Client) Shutdown() {
 		c.logf("Client.Shutdown done.")
 	}
 }
+
+// NodePublicKey returns the node public key currently in use. This is
+// used exclusively in tests.
+func (c *Client) TestOnlyNodePublicKey() wgcfg.Key {
+	priv := c.direct.GetPersist()
+	return priv.PrivateNodeKey.Public()
+}
+
+func (c *Client) TestOnlySetAuthKey(authkey string) {
+	c.direct.mu.Lock()
+	defer c.direct.mu.Unlock()
+	c.direct.authKey = authkey
+}
+
+func (c *Client) TestOnlyTimeNow() time.Time {
+	return c.timeNow()
+}

control/controlclient/controlclient_test.go

@@ -22,7 +22,7 @@ func fieldsOf(t reflect.Type) (fields []string) {
 
 func TestStatusEqual(t *testing.T) {
 	// Verify that the Equal method stays in sync with reality
-	equalHandles := []string{"LoginFinished", "Err", "URL", "Persist", "NetMap", "Hostinfo", "state"}
+	equalHandles := []string{"LoginFinished", "Err", "URL", "Persist", "NetMap", "Hostinfo", "State"}
 	if have := fieldsOf(reflect.TypeOf(Status{})); !reflect.DeepEqual(have, equalHandles) {
 		t.Errorf("Status.Equal check might be out of sync\nfields: %q\nhandled: %q\n",
 			have, equalHandles)
@@ -48,13 +48,13 @@ func TestStatusEqual(t *testing.T) {
 			true,
 		},
 		{
-			&Status{state: stateNew},
-			&Status{state: stateNew},
+			&Status{State: StateNew},
+			&Status{State: StateNew},
 			true,
 		},
 		{
-			&Status{state: stateNew},
-			&Status{state: stateAuthenticated},
+			&Status{State: StateNew},
+			&Status{State: StateAuthenticated},
 			false,
 		},
 		{
@@ -70,3 +70,10 @@ func TestStatusEqual(t *testing.T) {
 		}
 	}
 }
+
+func TestOSVersion(t *testing.T) {
+	if osVersion == nil {
+		t.Skip("not available for OS")
+	}
+	t.Logf("Got: %#q", osVersion())
+}

control/controlclient/direct.go

@@ -4,6 +4,8 @@
 
 package controlclient
 
+//go:generate go run tailscale.com/cmd/cloner -type=Persist -output=direct_clone.go
+
 import (
 	"bytes"
 	"context"
@@ -19,6 +21,8 @@ import (
 	"net/url"
 	"os"
 	"reflect"
+	"runtime"
+	"sort"
 	"strconv"
 	"strings"
 	"sync"
@@ -27,6 +31,9 @@ import (
 	"github.com/tailscale/wireguard-go/wgcfg"
 	"golang.org/x/crypto/nacl/box"
 	"golang.org/x/oauth2"
+	"inet.af/netaddr"
+	"tailscale.com/log/logheap"
+	"tailscale.com/net/netns"
 	"tailscale.com/net/tlsdial"
 	"tailscale.com/tailcfg"
 	"tailscale.com/types/logger"
@@ -83,6 +90,7 @@ type Direct struct {
 	newDecompressor func() (Decompressor, error)
 	keepAlive       bool
 	logf            logger.Logf
+	discoPubKey     tailcfg.DiscoKey
 
 	mu           sync.Mutex // mutex guards the following fields
 	serverKey    wgcfg.Key
@@ -90,9 +98,10 @@ type Direct struct {
 	authKey      string
 	tryingNewKey wgcfg.PrivateKey
 	expiry       *time.Time
-	hostinfo     *tailcfg.Hostinfo // always non-nil
-	endpoints    []string
-	localPort    uint16 // or zero to mean auto
+	// hostinfo is mutated in-place while mu is held.
+	hostinfo  *tailcfg.Hostinfo // always non-nil
+	endpoints []string
+	localPort uint16 // or zero to mean auto
 }
 
 type Options struct {
@@ -101,6 +110,7 @@ type Options struct {
 	AuthKey         string            // optional node auth key for auto registration
 	TimeNow         func() time.Time  // time.Now implementation used by Client
 	Hostinfo        *tailcfg.Hostinfo // non-nil passes ownership, nil means to use default using os.Hostname, etc
+	DiscoPublicKey  tailcfg.DiscoKey
 	NewDecompressor func() (Decompressor, error)
 	KeepAlive       bool
 	Logf            logger.Logf
@@ -133,7 +143,9 @@ func NewDirect(opts Options) (*Direct, error) {
 
 	httpc := opts.HTTPTestClient
 	if httpc == nil {
+		dialer := netns.NewDialer()
 		tr := http.DefaultTransport.(*http.Transport).Clone()
+		tr.DialContext = dialer.DialContext
 		tr.ForceAttemptHTTP2 = true
 		tr.TLSClientConfig = tlsdial.Config(serverURL.Host, tr.TLSClientConfig)
 		httpc = &http.Client{Transport: tr}
@@ -148,6 +160,7 @@ func NewDirect(opts Options) (*Direct, error) {
 		keepAlive:       opts.KeepAlive,
 		persist:         opts.Persist,
 		authKey:         opts.AuthKey,
+		discoPubKey:     opts.DiscoPublicKey,
 	}
 	if opts.Hostinfo == nil {
 		c.SetHostinfo(NewHostinfo())
@@ -157,12 +170,20 @@ func NewDirect(opts Options) (*Direct, error) {
 	return c, nil
 }
 
+var osVersion func() string // non-nil on some platforms
+
 func NewHostinfo() *tailcfg.Hostinfo {
 	hostname, _ := os.Hostname()
+	var osv string
+	if osVersion != nil {
+		osv = osVersion()
+	}
 	return &tailcfg.Hostinfo{
 		IPNVersion: version.LONG,
 		Hostname:   hostname,
 		OS:         version.OS(),
+		OSVersion:  osv,
+		GoArch:     runtime.GOARCH,
 	}
 }
 
@@ -257,6 +278,9 @@ func (c *Direct) doLogin(ctx context.Context, t *oauth2.Token, flags LoginFlags,
 	persist := c.persist
 	tryingNewKey := c.tryingNewKey
 	serverKey := c.serverKey
+	authKey := c.authKey
+	hostinfo := c.hostinfo.Clone()
+	backendLogID := hostinfo.BackendLogID
 	expired := c.expiry != nil && !c.expiry.IsZero() && c.expiry.Before(c.timeNow())
 	c.mu.Unlock()
 
@@ -313,7 +337,7 @@ func (c *Direct) doLogin(ctx context.Context, t *oauth2.Token, flags LoginFlags,
 	if tryingNewKey == (wgcfg.PrivateKey{}) {
 		log.Fatalf("tryingNewKey is empty, give up")
 	}
-	if c.hostinfo.BackendLogID == "" {
+	if backendLogID == "" {
 		err = errors.New("hostinfo: BackendLogID missing")
 		return regen, url, err
 	}
@@ -321,7 +345,7 @@ func (c *Direct) doLogin(ctx context.Context, t *oauth2.Token, flags LoginFlags,
 		Version:    1,
 		OldNodeKey: tailcfg.NodeKey(oldNodeKey),
 		NodeKey:    tailcfg.NodeKey(tryingNewKey.Public()),
-		Hostinfo:   c.hostinfo,
+		Hostinfo:   hostinfo,
 		Followup:   url,
 	}
 	c.logf("RegisterReq: onode=%v node=%v fup=%v",
@@ -330,7 +354,7 @@ func (c *Direct) doLogin(ctx context.Context, t *oauth2.Token, flags LoginFlags,
 	request.Auth.Oauth2Token = t
 	request.Auth.Provider = persist.Provider
 	request.Auth.LoginName = persist.LoginName
-	request.Auth.AuthKey = c.authKey
+	request.Auth.AuthKey = authKey
 	bodyData, err := encode(request, &serverKey, &persist.PrivateMachineKey)
 	if err != nil {
 		return regen, url, err
@@ -376,7 +400,7 @@ func (c *Direct) doLogin(ctx context.Context, t *oauth2.Token, flags LoginFlags,
 	//	- user is disabled
 
 	if resp.AuthURL != "" {
-		c.logf("AuthURL is %.20v...", resp.AuthURL)
+		c.logf("AuthURL is %v", resp.AuthURL)
 	} else {
 		c.logf("No AuthURL")
 	}
@@ -440,19 +464,18 @@ func (c *Direct) SetEndpoints(localPort uint16, endpoints []string) (changed boo
 	return c.newEndpoints(localPort, endpoints)
 }
 
-var debugNetmap, _ = strconv.ParseBool(os.Getenv("TS_DEBUG_NETMAP"))
-
 func (c *Direct) PollNetMap(ctx context.Context, maxPolls int, cb func(*NetworkMap)) error {
 	c.mu.Lock()
 	persist := c.persist
 	serverURL := c.serverURL
 	serverKey := c.serverKey
-	hostinfo := c.hostinfo
+	hostinfo := c.hostinfo.Clone()
+	backendLogID := hostinfo.BackendLogID
 	localPort := c.localPort
 	ep := append([]string(nil), c.endpoints...)
 	c.mu.Unlock()
 
-	if hostinfo.BackendLogID == "" {
+	if backendLogID == "" {
 		return errors.New("hostinfo: BackendLogID missing")
 	}
 
@@ -460,18 +483,21 @@ func (c *Direct) PollNetMap(ctx context.Context, maxPolls int, cb func(*NetworkM
 	c.logf("PollNetMap: stream=%v :%v %v", maxPolls, localPort, ep)
 
 	vlogf := logger.Discard
-	if debugNetmap {
+	if Debug.NetMap {
 		vlogf = c.logf
 	}
 
 	request := tailcfg.MapRequest{
-		Version:     4,
-		IncludeIPv6: includeIPv6(),
-		KeepAlive:   c.keepAlive,
-		NodeKey:     tailcfg.NodeKey(persist.PrivateNodeKey.Public()),
-		Endpoints:   ep,
-		Stream:      allowStream,
-		Hostinfo:    hostinfo,
+		Version:         4,
+		IncludeIPv6:     true,
+		DeltaPeers:      true,
+		KeepAlive:       c.keepAlive,
+		NodeKey:         tailcfg.NodeKey(persist.PrivateNodeKey.Public()),
+		DiscoKey:        c.discoPubKey,
+		Endpoints:       ep,
+		Stream:          allowStream,
+		Hostinfo:        hostinfo,
+		DebugForceDisco: Debug.ForceDisco,
 	}
 	if c.newDecompressor != nil {
 		request.Compress = "zstd"
@@ -540,6 +566,8 @@ func (c *Direct) PollNetMap(ctx context.Context, maxPolls int, cb func(*NetworkM
 		}
 	}()
 
+	var lastDERPMap *tailcfg.DERPMap
+
 	// If allowStream, then the server will use an HTTP long poll to
 	// return incremental results. There is always one response right
 	// away, followed by a delay, and eventually others.
@@ -547,6 +575,7 @@ func (c *Direct) PollNetMap(ctx context.Context, maxPolls int, cb func(*NetworkM
 	// the same format before just closing the connection.
 	// We can use this same read loop either way.
 	var msg []byte
+	var previousPeers []*tailcfg.Node // for delta-purposes
 	for i := 0; i < maxPolls || maxPolls < 0; i++ {
 		vlogf("netmap: starting size read after %v (poll %v)", time.Since(t0).Round(time.Millisecond), i)
 		var siz [4]byte
@@ -568,23 +597,51 @@ func (c *Direct) PollNetMap(ctx context.Context, maxPolls int, cb func(*NetworkM
 			vlogf("netmap: decode error: %v")
 			return err
 		}
+
 		if resp.KeepAlive {
 			vlogf("netmap: got keep-alive")
-			select {
-			case timeoutReset <- struct{}{}:
-				vlogf("netmap: sent keep-alive timer reset")
-			case <-ctx.Done():
-				c.logf("netmap: not resetting timer for keep-alive due to: %v", ctx.Err())
-				return ctx.Err()
-			}
+		} else {
+			vlogf("netmap: got new map")
+		}
+		select {
+		case timeoutReset <- struct{}{}:
+			vlogf("netmap: sent timer reset")
+		case <-ctx.Done():
+			c.logf("netmap: not resetting timer; context done: %v", ctx.Err())
+			return ctx.Err()
+		}
+		if resp.KeepAlive {
 			continue
 		}
-		vlogf("netmap: got new map")
+
+		undeltaPeers(&resp, previousPeers)
+		previousPeers = cloneNodes(resp.Peers) // defensive/lazy clone, since this escapes to who knows where
+
+		if resp.DERPMap != nil {
+			vlogf("netmap: new map contains DERP map")
+			lastDERPMap = resp.DERPMap
+		}
+		if resp.Debug != nil && resp.Debug.LogHeapPprof {
+			go logheap.LogHeap(resp.Debug.LogHeapURL)
+		}
+		// Temporarily (2020-06-29) support removing all but
+		// discovery-supporting nodes during development, for
+		// less noise.
+		if Debug.OnlyDisco {
+			filtered := resp.Peers[:0]
+			for _, p := range resp.Peers {
+				if !p.DiscoKey.IsZero() {
+					filtered = append(filtered, p)
+				}
+			}
+			resp.Peers = filtered
+		}
 
 		nm := &NetworkMap{
 			NodeKey:      tailcfg.NodeKey(persist.PrivateNodeKey.Public()),
 			PrivateKey:   persist.PrivateNodeKey,
 			Expiry:       resp.Node.KeyExpiry,
+			Name:         resp.Node.Name,
 			Addresses:    resp.Node.Addresses,
 			Peers:        resp.Peers,
 			LocalPort:    localPort,
@@ -592,10 +649,11 @@ func (c *Direct) PollNetMap(ctx context.Context, maxPolls int, cb func(*NetworkM
 			UserProfiles: make(map[tailcfg.UserID]tailcfg.UserProfile),
 			Domain:       resp.Domain,
 			Roles:        resp.Roles,
-			DNS:          resp.DNS,
-			DNSDomains:   resp.SearchPaths,
+			DNS:          resp.DNSConfig,
 			Hostinfo:     resp.Node.Hostinfo,
 			PacketFilter: c.parsePacketFilter(resp.PacketFilter),
+			DERPMap:      lastDERPMap,
+			Debug:        resp.Debug,
 		}
 		for _, profile := range resp.UserProfiles {
 			nm.UserProfiles[profile.ID] = profile
@@ -605,6 +663,15 @@ func (c *Direct) PollNetMap(ctx context.Context, maxPolls int, cb func(*NetworkM
 		} else {
 			nm.MachineStatus = tailcfg.MachineUnauthorized
 		}
+		if len(resp.DNS) > 0 {
+			nm.DNS.Nameservers = wgIPToNetaddr(resp.DNS)
+		}
+		if len(resp.SearchPaths) > 0 {
+			nm.DNS.Domains = resp.SearchPaths
+		}
+		if Debug.ProxyDNS {
+			nm.DNS.Proxied = true
+		}
 
 		// Printing the netmap can be extremely verbose, but is very
 		// handy for debugging. Let's limit how often we do it.
@@ -642,8 +709,10 @@ func decode(res *http.Response, v interface{}, serverKey *wgcfg.Key, mkey *wgcfg
 }
 
 func (c *Direct) decodeMsg(msg []byte, v interface{}) error {
+	c.mu.Lock()
 	mkey := c.persist.PrivateMachineKey
 	serverKey := c.serverKey
+	c.mu.Unlock()
 
 	decrypted, err := decryptMsg(msg, &serverKey, &mkey)
 	if err != nil {
@@ -653,7 +722,6 @@ func (c *Direct) decodeMsg(msg []byte, v interface{}) error {
 	if c.newDecompressor == nil {
 		b = decrypted
 	} else {
-		//decoder, err := zstd.NewReader(nil)
 		decoder, err := c.newDecompressor()
 		if err != nil {
 			return err
@@ -743,13 +811,145 @@ func loadServerKey(ctx context.Context, httpc *http.Client, serverURL string) (w
 	return key, nil
 }
 
-// includeIPv6 reports whether we should enable IPv6 for magicsock
-// connections. This is only here temporarily (2020-03-26) as a
-// opt-out in case there are problems.
-func includeIPv6() bool {
-	if e := os.Getenv("DEBUG_INCLUDE_IPV6"); e != "" {
-		v, _ := strconv.ParseBool(e)
-		return v
+func wgIPToNetaddr(ips []wgcfg.IP) (ret []netaddr.IP) {
+	for _, ip := range ips {
+		nip, ok := netaddr.FromStdIP(ip.IP())
+		if !ok {
+			panic(fmt.Sprintf("conversion of %s from wgcfg to netaddr IP failed", ip))
+		}
+		ret = append(ret, nip.Unmap())
+	}
+	return ret
+}
+
+// Debug contains temporary internal-only debug knobs.
+// They're unexported to not draw attention to them.
+var Debug = initDebug()
+
+type debug struct {
+	NetMap     bool
+	ProxyDNS   bool
+	OnlyDisco  bool
+	Disco      bool
+	ForceDisco bool // ask control server to not filter out our disco key
+}
+
+func initDebug() debug {
+	d := debug{
+		NetMap:     envBool("TS_DEBUG_NETMAP"),
+		ProxyDNS:   envBool("TS_DEBUG_PROXY_DNS"),
+		OnlyDisco:  os.Getenv("TS_DEBUG_USE_DISCO") == "only",
+		ForceDisco: os.Getenv("TS_DEBUG_USE_DISCO") == "only" || envBool("TS_DEBUG_USE_DISCO"),
+	}
+	if d.ForceDisco || os.Getenv("TS_DEBUG_USE_DISCO") == "" {
+		// This is now defaults to on.
+		d.Disco = true
+	}
+	return d
+}
+
+func envBool(k string) bool {
+	e := os.Getenv(k)
+	if e == "" {
+		return false
+	}
+	v, err := strconv.ParseBool(e)
+	if err != nil {
+		panic(fmt.Sprintf("invalid non-bool %q for env var %q", e, k))
+	}
+	return v
+}
+
+// undeltaPeers updates mapRes.Peers to be complete based on the provided previous peer list
+// and the PeersRemoved and PeersChanged fields in mapRes.
+// It then also nils out the delta fields.
+func undeltaPeers(mapRes *tailcfg.MapResponse, prev []*tailcfg.Node) {
+	if len(mapRes.Peers) > 0 {
+		// Not delta encoded.
+		if !nodesSorted(mapRes.Peers) {
+			log.Printf("netmap: undeltaPeers: MapResponse.Peers not sorted; sorting")
+			sortNodes(mapRes.Peers)
+		}
+		return
+	}
+
+	var removed map[tailcfg.NodeID]bool
+	if pr := mapRes.PeersRemoved; len(pr) > 0 {
+		removed = make(map[tailcfg.NodeID]bool, len(pr))
+		for _, id := range pr {
+			removed[id] = true
+		}
+	}
+	changed := mapRes.PeersChanged
+
+	if len(removed) == 0 && len(changed) == 0 {
+		// No changes fast path.
+		mapRes.Peers = prev
+		return
+	}
+
+	if !nodesSorted(changed) {
+		log.Printf("netmap: undeltaPeers: MapResponse.PeersChanged not sorted; sorting")
+		sortNodes(changed)
+	}
+	if !nodesSorted(prev) {
+		// Internal error (unrelated to the network) if we get here.
+		log.Printf("netmap: undeltaPeers: [unexpected] prev not sorted; sorting")
+		sortNodes(prev)
+	}
+
+	newFull := make([]*tailcfg.Node, 0, len(prev)-len(removed))
+	for len(prev) > 0 && len(changed) > 0 {
+		pID := prev[0].ID
+		cID := changed[0].ID
+		if removed[pID] {
+			prev = prev[1:]
+			continue
+		}
+		switch {
+		case pID < cID:
+			newFull = append(newFull, prev[0])
+			prev = prev[1:]
+		case pID == cID:
+			newFull = append(newFull, changed[0])
+			prev, changed = prev[1:], changed[1:]
+		case cID < pID:
+			newFull = append(newFull, changed[0])
+			changed = changed[1:]
+		}
+	}
+	newFull = append(newFull, changed...)
+	for _, n := range prev {
+		if !removed[n.ID] {
+			newFull = append(newFull, n)
+		}
+	}
+	sortNodes(newFull)
+	mapRes.Peers = newFull
+	mapRes.PeersChanged = nil
+	mapRes.PeersRemoved = nil
+}
+
+func nodesSorted(v []*tailcfg.Node) bool {
+	for i, n := range v {
+		if i > 0 && n.ID <= v[i-1].ID {
+			return false
+		}
 	}
 	return true
 }
+
+func sortNodes(v []*tailcfg.Node) {
+	sort.Slice(v, func(i, j int) bool { return v[i].ID < v[j].ID })
+}
+
+func cloneNodes(v1 []*tailcfg.Node) []*tailcfg.Node {
+	if v1 == nil {
+		return nil
+	}
+	v2 := make([]*tailcfg.Node, len(v1))
+	for i, n := range v1 {
+		v2[i] = n.Clone()
+	}
+	return v2
+}

control/controlclient/direct_clone.go

@@ -0,0 +1,20 @@
+// Copyright (c) 2020 Tailscale Inc & AUTHORS All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+// Code generated by tailscale.com/cmd/cloner -type Persist; DO NOT EDIT.
+
+package controlclient
+
+import ()
+
+// Clone makes a deep copy of Persist.
+// The result aliases no memory with the original.
+func (src *Persist) Clone() *Persist {
+	if src == nil {
+		return nil
+	}
+	dst := new(Persist)
+	*dst = *src
+	return dst
+}

control/controlclient/direct_test.go

@@ -2,363 +2,92 @@
 // Use of this source code is governed by a BSD-style
 // license that can be found in the LICENSE file.
 
-// +build depends_on_currently_unreleased
-
 package controlclient
 
 import (
-	"context"
-	"io/ioutil"
-	"net/http"
-	"net/http/cookiejar"
-	"net/http/httptest"
-	"os"
+	"fmt"
+	"reflect"
+	"strings"
 	"testing"
-	"time"
 
-	"github.com/klauspost/compress/zstd"
-	"github.com/tailscale/wireguard-go/wgcfg"
 	"tailscale.com/tailcfg"
-	"tailscale.io/control" // not yet released
 )
 
-// Test that when there are two controlclient connections using the
-// same credentials, the later one disconnects the earlier one.
-func TestClientsReusingKeys(t *testing.T) {
-	tmpdir, err := ioutil.TempDir("", "control-test-")
-	if err != nil {
-		t.Fatal(err)
+func TestUndeltaPeers(t *testing.T) {
+	n := func(id tailcfg.NodeID, name string) *tailcfg.Node {
+		return &tailcfg.Node{ID: id, Name: name}
 	}
-	var server *control.Server
-	httpsrv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
-		server.ServeHTTP(w, r)
-	}))
-	defer func() {
-		httpsrv.CloseClientConnections()
-		httpsrv.Close()
-		os.RemoveAll(tmpdir)
-	}()
-
-	httpc := httpsrv.Client()
-	httpc.Jar, err = cookiejar.New(nil)
-	if err != nil {
-		t.Fatal(err)
-	}
-
-	server, err = control.New(tmpdir, tmpdir, tmpdir, httpsrv.URL, true)
-	if err != nil {
-		t.Fatal(err)
-	}
-	server.QuietLogging = true
-
-	hi := NewHostinfo()
-	hi.FrontendLogID = "go-test-only"
-	hi.BackendLogID = "go-test-only"
-
-	// Let's test some nonempty extra hostinfo fields to make sure
-	// the server can handle them.
-	hi.RequestTags = []string{"tag:abc"}
-	cidr, err := wgcfg.ParseCIDR("1.2.3.4/24")
-	if err != nil {
-		t.Fatalf("ParseCIDR: %v", err)
-	}
-	hi.RoutableIPs = []wgcfg.CIDR{cidr}
-	hi.Services = []tailcfg.Service{
+	peers := func(nv ...*tailcfg.Node) []*tailcfg.Node { return nv }
+	tests := []struct {
+		name   string
+		mapRes *tailcfg.MapResponse
+		prev   []*tailcfg.Node
+		want   []*tailcfg.Node
+	}{
 		{
-			Proto:       tailcfg.TCP,
-			Port:        1234,
-			Description: "Description",
+			name: "full_peers",
+			mapRes: &tailcfg.MapResponse{
+				Peers: peers(n(1, "foo"), n(2, "bar")),
+			},
+			want: peers(n(1, "foo"), n(2, "bar")),
+		},
+		{
+			name: "full_peers_ignores_deltas",
+			mapRes: &tailcfg.MapResponse{
+				Peers:        peers(n(1, "foo"), n(2, "bar")),
+				PeersRemoved: []tailcfg.NodeID{2},
+			},
+			want: peers(n(1, "foo"), n(2, "bar")),
+		},
+		{
+			name: "add_and_update",
+			prev: peers(n(1, "foo"), n(2, "bar")),
+			mapRes: &tailcfg.MapResponse{
+				PeersChanged: peers(n(0, "zero"), n(2, "bar2"), n(3, "three")),
+			},
+			want: peers(n(0, "zero"), n(1, "foo"), n(2, "bar2"), n(3, "three")),
+		},
+		{
+			name: "remove",
+			prev: peers(n(1, "foo"), n(2, "bar")),
+			mapRes: &tailcfg.MapResponse{
+				PeersRemoved: []tailcfg.NodeID{1},
+			},
+			want: peers(n(2, "bar")),
+		},
+		{
+			name: "add_and_remove",
+			prev: peers(n(1, "foo"), n(2, "bar")),
+			mapRes: &tailcfg.MapResponse{
+				PeersChanged: peers(n(1, "foo2")),
+				PeersRemoved: []tailcfg.NodeID{2},
+			},
+			want: peers(n(1, "foo2")),
+		},
+		{
+			name:   "unchanged",
+			prev:   peers(n(1, "foo"), n(2, "bar")),
+			mapRes: &tailcfg.MapResponse{},
+			want:   peers(n(1, "foo"), n(2, "bar")),
 		},
 	}
-
-	c1, err := NewDirect(Options{
-		ServerURL:      httpsrv.URL,
-		HTTPTestClient: httpsrv.Client(),
-		//TimeNow:   s.control.TimeNow,
-		Logf: func(fmt string, args ...interface{}) {
-			t.Helper()
-			t.Logf("c1: "+fmt, args...)
-		},
-		Hostinfo: hi,
-	})
-	if err != nil {
-		t.Fatal(err)
-	}
-
-	// Use a cancelable context so that goroutines blocking in
-	// PollNetMap shut down when the test exits.
-	ctx, cancel := context.WithCancel(context.Background())
-	defer cancel()
-
-	// Execute c1's login flow: TryLogin to get an auth URL,
-	// postAuthURL to execute the (faked) OAuth segment of the flow,
-	// and WaitLoginURL to complete the login on the client end.
-	const user = "testuser1@tailscale.onmicrosoft.com"
-	authURL, err := c1.TryLogin(ctx, nil, 0)
-	if err != nil {
-		t.Fatal(err)
-	}
-	postAuthURL(t, ctx, httpc, user, authURL)
-	newURL, err := c1.WaitLoginURL(ctx, authURL)
-	if err != nil {
-		t.Fatal(err)
-	}
-	if newURL != "" {
-		t.Fatalf("unexpected newURL: %s", newURL)
-	}
-
-	// Start c1's netmap poll in parallel with the rest of the
-	// test. We're expecting it to block happily, invoking the no-op
-	// update function periodically, then exit once c2 starts its own
-	// poll below.
-	gotNetmap := make(chan struct{}, 1)
-	pollErrCh := make(chan error)
-	go func() {
-		pollErrCh <- c1.PollNetMap(ctx, -1, func(netMap *NetworkMap) {
-			select {
-			case gotNetmap <- struct{}{}:
-			default:
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			undeltaPeers(tt.mapRes, tt.prev)
+			if !reflect.DeepEqual(tt.mapRes.Peers, tt.want) {
+				t.Errorf("wrong results\n got: %s\nwant: %s", formatNodes(tt.mapRes.Peers), formatNodes(tt.want))
 			}
 		})
-	}()
-
-	select {
-	case <-gotNetmap:
-		t.Logf("c1: received initial netmap")
-	case err := <-pollErrCh:
-		t.Fatal(err)
-	case <-time.After(5 * time.Second):
-		t.Fatal("c1 did not receive an initial netmap")
-	}
-
-	// Connect c2, reusing c1's credentials. In other words, c2 *is*
-	// c1 from the server's perspective.
-	c2, err := NewDirect(Options{
-		ServerURL:      httpsrv.URL,
-		HTTPTestClient: httpsrv.Client(),
-		Logf: func(fmt string, args ...interface{}) {
-			t.Helper()
-			t.Logf("c2: "+fmt, args...)
-		},
-		Persist:  c1.GetPersist(),
-		Hostinfo: hi,
-		NewDecompressor: func() (Decompressor, error) {
-			return zstd.NewReader(nil)
-		},
-		KeepAlive: true,
-	})
-	if err != nil {
-		t.Fatal(err)
-	}
-	authURL, err = c2.TryLogin(ctx, nil, 0)
-	if err != nil {
-		t.Fatal(err)
-	}
-	// We don't expect to be given an authURL, our credentials from c1
-	// should still be good.
-	if authURL != "" {
-		t.Errorf("unexpected authURL %s", authURL)
-	}
-
-	// Request a single netmap, so this function returns promptly
-	// instead of blocking like c1's PollNetMap.
-	err = c2.PollNetMap(ctx, 1, func(netMap *NetworkMap) {})
-	if err != nil {
-		t.Fatal(err)
-	}
-
-	// Now that c2 connected and got a netmap, we expect c1's poll to
-	// have exited.
-	select {
-	case err := <-pollErrCh:
-		t.Logf("c1: netmap poll aborted as expected (%v)", err)
-	case <-time.After(5 * time.Second):
-		t.Fatal("first client poll failed to close")
 	}
 }
 
-func TestClientsReusingOldKey(t *testing.T) {
-	tmpdir, err := ioutil.TempDir("", "control-test-")
-	if err != nil {
-		t.Fatal(err)
-	}
-	var server *control.Server
-	httpsrv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
-		server.ServeHTTP(w, r)
-	}))
-	httpc := httpsrv.Client()
-	httpc.Jar, err = cookiejar.New(nil)
-	if err != nil {
-		t.Fatal(err)
-	}
-	server, err = control.New(tmpdir, tmpdir, tmpdir, httpsrv.URL, true)
-	if err != nil {
-		t.Fatal(err)
-	}
-	server.QuietLogging = true
-	defer func() {
-		httpsrv.CloseClientConnections()
-		httpsrv.Close()
-		os.RemoveAll(tmpdir)
-	}()
-
-	hi := NewHostinfo()
-	hi.FrontendLogID = "go-test-only"
-	hi.BackendLogID = "go-test-only"
-	genOpts := func() Options {
-		return Options{
-			ServerURL:      httpsrv.URL,
-			HTTPTestClient: httpc,
-			//TimeNow:   s.control.TimeNow,
-			Logf: func(fmt string, args ...interface{}) {
-				t.Helper()
-				t.Logf("c1: "+fmt, args...)
-			},
-			Hostinfo: hi,
+func formatNodes(nodes []*tailcfg.Node) string {
+	var sb strings.Builder
+	for i, n := range nodes {
+		if i > 0 {
+			sb.WriteString(", ")
 		}
+		fmt.Fprintf(&sb, "(%d, %q)", n.ID, n.Name)
 	}
-
-	// Login with a new node key. This requires authorization.
-	c1, err := NewDirect(genOpts())
-	if err != nil {
-		t.Fatal(err)
-	}
-	ctx, cancel := context.WithCancel(context.Background())
-	defer cancel()
-	authURL, err := c1.TryLogin(ctx, nil, 0)
-	if err != nil {
-		t.Fatal(err)
-	}
-	const user = "testuser1@tailscale.onmicrosoft.com"
-	postAuthURL(t, ctx, httpc, user, authURL)
-	newURL, err := c1.WaitLoginURL(ctx, authURL)
-	if err != nil {
-		t.Fatal(err)
-	}
-	if newURL != "" {
-		t.Fatalf("unexpected newURL: %s", newURL)
-	}
-
-	if err := c1.PollNetMap(ctx, 1, func(netMap *NetworkMap) {}); err != nil {
-		t.Fatal(err)
-	}
-
-	newPrivKey := func(t *testing.T) wgcfg.PrivateKey {
-		t.Helper()
-		k, err := wgcfg.NewPrivateKey()
-		if err != nil {
-			t.Fatal(err)
-		}
-		return k
-	}
-
-	// Replace the previous key with a new key.
-	persist1 := c1.GetPersist()
-	persist2 := Persist{
-		PrivateMachineKey: persist1.PrivateMachineKey,
-		OldPrivateNodeKey: persist1.PrivateNodeKey,
-		PrivateNodeKey:    newPrivKey(t),
-	}
-	opts := genOpts()
-	opts.Persist = persist2
-
-	c1, err = NewDirect(opts)
-	if err != nil {
-		t.Fatal(err)
-	}
-	if authURL, err := c1.TryLogin(ctx, nil, 0); err != nil {
-		t.Fatal(err)
-	} else if authURL == "" {
-		t.Fatal("expected authURL for reused oldNodeKey, got none")
-	} else {
-		postAuthURL(t, ctx, httpc, user, authURL)
-		if newURL, err := c1.WaitLoginURL(ctx, authURL); err != nil {
-			t.Fatal(err)
-		} else if newURL != "" {
-			t.Fatalf("unexpected newURL: %s", newURL)
-		}
-	}
-	if p := c1.GetPersist(); p.PrivateNodeKey != opts.Persist.PrivateNodeKey {
-		t.Error("unexpected node key change")
-	} else {
-		persist2 = p
-	}
-
-	// Here we simulate a client using using old persistent data.
-	// We use the key we have already replaced as the old node key.
-	// This requires the user to authenticate.
-	persist3 := Persist{
-		PrivateMachineKey: persist1.PrivateMachineKey,
-		OldPrivateNodeKey: persist1.PrivateNodeKey,
-		PrivateNodeKey:    newPrivKey(t),
-	}
-	opts = genOpts()
-	opts.Persist = persist3
-
-	c1, err = NewDirect(opts)
-	if err != nil {
-		t.Fatal(err)
-	}
-	if authURL, err := c1.TryLogin(ctx, nil, 0); err != nil {
-		t.Fatal(err)
-	} else if authURL == "" {
-		t.Fatal("expected authURL for reused oldNodeKey, got none")
-	} else {
-		postAuthURL(t, ctx, httpc, user, authURL)
-		if newURL, err := c1.WaitLoginURL(ctx, authURL); err != nil {
-			t.Fatal(err)
-		} else if newURL != "" {
-			t.Fatalf("unexpected newURL: %s", newURL)
-		}
-	}
-	if err := c1.PollNetMap(ctx, 1, func(netMap *NetworkMap) {}); err != nil {
-		t.Fatal(err)
-	}
-
-	// At this point, there should only be one node for the machine key
-	// registered as active in the server.
-	mkey := tailcfg.MachineKey(persist1.PrivateMachineKey.Public())
-	nodeIDs, err := server.DB().MachineNodes(mkey)
-	if err != nil {
-		t.Fatal(err)
-	}
-	if len(nodeIDs) != 1 {
-		t.Logf("active nodes for machine key %v:", mkey)
-		for i, nodeID := range nodeIDs {
-			nodeKey := server.DB().NodeKey(nodeID)
-			t.Logf("\tnode %d: id=%v, key=%v", i, nodeID, nodeKey)
-		}
-		t.Fatalf("want 1 active node for the client machine, got %d", len(nodeIDs))
-	}
-
-	// Now try the previous node key. It should fail.
-	opts = genOpts()
-	opts.Persist = persist2
-	c1, err = NewDirect(opts)
-	if err != nil {
-		t.Fatal(err)
-	}
-	// TODO(crawshaw): make this return an actual error.
-	// Have cfgdb track expired keys, and when an expired key is reused
-	// produce an error.
-	if authURL, err := c1.TryLogin(ctx, nil, 0); err != nil {
-		t.Fatal(err)
-	} else if authURL == "" {
-		t.Fatal("expected authURL for reused nodeKey, got none")
-	} else {
-		postAuthURL(t, ctx, httpc, user, authURL)
-		if newURL, err := c1.WaitLoginURL(ctx, authURL); err != nil {
-			t.Fatal(err)
-		} else if newURL != "" {
-			t.Fatalf("unexpected newURL: %s", newURL)
-		}
-	}
-	if err := c1.PollNetMap(ctx, 1, func(netMap *NetworkMap) {}); err != nil {
-		t.Fatal(err)
-	}
-	if nodeIDs, err := server.DB().MachineNodes(mkey); err != nil {
-		t.Fatal(err)
-	} else if len(nodeIDs) != 1 {
-		t.Fatalf("want 1 active node for the client machine, got %d", len(nodeIDs))
-	}
+	return sb.String()
 }

control/controlclient/hostinfo_linux.go

@@ -0,0 +1,87 @@
+// Copyright (c) 2020 Tailscale Inc & AUTHORS All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+// +build linux,!android
+
+package controlclient
+
+import (
+	"bytes"
+	"fmt"
+	"io"
+	"io/ioutil"
+	"strings"
+	"syscall"
+
+	"go4.org/mem"
+	"tailscale.com/util/lineread"
+)
+
+func init() {
+	osVersion = osVersionLinux
+}
+
+func osVersionLinux() string {
+	m := map[string]string{}
+	lineread.File("/etc/os-release", func(line []byte) error {
+		eq := bytes.IndexByte(line, '=')
+		if eq == -1 {
+			return nil
+		}
+		k, v := string(line[:eq]), strings.Trim(string(line[eq+1:]), `"`)
+		m[k] = v
+		return nil
+	})
+
+	var un syscall.Utsname
+	syscall.Uname(&un)
+
+	var attrBuf strings.Builder
+	attrBuf.WriteString("; kernel=")
+	for _, b := range un.Release {
+		if b == 0 {
+			break
+		}
+		attrBuf.WriteByte(byte(b))
+	}
+	if inContainer() {
+		attrBuf.WriteString("; container")
+	}
+	attr := attrBuf.String()
+
+	id := m["ID"]
+
+	switch id {
+	case "debian":
+		slurp, _ := ioutil.ReadFile("/etc/debian_version")
+		return fmt.Sprintf("Debian %s (%s)%s", bytes.TrimSpace(slurp), m["VERSION_CODENAME"], attr)
+	case "ubuntu":
+		return fmt.Sprintf("Ubuntu %s%s", m["VERSION"], attr)
+	case "", "centos": // CentOS 6 has no /etc/os-release, so its id is ""
+		if cr, _ := ioutil.ReadFile("/etc/centos-release"); len(cr) > 0 { // "CentOS release 6.10 (Final)
+			return fmt.Sprintf("%s%s", bytes.TrimSpace(cr), attr)
+		}
+		fallthrough
+	case "fedora", "rhel", "alpine":
+		// Their PRETTY_NAME is fine as-is for all versions I tested.
+		fallthrough
+	default:
+		if v := m["PRETTY_NAME"]; v != "" {
+			return fmt.Sprintf("%s%s", v, attr)
+		}
+	}
+	return fmt.Sprintf("Other%s", attr)
+}
+
+func inContainer() (ret bool) {
+	lineread.File("/proc/1/cgroup", func(line []byte) error {
+		if mem.Contains(mem.B(line), mem.S("/docker/")) ||
+			mem.Contains(mem.B(line), mem.S("/lxc/")) {
+			ret = true
+			return io.EOF // arbitrary non-nil error to stop loop
+		}
+		return nil
+	})
+	return
+}

control/controlclient/hostinfo_windows.go

@@ -0,0 +1,26 @@
+// Copyright (c) 2020 Tailscale Inc & AUTHORS All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package controlclient
+
+import (
+	"os/exec"
+	"strings"
+	"syscall"
+)
+
+func init() {
+	osVersion = osVersionWindows
+}
+
+func osVersionWindows() string {
+	cmd := exec.Command("cmd", "/c", "ver")
+	cmd.SysProcAttr = &syscall.SysProcAttr{HideWindow: true}
+	out, _ := cmd.Output() // "\nMicrosoft Windows [Version 10.0.19041.388]\n\n"
+	s := strings.TrimSpace(string(out))
+	s = strings.TrimPrefix(s, "Microsoft Windows [")
+	s = strings.TrimSuffix(s, "]")
+	s = strings.TrimPrefix(s, "Version ") // is this localized? do it last in case.
+	return s                              // "10.0.19041.388", ideally
+}

control/controlclient/netmap.go

@@ -5,34 +5,43 @@
 package controlclient
 
 import (
-	"bytes"
-	"encoding/base64"
 	"encoding/json"
 	"fmt"
-	"log"
+	"net"
+	"reflect"
+	"strconv"
 	"strings"
 	"time"
 
 	"github.com/tailscale/wireguard-go/wgcfg"
 	"tailscale.com/tailcfg"
+	"tailscale.com/types/logger"
 	"tailscale.com/wgengine/filter"
 )
 
 type NetworkMap struct {
 	// Core networking
 
-	NodeKey       tailcfg.NodeKey
-	PrivateKey    wgcfg.PrivateKey
-	Expiry        time.Time
+	NodeKey    tailcfg.NodeKey
+	PrivateKey wgcfg.PrivateKey
+	Expiry     time.Time
+	// Name is the DNS name assigned to this node.
+	Name          string
 	Addresses     []wgcfg.CIDR
 	LocalPort     uint16 // used for debugging
 	MachineStatus tailcfg.MachineStatus
-	Peers         []*tailcfg.Node
-	DNS           []wgcfg.IP
-	DNSDomains    []string
+	Peers         []*tailcfg.Node // sorted by Node.ID
+	DNS           tailcfg.DNSConfig
 	Hostinfo      tailcfg.Hostinfo
 	PacketFilter  filter.Matches
 
+	// DERPMap is the last DERP server map received. It's reused
+	// between updates and should not be modified.
+	DERPMap *tailcfg.DERPMap
+
+	// Debug knobs from control server for debug or feature gating.
+	Debug *tailcfg.Debug
+
 	// ACLs
 
 	User   tailcfg.UserID
@@ -45,92 +54,153 @@ type NetworkMap struct {
 	// TODO(crawshaw): Capabilities []tailcfg.Capability
 }
 
-func (n *NetworkMap) Equal(n2 *NetworkMap) bool {
-	// TODO(crawshaw): this is crude, but is an easy way to avoid bugs.
-	b, err := json.Marshal(n)
-	if err != nil {
-		panic(err)
-	}
-	b2, err := json.Marshal(n2)
-	if err != nil {
-		panic(err)
-	}
-	return bytes.Equal(b, b2)
-}
-
 func (nm NetworkMap) String() string {
 	return nm.Concise()
 }
 
 func (nm *NetworkMap) Concise() string {
 	buf := new(strings.Builder)
-	fmt.Fprintf(buf, "netmap: self: %v auth=%v :%v %v\n",
-		nm.NodeKey.ShortString(), nm.MachineStatus,
-		nm.LocalPort, nm.Addresses)
+
+	nm.printConciseHeader(buf)
 	for _, p := range nm.Peers {
-		aip := make([]string, len(p.AllowedIPs))
-		for i, a := range p.AllowedIPs {
-			s := fmt.Sprint(a)
-			if strings.HasSuffix(s, "/32") {
-				s = s[0 : len(s)-3]
-			}
-			aip[i] = s
-		}
-
-		ep := make([]string, len(p.Endpoints))
-		for i, e := range p.Endpoints {
-			// Align vertically on the ':' between IP and port
-			colon := strings.IndexByte(e, ':')
-			for colon > 0 && len(e)-colon < 6 {
-				e += " "
-				colon--
-			}
-			ep[i] = fmt.Sprintf("%21v", e)
-		}
-
-		derp := p.DERP
-		const derpPrefix = "127.3.3.40:"
-		if strings.HasPrefix(derp, derpPrefix) {
-			derp = "D" + derp[len(derpPrefix):]
-		}
-
-		// Most of the time, aip is just one element, so format the
-		// table to look good in that case. This will also make multi-
-		// subnet nodes stand out visually.
-		fmt.Fprintf(buf, " %v %-2v %-15v : %v\n",
-			p.Key.ShortString(), derp,
-			strings.Join(aip, " "),
-			strings.Join(ep, " "))
+		printPeerConcise(buf, p)
 	}
 	return buf.String()
 }
 
+// printConciseHeader prints a concise header line representing nm to buf.
+//
+// If this function is changed to access different fields of nm, keep
+// in equalConciseHeader in sync.
+func (nm *NetworkMap) printConciseHeader(buf *strings.Builder) {
+	fmt.Fprintf(buf, "netmap: self: %v auth=%v",
+		nm.NodeKey.ShortString(), nm.MachineStatus)
+	if nm.LocalPort != 0 {
+		fmt.Fprintf(buf, " port=%v", nm.LocalPort)
+	}
+	if nm.Debug != nil {
+		j, _ := json.Marshal(nm.Debug)
+		fmt.Fprintf(buf, " debug=%s", j)
+	}
+	fmt.Fprintf(buf, " %v", nm.Addresses)
+	buf.WriteByte('\n')
+}
+
+// equalConciseHeader reports whether a and b are equal for the fields
+// used by printConciseHeader.
+func (a *NetworkMap) equalConciseHeader(b *NetworkMap) bool {
+	if a.NodeKey != b.NodeKey ||
+		a.MachineStatus != b.MachineStatus ||
+		a.LocalPort != b.LocalPort ||
+		len(a.Addresses) != len(b.Addresses) {
+		return false
+	}
+	for i, a := range a.Addresses {
+		if b.Addresses[i] != a {
+			return false
+		}
+	}
+	return (a.Debug == nil && b.Debug == nil) || reflect.DeepEqual(a.Debug, b.Debug)
+}
+
+// printPeerConcise appends to buf a line repsenting the peer p.
+//
+// If this function is changed to access different fields of p, keep
+// in nodeConciseEqual in sync.
+func printPeerConcise(buf *strings.Builder, p *tailcfg.Node) {
+	aip := make([]string, len(p.AllowedIPs))
+	for i, a := range p.AllowedIPs {
+		s := strings.TrimSuffix(fmt.Sprint(a), "/32")
+		aip[i] = s
+	}
+
+	ep := make([]string, len(p.Endpoints))
+	for i, e := range p.Endpoints {
+		// Align vertically on the ':' between IP and port
+		colon := strings.IndexByte(e, ':')
+		spaces := 0
+		for colon > 0 && len(e)+spaces-colon < 6 {
+			spaces++
+			colon--
+		}
+		ep[i] = fmt.Sprintf("%21v", e+strings.Repeat(" ", spaces))
+	}
+
+	derp := p.DERP
+	const derpPrefix = "127.3.3.40:"
+	if strings.HasPrefix(derp, derpPrefix) {
+		derp = "D" + derp[len(derpPrefix):]
+	}
+	var discoShort string
+	if !p.DiscoKey.IsZero() {
+		discoShort = p.DiscoKey.ShortString() + " "
+	}
+
+	// Most of the time, aip is just one element, so format the
+	// table to look good in that case. This will also make multi-
+	// subnet nodes stand out visually.
+	fmt.Fprintf(buf, " %v %s%-2v %-15v : %v\n",
+		p.Key.ShortString(),
+		discoShort,
+		derp,
+		strings.Join(aip, " "),
+		strings.Join(ep, " "))
+}
+
+// nodeConciseEqual reports whether a and b are equal for the fields accessed by printPeerConcise.
+func nodeConciseEqual(a, b *tailcfg.Node) bool {
+	return a.Key == b.Key &&
+		a.DERP == b.DERP &&
+		a.DiscoKey == b.DiscoKey &&
+		eqCIDRsIgnoreNil(a.AllowedIPs, b.AllowedIPs) &&
+		eqStringsIgnoreNil(a.Endpoints, b.Endpoints)
+}
+
 func (b *NetworkMap) ConciseDiffFrom(a *NetworkMap) string {
-	out := []string{}
-	ra := strings.Split(a.Concise(), "\n")
-	rb := strings.Split(b.Concise(), "\n")
+	var diff strings.Builder
 
-	ma := map[string]struct{}{}
-	for _, s := range ra {
-		ma[s] = struct{}{}
+	// See if header (non-peers, "bare") part of the network map changed.
+	// If so, print its diff lines first.
+	if !a.equalConciseHeader(b) {
+		diff.WriteByte('-')
+		a.printConciseHeader(&diff)
+		diff.WriteByte('+')
+		b.printConciseHeader(&diff)
 	}
 
-	mb := map[string]struct{}{}
-	for _, s := range rb {
-		mb[s] = struct{}{}
-	}
-
-	for _, s := range ra {
-		if _, ok := mb[s]; !ok {
-			out = append(out, "-"+s)
+	aps, bps := a.Peers, b.Peers
+	for len(aps) > 0 && len(bps) > 0 {
+		pa, pb := aps[0], bps[0]
+		switch {
+		case pa.ID == pb.ID:
+			if !nodeConciseEqual(pa, pb) {
+				diff.WriteByte('-')
+				printPeerConcise(&diff, pa)
+				diff.WriteByte('+')
+				printPeerConcise(&diff, pb)
+			}
+			aps, bps = aps[1:], bps[1:]
+		case pa.ID > pb.ID:
+			// New peer in b.
+			diff.WriteByte('+')
+			printPeerConcise(&diff, pb)
+			bps = bps[1:]
+		case pb.ID > pa.ID:
+			// Deleted peer in b.
+			diff.WriteByte('-')
+			printPeerConcise(&diff, pa)
+			aps = aps[1:]
 		}
 	}
-	for _, s := range rb {
-		if _, ok := ma[s]; !ok {
-			out = append(out, "+"+s)
-		}
+	for _, pa := range aps {
+		diff.WriteByte('-')
+		printPeerConcise(&diff, pa)
 	}
-	return strings.Join(out, "\n")
+	for _, pb := range bps {
+		diff.WriteByte('+')
+		printPeerConcise(&diff, pb)
+	}
+	return diff.String()
 }
 
 func (nm *NetworkMap) JSON() string {
@@ -141,138 +211,126 @@ func (nm *NetworkMap) JSON() string {
 	return string(b)
 }
 
-const (
-	UAllowSingleHosts = 1 << iota
-	UAllowSubnetRoutes
-	UAllowDefaultRoute
-	UHackDefaultRoute
+// WGConfigFlags is a bitmask of flags to control the behavior of the
+// wireguard configuration generation done by NetMap.WGCfg.
+type WGConfigFlags int
 
-	UDefault = 0
+const (
+	AllowSingleHosts WGConfigFlags = 1 << iota
+	AllowSubnetRoutes
+	AllowDefaultRoute
+	HackDefaultRoute
 )
 
-// Several programs need to parse these arguments into uflags, so let's
-// centralize it here.
-func UFlagsHelper(uroutes, rroutes, droutes bool) int {
-	uflags := 0
-	if uroutes {
-		uflags |= UAllowSingleHosts
-	}
-	if rroutes {
-		uflags |= UAllowSubnetRoutes
-	}
-	if droutes {
-		uflags |= UAllowDefaultRoute
-	}
-	return uflags
-}
+// EndpointDiscoSuffix is appended to the hex representation of a peer's discovery key
+// and is then the sole wireguard endpoint for peers with a non-zero discovery key.
+// This form is then recognize by magicsock's CreateEndpoint.
+const EndpointDiscoSuffix = ".disco.tailscale:12345"
 
-// TODO(bradfitz): UAPI seems to only be used by the old confnode and
-// pingnode; delete this when those are deleted/rewritten?
-func (nm *NetworkMap) UAPI(uflags int, dnsOverride []wgcfg.IP) string {
-	wgcfg, err := nm.WGCfg(uflags, dnsOverride)
-	if err != nil {
-		log.Fatalf("WGCfg() failed unexpectedly: %v\n", err)
+// WGCfg returns the NetworkMaps's Wireguard configuration.
+func (nm *NetworkMap) WGCfg(logf logger.Logf, flags WGConfigFlags) (*wgcfg.Config, error) {
+	cfg := &wgcfg.Config{
+		Name:       "tailscale",
+		PrivateKey: nm.PrivateKey,
+		Addresses:  nm.Addresses,
+		ListenPort: nm.LocalPort,
+		Peers:      make([]wgcfg.Peer, 0, len(nm.Peers)),
 	}
-	s, err := wgcfg.ToUAPI()
-	if err != nil {
-		log.Fatalf("ToUAPI() failed unexpectedly: %v\n", err)
-	}
-	return s
-}
 
-func (nm *NetworkMap) WGCfg(uflags int, dnsOverride []wgcfg.IP) (*wgcfg.Config, error) {
-	s := nm._WireGuardConfig(uflags, dnsOverride, true)
-	return wgcfg.FromWgQuick(s, "tailscale")
-}
-
-// TODO(apenwarr): This mode is dangerous.
-// Discarding the extra endpoints is almost universally the wrong choice.
-// Except that plain wireguard can't handle a peer with multiple endpoints.
-// (Yet?)
-func (nm *NetworkMap) WireGuardConfigOneEndpoint(uflags int, dnsOverride []wgcfg.IP) string {
-	return nm._WireGuardConfig(uflags, dnsOverride, false)
-}
-
-func (nm *NetworkMap) _WireGuardConfig(uflags int, dnsOverride []wgcfg.IP, allEndpoints bool) string {
-	buf := new(strings.Builder)
-	fmt.Fprintf(buf, "[Interface]\n")
-	fmt.Fprintf(buf, "PrivateKey = %s\n", base64.StdEncoding.EncodeToString(nm.PrivateKey[:]))
-	if len(nm.Addresses) > 0 {
-		fmt.Fprintf(buf, "Address = ")
-		for i, cidr := range nm.Addresses {
-			if i > 0 {
-				fmt.Fprintf(buf, ", ")
-			}
-			fmt.Fprintf(buf, "%s", cidr)
-		}
-		fmt.Fprintf(buf, "\n")
-	}
-	fmt.Fprintf(buf, "ListenPort = %d\n", nm.LocalPort)
-	if len(dnsOverride) > 0 {
-		dnss := []string{}
-		for _, ip := range dnsOverride {
-			dnss = append(dnss, ip.String())
-		}
-		fmt.Fprintf(buf, "DNS = %s\n", strings.Join(dnss, ","))
-	}
-	fmt.Fprintf(buf, "\n")
-
-	for i, peer := range nm.Peers {
-		if (uflags&UAllowSingleHosts) == 0 && len(peer.AllowedIPs) < 2 {
-			log.Printf("wgcfg: %v skipping a single-host peer.\n", peer.Key.ShortString())
+	for _, peer := range nm.Peers {
+		if Debug.OnlyDisco && peer.DiscoKey.IsZero() {
 			continue
 		}
-		if i > 0 {
-			fmt.Fprintf(buf, "\n")
+		if (flags&AllowSingleHosts) == 0 && len(peer.AllowedIPs) < 2 {
+			logf("wgcfg: %v skipping a single-host peer.", peer.Key.ShortString())
+			continue
 		}
-		fmt.Fprintf(buf, "[Peer]\n")
-		fmt.Fprintf(buf, "PublicKey = %s\n", base64.StdEncoding.EncodeToString(peer.Key[:]))
-		var endpoints []string
-		if peer.DERP != "" {
-			endpoints = append(endpoints, peer.DERP)
+		cfg.Peers = append(cfg.Peers, wgcfg.Peer{
+			PublicKey: wgcfg.Key(peer.Key),
+		})
+		cpeer := &cfg.Peers[len(cfg.Peers)-1]
+		if peer.KeepAlive {
+			cpeer.PersistentKeepalive = 25 // seconds
 		}
-		endpoints = append(endpoints, peer.Endpoints...)
-		if len(endpoints) > 0 {
-			if len(endpoints) == 1 {
-				fmt.Fprintf(buf, "Endpoint = %s", endpoints[0])
-			} else if allEndpoints {
-				// TODO(apenwarr): This mode is incompatible.
-				// Normal wireguard clients don't know how to
-				// parse it (yet?)
-				fmt.Fprintf(buf, "Endpoint = %s",
-					strings.Join(endpoints, ","))
-			} else {
-				fmt.Fprintf(buf, "Endpoint = %s # other endpoints: %s",
-					endpoints[0],
-					strings.Join(endpoints[1:], ", "))
+
+		if !peer.DiscoKey.IsZero() {
+			if err := appendEndpoint(cpeer, fmt.Sprintf("%x%s", peer.DiscoKey[:], EndpointDiscoSuffix)); err != nil {
+				return nil, err
+			}
+			cpeer.Endpoints = []wgcfg.Endpoint{{Host: fmt.Sprintf("%x.disco.tailscale", peer.DiscoKey[:]), Port: 12345}}
+		} else {
+			if err := appendEndpoint(cpeer, peer.DERP); err != nil {
+				return nil, err
+			}
+			for _, ep := range peer.Endpoints {
+				if err := appendEndpoint(cpeer, ep); err != nil {
+					return nil, err
+				}
 			}
-			buf.WriteByte('\n')
 		}
-		var aips []string
 		for _, allowedIP := range peer.AllowedIPs {
-			aip := allowedIP.String()
 			if allowedIP.Mask == 0 {
-				if (uflags & UAllowDefaultRoute) == 0 {
-					log.Printf("wgcfg: %v skipping default route\n", peer.Key.ShortString())
+				if (flags & AllowDefaultRoute) == 0 {
+					logf("wgcfg: %v skipping default route", peer.Key.ShortString())
 					continue
 				}
-				if (uflags & UHackDefaultRoute) != 0 {
-					aip = "10.0.0.0/8"
-					log.Printf("wgcfg: %v converting default route => %v\n", peer.Key.ShortString(), aip)
+				if (flags & HackDefaultRoute) != 0 {
+					allowedIP = wgcfg.CIDR{IP: wgcfg.IPv4(10, 0, 0, 0), Mask: 8}
+					logf("wgcfg: %v converting default route => %v", peer.Key.ShortString(), allowedIP.String())
 				}
 			} else if allowedIP.Mask < 32 {
-				if (uflags & UAllowSubnetRoutes) == 0 {
-					log.Printf("wgcfg: %v skipping subnet route\n", peer.Key.ShortString())
+				if (flags & AllowSubnetRoutes) == 0 {
+					logf("wgcfg: %v skipping subnet route", peer.Key.ShortString())
 					continue
 				}
 			}
-			aips = append(aips, aip)
-		}
-		fmt.Fprintf(buf, "AllowedIPs = %s\n", strings.Join(aips, ", "))
-		if peer.KeepAlive {
-			fmt.Fprintf(buf, "PersistentKeepalive = 25\n")
+			cpeer.AllowedIPs = append(cpeer.AllowedIPs, allowedIP)
 		}
 	}
 
-	return buf.String()
+	return cfg, nil
+}
+
+func appendEndpoint(peer *wgcfg.Peer, epStr string) error {
+	if epStr == "" {
+		return nil
+	}
+	host, port, err := net.SplitHostPort(epStr)
+	if err != nil {
+		return fmt.Errorf("malformed endpoint %q for peer %v", epStr, peer.PublicKey.ShortString())
+	}
+	port16, err := strconv.ParseUint(port, 10, 16)
+	if err != nil {
+		return fmt.Errorf("invalid port in endpoint %q for peer %v", epStr, peer.PublicKey.ShortString())
+	}
+	peer.Endpoints = append(peer.Endpoints, wgcfg.Endpoint{Host: host, Port: uint16(port16)})
+	return nil
+}
+
+// eqStringsIgnoreNil reports whether a and b have the same length and
+// contents, but ignore whether a or b are nil.
+func eqStringsIgnoreNil(a, b []string) bool {
+	if len(a) != len(b) {
+		return false
+	}
+	for i, v := range a {
+		if v != b[i] {
+			return false
+		}
+	}
+	return true
+}
+
+// eqCIDRsIgnoreNil reports whether a and b have the same length and
+// contents, but ignore whether a or b are nil.
+func eqCIDRsIgnoreNil(a, b []wgcfg.CIDR) bool {
+	if len(a) != len(b) {
+		return false
+	}
+	for i, v := range a {
+		if v != b[i] {
+			return false
+		}
+	}
+	return true
 }

control/controlclient/netmap_test.go

@@ -0,0 +1,284 @@
+// Copyright (c) 2020 Tailscale Inc & AUTHORS All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package controlclient
+
+import (
+	"encoding/hex"
+	"testing"
+
+	"github.com/tailscale/wireguard-go/wgcfg"
+	"tailscale.com/tailcfg"
+)
+
+func testNodeKey(b byte) (ret tailcfg.NodeKey) {
+	for i := range ret {
+		ret[i] = b
+	}
+	return
+}
+
+func testDiscoKey(hexPrefix string) (ret tailcfg.DiscoKey) {
+	b, err := hex.DecodeString(hexPrefix)
+	if err != nil {
+		panic(err)
+	}
+	copy(ret[:], b)
+	return
+}
+
+func TestNetworkMapConcise(t *testing.T) {
+	for _, tt := range []struct {
+		name string
+		nm   *NetworkMap
+		want string
+	}{
+		{
+			name: "basic",
+			nm: &NetworkMap{
+				NodeKey: testNodeKey(1),
+				Peers: []*tailcfg.Node{
+					{
+						Key:       testNodeKey(2),
+						DERP:      "127.3.3.40:2",
+						Endpoints: []string{"192.168.0.100:12", "192.168.0.100:12354"},
+					},
+					{
+						Key:       testNodeKey(3),
+						DERP:      "127.3.3.40:4",
+						Endpoints: []string{"10.2.0.100:12", "10.1.0.100:12345"},
+					},
+				},
+			},
+			want: "netmap: self: [AQEBA] auth=machine-unknown []\n [AgICA] D2                 :    192.168.0.100:12     192.168.0.100:12354\n [AwMDA] D4                 :       10.2.0.100:12        10.1.0.100:12345\n",
+		},
+		{
+			name: "debug_non_nil",
+			nm: &NetworkMap{
+				NodeKey: testNodeKey(1),
+				Debug:   &tailcfg.Debug{},
+			},
+			want: "netmap: self: [AQEBA] auth=machine-unknown debug={} []\n",
+		},
+		{
+			name: "debug_values",
+			nm: &NetworkMap{
+				NodeKey: testNodeKey(1),
+				Debug:   &tailcfg.Debug{LogHeapPprof: true},
+			},
+			want: "netmap: self: [AQEBA] auth=machine-unknown debug={\"LogHeapPprof\":true} []\n",
+		},
+	} {
+		t.Run(tt.name, func(t *testing.T) {
+			var got string
+			n := int(testing.AllocsPerRun(1000, func() {
+				got = tt.nm.Concise()
+			}))
+			t.Logf("Allocs = %d", n)
+			if got != tt.want {
+				t.Errorf("Wrong output\n Got: %q\nWant: %q\n## Got (unescaped):\n%s\n## Want (unescaped):\n%s\n", got, tt.want, got, tt.want)
+			}
+		})
+	}
+}
+
+func TestConciseDiffFrom(t *testing.T) {
+	for _, tt := range []struct {
+		name string
+		a, b *NetworkMap
+		want string
+	}{
+		{
+			name: "no_change",
+			a: &NetworkMap{
+				NodeKey: testNodeKey(1),
+				Peers: []*tailcfg.Node{
+					{
+						Key:       testNodeKey(2),
+						DERP:      "127.3.3.40:2",
+						Endpoints: []string{"192.168.0.100:12", "192.168.0.100:12354"},
+					},
+				},
+			},
+			b: &NetworkMap{
+				NodeKey: testNodeKey(1),
+				Peers: []*tailcfg.Node{
+					{
+						Key:       testNodeKey(2),
+						DERP:      "127.3.3.40:2",
+						Endpoints: []string{"192.168.0.100:12", "192.168.0.100:12354"},
+					},
+				},
+			},
+			want: "",
+		},
+		{
+			name: "header_change",
+			a: &NetworkMap{
+				NodeKey: testNodeKey(1),
+				Peers: []*tailcfg.Node{
+					{
+						Key:       testNodeKey(2),
+						DERP:      "127.3.3.40:2",
+						Endpoints: []string{"192.168.0.100:12", "192.168.0.100:12354"},
+					},
+				},
+			},
+			b: &NetworkMap{
+				NodeKey: testNodeKey(2),
+				Peers: []*tailcfg.Node{
+					{
+						Key:       testNodeKey(2),
+						DERP:      "127.3.3.40:2",
+						Endpoints: []string{"192.168.0.100:12", "192.168.0.100:12354"},
+					},
+				},
+			},
+			want: "-netmap: self: [AQEBA] auth=machine-unknown []\n+netmap: self: [AgICA] auth=machine-unknown []\n",
+		},
+		{
+			name: "peer_add",
+			a: &NetworkMap{
+				NodeKey: testNodeKey(1),
+				Peers: []*tailcfg.Node{
+					{
+						ID:        2,
+						Key:       testNodeKey(2),
+						DERP:      "127.3.3.40:2",
+						Endpoints: []string{"192.168.0.100:12", "192.168.0.100:12354"},
+					},
+				},
+			},
+			b: &NetworkMap{
+				NodeKey: testNodeKey(1),
+				Peers: []*tailcfg.Node{
+					{
+						ID:        1,
+						Key:       testNodeKey(1),
+						DERP:      "127.3.3.40:1",
+						Endpoints: []string{"192.168.0.100:12", "192.168.0.100:12354"},
+					},
+					{
+						ID:        2,
+						Key:       testNodeKey(2),
+						DERP:      "127.3.3.40:2",
+						Endpoints: []string{"192.168.0.100:12", "192.168.0.100:12354"},
+					},
+					{
+						ID:        3,
+						Key:       testNodeKey(3),
+						DERP:      "127.3.3.40:3",
+						Endpoints: []string{"192.168.0.100:12", "192.168.0.100:12354"},
+					},
+				},
+			},
+			want: "+ [AQEBA] D1                 :    192.168.0.100:12     192.168.0.100:12354\n+ [AwMDA] D3                 :    192.168.0.100:12     192.168.0.100:12354\n",
+		},
+		{
+			name: "peer_remove",
+			a: &NetworkMap{
+				NodeKey: testNodeKey(1),
+				Peers: []*tailcfg.Node{
+					{
+						ID:        1,
+						Key:       testNodeKey(1),
+						DERP:      "127.3.3.40:1",
+						Endpoints: []string{"192.168.0.100:12", "192.168.0.100:12354"},
+					},
+					{
+						ID:        2,
+						Key:       testNodeKey(2),
+						DERP:      "127.3.3.40:2",
+						Endpoints: []string{"192.168.0.100:12", "192.168.0.100:12354"},
+					},
+					{
+						ID:        3,
+						Key:       testNodeKey(3),
+						DERP:      "127.3.3.40:3",
+						Endpoints: []string{"192.168.0.100:12", "192.168.0.100:12354"},
+					},
+				},
+			},
+			b: &NetworkMap{
+				NodeKey: testNodeKey(1),
+				Peers: []*tailcfg.Node{
+					{
+						ID:        2,
+						Key:       testNodeKey(2),
+						DERP:      "127.3.3.40:2",
+						Endpoints: []string{"192.168.0.100:12", "192.168.0.100:12354"},
+					},
+				},
+			},
+			want: "- [AQEBA] D1                 :    192.168.0.100:12     192.168.0.100:12354\n- [AwMDA] D3                 :    192.168.0.100:12     192.168.0.100:12354\n",
+		},
+		{
+			name: "peer_port_change",
+			a: &NetworkMap{
+				NodeKey: testNodeKey(1),
+				Peers: []*tailcfg.Node{
+					{
+						ID:        2,
+						Key:       testNodeKey(2),
+						DERP:      "127.3.3.40:2",
+						Endpoints: []string{"192.168.0.100:12", "1.1.1.1:1"},
+					},
+				},
+			},
+			b: &NetworkMap{
+				NodeKey: testNodeKey(1),
+				Peers: []*tailcfg.Node{
+					{
+						ID:        2,
+						Key:       testNodeKey(2),
+						DERP:      "127.3.3.40:2",
+						Endpoints: []string{"192.168.0.100:12", "1.1.1.1:2"},
+					},
+				},
+			},
+			want: "- [AgICA] D2                 :    192.168.0.100:12             1.1.1.1:1  \n+ [AgICA] D2                 :    192.168.0.100:12             1.1.1.1:2  \n",
+		},
+		{
+			name: "disco_key_only_change",
+			a: &NetworkMap{
+				NodeKey: testNodeKey(1),
+				Peers: []*tailcfg.Node{
+					{
+						ID:         2,
+						Key:        testNodeKey(2),
+						DERP:       "127.3.3.40:2",
+						Endpoints:  []string{"192.168.0.100:41641", "1.1.1.1:41641"},
+						DiscoKey:   testDiscoKey("f00f00f00f"),
+						AllowedIPs: []wgcfg.CIDR{{IP: wgcfg.IPv4(100, 102, 103, 104), Mask: 32}},
+					},
+				},
+			},
+			b: &NetworkMap{
+				NodeKey: testNodeKey(1),
+				Peers: []*tailcfg.Node{
+					{
+						ID:         2,
+						Key:        testNodeKey(2),
+						DERP:       "127.3.3.40:2",
+						Endpoints:  []string{"192.168.0.100:41641", "1.1.1.1:41641"},
+						DiscoKey:   testDiscoKey("ba4ba4ba4b"),
+						AllowedIPs: []wgcfg.CIDR{{IP: wgcfg.IPv4(100, 102, 103, 104), Mask: 32}},
+					},
+				},
+			},
+			want: "- [AgICA] d:f00f00f00f000000 D2 100.102.103.104 :   192.168.0.100:41641         1.1.1.1:41641\n+ [AgICA] d:ba4ba4ba4b000000 D2 100.102.103.104 :   192.168.0.100:41641         1.1.1.1:41641\n",
+		},
+	} {
+		t.Run(tt.name, func(t *testing.T) {
+			var got string
+			n := int(testing.AllocsPerRun(50, func() {
+				got = tt.b.ConciseDiffFrom(tt.a)
+			}))
+			t.Logf("Allocs = %d", n)
+			if got != tt.want {
+				t.Errorf("Wrong output\n Got: %q\nWant: %q\n## Got (unescaped):\n%s\n## Want (unescaped):\n%s\n", got, tt.want, got, tt.want)
+			}
+		})
+	}
+}

derp/derp.go

@@ -32,10 +32,11 @@ const MaxPacketSize = 64 << 10
 const magic = "DERP🔑" // 8 bytes: 0x44 45 52 50 f0 9f 94 91
 
 const (
-	nonceLen   = 24
-	keyLen     = 32
-	maxInfoLen = 1 << 20
-	keepAlive  = 60 * time.Second
+	nonceLen       = 24
+	frameHeaderLen = 1 + 4 // frameType byte + 4 byte length
+	keyLen         = 32
+	maxInfoLen     = 1 << 20
+	keepAlive      = 60 * time.Second
 )
 
 // protocolVersion is bumped whenever there's a wire-incompatible change.
@@ -71,6 +72,7 @@ const (
 	frameClientInfo    = frameType(0x02) // 32B pub key + 24B nonce + naclbox(json)
 	frameServerInfo    = frameType(0x03) // 24B nonce + naclbox(json)
 	frameSendPacket    = frameType(0x04) // 32B dest pub key + packet bytes
+	frameForwardPacket = frameType(0x0a) // 32B src pub key + 32B dst pub key + packet bytes
 	frameRecvPacket    = frameType(0x05) // v0/1: packet bytes, v2: 32B src pub key + packet bytes
 	frameKeepAlive     = frameType(0x06) // no payload, no-op (to be replaced with ping/pong)
 	frameNotePreferred = frameType(0x07) // 1 byte payload: 0x01 or 0x00 for whether this is client's home node
@@ -81,6 +83,24 @@ const (
 	// framePeerGone to B so B can forget that a reverse path
 	// exists on that connection to get back to A.
 	framePeerGone = frameType(0x08) // 32B pub key of peer that's gone
+
+	// framePeerPresent is like framePeerGone, but for other
+	// members of the DERP region when they're meshed up together.
+	framePeerPresent = frameType(0x09) // 32B pub key of peer that's connected
+
+	// frameWatchConns is how one DERP node in a regional mesh
+	// subscribes to the others in the region.
+	// There's no payload. If the sender doesn't have permission, the connection
+	// is closed. Otherwise, the client is initially flooded with
+	// framePeerPresent for all connected nodes, and then a stream of
+	// framePeerPresent & framePeerGone has peers connect and disconnect.
+	frameWatchConns = frameType(0x10)
+
+	// frameClosePeer is a privileged frame type (requires the
+	// mesh key for now) that closes the provided peer's
+	// connection. (To be used for cluster load balancing
+	// purposes, when clients end up on a non-ideal node)
+	frameClosePeer = frameType(0x11) // 32B pub key of peer to close.
 )
 
 var bin = binary.BigEndian

derp/derp_client.go

@@ -19,6 +19,7 @@ import (
 	"tailscale.com/types/logger"
 )
 
+// Client is a DERP client.
 type Client struct {
 	serverKey    key.Public // of the DERP server; not a machine or node key
 	privateKey   key.Private
@@ -27,13 +28,48 @@ type Client struct {
 	logf         logger.Logf
 	nc           Conn
 	br           *bufio.Reader
+	meshKey      string
 
-	wmu     sync.Mutex // hold while writing to bw
-	bw      *bufio.Writer
+	wmu sync.Mutex // hold while writing to bw
+	bw  *bufio.Writer
+
+	// Owned by Recv:
+	peeked  int   // bytes to discard on next Recv
 	readErr error // sticky read error
 }
 
-func NewClient(privateKey key.Private, nc Conn, brw *bufio.ReadWriter, logf logger.Logf) (*Client, error) {
+// ClientOpt is an option passed to NewClient.
+type ClientOpt interface {
+	update(*clientOpt)
+}
+
+type clientOptFunc func(*clientOpt)
+
+func (f clientOptFunc) update(o *clientOpt) { f(o) }
+
+// clientOpt are the options passed to newClient.
+type clientOpt struct {
+	MeshKey string
+}
+
+// MeshKey returns a ClientOpt to pass to the DERP server during connect to get
+// access to join the mesh.
+//
+// An empty key means to not use a mesh key.
+func MeshKey(key string) ClientOpt { return clientOptFunc(func(o *clientOpt) { o.MeshKey = key }) }
+
+func NewClient(privateKey key.Private, nc Conn, brw *bufio.ReadWriter, logf logger.Logf, opts ...ClientOpt) (*Client, error) {
+	var opt clientOpt
+	for _, o := range opts {
+		if o == nil {
+			return nil, errors.New("nil ClientOpt")
+		}
+		o.update(&opt)
+	}
+	return newClient(privateKey, nc, brw, logf, opt)
+}
+
+func newClient(privateKey key.Private, nc Conn, brw *bufio.ReadWriter, logf logger.Logf, opt clientOpt) (*Client, error) {
 	c := &Client{
 		privateKey: privateKey,
 		publicKey:  privateKey.Public(),
@@ -41,8 +77,8 @@ func NewClient(privateKey key.Private, nc Conn, brw *bufio.ReadWriter, logf logg
 		nc:         nc,
 		br:         brw.Reader,
 		bw:         brw.Writer,
+		meshKey:    opt.MeshKey,
 	}
-
 	if err := c.recvServerKey(); err != nil {
 		return nil, fmt.Errorf("derp.Client: failed to receive server key: %v", err)
 	}
@@ -109,6 +145,12 @@ func (c *Client) recvServerInfo() (*serverInfo, error) {
 
 type clientInfo struct {
 	Version int // `json:"version,omitempty"`
+
+	// MeshKey optionally specifies a pre-shared key used by
+	// trusted clients.  It's required to subscribe to the
+	// connection list & forward packets. It's empty for regular
+	// users.
+	MeshKey string // `json:"meshKey,omitempty"`
 }
 
 func (c *Client) sendClientKey() error {
@@ -116,7 +158,10 @@ func (c *Client) sendClientKey() error {
 	if _, err := crand.Read(nonce[:]); err != nil {
 		return err
 	}
-	msg, err := json.Marshal(clientInfo{Version: protocolVersion})
+	msg, err := json.Marshal(clientInfo{
+		Version: protocolVersion,
+		MeshKey: c.meshKey,
+	})
 	if err != nil {
 		return err
 	}
@@ -129,6 +174,9 @@ func (c *Client) sendClientKey() error {
 	return writeFrame(c.bw, frameClientInfo, buf)
 }
 
+// ServerPublicKey returns the server's public key.
+func (c *Client) ServerPublicKey() key.Public { return c.serverKey }
+
 // Send sends a packet to the Tailscale node identified by dstKey.
 //
 // It is an error if the packet is larger than 64KB.
@@ -160,6 +208,40 @@ func (c *Client) send(dstKey key.Public, pkt []byte) (ret error) {
 	return c.bw.Flush()
 }
 
+func (c *Client) ForwardPacket(srcKey, dstKey key.Public, pkt []byte) (err error) {
+	defer func() {
+		if err != nil {
+			err = fmt.Errorf("derp.ForwardPacket: %w", err)
+		}
+	}()
+
+	if len(pkt) > MaxPacketSize {
+		return fmt.Errorf("packet too big: %d", len(pkt))
+	}
+
+	c.wmu.Lock()
+	defer c.wmu.Unlock()
+
+	timer := time.AfterFunc(5*time.Second, c.writeTimeoutFired)
+	defer timer.Stop()
+
+	if err := writeFrameHeader(c.bw, frameForwardPacket, uint32(keyLen*2+len(pkt))); err != nil {
+		return err
+	}
+	if _, err := c.bw.Write(srcKey[:]); err != nil {
+		return err
+	}
+	if _, err := c.bw.Write(dstKey[:]); err != nil {
+		return err
+	}
+	if _, err := c.bw.Write(pkt); err != nil {
+		return err
+	}
+	return c.bw.Flush()
+}
+
+func (c *Client) writeTimeoutFired() { c.nc.Close() }
+
 // NotePreferred sends a packet that tells the server whether this
 // client is the user's preferred server. This is only used in the
 // server for stats.
@@ -186,6 +268,25 @@ func (c *Client) NotePreferred(preferred bool) (err error) {
 	return c.bw.Flush()
 }
 
+// WatchConnectionChanges sends a request to subscribe to the peer's connection list.
+// It's a fatal error if the client wasn't created using MeshKey.
+func (c *Client) WatchConnectionChanges() error {
+	c.wmu.Lock()
+	defer c.wmu.Unlock()
+	if err := writeFrameHeader(c.bw, frameWatchConns, 0); err != nil {
+		return err
+	}
+	return c.bw.Flush()
+}
+
+// ClosePeer asks the server to close target's TCP connection.
+// It's a fatal error if the client wasn't created using MeshKey.
+func (c *Client) ClosePeer(target key.Public) error {
+	c.wmu.Lock()
+	defer c.wmu.Unlock()
+	return writeFrame(c.bw, frameClosePeer, target[:])
+}
+
 // ReceivedMessage represents a type returned by Client.Recv. Unless
 // otherwise documented, the returned message aliases the byte slice
 // provided to Recv and thus the message is only as good as that
@@ -211,11 +312,23 @@ type PeerGoneMessage key.Public
 
 func (PeerGoneMessage) msg() {}
 
+// PeerPresentMessage is a ReceivedMessage that indicates that the client
+// is connected to the server. (Only used by trusted mesh clients)
+type PeerPresentMessage key.Public
+
+func (PeerPresentMessage) msg() {}
+
 // Recv reads a message from the DERP server.
-// The provided buffer must be large enough to receive a complete packet,
-// which in practice are are 1.5-4 KB, but can be up to 64 KB.
+//
+// The returned message may alias memory owned by the Client; it
+// should only be accessed until the next call to Client.
+//
 // Once Recv returns an error, the Client is dead forever.
-func (c *Client) Recv(b []byte) (m ReceivedMessage, err error) {
+func (c *Client) Recv() (m ReceivedMessage, err error) {
+	return c.recvTimeout(120 * time.Second)
+}
+
+func (c *Client) recvTimeout(timeout time.Duration) (m ReceivedMessage, err error) {
 	if c.readErr != nil {
 		return nil, c.readErr
 	}
@@ -227,11 +340,45 @@ func (c *Client) Recv(b []byte) (m ReceivedMessage, err error) {
 	}()
 
 	for {
-		c.nc.SetReadDeadline(time.Now().Add(120 * time.Second))
-		t, n, err := readFrame(c.br, 1<<20, b)
+		c.nc.SetReadDeadline(time.Now().Add(timeout))
+
+		// Discard any peeked bytes from a previous Recv call.
+		if c.peeked != 0 {
+			if n, err := c.br.Discard(c.peeked); err != nil || n != c.peeked {
+				// Documented to never fail, but might as well check.
+				return nil, fmt.Errorf("bufio.Reader.Discard(%d bytes): got %v, %v", c.peeked, n, err)
+			}
+			c.peeked = 0
+		}
+
+		t, n, err := readFrameHeader(c.br)
 		if err != nil {
 			return nil, err
 		}
+		if n > 1<<20 {
+			return nil, fmt.Errorf("unexpectedly large frame of %d bytes returned", n)
+		}
+
+		var b []byte // frame payload (past the 5 byte header)
+
+		// If the frame fits in our bufio.Reader buffer, just use it.
+		// In practice it's 4KB (from derphttp.Client's bufio.NewReader(httpConn)) and
+		// in practive, WireGuard packets (and thus DERP frames) are under 1.5KB.
+		// So This is the common path.
+		if int(n) <= c.br.Size() {
+			b, err = c.br.Peek(int(n))
+			c.peeked = int(n)
+		} else {
+			// But if for some reason we read a large DERP message (which isn't necessarily
+			// a Wireguard packet), then just allocate memory for it.
+			// TODO(bradfitz): use a pool if large frames ever happen in practice.
+			b = make([]byte, n)
+			_, err = io.ReadFull(c.br, b)
+		}
+		if err != nil {
+			return nil, err
+		}
+
 		switch t {
 		default:
 			continue
@@ -248,6 +395,15 @@ func (c *Client) Recv(b []byte) (m ReceivedMessage, err error) {
 			copy(pg[:], b[:keyLen])
 			return pg, nil
 
+		case framePeerPresent:
+			if n < keyLen {
+				c.logf("[unexpected] dropping short peerPresent frame from DERP server")
+				continue
+			}
+			var pg PeerPresentMessage
+			copy(pg[:], b[:keyLen])
+			return pg, nil
+
 		case frameRecvPacket:
 			var rp ReceivedPacket
 			if c.protoVersion < protocolSrcAddrs {

derp/derp_server.go

@@ -20,6 +20,7 @@ import (
 	"os"
 	"runtime"
 	"strconv"
+	"strings"
 	"sync"
 	"time"
 
@@ -28,6 +29,7 @@ import (
 	"tailscale.com/metrics"
 	"tailscale.com/types/key"
 	"tailscale.com/types/logger"
+	"tailscale.com/version"
 )
 
 var debug, _ = strconv.ParseBool(os.Getenv("DERP_DEBUG_LOGS"))
@@ -37,6 +39,13 @@ const (
 	writeTimeout            = 2 * time.Second
 )
 
+const host64bit = (^uint(0) >> 32) & 1 // 1 on 64-bit, 0 on 32-bit
+
+// pad32bit is 4 on 32-bit machines and 0 on 64-bit.
+// It exists so the Server struct's atomic fields can be aligned to 8
+// byte boundaries. (As tested by GOARCH=386 go test, etc)
+const pad32bit = 4 - host64bit*4 // 0 on 64-bit, 4 on 32-bit
+
 // Server is a DERP server.
 type Server struct {
 	// WriteTimeout, if non-zero, specifies how long to wait
@@ -47,31 +56,62 @@ type Server struct {
 	publicKey  key.Public
 	logf       logger.Logf
 	memSys0    uint64 // runtime.MemStats.Sys at start (or early-ish)
+	meshKey    string
 
 	// Counters:
-	packetsSent, bytesSent  expvar.Int
-	packetsRecv, bytesRecv  expvar.Int
-	packetsDropped          expvar.Int
-	packetsDroppedReason    metrics.LabelMap
-	packetsDroppedUnknown   *expvar.Int // unknown dst pubkey
-	packetsDroppedGone      *expvar.Int // dst conn shutting down
-	packetsDroppedQueueHead *expvar.Int // queue full, drop head packet
-	packetsDroppedQueueTail *expvar.Int // queue full, drop tail packet
-	packetsDroppedWrite     *expvar.Int // error writing to dst conn
-	peerGoneFrames          expvar.Int  // number of peer gone frames sent
-	accepts                 expvar.Int
-	curClients              expvar.Int
-	curHomeClients          expvar.Int // ones with preferred
-	clientsReplaced         expvar.Int
-	unknownFrames           expvar.Int
-	homeMovesIn             expvar.Int // established clients announce home server moves in
-	homeMovesOut            expvar.Int // established clients announce home server moves out
+	_                        [pad32bit]byte
+	packetsSent, bytesSent   expvar.Int
+	packetsRecv, bytesRecv   expvar.Int
+	packetsDropped           expvar.Int
+	packetsDroppedReason     metrics.LabelMap
+	packetsDroppedUnknown    *expvar.Int // unknown dst pubkey
+	packetsDroppedFwdUnknown *expvar.Int // unknown dst pubkey on forward
+	packetsDroppedGone       *expvar.Int // dst conn shutting down
+	packetsDroppedQueueHead  *expvar.Int // queue full, drop head packet
+	packetsDroppedQueueTail  *expvar.Int // queue full, drop tail packet
+	packetsDroppedWrite      *expvar.Int // error writing to dst conn
+	_                        [pad32bit]byte
+	packetsForwardedOut      expvar.Int
+	packetsForwardedIn       expvar.Int
+	peerGoneFrames           expvar.Int // number of peer gone frames sent
+	accepts                  expvar.Int
+	curClients               expvar.Int
+	curHomeClients           expvar.Int // ones with preferred
+	clientsReplaced          expvar.Int
+	unknownFrames            expvar.Int
+	homeMovesIn              expvar.Int // established clients announce home server moves in
+	homeMovesOut             expvar.Int // established clients announce home server moves out
+	multiForwarderCreated    expvar.Int
+	multiForwarderDeleted    expvar.Int
+	removePktForwardOther    expvar.Int
 
 	mu          sync.Mutex
 	closed      bool
 	netConns    map[Conn]chan struct{} // chan is closed when conn closes
 	clients     map[key.Public]*sclient
 	clientsEver map[key.Public]bool // never deleted from, for stats; fine for now
+	watchers    map[*sclient]bool   // mesh peer -> true
+	// clientsMesh tracks all clients in the cluster, both locally
+	// and to mesh peers.  If the value is nil, that means the
+	// peer is only local (and thus in the clients Map, but not
+	// remote). If the value is non-nil, it's remote (+ maybe also
+	// local).
+	clientsMesh map[key.Public]PacketForwarder
+	// sentTo tracks which peers have sent to which other peers,
+	// and at which connection number. This isn't on sclient
+	// because it includes intra-region forwarded packets as the
+	// src.
+	sentTo map[key.Public]map[key.Public]int64 // src => dst => dst's latest sclient.connNum
+}
+
+// PacketForwarder is something that can forward packets.
+//
+// It's mostly an inteface for circular dependency reasons; the
+// typical implementation is derphttp.Client. The other implementation
+// is a multiForwarder, which this package creates as needed if a
+// public key gets more than one PacketForwarder registered for it.
+type PacketForwarder interface {
+	ForwardPacket(src, dst key.Public, payload []byte) error
 }
 
 // Conn is the subset of the underlying net.Conn the DERP Server needs.
@@ -97,12 +137,16 @@ func NewServer(privateKey key.Private, logf logger.Logf) *Server {
 		publicKey:            privateKey.Public(),
 		logf:                 logf,
 		packetsDroppedReason: metrics.LabelMap{Label: "reason"},
-		clients:              make(map[key.Public]*sclient),
-		clientsEver:          make(map[key.Public]bool),
-		netConns:             make(map[Conn]chan struct{}),
+		clients:              map[key.Public]*sclient{},
+		clientsEver:          map[key.Public]bool{},
+		clientsMesh:          map[key.Public]PacketForwarder{},
+		netConns:             map[Conn]chan struct{}{},
 		memSys0:              ms.Sys,
+		watchers:             map[*sclient]bool{},
+		sentTo:               map[key.Public]map[key.Public]int64{},
 	}
 	s.packetsDroppedUnknown = s.packetsDroppedReason.Get("unknown_dest")
+	s.packetsDroppedFwdUnknown = s.packetsDroppedReason.Get("unknown_dest_on_fwd")
 	s.packetsDroppedGone = s.packetsDroppedReason.Get("gone")
 	s.packetsDroppedQueueHead = s.packetsDroppedReason.Get("queue_head")
 	s.packetsDroppedQueueTail = s.packetsDroppedReason.Get("queue_tail")
@@ -110,6 +154,26 @@ func NewServer(privateKey key.Private, logf logger.Logf) *Server {
 	return s
 }
 
+// SetMesh sets the pre-shared key that regional DERP servers used to mesh
+// amongst themselves.
+//
+// It must be called before serving begins.
+func (s *Server) SetMeshKey(v string) {
+	s.meshKey = v
+}
+
+// HasMeshKey reports whether the server is configured with a mesh key.
+func (s *Server) HasMeshKey() bool { return s.meshKey != "" }
+
+// MeshKey returns the configured mesh key, if any.
+func (s *Server) MeshKey() string { return s.meshKey }
+
+// PrivateKey returns the server's private key.
+func (s *Server) PrivateKey() key.Private { return s.privateKey }
+
+// PublicKey returns the server's public key.
+func (s *Server) PublicKey() key.Public { return s.publicKey }
+
 // Close closes the server and waits for the connections to disconnect.
 func (s *Server) Close() error {
 	s.mu.Lock()
@@ -187,7 +251,23 @@ func (s *Server) registerClient(c *sclient) {
 	}
 	s.clients[c.key] = c
 	s.clientsEver[c.key] = true
+	if _, ok := s.clientsMesh[c.key]; !ok {
+		s.clientsMesh[c.key] = nil // just for varz of total users in cluster
+	}
 	s.curClients.Add(1)
+	s.broadcastPeerStateChangeLocked(c.key, true)
+}
+
+// broadcastPeerStateChangeLocked enqueues a message to all watchers
+// (other DERP nodes in the region, or trusted clients) that peer's
+// presence changed.
+//
+// s.mu must be held.
+func (s *Server) broadcastPeerStateChangeLocked(peer key.Public, present bool) {
+	for w := range s.watchers {
+		w.peerStateChange = append(w.peerStateChange, peerConnState{peer: peer, present: present})
+		go w.requestMeshUpdate()
+	}
 }
 
 // unregisterClient removes a client from the server.
@@ -198,30 +278,66 @@ func (s *Server) unregisterClient(c *sclient) {
 	if cur == c {
 		c.logf("removing connection")
 		delete(s.clients, c.key)
+		if v, ok := s.clientsMesh[c.key]; ok && v == nil {
+			delete(s.clientsMesh, c.key)
+			s.notePeerGoneFromRegionLocked(c.key)
+		}
+		s.broadcastPeerStateChangeLocked(c.key, false)
+	}
+	if c.canMesh {
+		delete(s.watchers, c)
 	}
 
 	s.curClients.Add(-1)
 	if c.preferred {
 		s.curHomeClients.Add(-1)
 	}
+}
+
+// notePeerGoneFromRegionLocked sends peerGone frames to parties that
+// key has sent to previously (whether those sends were from a local
+// client or forwarded).  It must only be called after the key has
+// been removed from clientsMesh.
+func (s *Server) notePeerGoneFromRegionLocked(key key.Public) {
+	if _, ok := s.clientsMesh[key]; ok {
+		panic("usage")
+	}
 
 	// Find still-connected peers and either notify that we've gone away
 	// so they can drop their route entries to us (issue 150)
 	// or move them over to the active client (in case a replaced client
 	// connection is being unregistered).
-	for pubKey, connNum := range c.sentTo {
+	for pubKey, connNum := range s.sentTo[key] {
 		if peer, ok := s.clients[pubKey]; ok && peer.connNum == connNum {
-			if cur == c {
-				go peer.requestPeerGoneWrite(c.key)
-			} else {
-				// Only if the current client has not already accepted a newer
-				// connection from the peer.
-				if _, ok := cur.sentTo[pubKey]; !ok {
-					cur.sentTo[pubKey] = connNum
-				}
-			}
+			go peer.requestPeerGoneWrite(key)
 		}
 	}
+	delete(s.sentTo, key)
+}
+
+func (s *Server) addWatcher(c *sclient) {
+	if !c.canMesh {
+		panic("invariant: addWatcher called without permissions")
+	}
+
+	if c.key == s.publicKey {
+		// We're connecting to ourself. Do nothing.
+		return
+	}
+
+	s.mu.Lock()
+	defer s.mu.Unlock()
+
+	// Queue messages for each already-connected client.
+	for peer := range s.clients {
+		c.peerStateChange = append(c.peerStateChange, peerConnState{peer: peer, present: true})
+	}
+
+	// And enroll the watcher in future updates (of both
+	// connections & disconnections).
+	s.watchers[c] = true
+
+	go c.requestMeshUpdate()
 }
 
 func (s *Server) accept(nc Conn, brw *bufio.ReadWriter, remoteAddr string, connNum int64) error {
@@ -258,7 +374,10 @@ func (s *Server) accept(nc Conn, brw *bufio.ReadWriter, remoteAddr string, connN
 		connectedAt: time.Now(),
 		sendQueue:   make(chan pkt, perClientSendQueueDepth),
 		peerGone:    make(chan key.Public),
-		sentTo:      make(map[key.Public]int64),
+		canMesh:     clientInfo.MeshKey != "" && clientInfo.MeshKey == s.meshKey,
+	}
+	if c.canMesh {
+		c.meshUpdate = make(chan struct{})
 	}
 	if clientInfo != nil {
 		c.info = *clientInfo
@@ -307,6 +426,12 @@ func (c *sclient) run(ctx context.Context) error {
 			err = c.handleFrameNotePreferred(ft, fl)
 		case frameSendPacket:
 			err = c.handleFrameSendPacket(ft, fl)
+		case frameForwardPacket:
+			err = c.handleFrameForwardPacket(ft, fl)
+		case frameWatchConns:
+			err = c.handleFrameWatchConns(ft, fl)
+		case frameClosePeer:
+			err = c.handleFrameClosePeer(ft, fl)
 		default:
 			err = c.handleUnknownFrame(ft, fl)
 		}
@@ -333,6 +458,92 @@ func (c *sclient) handleFrameNotePreferred(ft frameType, fl uint32) error {
 	return nil
 }
 
+func (c *sclient) handleFrameWatchConns(ft frameType, fl uint32) error {
+	if fl != 0 {
+		return fmt.Errorf("handleFrameWatchConns wrong size")
+	}
+	if !c.canMesh {
+		return fmt.Errorf("insufficient permissions")
+	}
+	c.s.addWatcher(c)
+	return nil
+}
+
+func (c *sclient) handleFrameClosePeer(ft frameType, fl uint32) error {
+	if fl != keyLen {
+		return fmt.Errorf("handleFrameClosePeer wrong size")
+	}
+	if !c.canMesh {
+		return fmt.Errorf("insufficient permissions")
+	}
+	var targetKey key.Public
+	if _, err := io.ReadFull(c.br, targetKey[:]); err != nil {
+		return err
+	}
+	s := c.s
+
+	s.mu.Lock()
+	defer s.mu.Unlock()
+
+	if target, ok := s.clients[targetKey]; ok {
+		c.logf("frameClosePeer closing peer %x", targetKey)
+		go target.nc.Close()
+	} else {
+		c.logf("frameClosePeer failed to find peer %x", targetKey)
+	}
+
+	return nil
+}
+
+// handleFrameForwardPacket reads a "forward packet" frame from the client
+// (which must be a trusted client, a peer in our mesh).
+func (c *sclient) handleFrameForwardPacket(ft frameType, fl uint32) error {
+	if !c.canMesh {
+		return fmt.Errorf("insufficient permissions")
+	}
+	s := c.s
+
+	srcKey, dstKey, contents, err := s.recvForwardPacket(c.br, fl)
+	if err != nil {
+		return fmt.Errorf("client %x: recvForwardPacket: %v", c.key, err)
+	}
+	s.packetsForwardedIn.Add(1)
+
+	s.mu.Lock()
+	dst := s.clients[dstKey]
+	if dst != nil {
+		s.notePeerSendLocked(srcKey, dst)
+	}
+	s.mu.Unlock()
+
+	if dst == nil {
+		s.packetsDropped.Add(1)
+		s.packetsDroppedFwdUnknown.Add(1)
+		if debug {
+			c.logf("dropping forwarded packet for unknown %x", dstKey)
+		}
+		return nil
+	}
+
+	return c.sendPkt(dst, pkt{
+		bs:  contents,
+		src: srcKey,
+	})
+}
+
+// notePeerSendLocked records that src sent to dst.  We keep track of
+// that so when src disconnects, we can tell dst (if it's still
+// around) that src is gone (a peerGone frame).
+func (s *Server) notePeerSendLocked(src key.Public, dst *sclient) {
+	m, ok := s.sentTo[src]
+	if !ok {
+		m = map[key.Public]int64{}
+		s.sentTo[src] = m
+	}
+	m[dst.key] = dst.connNum
+}
+
+// handleFrameSendPacket reads a "send packet" frame from the client.
 func (c *sclient) handleFrameSendPacket(ft frameType, fl uint32) error {
 	s := c.s
 
@@ -341,17 +552,25 @@ func (c *sclient) handleFrameSendPacket(ft frameType, fl uint32) error {
 		return fmt.Errorf("client %x: recvPacket: %v", c.key, err)
 	}
 
+	var fwd PacketForwarder
 	s.mu.Lock()
 	dst := s.clients[dstKey]
-	if dst != nil {
-		// Track that we've sent to this peer, so if/when we
-		// disconnect first, the server can inform all our old
-		// recipients that we're gone. (Issue 150 optimization)
-		c.sentTo[dstKey] = dst.connNum
+	if dst == nil {
+		fwd = s.clientsMesh[dstKey]
+	} else {
+		s.notePeerSendLocked(c.key, dst)
 	}
 	s.mu.Unlock()
 
 	if dst == nil {
+		if fwd != nil {
+			s.packetsForwardedOut.Add(1)
+			if err := fwd.ForwardPacket(c.key, dstKey, contents); err != nil {
+				// TODO:
+				return nil
+			}
+			return nil
+		}
 		s.packetsDropped.Add(1)
 		s.packetsDroppedUnknown.Add(1)
 		if debug {
@@ -366,6 +585,13 @@ func (c *sclient) handleFrameSendPacket(ft frameType, fl uint32) error {
 	if dst.info.Version >= protocolSrcAddrs {
 		p.src = c.key
 	}
+	return c.sendPkt(dst, p)
+}
+
+func (c *sclient) sendPkt(dst *sclient, p pkt) error {
+	s := c.s
+	dstKey := dst.key
+
 	// Attempt to queue for sending up to 3 times. On each attempt, if
 	// the queue is full, try to drop from queue head to prioritize
 	// fresher packets.
@@ -418,6 +644,16 @@ func (c *sclient) requestPeerGoneWrite(peer key.Public) {
 	}
 }
 
+func (c *sclient) requestMeshUpdate() {
+	if !c.canMesh {
+		panic("unexpected requestMeshUpdate")
+	}
+	select {
+	case c.meshUpdate <- struct{}{}:
+	case <-c.done:
+	}
+}
+
 func (s *Server) verifyClient(clientKey key.Public, info *clientInfo) error {
 	// TODO(crawshaw): implement policy constraints on who can use the DERP server
 	// TODO(bradfitz): ... and at what rate.
@@ -464,60 +700,86 @@ func (s *Server) sendServerInfo(bw *bufio.Writer, clientKey key.Public) error {
 func (s *Server) recvClientKey(br *bufio.Reader) (clientKey key.Public, info *clientInfo, err error) {
 	fl, err := readFrameTypeHeader(br, frameClientInfo)
 	if err != nil {
-		return key.Public{}, nil, err
+		return zpub, nil, err
 	}
 	const minLen = keyLen + nonceLen
 	if fl < minLen {
-		return key.Public{}, nil, errors.New("short client info")
+		return zpub, nil, errors.New("short client info")
 	}
 	// We don't trust the client at all yet, so limit its input size to limit
 	// things like JSON resource exhausting (http://github.com/golang/go/issues/31789).
 	if fl > 256<<10 {
-		return key.Public{}, nil, errors.New("long client info")
+		return zpub, nil, errors.New("long client info")
 	}
 	if _, err := io.ReadFull(br, clientKey[:]); err != nil {
-		return key.Public{}, nil, err
+		return zpub, nil, err
 	}
 	var nonce [24]byte
 	if _, err := io.ReadFull(br, nonce[:]); err != nil {
-		return key.Public{}, nil, fmt.Errorf("nonce: %v", err)
+		return zpub, nil, fmt.Errorf("nonce: %v", err)
 	}
 	msgLen := int(fl - minLen)
 	msgbox := make([]byte, msgLen)
 	if _, err := io.ReadFull(br, msgbox); err != nil {
-		return key.Public{}, nil, fmt.Errorf("msgbox: %v", err)
+		return zpub, nil, fmt.Errorf("msgbox: %v", err)
 	}
 	msg, ok := box.Open(nil, msgbox, &nonce, (*[32]byte)(&clientKey), s.privateKey.B32())
 	if !ok {
-		return key.Public{}, nil, fmt.Errorf("msgbox: cannot open len=%d with client key %x", msgLen, clientKey[:])
+		return zpub, nil, fmt.Errorf("msgbox: cannot open len=%d with client key %x", msgLen, clientKey[:])
 	}
 	info = new(clientInfo)
 	if err := json.Unmarshal(msg, info); err != nil {
-		return key.Public{}, nil, fmt.Errorf("msg: %v", err)
+		return zpub, nil, fmt.Errorf("msg: %v", err)
 	}
 	return clientKey, info, nil
 }
 
 func (s *Server) recvPacket(br *bufio.Reader, frameLen uint32) (dstKey key.Public, contents []byte, err error) {
 	if frameLen < keyLen {
-		return key.Public{}, nil, errors.New("short send packet frame")
+		return zpub, nil, errors.New("short send packet frame")
 	}
 	if _, err := io.ReadFull(br, dstKey[:]); err != nil {
-		return key.Public{}, nil, err
+		return zpub, nil, err
 	}
 	packetLen := frameLen - keyLen
 	if packetLen > MaxPacketSize {
-		return key.Public{}, nil, fmt.Errorf("data packet longer (%d) than max of %v", packetLen, MaxPacketSize)
+		return zpub, nil, fmt.Errorf("data packet longer (%d) than max of %v", packetLen, MaxPacketSize)
 	}
 	contents = make([]byte, packetLen)
 	if _, err := io.ReadFull(br, contents); err != nil {
-		return key.Public{}, nil, err
+		return zpub, nil, err
 	}
 	s.packetsRecv.Add(1)
 	s.bytesRecv.Add(int64(len(contents)))
 	return dstKey, contents, nil
 }
 
+// zpub is the key.Public zero value.
+var zpub key.Public
+
+func (s *Server) recvForwardPacket(br *bufio.Reader, frameLen uint32) (srcKey, dstKey key.Public, contents []byte, err error) {
+	if frameLen < keyLen*2 {
+		return zpub, zpub, nil, errors.New("short send packet frame")
+	}
+	if _, err := io.ReadFull(br, srcKey[:]); err != nil {
+		return zpub, zpub, nil, err
+	}
+	if _, err := io.ReadFull(br, dstKey[:]); err != nil {
+		return zpub, zpub, nil, err
+	}
+	packetLen := frameLen - keyLen*2
+	if packetLen > MaxPacketSize {
+		return zpub, zpub, nil, fmt.Errorf("data packet longer (%d) than max of %v", packetLen, MaxPacketSize)
+	}
+	contents = make([]byte, packetLen)
+	if _, err := io.ReadFull(br, contents); err != nil {
+		return zpub, zpub, nil, err
+	}
+	// TODO: was s.packetsRecv.Add(1)
+	// TODO: was s.bytesRecv.Add(int64(len(contents)))
+	return srcKey, dstKey, contents, nil
+}
+
 // sclient is a client connection to the server.
 //
 // (The "s" prefix is to more explicitly distinguish it from Client in derp_client.go)
@@ -532,7 +794,9 @@ type sclient struct {
 	done       <-chan struct{} // closed when connection closes
 	remoteAddr string          // usually ip:port from net.Conn.RemoteAddr().String()
 	sendQueue  chan pkt        // packets queued to this client; never closed
-	peerGone   chan key.Public // write request that a previous sender has disconnected
+	peerGone   chan key.Public // write request that a previous sender has disconnected (not used by mesh peers)
+	meshUpdate chan struct{}   // write request to write peerStateChange
+	canMesh    bool            // clientInfo had correct mesh token for inter-region routing
 
 	// Owned by run, not thread-safe.
 	br          *bufio.Reader
@@ -542,11 +806,20 @@ type sclient struct {
 	// Owned by sender, not thread-safe.
 	bw *bufio.Writer
 
-	// Guarded by s.mu.
+	// Guarded by s.mu
 	//
-	// sentTo tracks all the peers this client has ever sent a packet to, and at which
-	// connection number.
-	sentTo map[key.Public]int64 // recipient => rcpt's latest sclient.connNum
+	// peerStateChange is used by mesh peers (a set of regional
+	// DERP servers) and contains records that need to be sent to
+	// the client for them to update their map of who's connected
+	// to this node.
+	peerStateChange []peerConnState
+}
+
+// peerConnState represents whether a peer is connected to the server
+// or not.
+type peerConnState struct {
+	peer    key.Public
+	present bool
 }
 
 // pkt is a request to write a data frame to an sclient.
@@ -628,6 +901,9 @@ func (c *sclient) sendLoop(ctx context.Context) error {
 		case peer := <-c.peerGone:
 			werr = c.sendPeerGone(peer)
 			continue
+		case <-c.meshUpdate:
+			werr = c.sendMeshUpdates()
+			continue
 		case msg := <-c.sendQueue:
 			werr = c.sendPacket(msg.src, msg.bs)
 			continue
@@ -648,6 +924,9 @@ func (c *sclient) sendLoop(ctx context.Context) error {
 			return nil
 		case peer := <-c.peerGone:
 			werr = c.sendPeerGone(peer)
+		case <-c.meshUpdate:
+			werr = c.sendMeshUpdates()
+			continue
 		case msg := <-c.sendQueue:
 			werr = c.sendPacket(msg.src, msg.bs)
 		case <-keepAliveTick.C:
@@ -677,6 +956,59 @@ func (c *sclient) sendPeerGone(peer key.Public) error {
 	return err
 }
 
+// sendPeerPresent sends a peerPresent frame, without flushing.
+func (c *sclient) sendPeerPresent(peer key.Public) error {
+	c.setWriteDeadline()
+	if err := writeFrameHeader(c.bw, framePeerPresent, keyLen); err != nil {
+		return err
+	}
+	_, err := c.bw.Write(peer[:])
+	return err
+}
+
+// sendMeshUpdates drains as many mesh peerStateChange entries as
+// possible into the write buffer WITHOUT flushing or otherwise
+// blocking (as it holds c.s.mu while working). If it can't drain them
+// all, it schedules itself to be called again in the future.
+func (c *sclient) sendMeshUpdates() error {
+	c.s.mu.Lock()
+	defer c.s.mu.Unlock()
+
+	writes := 0
+	for _, pcs := range c.peerStateChange {
+		if c.bw.Available() <= frameHeaderLen+keyLen {
+			break
+		}
+		var err error
+		if pcs.present {
+			err = c.sendPeerPresent(pcs.peer)
+		} else {
+			err = c.sendPeerGone(pcs.peer)
+		}
+		if err != nil {
+			// Shouldn't happen, though, as we're writing
+			// into available buffer space, not the
+			// network.
+			return err
+		}
+		writes++
+	}
+
+	remain := copy(c.peerStateChange, c.peerStateChange[writes:])
+	c.peerStateChange = c.peerStateChange[:remain]
+
+	// Did we manage to write them all into the bufio buffer without flushing?
+	if len(c.peerStateChange) == 0 {
+		if cap(c.peerStateChange) > 16 {
+			c.peerStateChange = nil
+		}
+	} else {
+		// Didn't finish in the buffer space provided; schedule a future run.
+		go c.requestMeshUpdate()
+	}
+	return nil
+}
+
 // sendPacket writes contents to the client in a RecvPacket frame. If
 // srcKey.IsZero, uses the old DERPv1 framing format, otherwise uses
 // DERPv2. The bytes of contents are only valid until this function
@@ -716,6 +1048,114 @@ func (c *sclient) sendPacket(srcKey key.Public, contents []byte) (err error) {
 	return err
 }
 
+// AddPacketForwarder registers fwd as a packet forwarder for dst.
+// fwd must be comparable.
+func (s *Server) AddPacketForwarder(dst key.Public, fwd PacketForwarder) {
+	s.mu.Lock()
+	defer s.mu.Unlock()
+	if prev, ok := s.clientsMesh[dst]; ok {
+		if prev == fwd {
+			// Duplicate registration of same forwarder. Ignore.
+			return
+		}
+		if m, ok := prev.(multiForwarder); ok {
+			if _, ok := m[fwd]; !ok {
+				// Duplicate registration of same forwarder in set; ignore.
+				return
+			}
+			m[fwd] = m.maxVal() + 1
+			return
+		}
+		if prev != nil {
+			// Otherwise, the existing value is not a set,
+			// not a dup, and not local-only (nil) so make
+			// it a set.
+			fwd = multiForwarder{
+				prev: 1, // existed 1st, higher priority
+				fwd:  2, // the passed in fwd is in 2nd place
+			}
+			s.multiForwarderCreated.Add(1)
+		}
+	}
+	s.clientsMesh[dst] = fwd
+}
+
+// RemovePacketForwarder removes fwd as a packet forwarder for dst.
+// fwd must be comparable.
+func (s *Server) RemovePacketForwarder(dst key.Public, fwd PacketForwarder) {
+	s.mu.Lock()
+	defer s.mu.Unlock()
+	v, ok := s.clientsMesh[dst]
+	if !ok {
+		return
+	}
+	if m, ok := v.(multiForwarder); ok {
+		if len(m) < 2 {
+			panic("unexpected")
+		}
+		delete(m, fwd)
+		// If fwd was in m and we no longer need to be a
+		// multiForwarder, replace the entry with the
+		// remaining PacketForwarder.
+		if len(m) == 1 {
+			var remain PacketForwarder
+			for k := range m {
+				remain = k
+			}
+			s.clientsMesh[dst] = remain
+			s.multiForwarderDeleted.Add(1)
+		}
+		return
+	}
+	if v != fwd {
+		s.removePktForwardOther.Add(1)
+		// Delete of an entry that wasn't in the
+		// map. Harmless, so ignore.
+		// (This might happen if a user is moving around
+		// between nodes and/or the server sent duplicate
+		// connection change broadcasts.)
+		return
+	}
+
+	if _, isLocal := s.clients[dst]; isLocal {
+		s.clientsMesh[dst] = nil
+	} else {
+		delete(s.clientsMesh, dst)
+		s.notePeerGoneFromRegionLocked(dst)
+	}
+}
+
+// multiForwarder is a PacketForwarder that represents a set of
+// forwarding options. It's used in the rare cases that a client is
+// connected to multiple DERP nodes in a region. That shouldn't really
+// happen except for perhaps during brief moments while the client is
+// reconfiguring, in which case we don't want to forget where the
+// client is. The map value is unique connection number; the lowest
+// one has been seen the longest. It's used to make sure we forward
+// packets consistently to the same node and don't pick randomly.
+type multiForwarder map[PacketForwarder]uint8
+
+func (m multiForwarder) maxVal() (max uint8) {
+	for _, v := range m {
+		if v > max {
+			max = v
+		}
+	}
+	return
+}
+
+func (m multiForwarder) ForwardPacket(src, dst key.Public, payload []byte) error {
+	var fwd PacketForwarder
+	var lowest uint8
+	for k, v := range m {
+		if fwd == nil || v < lowest {
+			fwd = k
+			lowest = v
+		}
+	}
+	return fwd.ForwardPacket(src, dst, payload)
+}
+
 func (s *Server) expVarFunc(f func() interface{}) expvar.Func {
 	return expvar.Func(func() interface{} {
 		s.mu.Lock()
@@ -729,8 +1169,12 @@ func (s *Server) ExpVar() expvar.Var {
 	m := new(metrics.Set)
 	m.Set("counter_unique_clients_ever", s.expVarFunc(func() interface{} { return len(s.clientsEver) }))
 	m.Set("gauge_memstats_sys0", expvar.Func(func() interface{} { return int64(s.memSys0) }))
-	m.Set("gauge_current_connnections", &s.curClients)
-	m.Set("gauge_current_home_connnections", &s.curHomeClients)
+	m.Set("gauge_watchers", s.expVarFunc(func() interface{} { return len(s.watchers) }))
+	m.Set("gauge_current_connections", &s.curClients)
+	m.Set("gauge_current_home_connections", &s.curHomeClients)
+	m.Set("gauge_clients_total", expvar.Func(func() interface{} { return len(s.clientsMesh) }))
+	m.Set("gauge_clients_local", expvar.Func(func() interface{} { return len(s.clients) }))
+	m.Set("gauge_clients_remote", expvar.Func(func() interface{} { return len(s.clientsMesh) - len(s.clients) }))
 	m.Set("accepts", &s.accepts)
 	m.Set("clients_replaced", &s.clientsReplaced)
 	m.Set("bytes_received", &s.bytesRecv)
@@ -743,5 +1187,52 @@ func (s *Server) ExpVar() expvar.Var {
 	m.Set("home_moves_in", &s.homeMovesIn)
 	m.Set("home_moves_out", &s.homeMovesOut)
 	m.Set("peer_gone_frames", &s.peerGoneFrames)
+	m.Set("packets_forwarded_out", &s.packetsForwardedOut)
+	m.Set("packets_forwarded_in", &s.packetsForwardedIn)
+	m.Set("multiforwarder_created", &s.multiForwarderCreated)
+	m.Set("multiforwarder_deleted", &s.multiForwarderDeleted)
+	m.Set("packet_forwarder_delete_other_value", &s.removePktForwardOther)
+	var expvarVersion expvar.String
+	expvarVersion.Set(version.LONG)
+	m.Set("version", &expvarVersion)
 	return m
 }
+
+func (s *Server) ConsistencyCheck() error {
+	s.mu.Lock()
+	defer s.mu.Unlock()
+
+	var errs []string
+
+	var nilMeshNotInClient int
+	for k, f := range s.clientsMesh {
+		if f == nil {
+			if _, ok := s.clients[k]; !ok {
+				nilMeshNotInClient++
+			}
+		}
+	}
+	if nilMeshNotInClient != 0 {
+		errs = append(errs, fmt.Sprintf("%d s.clientsMesh keys not in s.clients", nilMeshNotInClient))
+	}
+
+	var clientNotInMesh int
+	for k := range s.clients {
+		if _, ok := s.clientsMesh[k]; !ok {
+			clientNotInMesh++
+		}
+	}
+	if clientNotInMesh != 0 {
+		errs = append(errs, fmt.Sprintf("%d s.clients keys not in s.clientsMesh", clientNotInMesh))
+	}
+
+	if s.curClients.Value() != int64(len(s.clients)) {
+		errs = append(errs, fmt.Sprintf("expvar connections = %d != clients map says of %d",
+			s.curClients.Value(),
+			len(s.clients)))
+	}
+	if len(errs) == 0 {
+		return nil
+	}
+	return errors.New(strings.Join(errs, ", "))
+}

derp/derp_test.go

@@ -13,17 +13,20 @@ import (
 	"fmt"
 	"io"
 	"net"
+	"reflect"
+	"sync"
 	"testing"
 	"time"
 
 	"tailscale.com/net/nettest"
 	"tailscale.com/types/key"
+	"tailscale.com/types/logger"
 )
 
-func newPrivateKey(t *testing.T) (k key.Private) {
-	t.Helper()
+func newPrivateKey(tb testing.TB) (k key.Private) {
+	tb.Helper()
 	if _, err := crand.Read(k[:]); err != nil {
-		t.Fatal(err)
+		tb.Fatal(err)
 	}
 	return
 }
@@ -87,8 +90,7 @@ func TestSendRecv(t *testing.T) {
 	for i := 0; i < numClients; i++ {
 		go func(i int) {
 			for {
-				b := make([]byte, 1<<16)
-				m, err := clients[i].Recv(b)
+				m, err := clients[i].Recv()
 				if err != nil {
 					errCh <- err
 					return
@@ -103,7 +105,7 @@ func TestSendRecv(t *testing.T) {
 					if m.Source.IsZero() {
 						t.Errorf("zero Source address in ReceivedPacket")
 					}
-					recvChs[i] <- m.Data
+					recvChs[i] <- append([]byte(nil), m.Data...)
 				}
 			}
 		}(i)
@@ -256,8 +258,7 @@ func TestSendFreeze(t *testing.T) {
 	recv := func(name string, client *Client) {
 		ch := chs(name)
 		for {
-			b := make([]byte, 1<<9)
-			m, err := client.Recv(b)
+			m, err := client.Recv()
 			if err != nil {
 				errCh <- fmt.Errorf("%s: %w", name, err)
 				return
@@ -391,3 +392,414 @@ func TestSendFreeze(t *testing.T) {
 		}
 	}
 }
+
+type testServer struct {
+	s    *Server
+	ln   net.Listener
+	logf logger.Logf
+
+	mu      sync.Mutex
+	pubName map[key.Public]string
+	clients map[*testClient]bool
+}
+
+func (ts *testServer) addTestClient(c *testClient) {
+	ts.mu.Lock()
+	defer ts.mu.Unlock()
+	ts.clients[c] = true
+}
+
+func (ts *testServer) addKeyName(k key.Public, name string) {
+	ts.mu.Lock()
+	defer ts.mu.Unlock()
+	ts.pubName[k] = name
+	ts.logf("test adding named key %q for %x", name, k)
+}
+
+func (ts *testServer) keyName(k key.Public) string {
+	ts.mu.Lock()
+	defer ts.mu.Unlock()
+	if name, ok := ts.pubName[k]; ok {
+		return name
+	}
+	return k.ShortString()
+}
+
+func (ts *testServer) close(t *testing.T) error {
+	ts.ln.Close()
+	ts.s.Close()
+	for c := range ts.clients {
+		c.close(t)
+	}
+	return nil
+}
+
+func newTestServer(t *testing.T) *testServer {
+	t.Helper()
+	logf := logger.WithPrefix(t.Logf, "derp-server: ")
+	s := NewServer(newPrivateKey(t), logf)
+	s.SetMeshKey("mesh-key")
+	ln, err := net.Listen("tcp", "127.0.0.1:0")
+	if err != nil {
+		t.Fatal(err)
+	}
+	go func() {
+		i := 0
+		for {
+			i++
+			c, err := ln.Accept()
+			if err != nil {
+				return
+			}
+			// TODO: register c in ts so Close also closes it?
+			go func(i int) {
+				brwServer := bufio.NewReadWriter(bufio.NewReader(c), bufio.NewWriter(c))
+				go s.Accept(c, brwServer, fmt.Sprintf("test-client-%d", i))
+			}(i)
+		}
+	}()
+	return &testServer{
+		s:       s,
+		ln:      ln,
+		logf:    logf,
+		clients: map[*testClient]bool{},
+		pubName: map[key.Public]string{},
+	}
+}
+
+type testClient struct {
+	name   string
+	c      *Client
+	nc     net.Conn
+	pub    key.Public
+	ts     *testServer
+	closed bool
+}
+
+func newTestClient(t *testing.T, ts *testServer, name string, newClient func(net.Conn, key.Private, logger.Logf) (*Client, error)) *testClient {
+	t.Helper()
+	nc, err := net.Dial("tcp", ts.ln.Addr().String())
+	if err != nil {
+		t.Fatal(err)
+	}
+	key := newPrivateKey(t)
+	ts.addKeyName(key.Public(), name)
+	c, err := newClient(nc, key, logger.WithPrefix(t.Logf, "client-"+name+": "))
+	if err != nil {
+		t.Fatal(err)
+	}
+	tc := &testClient{
+		name: name,
+		nc:   nc,
+		c:    c,
+		ts:   ts,
+		pub:  key.Public(),
+	}
+	ts.addTestClient(tc)
+	return tc
+}
+
+func newRegularClient(t *testing.T, ts *testServer, name string) *testClient {
+	return newTestClient(t, ts, name, func(nc net.Conn, priv key.Private, logf logger.Logf) (*Client, error) {
+		brw := bufio.NewReadWriter(bufio.NewReader(nc), bufio.NewWriter(nc))
+		return NewClient(priv, nc, brw, logf)
+	})
+}
+
+func newTestWatcher(t *testing.T, ts *testServer, name string) *testClient {
+	return newTestClient(t, ts, name, func(nc net.Conn, priv key.Private, logf logger.Logf) (*Client, error) {
+		brw := bufio.NewReadWriter(bufio.NewReader(nc), bufio.NewWriter(nc))
+		c, err := NewClient(priv, nc, brw, logf, MeshKey("mesh-key"))
+		if err != nil {
+			return nil, err
+		}
+		if err := c.WatchConnectionChanges(); err != nil {
+			return nil, err
+		}
+		return c, nil
+	})
+}
+
+func (tc *testClient) wantPresent(t *testing.T, peers ...key.Public) {
+	t.Helper()
+	want := map[key.Public]bool{}
+	for _, k := range peers {
+		want[k] = true
+	}
+
+	for {
+		m, err := tc.c.recvTimeout(time.Second)
+		if err != nil {
+			t.Fatal(err)
+		}
+		switch m := m.(type) {
+		case PeerPresentMessage:
+			got := key.Public(m)
+			if !want[got] {
+				t.Fatalf("got peer present for %v; want present for %v", tc.ts.keyName(got), logger.ArgWriter(func(bw *bufio.Writer) {
+					for _, pub := range peers {
+						fmt.Fprintf(bw, "%s ", tc.ts.keyName(pub))
+					}
+				}))
+			}
+			delete(want, got)
+			if len(want) == 0 {
+				return
+			}
+		default:
+			t.Fatalf("unexpected message type %T", m)
+		}
+	}
+}
+
+func (tc *testClient) wantGone(t *testing.T, peer key.Public) {
+	t.Helper()
+	m, err := tc.c.recvTimeout(time.Second)
+	if err != nil {
+		t.Fatal(err)
+	}
+	switch m := m.(type) {
+	case PeerGoneMessage:
+		got := key.Public(m)
+		if peer != got {
+			t.Errorf("got gone message for %v; want gone for %v", tc.ts.keyName(got), tc.ts.keyName(peer))
+		}
+	default:
+		t.Fatalf("unexpected message type %T", m)
+	}
+}
+
+func (c *testClient) close(t *testing.T) {
+	t.Helper()
+	if c.closed {
+		return
+	}
+	c.closed = true
+	t.Logf("closing client %q (%x)", c.name, c.pub)
+	c.nc.Close()
+}
+
+// TestWatch tests the connection watcher mechanism used by regional
+// DERP nodes to mesh up with each other.
+func TestWatch(t *testing.T) {
+	ts := newTestServer(t)
+	defer ts.close(t)
+
+	w1 := newTestWatcher(t, ts, "w1")
+	w1.wantPresent(t, w1.pub)
+
+	c1 := newRegularClient(t, ts, "c1")
+	w1.wantPresent(t, c1.pub)
+
+	c2 := newRegularClient(t, ts, "c2")
+	w1.wantPresent(t, c2.pub)
+
+	w2 := newTestWatcher(t, ts, "w2")
+	w1.wantPresent(t, w2.pub)
+	w2.wantPresent(t, w1.pub, w2.pub, c1.pub, c2.pub)
+
+	c3 := newRegularClient(t, ts, "c3")
+	w1.wantPresent(t, c3.pub)
+	w2.wantPresent(t, c3.pub)
+
+	c2.close(t)
+	w1.wantGone(t, c2.pub)
+	w2.wantGone(t, c2.pub)
+
+	w3 := newTestWatcher(t, ts, "w3")
+	w1.wantPresent(t, w3.pub)
+	w2.wantPresent(t, w3.pub)
+	w3.wantPresent(t, c1.pub, c3.pub, w1.pub, w2.pub, w3.pub)
+
+	c1.close(t)
+	w1.wantGone(t, c1.pub)
+	w2.wantGone(t, c1.pub)
+	w3.wantGone(t, c1.pub)
+}
+
+type testFwd int
+
+func (testFwd) ForwardPacket(key.Public, key.Public, []byte) error { panic("not called in tests") }
+
+func pubAll(b byte) (ret key.Public) {
+	for i := range ret {
+		ret[i] = b
+	}
+	return
+}
+
+func TestForwarderRegistration(t *testing.T) {
+	s := &Server{
+		clients:     make(map[key.Public]*sclient),
+		clientsMesh: map[key.Public]PacketForwarder{},
+	}
+	want := func(want map[key.Public]PacketForwarder) {
+		t.Helper()
+		if got := s.clientsMesh; !reflect.DeepEqual(got, want) {
+			t.Fatalf("mismatch\n got: %v\nwant: %v\n", got, want)
+		}
+	}
+	wantCounter := func(c *expvar.Int, want int) {
+		t.Helper()
+		if got := c.Value(); got != int64(want) {
+			t.Errorf("counter = %v; want %v", got, want)
+		}
+	}
+
+	u1 := pubAll(1)
+	u2 := pubAll(2)
+	u3 := pubAll(3)
+
+	s.AddPacketForwarder(u1, testFwd(1))
+	s.AddPacketForwarder(u2, testFwd(2))
+	want(map[key.Public]PacketForwarder{
+		u1: testFwd(1),
+		u2: testFwd(2),
+	})
+
+	// Verify a remove of non-registered forwarder is no-op.
+	s.RemovePacketForwarder(u2, testFwd(999))
+	want(map[key.Public]PacketForwarder{
+		u1: testFwd(1),
+		u2: testFwd(2),
+	})
+
+	// Verify a remove of non-registered user is no-op.
+	s.RemovePacketForwarder(u3, testFwd(1))
+	want(map[key.Public]PacketForwarder{
+		u1: testFwd(1),
+		u2: testFwd(2),
+	})
+
+	// Actual removal.
+	s.RemovePacketForwarder(u2, testFwd(2))
+	want(map[key.Public]PacketForwarder{
+		u1: testFwd(1),
+	})
+
+	// Adding a dup for a user.
+	wantCounter(&s.multiForwarderCreated, 0)
+	s.AddPacketForwarder(u1, testFwd(100))
+	want(map[key.Public]PacketForwarder{
+		u1: multiForwarder{
+			testFwd(1):   1,
+			testFwd(100): 2,
+		},
+	})
+	wantCounter(&s.multiForwarderCreated, 1)
+
+	// Removing a forwarder in a multi set that doesn't exist; does nothing.
+	s.RemovePacketForwarder(u1, testFwd(55))
+	want(map[key.Public]PacketForwarder{
+		u1: multiForwarder{
+			testFwd(1):   1,
+			testFwd(100): 2,
+		},
+	})
+
+	// Removing a forwarder in a multi set that does exist should collapse it away
+	// from being a multiForwarder.
+	wantCounter(&s.multiForwarderDeleted, 0)
+	s.RemovePacketForwarder(u1, testFwd(1))
+	want(map[key.Public]PacketForwarder{
+		u1: testFwd(100),
+	})
+	wantCounter(&s.multiForwarderDeleted, 1)
+
+	// Removing an entry for a client that's still connected locally should result
+	// in a nil forwarder.
+	u1c := &sclient{
+		key:  u1,
+		logf: logger.Discard,
+	}
+	s.clients[u1] = u1c
+	s.RemovePacketForwarder(u1, testFwd(100))
+	want(map[key.Public]PacketForwarder{
+		u1: nil,
+	})
+
+	// But once that client disconnects, it should go away.
+	s.unregisterClient(u1c)
+	want(map[key.Public]PacketForwarder{})
+
+	// But if it already has a forwarder, it's not removed.
+	s.AddPacketForwarder(u1, testFwd(2))
+	s.unregisterClient(u1c)
+	want(map[key.Public]PacketForwarder{
+		u1: testFwd(2),
+	})
+
+	// Now pretend u1 was already connected locally (so clientsMesh[u1] is nil), and then we heard
+	// that they're also connected to a peer of ours. That sholdn't transition the forwarder
+	// from nil to the new one, not a multiForwarder.
+	s.clients[u1] = u1c
+	s.clientsMesh[u1] = nil
+	want(map[key.Public]PacketForwarder{
+		u1: nil,
+	})
+	s.AddPacketForwarder(u1, testFwd(3))
+	want(map[key.Public]PacketForwarder{
+		u1: testFwd(3),
+	})
+}
+
+func BenchmarkSendRecv(b *testing.B) {
+	for _, size := range []int{10, 100, 1000, 10000} {
+		b.Run(fmt.Sprintf("msgsize=%d", size), func(b *testing.B) { benchmarkSendRecvSize(b, size) })
+	}
+}
+
+func benchmarkSendRecvSize(b *testing.B, packetSize int) {
+	serverPrivateKey := newPrivateKey(b)
+	s := NewServer(serverPrivateKey, logger.Discard)
+	defer s.Close()
+
+	key := newPrivateKey(b)
+	clientKey := key.Public()
+
+	ln, err := net.Listen("tcp", "127.0.0.1:0")
+	if err != nil {
+		b.Fatal(err)
+	}
+	defer ln.Close()
+
+	connOut, err := net.Dial("tcp", ln.Addr().String())
+	if err != nil {
+		b.Fatal(err)
+	}
+	defer connOut.Close()
+
+	connIn, err := ln.Accept()
+	if err != nil {
+		b.Fatal(err)
+	}
+	defer connIn.Close()
+
+	brwServer := bufio.NewReadWriter(bufio.NewReader(connIn), bufio.NewWriter(connIn))
+	go s.Accept(connIn, brwServer, "test-client")
+
+	brw := bufio.NewReadWriter(bufio.NewReader(connOut), bufio.NewWriter(connOut))
+	client, err := NewClient(key, connOut, brw, logger.Discard)
+	if err != nil {
+		b.Fatalf("client: %v", err)
+	}
+
+	go func() {
+		for {
+			_, err := client.Recv()
+			if err != nil {
+				return
+			}
+		}
+	}()
+
+	msg := make([]byte, packetSize)
+	b.SetBytes(int64(len(msg)))
+	b.ReportAllocs()
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		if err := client.Send(clientKey, msg); err != nil {
+			b.Fatal(err)
+		}
+	}
+}

derp/derphttp/derphttp_client.go

@@ -24,9 +24,12 @@ import (
 	"sync"
 	"time"
 
+	"inet.af/netaddr"
 	"tailscale.com/derp"
 	"tailscale.com/net/dnscache"
+	"tailscale.com/net/netns"
 	"tailscale.com/net/tlsdial"
+	"tailscale.com/tailcfg"
 	"tailscale.com/types/key"
 	"tailscale.com/types/logger"
 )
@@ -40,23 +43,49 @@ import (
 type Client struct {
 	TLSConfig *tls.Config        // optional; nil means default
 	DNSCache  *dnscache.Resolver // optional; nil means no caching
+	MeshKey   string             // optional; for trusted clients
 
 	privateKey key.Private
 	logf       logger.Logf
-	url        *url.URL
+
+	// Either url or getRegion is non-nil:
+	url       *url.URL
+	getRegion func() *tailcfg.DERPRegion
 
 	ctx       context.Context // closed via cancelCtx in Client.Close
 	cancelCtx context.CancelFunc
 
-	mu        sync.Mutex
-	preferred bool
-	closed    bool
-	netConn   io.Closer
-	client    *derp.Client
+	mu           sync.Mutex
+	preferred    bool
+	closed       bool
+	netConn      io.Closer
+	client       *derp.Client
+	connGen      int // incremented once per new connection; valid values are >0
+	serverPubKey key.Public
+}
+
+// NewRegionClient returns a new DERP-over-HTTP client. It connects lazily.
+// To trigger a connection, use Connect.
+func NewRegionClient(privateKey key.Private, logf logger.Logf, getRegion func() *tailcfg.DERPRegion) *Client {
+	ctx, cancel := context.WithCancel(context.Background())
+	c := &Client{
+		privateKey: privateKey,
+		logf:       logf,
+		getRegion:  getRegion,
+		ctx:        ctx,
+		cancelCtx:  cancel,
+	}
+	return c
+}
+
+// NewNetcheckClient returns a Client that's only able to have its DialRegion method called.
+// It's used by the netcheck package.
+func NewNetcheckClient(logf logger.Logf) *Client {
+	return &Client{logf: logf}
 }
 
 // NewClient returns a new DERP-over-HTTP client. It connects lazily.
-// To trigger a connection use Connect.
+// To trigger a connection, use Connect.
 func NewClient(privateKey key.Private, serverURL string, logf logger.Logf) (*Client, error) {
 	u, err := url.Parse(serverURL)
 	if err != nil {
@@ -65,6 +94,7 @@ func NewClient(privateKey key.Private, serverURL string, logf logger.Logf) (*Cli
 	if urlPort(u) == "" {
 		return nil, fmt.Errorf("derphttp.NewClient: invalid URL scheme %q", u.Scheme)
 	}
+
 	ctx, cancel := context.WithCancel(context.Background())
 	c := &Client{
 		privateKey: privateKey,
@@ -79,10 +109,20 @@ func NewClient(privateKey key.Private, serverURL string, logf logger.Logf) (*Cli
 // Connect connects or reconnects to the server, unless already connected.
 // It returns nil if there was already a good connection, or if one was made.
 func (c *Client) Connect(ctx context.Context) error {
-	_, err := c.connect(ctx, "derphttp.Client.Connect")
+	_, _, err := c.connect(ctx, "derphttp.Client.Connect")
 	return err
 }
 
+// ServerPublicKey returns the server's public key.
+//
+// It only returns a non-zero value once a connection has succeeded
+// from an earlier call.
+func (c *Client) ServerPublicKey() key.Public {
+	c.mu.Lock()
+	defer c.mu.Unlock()
+	return c.serverPubKey
+}
+
 func urlPort(u *url.URL) string {
 	if p := u.Port(); p != "" {
 		return p
@@ -96,18 +136,45 @@ func urlPort(u *url.URL) string {
 	return ""
 }
 
-func (c *Client) connect(ctx context.Context, caller string) (client *derp.Client, err error) {
+func (c *Client) targetString(reg *tailcfg.DERPRegion) string {
+	if c.url != nil {
+		return c.url.String()
+	}
+	return fmt.Sprintf("region %d (%v)", reg.RegionID, reg.RegionCode)
+}
+
+func (c *Client) useHTTPS() bool {
+	if c.url != nil && c.url.Scheme == "http" {
+		return false
+	}
+	return true
+}
+
+// tlsServerName returns the tls.Config.ServerName value (for the TLS ClientHello).
+func (c *Client) tlsServerName(node *tailcfg.DERPNode) string {
+	if c.url != nil {
+		return c.url.Host
+	}
+	return node.HostName
+}
+
+func (c *Client) urlString(node *tailcfg.DERPNode) string {
+	if c.url != nil {
+		return c.url.String()
+	}
+	return fmt.Sprintf("https://%s/derp", node.HostName)
+}
+
+func (c *Client) connect(ctx context.Context, caller string) (client *derp.Client, connGen int, err error) {
 	c.mu.Lock()
 	defer c.mu.Unlock()
 	if c.closed {
-		return nil, ErrClientClosed
+		return nil, 0, ErrClientClosed
 	}
 	if c.client != nil {
-		return c.client, nil
+		return c.client, c.connGen, nil
 	}
 
-	c.logf("%s: connecting to %v", caller, c.url)
-
 	// timeout is the fallback maximum time (if ctx doesn't limit
 	// it further) to do all of: DNS + TCP + TLS + HTTP Upgrade +
 	// DERP upgrade.
@@ -127,38 +194,42 @@ func (c *Client) connect(ctx context.Context, caller string) (client *derp.Clien
 	}()
 	defer cancel()
 
+	var reg *tailcfg.DERPRegion // nil when using c.url to dial
+	if c.getRegion != nil {
+		reg = c.getRegion()
+		if reg == nil {
+			return nil, 0, errors.New("DERP region not available")
+		}
+	}
+
 	var tcpConn net.Conn
+
 	defer func() {
 		if err != nil {
 			if ctx.Err() != nil {
 				err = fmt.Errorf("%v: %v", ctx.Err(), err)
 			}
-			err = fmt.Errorf("%s connect to %v: %v", caller, c.url, err)
+			err = fmt.Errorf("%s connect to %v: %v", caller, c.targetString(reg), err)
 			if tcpConn != nil {
 				go tcpConn.Close()
 			}
 		}
 	}()
 
-	host := c.url.Hostname()
-	hostOrIP := host
-
-	var d net.Dialer
-
-	if c.DNSCache != nil {
-		ip, err := c.DNSCache.LookupIP(ctx, host)
-		if err != nil {
-			return nil, err
-		}
-		hostOrIP = ip.String()
+	var node *tailcfg.DERPNode // nil when using c.url to dial
+	if c.url != nil {
+		c.logf("%s: connecting to %v", caller, c.url)
+		tcpConn, err = c.dialURL(ctx)
+	} else {
+		c.logf("%s: connecting to derp-%d (%v)", caller, reg.RegionID, reg.RegionCode)
+		tcpConn, node, err = c.dialRegion(ctx, reg)
 	}
-
-	tcpConn, err = d.DialContext(ctx, "tcp", net.JoinHostPort(hostOrIP, urlPort(c.url)))
 	if err != nil {
-		return nil, fmt.Errorf("dial of %q: %v", host, err)
+		return nil, 0, err
 	}
 
-	// Now that we have a TCP connection, force close it.
+	// Now that we have a TCP connection, force close it if the
+	// TLS handshake + DERP setup takes too long.
 	done := make(chan struct{})
 	defer close(done)
 	go func() {
@@ -182,56 +253,235 @@ func (c *Client) connect(ctx context.Context, caller string) (client *derp.Clien
 	}()
 
 	var httpConn net.Conn // a TCP conn or a TLS conn; what we speak HTTP to
-	if c.url.Scheme == "https" {
-		httpConn = tls.Client(tcpConn, tlsdial.Config(c.url.Host, c.TLSConfig))
+	if c.useHTTPS() {
+		httpConn = c.tlsClient(tcpConn, node)
 	} else {
 		httpConn = tcpConn
 	}
 
 	brw := bufio.NewReadWriter(bufio.NewReader(httpConn), bufio.NewWriter(httpConn))
 
-	req, err := http.NewRequest("GET", c.url.String(), nil)
+	req, err := http.NewRequest("GET", c.urlString(node), nil)
 	if err != nil {
-		return nil, err
+		return nil, 0, err
 	}
 	req.Header.Set("Upgrade", "DERP")
 	req.Header.Set("Connection", "Upgrade")
 
 	if err := req.Write(brw); err != nil {
-		return nil, err
+		return nil, 0, err
 	}
 	if err := brw.Flush(); err != nil {
-		return nil, err
+		return nil, 0, err
 	}
 
 	resp, err := http.ReadResponse(brw.Reader, req)
 	if err != nil {
-		return nil, err
+		return nil, 0, err
 	}
 	if resp.StatusCode != http.StatusSwitchingProtocols {
 		b, _ := ioutil.ReadAll(resp.Body)
 		resp.Body.Close()
-		return nil, fmt.Errorf("GET failed: %v: %s", err, b)
+		return nil, 0, fmt.Errorf("GET failed: %v: %s", err, b)
 	}
 
-	derpClient, err := derp.NewClient(c.privateKey, httpConn, brw, c.logf)
+	derpClient, err := derp.NewClient(c.privateKey, httpConn, brw, c.logf, derp.MeshKey(c.MeshKey))
 	if err != nil {
-		return nil, err
+		return nil, 0, err
 	}
 	if c.preferred {
 		if err := derpClient.NotePreferred(true); err != nil {
 			go httpConn.Close()
+			return nil, 0, err
+		}
+	}
+
+	c.serverPubKey = derpClient.ServerPublicKey()
+	c.client = derpClient
+	c.netConn = tcpConn
+	c.connGen++
+	return c.client, c.connGen, nil
+}
+
+func (c *Client) dialURL(ctx context.Context) (net.Conn, error) {
+	host := c.url.Hostname()
+	hostOrIP := host
+
+	dialer := netns.NewDialer()
+
+	if c.DNSCache != nil {
+		ip, err := c.DNSCache.LookupIP(ctx, host)
+		if err == nil {
+			hostOrIP = ip.String()
+		}
+		if err != nil && netns.IsSOCKSDialer(dialer) {
+			// Return an error if we're not using a dial
+			// proxy that can do DNS lookups for us.
 			return nil, err
 		}
 	}
 
-	c.client = derpClient
-	c.netConn = tcpConn
-	return c.client, nil
+	tcpConn, err := dialer.DialContext(ctx, "tcp", net.JoinHostPort(hostOrIP, urlPort(c.url)))
+	if err != nil {
+		return nil, fmt.Errorf("dial of %v: %v", host, err)
+	}
+	return tcpConn, nil
+}
+
+// dialRegion returns a TCP connection to the provided region, trying
+// each node in order (with dialNode) until one connects or ctx is
+// done.
+func (c *Client) dialRegion(ctx context.Context, reg *tailcfg.DERPRegion) (net.Conn, *tailcfg.DERPNode, error) {
+	if len(reg.Nodes) == 0 {
+		return nil, nil, fmt.Errorf("no nodes for %s", c.targetString(reg))
+	}
+	var firstErr error
+	for _, n := range reg.Nodes {
+		if n.STUNOnly {
+			if firstErr == nil {
+				firstErr = fmt.Errorf("no non-STUNOnly nodes for %s", c.targetString(reg))
+			}
+			continue
+		}
+		c, err := c.dialNode(ctx, n)
+		if err == nil {
+			return c, n, nil
+		}
+		if firstErr == nil {
+			firstErr = err
+		}
+	}
+	return nil, nil, firstErr
+}
+
+func (c *Client) tlsClient(nc net.Conn, node *tailcfg.DERPNode) *tls.Conn {
+	tlsConf := tlsdial.Config(c.tlsServerName(node), c.TLSConfig)
+	if node != nil {
+		if node.DERPTestPort != 0 {
+			tlsConf.InsecureSkipVerify = true
+		}
+		if node.CertName != "" {
+			tlsdial.SetConfigExpectedCert(tlsConf, node.CertName)
+		}
+	}
+	return tls.Client(nc, tlsConf)
+}
+
+func (c *Client) DialRegionTLS(ctx context.Context, reg *tailcfg.DERPRegion) (tlsConn *tls.Conn, connClose io.Closer, err error) {
+	tcpConn, node, err := c.dialRegion(ctx, reg)
+	if err != nil {
+		return nil, nil, err
+	}
+	done := make(chan bool) // unbufferd
+	defer close(done)
+
+	tlsConn = c.tlsClient(tcpConn, node)
+	go func() {
+		select {
+		case <-done:
+		case <-ctx.Done():
+			tcpConn.Close()
+		}
+	}()
+	err = tlsConn.Handshake()
+	if err != nil {
+		return nil, nil, err
+	}
+	select {
+	case done <- true:
+		return tlsConn, tcpConn, nil
+	case <-ctx.Done():
+		return nil, nil, ctx.Err()
+	}
+}
+
+func (c *Client) dialContext(ctx context.Context, proto, addr string) (net.Conn, error) {
+	return netns.NewDialer().DialContext(ctx, proto, addr)
+}
+
+// shouldDialProto reports whether an explicitly provided IPv4 or IPv6
+// address (given in s) is valid. An empty value means to dial, but to
+// use DNS. The predicate function reports whether the non-empty
+// string s contained a valid IP address of the right family.
+func shouldDialProto(s string, pred func(netaddr.IP) bool) bool {
+	if s == "" {
+		return true
+	}
+	ip, _ := netaddr.ParseIP(s)
+	return pred(ip)
+}
+
+const dialNodeTimeout = 1500 * time.Millisecond
+
+// dialNode returns a TCP connection to node n, racing IPv4 and IPv6
+// (both as applicable) against each other.
+// A node is only given dialNodeTimeout to connect.
+//
+// TODO(bradfitz): longer if no options remain perhaps? ...  Or longer
+// overall but have dialRegion start overlapping races?
+func (c *Client) dialNode(ctx context.Context, n *tailcfg.DERPNode) (net.Conn, error) {
+	type res struct {
+		c   net.Conn
+		err error
+	}
+	resc := make(chan res) // must be unbuffered
+	ctx, cancel := context.WithTimeout(ctx, dialNodeTimeout)
+	defer cancel()
+
+	nwait := 0
+	startDial := func(dstPrimary, proto string) {
+		nwait++
+		go func() {
+			dst := dstPrimary
+			if dst == "" {
+				dst = n.HostName
+			}
+			port := "443"
+			if n.DERPTestPort != 0 {
+				port = fmt.Sprint(n.DERPTestPort)
+			}
+			c, err := c.dialContext(ctx, proto, net.JoinHostPort(dst, port))
+			select {
+			case resc <- res{c, err}:
+			case <-ctx.Done():
+				if c != nil {
+					c.Close()
+				}
+			}
+		}()
+	}
+	if shouldDialProto(n.IPv4, netaddr.IP.Is4) {
+		startDial(n.IPv4, "tcp4")
+	}
+	if shouldDialProto(n.IPv6, netaddr.IP.Is6) {
+		startDial(n.IPv6, "tcp6")
+	}
+	if nwait == 0 {
+		return nil, errors.New("both IPv4 and IPv6 are explicitly disabled for node")
+	}
+
+	var firstErr error
+	for {
+		select {
+		case res := <-resc:
+			nwait--
+			if res.err == nil {
+				return res.c, nil
+			}
+			if firstErr == nil {
+				firstErr = res.err
+			}
+			if nwait == 0 {
+				return nil, firstErr
+			}
+		case <-ctx.Done():
+			return nil, ctx.Err()
+		}
+	}
 }
 
 func (c *Client) Send(dstKey key.Public, b []byte) error {
-	client, err := c.connect(context.TODO(), "derphttp.Client.Send")
+	client, _, err := c.connect(context.TODO(), "derphttp.Client.Send")
 	if err != nil {
 		return err
 	}
@@ -241,6 +491,17 @@ func (c *Client) Send(dstKey key.Public, b []byte) error {
 	return err
 }
 
+func (c *Client) ForwardPacket(from, to key.Public, b []byte) error {
+	client, _, err := c.connect(context.TODO(), "derphttp.Client.ForwardPacket")
+	if err != nil {
+		return err
+	}
+	if err := client.ForwardPacket(from, to, b); err != nil {
+		c.closeForReconnect(client)
+	}
+	return err
+}
+
 // NotePreferred notes whether this Client is the caller's preferred
 // (home) DERP node. It's only used for stats.
 func (c *Client) NotePreferred(v bool) {
@@ -260,18 +521,58 @@ func (c *Client) NotePreferred(v bool) {
 	}
 }
 
-func (c *Client) Recv(b []byte) (derp.ReceivedMessage, error) {
-	client, err := c.connect(context.TODO(), "derphttp.Client.Recv")
+// WatchConnectionChanges sends a request to subscribe to
+// notifications about clients connecting & disconnecting.
+//
+// Only trusted connections (using MeshKey) are allowed to use this.
+func (c *Client) WatchConnectionChanges() error {
+	client, _, err := c.connect(context.TODO(), "derphttp.Client.WatchConnectionChanges")
 	if err != nil {
-		return nil, err
+		return err
 	}
-	m, err := client.Recv(b)
+	err = client.WatchConnectionChanges()
 	if err != nil {
 		c.closeForReconnect(client)
 	}
+	return err
+}
+
+// ClosePeer asks the server to close target's TCP connection.
+//
+// Only trusted connections (using MeshKey) are allowed to use this.
+func (c *Client) ClosePeer(target key.Public) error {
+	client, _, err := c.connect(context.TODO(), "derphttp.Client.ClosePeer")
+	if err != nil {
+		return err
+	}
+	err = client.ClosePeer(target)
+	if err != nil {
+		c.closeForReconnect(client)
+	}
+	return err
+}
+
+// Recv reads a message from c. The returned message may alias memory from Client.
+// The message should only be used until the next Client call.
+func (c *Client) Recv() (derp.ReceivedMessage, error) {
+	m, _, err := c.RecvDetail()
 	return m, err
 }
 
+// RecvDetail is like Recv, but additional returns the connection generation on each message.
+// The connGen value is incremented every time the derphttp.Client reconnects to the server.
+func (c *Client) RecvDetail() (m derp.ReceivedMessage, connGen int, err error) {
+	client, connGen, err := c.connect(context.TODO(), "derphttp.Client.Recv")
+	if err != nil {
+		return nil, 0, err
+	}
+	m, err = client.Recv()
+	if err != nil {
+		c.closeForReconnect(client)
+	}
+	return m, connGen, err
+}
+
 // Close closes the client. It will not automatically reconnect after
 // being closed.
 func (c *Client) Close() error {

derp/derphttp/derphttp_test.go

@@ -93,8 +93,7 @@ func TestSendRecv(t *testing.T) {
 					return
 				default:
 				}
-				b := make([]byte, 1<<16)
-				m, err := c.Recv(b)
+				m, err := c.Recv()
 				if err != nil {
 					t.Logf("client%d: %v", i, err)
 					break
@@ -106,7 +105,7 @@ func TestSendRecv(t *testing.T) {
 				case derp.PeerGoneMessage:
 					// Ignore.
 				case derp.ReceivedPacket:
-					recvChs[i] <- m.Data
+					recvChs[i] <- append([]byte(nil), m.Data...)
 				}
 			}
 		}(i)

derp/derphttp/mesh_client.go

@@ -0,0 +1,122 @@
+// Copyright (c) 2020 Tailscale Inc & AUTHORS All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package derphttp
+
+import (
+	"sync"
+	"time"
+
+	"tailscale.com/derp"
+	"tailscale.com/types/key"
+)
+
+// RunWatchConnectionLoop loops forever, sending WatchConnectionChanges and subscribing to
+// connection changes.
+//
+// If the server's public key is ignoreServerKey, RunWatchConnectionLoop returns.
+//
+// Otherwise, the add and remove funcs are called as clients come & go.
+func (c *Client) RunWatchConnectionLoop(ignoreServerKey key.Public, add, remove func(key.Public)) {
+	logf := c.logf
+	const retryInterval = 5 * time.Second
+	const statusInterval = 10 * time.Second
+	var (
+		mu              sync.Mutex
+		present         = map[key.Public]bool{}
+		loggedConnected = false
+	)
+	clear := func() {
+		mu.Lock()
+		defer mu.Unlock()
+		if len(present) == 0 {
+			return
+		}
+		logf("reconnected; clearing %d forwarding mappings", len(present))
+		for k := range present {
+			remove(k)
+		}
+		present = map[key.Public]bool{}
+	}
+	lastConnGen := 0
+	lastStatus := time.Now()
+	logConnectedLocked := func() {
+		if loggedConnected {
+			return
+		}
+		logf("connected; %d peers", len(present))
+		loggedConnected = true
+	}
+
+	const logConnectedDelay = 200 * time.Millisecond
+	timer := time.AfterFunc(2*time.Second, func() {
+		mu.Lock()
+		defer mu.Unlock()
+		logConnectedLocked()
+	})
+	defer timer.Stop()
+
+	updatePeer := func(k key.Public, isPresent bool) {
+		if isPresent {
+			add(k)
+		} else {
+			remove(k)
+		}
+
+		mu.Lock()
+		defer mu.Unlock()
+		if isPresent {
+			present[k] = true
+			if !loggedConnected {
+				timer.Reset(logConnectedDelay)
+			}
+		} else {
+			// If we got a peerGone message, that means the initial connection's
+			// flood of peerPresent messages is done, so we can log already:
+			logConnectedLocked()
+			delete(present, k)
+		}
+	}
+
+	for {
+		err := c.WatchConnectionChanges()
+		if err != nil {
+			clear()
+			logf("WatchConnectionChanges: %v", err)
+			time.Sleep(retryInterval)
+			continue
+		}
+
+		if c.ServerPublicKey() == ignoreServerKey {
+			logf("detected self-connect; ignoring host")
+			return
+		}
+		for {
+			m, connGen, err := c.RecvDetail()
+			if err != nil {
+				clear()
+				logf("Recv: %v", err)
+				time.Sleep(retryInterval)
+				break
+			}
+			if connGen != lastConnGen {
+				lastConnGen = connGen
+				clear()
+			}
+			switch m := m.(type) {
+			case derp.PeerPresentMessage:
+				updatePeer(key.Public(m), true)
+			case derp.PeerGoneMessage:
+				updatePeer(key.Public(m), false)
+			default:
+				continue
+			}
+			if now := time.Now(); now.Sub(lastStatus) > statusInterval {
+				lastStatus = now
+				logf("%d peers", len(present))
+			}
+		}
+	}
+
+}

derp/derpmap/derpmap.go

@@ -7,151 +7,59 @@ package derpmap
 
 import (
 	"fmt"
-	"net"
+	"strings"
 
-	"tailscale.com/types/structs"
+	"tailscale.com/tailcfg"
 )
 
-// World is a set of DERP server.
-type World struct {
-	servers []*Server
-	ids     []int
-	byID    map[int]*Server
-	stun4   []string
-	stun6   []string
-}
-
-func (w *World) IDs() []int                { return w.ids }
-func (w *World) STUN4() []string           { return w.stun4 }
-func (w *World) STUN6() []string           { return w.stun6 }
-func (w *World) ServerByID(id int) *Server { return w.byID[id] }
-
-// LocationOfID returns the geographic name of a node, if present.
-func (w *World) LocationOfID(id int) string {
-	if s, ok := w.byID[id]; ok {
-		return s.Geo
-	}
-	return ""
-}
-
-func (w *World) NodeIDOfSTUNServer(server string) int {
-	// TODO: keep reverse map? Small enough to not matter for now.
-	for _, s := range w.servers {
-		if s.STUN4 == server || s.STUN6 == server {
-			return s.ID
-		}
-	}
-	return 0
-}
-
-// ForeachServer calls fn for each DERP server, in an unspecified order.
-func (w *World) ForeachServer(fn func(*Server)) {
-	for _, s := range w.byID {
-		fn(s)
+func derpNode(suffix, v4, v6 string) *tailcfg.DERPNode {
+	return &tailcfg.DERPNode{
+		Name:     suffix, // updated later
+		RegionID: 0,      // updated later
+		IPv4:     v4,
+		IPv6:     v6,
 	}
 }
 
-// Prod returns the production DERP nodes.
-func Prod() *World {
-	return prod
+func derpRegion(id int, code string, nodes ...*tailcfg.DERPNode) *tailcfg.DERPRegion {
+	region := &tailcfg.DERPRegion{
+		RegionID:   id,
+		RegionCode: code,
+		Nodes:      nodes,
+	}
+	for _, n := range nodes {
+		n.Name = fmt.Sprintf("%d%s", id, n.Name)
+		n.RegionID = id
+		n.HostName = fmt.Sprintf("derp%s.tailscale.com", strings.TrimSuffix(n.Name, "a"))
+	}
+	return region
 }
 
-func NewTestWorld(stun ...string) *World {
-	w := &World{}
-	for i, s := range stun {
-		w.add(&Server{
-			ID:    i + 1,
-			Geo:   fmt.Sprintf("Testopolis-%d", i+1),
-			STUN4: s,
-		})
-	}
-	return w
-}
-
-func NewTestWorldWith(servers ...*Server) *World {
-	w := &World{}
-	for _, s := range servers {
-		w.add(s)
-	}
-	return w
-}
-
-var prod = new(World) // ... a dazzling place I never knew
-
-func addProd(id int, geo string) {
-	prod.add(&Server{
-		ID:        id,
-		Geo:       geo,
-		HostHTTPS: fmt.Sprintf("derp%v.tailscale.com", id),
-		STUN4:     fmt.Sprintf("derp%v.tailscale.com:3478", id),
-		STUN6:     fmt.Sprintf("derp%v-v6.tailscale.com:3478", id),
-	})
-}
-
-func (w *World) add(s *Server) {
-	if s.ID == 0 {
-		panic("ID required")
-	}
-	if _, dup := w.byID[s.ID]; dup {
-		panic("duplicate prod server")
-	}
-	if w.byID == nil {
-		w.byID = make(map[int]*Server)
-	}
-	w.byID[s.ID] = s
-	w.ids = append(w.ids, s.ID)
-	w.servers = append(w.servers, s)
-	if s.STUN4 != "" {
-		w.stun4 = append(w.stun4, s.STUN4)
-		if _, _, err := net.SplitHostPort(s.STUN4); err != nil {
-			panic("not a host:port: " + s.STUN4)
-		}
-	}
-	if s.STUN6 != "" {
-		w.stun6 = append(w.stun6, s.STUN6)
-		if _, _, err := net.SplitHostPort(s.STUN6); err != nil {
-			panic("not a host:port: " + s.STUN6)
-		}
+// Prod returns Tailscale's map of relay servers.
+//
+// This list is only used by cmd/tailscale's netcheck subcommand. In
+// normal operation the Tailscale nodes get this sent to them from the
+// control server.
+//
+// This list is subject to change and should not be relied on.
+func Prod() *tailcfg.DERPMap {
+	return &tailcfg.DERPMap{
+		Regions: map[int]*tailcfg.DERPRegion{
+			1: derpRegion(1, "nyc",
+				derpNode("a", "159.89.225.99", "2604:a880:400:d1::828:b001"),
+			),
+			2: derpRegion(2, "sfo",
+				derpNode("a", "167.172.206.31", "2604:a880:2:d1::c5:7001"),
+			),
+			3: derpRegion(3, "sin",
+				derpNode("a", "68.183.179.66", "2400:6180:0:d1::67d:8001"),
+			),
+			4: derpRegion(4, "fra",
+				derpNode("a", "167.172.182.26", "2a03:b0c0:3:e0::36e:9001"),
+			),
+			5: derpRegion(5, "syd",
+				derpNode("a", "103.43.75.49", "2001:19f0:5801:10b7:5400:2ff:feaa:284c"),
+			),
+		},
 	}
 }
-
-func init() {
-	addProd(1, "New York")
-	addProd(2, "San Francisco")
-	addProd(3, "Singapore")
-	addProd(4, "Frankfurt")
-	addProd(5, "Sydney")
-}
-
-// Server is configuration for a DERP server.
-type Server struct {
-	_ structs.Incomparable
-
-	ID int
-
-	// HostHTTPS is the HTTPS hostname.
-	HostHTTPS string
-
-	// STUN4 is the host:port of the IPv4 STUN server on this DERP
-	// node. Required.
-	STUN4 string
-
-	// STUN6 optionally provides the IPv6 host:port of the STUN
-	// server on the DERP node.
-	// It should be an IPv6-only address for now. (We currently make lazy
-	// assumptions that the server names are unique.)
-	STUN6 string
-
-	// Geo is a human-readable geographic region name of this server.
-	Geo string
-}
-
-func (s *Server) String() string {
-	if s == nil {
-		return "<nil *derpmap.Server>"
-	}
-	if s.Geo != "" {
-		return fmt.Sprintf("%v (%v)", s.HostHTTPS, s.Geo)
-	}
-	return s.HostHTTPS
-}

disco/disco.go

@@ -0,0 +1,179 @@
+// Copyright (c) 2020 Tailscale Inc & AUTHORS All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+// Package disco contains the discovery message types.
+//
+// A discovery message is:
+//
+// Header:
+//     magic          [6]byte  // “TS💬” (0x54 53 f0 9f 92 ac)
+//     senderDiscoPub [32]byte // nacl public key
+//     nonce          [24]byte
+//
+// The recipient then decrypts the bytes following (the nacl secretbox)
+// and then the inner payload structure is:
+//
+//     messageType    byte  (the MessageType constants below)
+//     messageVersion byte  (0 for now; but always ignore bytes at the end)
+//     message-paylod [...]byte
+package disco
+
+import (
+	"encoding/binary"
+	"errors"
+	"fmt"
+	"net"
+
+	"inet.af/netaddr"
+)
+
+// Magic is the 6 byte header of all discovery messages.
+const Magic = "TS💬" // 6 bytes: 0x54 53 f0 9f 92 ac
+
+const keyLen = 32
+
+// NonceLen is the length of the nonces used by nacl secretboxes.
+const NonceLen = 24
+
+type MessageType byte
+
+const (
+	TypePing        = MessageType(0x01)
+	TypePong        = MessageType(0x02)
+	TypeCallMeMaybe = MessageType(0x03)
+)
+
+const v0 = byte(0)
+
+var errShort = errors.New("short message")
+
+// LooksLikeDiscoWrapper reports whether p looks like it's a packet
+// containing an encrypted disco message.
+func LooksLikeDiscoWrapper(p []byte) bool {
+	if len(p) < len(Magic)+keyLen+NonceLen {
+		return false
+	}
+	return string(p[:len(Magic)]) == Magic
+}
+
+// Parse parses the encrypted part of the message from inside the
+// nacl secretbox.
+func Parse(p []byte) (Message, error) {
+	if len(p) < 2 {
+		return nil, errShort
+	}
+	t, ver, p := MessageType(p[0]), p[1], p[2:]
+	switch t {
+	case TypePing:
+		return parsePing(ver, p)
+	case TypePong:
+		return parsePong(ver, p)
+	case TypeCallMeMaybe:
+		return CallMeMaybe{}, nil
+	default:
+		return nil, fmt.Errorf("unknown message type 0x%02x", byte(t))
+	}
+}
+
+// Message a discovery message.
+type Message interface {
+	// AppendMarshal appends the message's marshaled representation.
+	AppendMarshal([]byte) []byte
+}
+
+// appendMsgHeader appends two bytes (for t and ver) and then also
+// dataLen bytes to b, returning the appended slice in all. The
+// returned data slice is a subslice of all with just dataLen bytes of
+// where the caller will fill in the data.
+func appendMsgHeader(b []byte, t MessageType, ver uint8, dataLen int) (all, data []byte) {
+	// TODO: optimize this?
+	all = append(b, make([]byte, dataLen+2)...)
+	all[len(b)] = byte(t)
+	all[len(b)+1] = ver
+	data = all[len(b)+2:]
+	return
+}
+
+type Ping struct {
+	TxID [12]byte
+}
+
+func (m *Ping) AppendMarshal(b []byte) []byte {
+	ret, d := appendMsgHeader(b, TypePing, v0, 12)
+	copy(d, m.TxID[:])
+	return ret
+}
+
+func parsePing(ver uint8, p []byte) (m *Ping, err error) {
+	if len(p) < 12 {
+		return nil, errShort
+	}
+	m = new(Ping)
+	copy(m.TxID[:], p)
+	return m, nil
+}
+
+// CallMeMaybe is a message sent only over DERP to request that the recipient try
+// to open up a magicsock path back to the sender.
+//
+// The sender should've already sent UDP packets to the peer to open
+// up the stateful firewall mappings inbound.
+//
+// The recipient may choose to not open a path back, if it's already
+// happy with its path. But usually it will.
+type CallMeMaybe struct{}
+
+func (CallMeMaybe) AppendMarshal(b []byte) []byte {
+	ret, _ := appendMsgHeader(b, TypeCallMeMaybe, v0, 0)
+	return ret
+}
+
+// Pong is a response a Ping.
+//
+// It includes the sender's source IP + port, so it's effectively a
+// STUN response.
+type Pong struct {
+	TxID [12]byte
+	Src  netaddr.IPPort // 18 bytes (16+2) on the wire; v4-mapped ipv6 for IPv4
+}
+
+const pongLen = 12 + 16 + 2
+
+func (m *Pong) AppendMarshal(b []byte) []byte {
+	ret, d := appendMsgHeader(b, TypePong, v0, pongLen)
+	d = d[copy(d, m.TxID[:]):]
+	ip16 := m.Src.IP.As16()
+	d = d[copy(d, ip16[:]):]
+	binary.BigEndian.PutUint16(d, m.Src.Port)
+	return ret
+}
+
+func parsePong(ver uint8, p []byte) (m *Pong, err error) {
+	if len(p) < pongLen {
+		return nil, errShort
+	}
+	m = new(Pong)
+	copy(m.TxID[:], p)
+	p = p[12:]
+
+	m.Src.IP, _ = netaddr.FromStdIP(net.IP(p[:16]))
+	p = p[16:]
+
+	m.Src.Port = binary.BigEndian.Uint16(p)
+	return m, nil
+}
+
+// MessageSummary returns a short summary of m for logging purposes.
+func MessageSummary(m Message) string {
+	switch m := m.(type) {
+	case *Ping:
+		return fmt.Sprintf("ping tx=%x", m.TxID[:6])
+	case *Pong:
+		return fmt.Sprintf("pong tx=%x", m.TxID[:6])
+	case CallMeMaybe:
+		return "call-me-maybe"
+	default:
+		return fmt.Sprintf("%#v", m)
+	}
+}

disco/disco_test.go

@@ -0,0 +1,82 @@
+// Copyright (c) 2020 Tailscale Inc & AUTHORS All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package disco
+
+import (
+	"fmt"
+	"reflect"
+	"strings"
+	"testing"
+
+	"inet.af/netaddr"
+)
+
+func TestMarshalAndParse(t *testing.T) {
+	tests := []struct {
+		name string
+		want string
+		m    Message
+	}{
+		{
+			name: "ping",
+			m: &Ping{
+				TxID: [12]byte{1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12},
+			},
+			want: "01 00 01 02 03 04 05 06 07 08 09 0a 0b 0c",
+		},
+		{
+			name: "pong",
+			m: &Pong{
+				TxID: [12]byte{1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12},
+				Src:  mustIPPort("2.3.4.5:1234"),
+			},
+			want: "02 00 01 02 03 04 05 06 07 08 09 0a 0b 0c 00 00 00 00 00 00 00 00 00 00 ff ff 02 03 04 05 04 d2",
+		},
+		{
+			name: "pongv6",
+			m: &Pong{
+				TxID: [12]byte{1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12},
+				Src:  mustIPPort("[fed0::12]:6666"),
+			},
+			want: "02 00 01 02 03 04 05 06 07 08 09 0a 0b 0c fe d0 00 00 00 00 00 00 00 00 00 00 00 00 00 12 1a 0a",
+		},
+		{
+			name: "call_me_maybe",
+			m:    CallMeMaybe{},
+			want: "03 00",
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			foo := []byte("foo")
+			got := string(tt.m.AppendMarshal(foo))
+			if !strings.HasPrefix(got, "foo") {
+				t.Fatalf("didn't start with foo: got %q", got)
+			}
+			got = strings.TrimPrefix(got, "foo")
+
+			gotHex := fmt.Sprintf("% x", got)
+			if gotHex != tt.want {
+				t.Fatalf("wrong marshal\n got: %s\nwant: %s\n", gotHex, tt.want)
+			}
+
+			back, err := Parse([]byte(got))
+			if err != nil {
+				t.Fatalf("parse back: %v", err)
+			}
+			if !reflect.DeepEqual(back, tt.m) {
+				t.Errorf("message in %+v doesn't match Parse back result %+v", tt.m, back)
+			}
+		})
+	}
+}
+
+func mustIPPort(s string) netaddr.IPPort {
+	ipp, err := netaddr.ParseIPPort(s)
+	if err != nil {
+		panic(err)
+	}
+	return ipp
+}

go.mod

@@ -1,6 +1,6 @@
 module tailscale.com
 
-go 1.13
+go 1.14
 
 require (
 	github.com/anmitsu/go-shlex v0.0.0-20161002113705-648efa622239 // indirect
@@ -9,25 +9,30 @@ require (
 	github.com/flynn/go-shlex v0.0.0-20150515145356-3f9db97f8568 // indirect
 	github.com/gliderlabs/ssh v0.2.2
 	github.com/go-ole/go-ole v1.2.4
+	github.com/godbus/dbus/v5 v5.0.3
 	github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e
 	github.com/google/go-cmp v0.4.0
 	github.com/goreleaser/nfpm v1.1.10
-	github.com/klauspost/compress v1.9.8
+	github.com/jsimonetti/rtnetlink v0.0.0-20200117123717-f846d4f6c1f4
+	github.com/klauspost/compress v1.10.10
 	github.com/kr/pty v1.1.1
 	github.com/mdlayher/netlink v1.1.0
+	github.com/miekg/dns v1.1.30
 	github.com/pborman/getopt v0.0.0-20190409184431-ee0cd42419d3
 	github.com/peterbourgon/ff/v2 v2.0.0
 	github.com/tailscale/winipcfg-go v0.0.0-20200413171540-609dcf2df55f
-	github.com/tailscale/wireguard-go v0.0.0-20200424121617-8d10f231531a
+	github.com/tailscale/wireguard-go v0.0.0-20200806235025-91988cfbaa3a
+	github.com/tcnksm/go-httpstat v0.2.0
 	github.com/toqueteos/webbrowser v1.2.0
-	go4.org/mem v0.0.0-20200411205429-f77f31c81751
-	golang.org/x/crypto v0.0.0-20200317142112-1b76d66859c6
-	golang.org/x/net v0.0.0-20200301022130-244492dfa37a // indirect
+	go4.org/mem v0.0.0-20200706164138-185c595c3ecc
+	golang.org/x/crypto v0.0.0-20200429183012-4b2356b1ed79
+	golang.org/x/net v0.0.0-20200501053045-e0ff5e5a1de5
 	golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d
 	golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e
-	golang.org/x/sys v0.0.0-20200501052902-10377860bb8e
+	golang.org/x/sys v0.0.0-20200501145240-bc7a7d42d5c3
 	golang.org/x/time v0.0.0-20191024005414-555d28b269f0
-	gortc.io/stun v1.22.1
-	inet.af/netaddr v0.0.0-20200417213433-f9e5bcc2d6ea
+	golang.org/x/tools v0.0.0-20191216052735-49a3e744a425
+	honnef.co/go/tools v0.0.1-2020.1.4
+	inet.af/netaddr v0.0.0-20200810144936-56928fe48a98
 	rsc.io/goversion v1.2.0
 )

go.sum

@@ -30,6 +30,8 @@ github.com/gliderlabs/ssh v0.2.2 h1:6zsha5zo/TWhRhwqCD3+EarCAgZ2yN28ipRnGPnwkI0=
 github.com/gliderlabs/ssh v0.2.2/go.mod h1:U7qILu1NlMHj9FlMhZLlkCdDnU1DBEAqr0aevW3Awn0=
 github.com/go-ole/go-ole v1.2.4 h1:nNBDSCOigTSiarFpYE9J/KtEA1IOW4CNeqT9TQDqCxI=
 github.com/go-ole/go-ole v1.2.4/go.mod h1:XCwSNxSkXRo4vlyPy93sltvi/qJq0jqQhjqQNIwKuxM=
+github.com/godbus/dbus/v5 v5.0.3 h1:ZqHaoEF7TBzh4jzPmqVhE/5A1z9of6orkAe5uHoAeME=
+github.com/godbus/dbus/v5 v5.0.3/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA=
 github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e h1:1r7pUrabqp18hOBcwBwiTsbnFeTZHV9eER/QT5JVZxY=
 github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
 github.com/golang/protobuf v1.2.0 h1:P3YflyNX/ehuJFLhxviNdFxQPkGK5cDcApsge1SqnvM=
@@ -38,6 +40,7 @@ github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5a
 github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
 github.com/google/go-cmp v0.4.0 h1:xsAVV57WRhGj6kEIi8ReJzQlHHqcBYCElAvkovg3B/4=
 github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
+github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI=
 github.com/google/rpmpack v0.0.0-20191226140753-aa36bfddb3a0 h1:BW6OvS3kpT5UEPbCZ+KyX/OB4Ks9/MNMhWjqPPkZxsE=
 github.com/google/rpmpack v0.0.0-20191226140753-aa36bfddb3a0/go.mod h1:RaTPr0KUf2K7fnZYLNDrr8rxAamWs3iNywJLtQ2AzBg=
 github.com/goreleaser/nfpm v1.1.10 h1:0nwzKUJTcygNxTzVKq2Dh9wpVP1W2biUH6SNKmoxR3w=
@@ -47,8 +50,9 @@ github.com/imdario/mergo v0.3.8/go.mod h1:2EnlNZ0deacrJVfApfmtdGgDfMuh/nq6Ok1EcJ
 github.com/jsimonetti/rtnetlink v0.0.0-20190606172950-9527aa82566a/go.mod h1:Oz+70psSo5OFh8DBl0Zv2ACw7Esh6pPUphlvZG9x7uw=
 github.com/jsimonetti/rtnetlink v0.0.0-20200117123717-f846d4f6c1f4 h1:nwOc1YaOrYJ37sEBrtWZrdqzK22hiJs3GpDmP3sR2Yw=
 github.com/jsimonetti/rtnetlink v0.0.0-20200117123717-f846d4f6c1f4/go.mod h1:WGuG/smIU4J/54PblvSbh+xvCZmpJnFgr3ds6Z55XMQ=
-github.com/klauspost/compress v1.9.8 h1:VMAMUUOh+gaxKTMk+zqbjsSjsIcUcL/LF4o63i82QyA=
-github.com/klauspost/compress v1.9.8/go.mod h1:RyIbtBH6LamlWaDj8nUwkbUhJ87Yi3uG0guNDohfE1A=
+github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck=
+github.com/klauspost/compress v1.10.10 h1:a/y8CglcM7gLGYmlbP/stPE5sR3hbhFRUjCBfd/0B3I=
+github.com/klauspost/compress v1.10.10/go.mod h1:aoV0uJVorq1K+umq18yTdKaF57EivdYsUV+/s2qKfXs=
 github.com/kr/pretty v0.1.0 h1:L/CwN0zerZDmRFUapSPitk6f+Q3+0za1rQkzVuMiMFI=
 github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo=
 github.com/kr/pty v1.1.1 h1:VkoXIwSboBpnk99O/KFauAEILuNHv5DVFKZMBN/gUgw=
@@ -61,6 +65,8 @@ github.com/mdlayher/netlink v0.0.0-20190409211403-11939a169225/go.mod h1:eQB3mZE
 github.com/mdlayher/netlink v1.0.0/go.mod h1:KxeJAFOFLG6AjpyDkQ/iIhxygIUKD+vcwqcnu43w/+M=
 github.com/mdlayher/netlink v1.1.0 h1:mpdLgm+brq10nI9zM1BpX1kpDbh3NLl3RSnVq6ZSkfg=
 github.com/mdlayher/netlink v1.1.0/go.mod h1:H4WCitaheIsdF9yOYu8CFmCgQthAPIWZmcKp9uZHgmY=
+github.com/miekg/dns v1.1.30 h1:Qww6FseFn8PRfw07jueqIXqodm0JKiiKuK0DeXSqfyo=
+github.com/miekg/dns v1.1.30/go.mod h1:KNUDUusw/aVsxyTYZM1oqvCicbwhgbNgztCETuNZ7xM=
 github.com/op/go-logging v0.0.0-20160315200505-970db520ece7 h1:lDH9UUVJtmYCjyT0CI4q8xvlXPxeZ0gYCVvWbmPlp88=
 github.com/op/go-logging v0.0.0-20160315200505-970db520ece7/go.mod h1:HzydrMdWErDVzsI23lYNej1Htcns9BCg93Dk0bBINWk=
 github.com/pborman/getopt v0.0.0-20190409184431-ee0cd42419d3 h1:YtFkrqsMEj7YqpIhRteVxJxCeC3jJBieuLr0d4C4rSA=
@@ -72,6 +78,7 @@ github.com/pkg/errors v0.8.1 h1:iURUrRGxPUNPdy5/HRSm+Yj6okJ6UtLINN0Q9M4+h3I=
 github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
 github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
 github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
+github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4=
 github.com/sassoftware/go-rpmutils v0.0.0-20190420191620-a8f1baeba37b h1:+gCnWOZV8Z/8jehJ2CdqB47Z3S+SREmQcuXkRFLNsiI=
 github.com/sassoftware/go-rpmutils v0.0.0-20190420191620-a8f1baeba37b/go.mod h1:am+Fp8Bt506lA3Rk3QCmSqmYmLMnPDhdDUcosQCAx+I=
 github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
@@ -79,35 +86,43 @@ github.com/stretchr/testify v1.4.0 h1:2E4SXV/wtOkTonXsotYi4li6zVWxYlZuYNCXe9XRJy
 github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4=
 github.com/tailscale/winipcfg-go v0.0.0-20200413171540-609dcf2df55f h1:uFj5bslHsMzxIM8UTjAhq4VXeo6GfNW91rpoh/WMJaY=
 github.com/tailscale/winipcfg-go v0.0.0-20200413171540-609dcf2df55f/go.mod h1:x880GWw5fvrl2DVTQ04ttXQD4DuppTt1Yz6wLibbjNE=
-github.com/tailscale/wireguard-go v0.0.0-20200424121617-8d10f231531a h1:HMkTFyhcvZaKf7+7T76rks4HqB83fptUemBIfLGI6TM=
-github.com/tailscale/wireguard-go v0.0.0-20200424121617-8d10f231531a/go.mod h1:JPm5cTfu1K+qDFRbiHy0sOlHUylYQbpl356sdYFD8V4=
+github.com/tailscale/wireguard-go v0.0.0-20200806235025-91988cfbaa3a h1:dQEgNpoOJf+8MswlvXJicb8ZDQqZAGe8f/WfzbDMvtE=
+github.com/tailscale/wireguard-go v0.0.0-20200806235025-91988cfbaa3a/go.mod h1:WXq+IkSOJGIgfF1XW+4z4oW+LX/TXzU9DcKlT5EZLi4=
+github.com/tcnksm/go-httpstat v0.2.0 h1:rP7T5e5U2HfmOBmZzGgGZjBQ5/GluWUylujl0tJ04I0=
+github.com/tcnksm/go-httpstat v0.2.0/go.mod h1:s3JVJFtQxtBEBC9dwcdTTXS9xFnM3SXAZwPG41aurT8=
 github.com/toqueteos/webbrowser v1.2.0 h1:tVP/gpK69Fx+qMJKsLE7TD8LuGWPnEV71wBN9rrstGQ=
 github.com/toqueteos/webbrowser v1.2.0/go.mod h1:XWoZq4cyp9WeUeak7w7LXRUQf1F1ATJMir8RTqb4ayM=
 github.com/ulikunitz/xz v0.5.6 h1:jGHAfXawEGZQ3blwU5wnWKQJvAraT7Ftq9EXjnXYgt8=
 github.com/ulikunitz/xz v0.5.6/go.mod h1:2bypXElzHzzJZwzH67Y6wb67pO62Rzfn7BSiF4ABRW8=
 github.com/xi2/xz v0.0.0-20171230120015-48954b6210f8 h1:nIPpBwaJSVYIxUFsDv3M8ofmx9yWTog9BfvIu0q41lo=
 github.com/xi2/xz v0.0.0-20171230120015-48954b6210f8/go.mod h1:HUYIGzjTL3rfEspMxjDjgmT5uz5wzYJKVo23qUhYTos=
-go4.org/mem v0.0.0-20200411205429-f77f31c81751 h1:sgGPu7KkyLjyOYOwKFHCtnfosdSuM5q2Gud23Y/+nzw=
-go4.org/mem v0.0.0-20200411205429-f77f31c81751/go.mod h1:NEYvpHWemiG/E5UWfaN5QAIGZeT1sa0Z2UNk6oeMb/k=
+go4.org/mem v0.0.0-20200706164138-185c595c3ecc h1:paujszgN6SpsO/UsXC7xax3gQAKz/XQKCYZLQdU34Tw=
+go4.org/mem v0.0.0-20200706164138-185c595c3ecc/go.mod h1:NEYvpHWemiG/E5UWfaN5QAIGZeT1sa0Z2UNk6oeMb/k=
 golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
+golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
 golang.org/x/crypto v0.0.0-20191002192127-34f69633bfdc/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
-golang.org/x/crypto v0.0.0-20200317142112-1b76d66859c6 h1:TjszyFsQsyZNHwdVdZ5m7bjmreu0znc2kRYsEml9/Ww=
-golang.org/x/crypto v0.0.0-20200317142112-1b76d66859c6/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
+golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
+golang.org/x/crypto v0.0.0-20200429183012-4b2356b1ed79 h1:IaQbIIB2X/Mp/DKctl6ROxz1KyMlKp4uyvL6+kQ7C88=
+golang.org/x/crypto v0.0.0-20200429183012-4b2356b1ed79/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
+golang.org/x/mod v0.0.0-20190513183733-4bf6d317e70e/go.mod h1:mXi4GBBbnImb6dmsKGUJ2LatrhH/nqhxcFungHvyanc=
+golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg=
 golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/net v0.0.0-20190108225652-1e06a53dbb7e/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
 golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
+golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
 golang.org/x/net v0.0.0-20190827160401-ba9fcec4b297/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
-golang.org/x/net v0.0.0-20191003171128-d98b1b443823/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
+golang.org/x/net v0.0.0-20190923162816-aa69164e4478/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
 golang.org/x/net v0.0.0-20191007182048-72f939374954/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
 golang.org/x/net v0.0.0-20200202094626-16171245cfb2 h1:CCH4IOTTfewWjGOlSp+zGcjutRKlBEZQ6wTn8ozI/nI=
 golang.org/x/net v0.0.0-20200202094626-16171245cfb2/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
-golang.org/x/net v0.0.0-20200301022130-244492dfa37a h1:GuSPYbZzB5/dcLNCwLQLsg3obCJtX9IJhpXkvY7kzk0=
-golang.org/x/net v0.0.0-20200301022130-244492dfa37a/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
+golang.org/x/net v0.0.0-20200501053045-e0ff5e5a1de5 h1:WQ8q63x+f/zpC8Ac1s9wLElVoHhm32p6tudrU72n1QA=
+golang.org/x/net v0.0.0-20200501053045-e0ff5e5a1de5/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A=
 golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d h1:TzXSXBo42m9gQenoE3b9BGiEpg5IG2JkU5FkPIawgtw=
 golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
 golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4 h1:YUO/7uOKsKeq9UokNS62b8FYywz3ker1l1vDZRCRefw=
 golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e h1:vcxGaoTs7kV8m5Np9uUNQin4BrLOthgV7252N8V+FwY=
 golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
@@ -117,34 +132,43 @@ golang.org/x/sys v0.0.0-20190405154228-4b34438f7a67/go.mod h1:h1NjWce9XRLGQEsW7w
 golang.org/x/sys v0.0.0-20190411185658-b44545bcd369/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20190826190057-c7b8b68b1456/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20191003212358-c178f38b412c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20190924154521-2837fb4f24fe/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20191008105621-543471e840be/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20200202164722-d101bd2416d5 h1:LfCXLvNmTYH9kEmVgqbnsWfruoXZIrh4YBgqVHtDvw0=
 golang.org/x/sys v0.0.0-20200202164722-d101bd2416d5/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20200317113312-5766fd39f98d h1:62ap6LNOjDU6uGmKXHJbSfciMoV+FeI1sRXx/pLDL44=
-golang.org/x/sys v0.0.0-20200317113312-5766fd39f98d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20200501052902-10377860bb8e h1:hq86ru83GdWTlfQFZGO4nZJTU4Bs2wfHl8oFHRaXsfc=
-golang.org/x/sys v0.0.0-20200501052902-10377860bb8e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200501145240-bc7a7d42d5c3 h1:5B6i6EAiSYyejWfvc5Rc9BbI3rzIsrrXfAQBWnYfn+w=
+golang.org/x/sys v0.0.0-20200501145240-bc7a7d42d5c3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
 golang.org/x/text v0.3.2 h1:tW2bmiBqwgJj/UpqtC8EpXEZVYOwU0yG4iWbprSVAcs=
 golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk=
 golang.org/x/time v0.0.0-20191024005414-555d28b269f0 h1:/5xXl8Y5W96D+TtHSlonuFqGHIWVuyCkGJLwGh9JJFs=
 golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
 golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
+golang.org/x/tools v0.0.0-20191130070609-6e064ea0cf2d h1:/iIZNFGxc/a7C3yWjGcnboV+Tkc7mxr+p6fDztwoxuM=
+golang.org/x/tools v0.0.0-20191130070609-6e064ea0cf2d/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
+golang.org/x/tools v0.0.0-20191216052735-49a3e744a425 h1:VvQyQJN0tSuecqgcIxMWnnfG5kSmgy9KZR9sW3W5QeA=
+golang.org/x/tools v0.0.0-20191216052735-49a3e744a425/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=
+golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
+golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
 golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543 h1:E7g+9GITq07hpfrRu66IVDexMakfv52eLZ2CXBWiKr4=
 golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
 google.golang.org/appengine v1.4.0 h1:/wp5JvzpHIxhs/dumFmF7BXTf3Z+dd4uXta4kVyO508=
 google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4=
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
+gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
 gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15 h1:YR8cESwS4TdDjEe65xsg0ogRM/Nc3DYOhEAlW+xobZo=
 gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
+gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI=
 gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
 gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
 gopkg.in/yaml.v2 v2.2.7 h1:VUgggvou5XRW9mHwD/yXxIYSMtY0zoKQf/v226p2nyo=
 gopkg.in/yaml.v2 v2.2.7/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
-gortc.io/stun v1.22.1 h1:96mOdDATYRqhYB+TZdenWBg4CzL2Ye5kPyBXQ8KAB+8=
-gortc.io/stun v1.22.1/go.mod h1:XD5lpONVyjvV3BgOyJFNo0iv6R2oZB4L+weMqxts+zg=
-inet.af/netaddr v0.0.0-20200417213433-f9e5bcc2d6ea h1:DpXewrGVf9+vvYQFrNGj9v34bXMuTVQv+2wuULTNV8I=
-inet.af/netaddr v0.0.0-20200417213433-f9e5bcc2d6ea/go.mod h1:qqYzz/2whtrbWJvt+DNWQyvekNN4ePQZcg2xc2/Yjww=
+honnef.co/go/tools v0.0.1-2020.1.4 h1:UoveltGrhghAA7ePc+e+QYDHXrBps2PqFZiHkGR/xK8=
+honnef.co/go/tools v0.0.1-2020.1.4/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k=
+inet.af/netaddr v0.0.0-20200718043157-99321d6ad24c h1:si3Owrfem175Ry6gKqnh59eOXxDojyBTIHxUKuvK/Eo=
+inet.af/netaddr v0.0.0-20200718043157-99321d6ad24c/go.mod h1:qqYzz/2whtrbWJvt+DNWQyvekNN4ePQZcg2xc2/Yjww=
+inet.af/netaddr v0.0.0-20200810144936-56928fe48a98 h1:bWyWDZP0l6VnQ1TDKf6yNwuiEDV6Q3q1Mv34m+lzT1I=
+inet.af/netaddr v0.0.0-20200810144936-56928fe48a98/go.mod h1:qqYzz/2whtrbWJvt+DNWQyvekNN4ePQZcg2xc2/Yjww=
 rsc.io/goversion v1.2.0 h1:SPn+NLTiAG7w30IRK/DKp1BjvpWabYgxlLp/+kx5J8w=
 rsc.io/goversion v1.2.0/go.mod h1:Eih9y/uIBS3ulggl7KNJ09xGSLcuNaLgmvvqa07sgfo=

internal/deepprint/deepprint.go

@@ -0,0 +1,103 @@
+// Copyright (c) 2020 Tailscale Inc & AUTHORS All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+// Package deepprint walks a Go value recursively, in a predictable
+// order, without looping, and prints each value out to a given
+// Writer, which is assumed to be a hash.Hash, as this package doesn't
+// format things nicely.
+//
+// This is intended as a lighter version of go-spew, etc. We don't need its
+// features when our writer is just a hash.
+package deepprint
+
+import (
+	"crypto/sha256"
+	"fmt"
+	"io"
+	"reflect"
+)
+
+func Hash(v ...interface{}) string {
+	h := sha256.New()
+	Print(h, v)
+	return fmt.Sprintf("%x", h.Sum(nil))
+}
+
+// UpdateHash sets last to the hash of v and reports whether its value changed.
+func UpdateHash(last *string, v ...interface{}) (changed bool) {
+	sig := Hash(v)
+	if *last != sig {
+		*last = sig
+		return true
+	}
+	return false
+}
+
+func Print(w io.Writer, v ...interface{}) {
+	print(w, reflect.ValueOf(v), make(map[uintptr]bool))
+}
+
+func print(w io.Writer, v reflect.Value, visited map[uintptr]bool) {
+	if !v.IsValid() {
+		return
+	}
+	switch v.Kind() {
+	default:
+		panic(fmt.Sprintf("unhandled kind %v for type %v", v.Kind(), v.Type()))
+	case reflect.Ptr:
+		ptr := v.Pointer()
+		if visited[ptr] {
+			return
+		}
+		visited[ptr] = true
+		print(w, v.Elem(), visited)
+		return
+	case reflect.Struct:
+		fmt.Fprintf(w, "struct{\n")
+		t := v.Type()
+		for i, n := 0, v.NumField(); i < n; i++ {
+			sf := t.Field(i)
+			fmt.Fprintf(w, "%s: ", sf.Name)
+			print(w, v.Field(i), visited)
+			fmt.Fprintf(w, "\n")
+		}
+	case reflect.Slice, reflect.Array:
+		if v.Type().Elem().Kind() == reflect.Uint8 && v.CanInterface() {
+			fmt.Fprintf(w, "%q", v.Interface())
+			return
+		}
+		fmt.Fprintf(w, "[%d]{\n", v.Len())
+		for i, ln := 0, v.Len(); i < ln; i++ {
+			fmt.Fprintf(w, " [%d]: ", i)
+			print(w, v.Index(i), visited)
+			fmt.Fprintf(w, "\n")
+		}
+		fmt.Fprintf(w, "}\n")
+	case reflect.Interface:
+		print(w, v.Elem(), visited)
+	case reflect.Map:
+		sm := newSortedMap(v)
+		fmt.Fprintf(w, "map[%d]{\n", len(sm.Key))
+		for i, k := range sm.Key {
+			print(w, k, visited)
+			fmt.Fprintf(w, ": ")
+			print(w, sm.Value[i], visited)
+			fmt.Fprintf(w, "\n")
+		}
+		fmt.Fprintf(w, "}\n")
+
+	case reflect.String:
+		fmt.Fprintf(w, "%s", v.String())
+	case reflect.Bool:
+		fmt.Fprintf(w, "%v", v.Bool())
+	case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64:
+		fmt.Fprintf(w, "%v", v.Int())
+	case reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64, reflect.Uintptr:
+		fmt.Fprintf(w, "%v", v.Uint())
+	case reflect.Float32, reflect.Float64:
+		fmt.Fprintf(w, "%v", v.Float())
+	case reflect.Complex64, reflect.Complex128:
+		fmt.Fprintf(w, "%v", v.Complex())
+	}
+}

internal/deepprint/deepprint_test.go

@@ -0,0 +1,71 @@
+// Copyright (c) 2020 Tailscale Inc & AUTHORS All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package deepprint
+
+import (
+	"bytes"
+	"testing"
+
+	"github.com/tailscale/wireguard-go/wgcfg"
+	"inet.af/netaddr"
+	"tailscale.com/wgengine/router"
+	"tailscale.com/wgengine/router/dns"
+)
+
+func TestDeepPrint(t *testing.T) {
+	// v contains the types of values we care about for our current callers.
+	// Mostly we're just testing that we don't panic on handled types.
+	v := getVal()
+
+	var buf bytes.Buffer
+	Print(&buf, v)
+	t.Logf("Got: %s", buf.Bytes())
+
+	hash1 := Hash(v)
+	t.Logf("hash: %v", hash1)
+	for i := 0; i < 20; i++ {
+		hash2 := Hash(getVal())
+		if hash1 != hash2 {
+			t.Error("second hash didn't match")
+		}
+	}
+}
+
+func getVal() []interface{} {
+	return []interface{}{
+		&wgcfg.Config{
+			Name:       "foo",
+			Addresses:  []wgcfg.CIDR{{Mask: 5, IP: wgcfg.IP{Addr: [16]byte{3: 3}}}},
+			ListenPort: 5,
+			Peers: []wgcfg.Peer{
+				{
+					Endpoints: []wgcfg.Endpoint{
+						{
+							Host: "foo",
+							Port: 5,
+						},
+					},
+				},
+			},
+		},
+		&router.Config{
+			DNS: dns.Config{
+				Nameservers: []netaddr.IP{netaddr.IPv4(8, 8, 8, 8)},
+				Domains:     []string{"tailscale.net"},
+			},
+		},
+		map[string]string{
+			"key1": "val1",
+			"key2": "val2",
+			"key3": "val3",
+			"key4": "val4",
+			"key5": "val5",
+			"key6": "val6",
+			"key7": "val7",
+			"key8": "val8",
+			"key9": "val9",
+		},
+	}
+}

internal/deepprint/fmtsort.go

@@ -0,0 +1,224 @@
+// Copyright (c) 2020 Tailscale Inc & AUTHORS All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+// and
+
+// Copyright 2018 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+// This is a slightly modified fork of Go's src/internal/fmtsort/sort.go
+
+package deepprint
+
+import (
+	"reflect"
+	"sort"
+)
+
+// Note: Throughout this package we avoid calling reflect.Value.Interface as
+// it is not always legal to do so and it's easier to avoid the issue than to face it.
+
+// sortedMap represents a map's keys and values. The keys and values are
+// aligned in index order: Value[i] is the value in the map corresponding to Key[i].
+type sortedMap struct {
+	Key   []reflect.Value
+	Value []reflect.Value
+}
+
+func (o *sortedMap) Len() int           { return len(o.Key) }
+func (o *sortedMap) Less(i, j int) bool { return compare(o.Key[i], o.Key[j]) < 0 }
+func (o *sortedMap) Swap(i, j int) {
+	o.Key[i], o.Key[j] = o.Key[j], o.Key[i]
+	o.Value[i], o.Value[j] = o.Value[j], o.Value[i]
+}
+
+// Sort accepts a map and returns a sortedMap that has the same keys and
+// values but in a stable sorted order according to the keys, modulo issues
+// raised by unorderable key values such as NaNs.
+//
+// The ordering rules are more general than with Go's < operator:
+//
+//  - when applicable, nil compares low
+//  - ints, floats, and strings order by <
+//  - NaN compares less than non-NaN floats
+//  - bool compares false before true
+//  - complex compares real, then imag
+//  - pointers compare by machine address
+//  - channel values compare by machine address
+//  - structs compare each field in turn
+//  - arrays compare each element in turn.
+//    Otherwise identical arrays compare by length.
+//  - interface values compare first by reflect.Type describing the concrete type
+//    and then by concrete value as described in the previous rules.
+//
+func newSortedMap(mapValue reflect.Value) *sortedMap {
+	if mapValue.Type().Kind() != reflect.Map {
+		return nil
+	}
+	// Note: this code is arranged to not panic even in the presence
+	// of a concurrent map update. The runtime is responsible for
+	// yelling loudly if that happens. See issue 33275.
+	n := mapValue.Len()
+	key := make([]reflect.Value, 0, n)
+	value := make([]reflect.Value, 0, n)
+	iter := mapValue.MapRange()
+	for iter.Next() {
+		key = append(key, iter.Key())
+		value = append(value, iter.Value())
+	}
+	sorted := &sortedMap{
+		Key:   key,
+		Value: value,
+	}
+	sort.Stable(sorted)
+	return sorted
+}
+
+// compare compares two values of the same type. It returns -1, 0, 1
+// according to whether a > b (1), a == b (0), or a < b (-1).
+// If the types differ, it returns -1.
+// See the comment on Sort for the comparison rules.
+func compare(aVal, bVal reflect.Value) int {
+	aType, bType := aVal.Type(), bVal.Type()
+	if aType != bType {
+		return -1 // No good answer possible, but don't return 0: they're not equal.
+	}
+	switch aVal.Kind() {
+	case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64:
+		a, b := aVal.Int(), bVal.Int()
+		switch {
+		case a < b:
+			return -1
+		case a > b:
+			return 1
+		default:
+			return 0
+		}
+	case reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64, reflect.Uintptr:
+		a, b := aVal.Uint(), bVal.Uint()
+		switch {
+		case a < b:
+			return -1
+		case a > b:
+			return 1
+		default:
+			return 0
+		}
+	case reflect.String:
+		a, b := aVal.String(), bVal.String()
+		switch {
+		case a < b:
+			return -1
+		case a > b:
+			return 1
+		default:
+			return 0
+		}
+	case reflect.Float32, reflect.Float64:
+		return floatCompare(aVal.Float(), bVal.Float())
+	case reflect.Complex64, reflect.Complex128:
+		a, b := aVal.Complex(), bVal.Complex()
+		if c := floatCompare(real(a), real(b)); c != 0 {
+			return c
+		}
+		return floatCompare(imag(a), imag(b))
+	case reflect.Bool:
+		a, b := aVal.Bool(), bVal.Bool()
+		switch {
+		case a == b:
+			return 0
+		case a:
+			return 1
+		default:
+			return -1
+		}
+	case reflect.Ptr:
+		a, b := aVal.Pointer(), bVal.Pointer()
+		switch {
+		case a < b:
+			return -1
+		case a > b:
+			return 1
+		default:
+			return 0
+		}
+	case reflect.Chan:
+		if c, ok := nilCompare(aVal, bVal); ok {
+			return c
+		}
+		ap, bp := aVal.Pointer(), bVal.Pointer()
+		switch {
+		case ap < bp:
+			return -1
+		case ap > bp:
+			return 1
+		default:
+			return 0
+		}
+	case reflect.Struct:
+		for i := 0; i < aVal.NumField(); i++ {
+			if c := compare(aVal.Field(i), bVal.Field(i)); c != 0 {
+				return c
+			}
+		}
+		return 0
+	case reflect.Array:
+		for i := 0; i < aVal.Len(); i++ {
+			if c := compare(aVal.Index(i), bVal.Index(i)); c != 0 {
+				return c
+			}
+		}
+		return 0
+	case reflect.Interface:
+		if c, ok := nilCompare(aVal, bVal); ok {
+			return c
+		}
+		c := compare(reflect.ValueOf(aVal.Elem().Type()), reflect.ValueOf(bVal.Elem().Type()))
+		if c != 0 {
+			return c
+		}
+		return compare(aVal.Elem(), bVal.Elem())
+	default:
+		// Certain types cannot appear as keys (maps, funcs, slices), but be explicit.
+		panic("bad type in compare: " + aType.String())
+	}
+}
+
+// nilCompare checks whether either value is nil. If not, the boolean is false.
+// If either value is nil, the boolean is true and the integer is the comparison
+// value. The comparison is defined to be 0 if both are nil, otherwise the one
+// nil value compares low. Both arguments must represent a chan, func,
+// interface, map, pointer, or slice.
+func nilCompare(aVal, bVal reflect.Value) (int, bool) {
+	if aVal.IsNil() {
+		if bVal.IsNil() {
+			return 0, true
+		}
+		return -1, true
+	}
+	if bVal.IsNil() {
+		return 1, true
+	}
+	return 0, false
+}
+
+// floatCompare compares two floating-point values. NaNs compare low.
+func floatCompare(a, b float64) int {
+	switch {
+	case isNaN(a):
+		return -1 // No good answer if b is a NaN so don't bother checking.
+	case isNaN(b):
+		return 1
+	case a < b:
+		return -1
+	case a > b:
+		return 1
+	}
+	return 0
+}
+
+func isNaN(a float64) bool {
+	return a != a
+}

ipn/backend.go

@@ -5,8 +5,10 @@
 package ipn
 
 import (
+	"net/http"
 	"time"
 
+	"golang.org/x/oauth2"
 	"tailscale.com/control/controlclient"
 	"tailscale.com/ipn/ipnstate"
 	"tailscale.com/tailcfg"
@@ -26,6 +28,10 @@ const (
 	Running
 )
 
+// GoogleIDToken Type is the oauth2.Token.TokenType for the Google
+// ID tokens used by the Android client.
+const GoogleIDTokenType = "ts_android_google_login"
+
 func (s State) String() string {
 	return [...]string{"NoState", "NeedsLogin", "NeedsMachineAuth",
 		"Stopped", "Starting", "Running"}[s]
@@ -39,8 +45,6 @@ type EngineStatus struct {
 	LivePeers      map[tailcfg.NodeKey]wgengine.PeerStatus
 }
 
-type NetworkMap = controlclient.NetworkMap
-
 // Notify is a communication from a backend (e.g. tailscaled) to a frontend
 // (cmd/tailscale, iOS, macOS, Win Tasktray).
 // In any given notification, any or all of these may be nil, meaning
@@ -48,16 +52,22 @@ type NetworkMap = controlclient.NetworkMap
 // They are JSON-encoded on the wire, despite the lack of struct tags.
 type Notify struct {
 	_             structs.Incomparable
-	Version       string           // version number of IPN backend
-	ErrMessage    *string          // critical error message, if any
-	LoginFinished *empty.Message   // event: non-nil when login process succeeded
-	State         *State           // current IPN state has changed
-	Prefs         *Prefs           // preferences were changed
-	NetMap        *NetworkMap      // new netmap received
-	Engine        *EngineStatus    // wireguard engine stats
-	Status        *ipnstate.Status // full status
-	BrowseToURL   *string          // UI should open a browser right now
-	BackendLogID  *string          // public logtail id used by backend
+	Version       string                    // version number of IPN backend
+	ErrMessage    *string                   // critical error message, if any
+	LoginFinished *empty.Message            // event: non-nil when login process succeeded
+	State         *State                    // current IPN state has changed
+	Prefs         *Prefs                    // preferences were changed
+	NetMap        *controlclient.NetworkMap // new netmap received
+	Engine        *EngineStatus             // wireguard engine stats
+	Status        *ipnstate.Status          // full status
+	BrowseToURL   *string                   // UI should open a browser right now
+	BackendLogID  *string                   // public logtail id used by backend
+
+	// LocalTCPPort, if non-nil, informs the UI frontend which
+	// (non-zero) localhost TCP port it's listening on.
+	// This is currently only used by Tailscale when run in the
+	// macOS Network Extension.
+	LocalTCPPort *uint16 `json:",omitempty"`
 
 	// type is mirrored in xcode/Shared/IPN.swift
 }
@@ -92,10 +102,10 @@ type Options struct {
 	//  - StateKey!="" && Prefs!=nil: like the previous case, but do
 	//    an initial overwrite of backend state with Prefs.
 	StateKey StateKey
+	Prefs    *Prefs
 	// AuthKey is an optional node auth key used to authorize a
 	// new node key without user interaction.
 	AuthKey string
-	Prefs   *Prefs
 	// LegacyConfigPath optionally specifies the old-style relaynode
 	// relay.conf location. If both LegacyConfigPath and StateKey are
 	// specified and the requested state doesn't exist in the backend
@@ -106,6 +116,9 @@ type Options struct {
 	LegacyConfigPath string
 	// Notify is called when backend events happen.
 	Notify func(Notify) `json:"-"`
+	// HTTPTestClient is an optional HTTP client to pass to controlclient
+	// (for tests only).
+	HTTPTestClient *http.Client
 }
 
 // Backend is the interface between Tailscale frontends
@@ -121,6 +134,8 @@ type Backend interface {
 	// flow. This should trigger a new BrowseToURL notification
 	// eventually.
 	StartLoginInteractive()
+	// Login logs in with an OAuth2 token.
+	Login(token *oauth2.Token)
 	// Logout terminates the current login session and stops the
 	// wireguard engine.
 	Logout()

ipn/e2e_test.go

@@ -1,279 +0,0 @@
-// Copyright (c) 2020 Tailscale Inc & AUTHORS All rights reserved.
-// Use of this source code is governed by a BSD-style
-// license that can be found in the LICENSE file.
-
-// +build depends_on_currently_unreleased
-
-package ipn
-
-import (
-	"bytes"
-	"io/ioutil"
-	"net/http"
-	"net/http/cookiejar"
-	"net/http/httptest"
-	"net/url"
-	"os"
-	"strings"
-	"testing"
-	"time"
-
-	"github.com/tailscale/wireguard-go/tun/tuntest"
-	"github.com/tailscale/wireguard-go/wgcfg"
-	"tailscale.com/control/controlclient"
-	"tailscale.com/tailcfg"
-	"tailscale.com/tstest"
-	"tailscale.com/wgengine"
-	"tailscale.com/wgengine/magicsock"
-	"tailscale.com/wgengine/router"
-	"tailscale.io/control" // not yet released
-)
-
-func init() {
-	// Hacky way to signal to magicsock for now not to bind on the
-	// unspecified address. TODO(bradfitz): clean up wgengine's
-	// constructors.
-	os.Setenv("IN_TS_TEST", "1")
-}
-
-func TestIPN(t *testing.T) {
-	tstest.FixLogs(t)
-	defer tstest.UnfixLogs(t)
-
-	// Turn off STUN for the test to make it hermetic.
-	// TODO(crawshaw): add a test that runs against a local STUN server.
-	magicsock.DisableSTUNForTesting = true
-	defer func() { magicsock.DisableSTUNForTesting = false }()
-
-	// TODO(apenwarr): Make resource checks actually pass.
-	// They don't right now, because (at least) wgengine doesn't fully
-	// shut down.
-	//	rc := tstest.NewResourceCheck()
-	//	defer rc.Assert(t)
-
-	var ctl *control.Server
-
-	ctlHandler := func(w http.ResponseWriter, r *http.Request) {
-		ctl.ServeHTTP(w, r)
-	}
-	https := httptest.NewServer(http.HandlerFunc(ctlHandler))
-	serverURL := https.URL
-	defer https.Close()
-	defer https.CloseClientConnections()
-
-	tmpdir, err := ioutil.TempDir("", "ipntest")
-	if err != nil {
-		t.Fatalf("create tempdir: %v\n", err)
-	}
-	ctl, err = control.New(tmpdir, tmpdir, tmpdir, serverURL, true)
-	if err != nil {
-		t.Fatalf("create control server: %v\n", ctl)
-	}
-	if _, err := ctl.DB().FindOrCreateUser("google", "test1@example.com", "", ""); err != nil {
-		t.Fatal(err)
-	}
-
-	n1 := newNode(t, "n1", https, false)
-	defer n1.Backend.Shutdown()
-	n1.Backend.StartLoginInteractive()
-
-	n2 := newNode(t, "n2", https, true)
-	defer n2.Backend.Shutdown()
-	n2.Backend.StartLoginInteractive()
-
-	t.Run("login", func(t *testing.T) {
-		var s1, s2 State
-		for {
-			t.Logf("\n\nn1.state=%v n2.state=%v\n\n", s1, s2)
-
-			// TODO(crawshaw): switch from || to &&. To do this we need to
-			// transmit some data so that the handshake completes on both
-			// sides. (Because handshakes are 1RTT, it is the data
-			// transmission that completes the handshake.)
-			if s1 == Running || s2 == Running {
-				// TODO(apenwarr): ensure state sequence.
-				// Right now we'll just exit as soon as
-				// state==Running, even if the backend is lying or
-				// something. Not a great test.
-				break
-			}
-
-			select {
-			case n := <-n1.NotifyCh:
-				t.Logf("n1n: %v\n", n)
-				if n.State != nil {
-					s1 = *n.State
-					if s1 == NeedsMachineAuth {
-						authNode(t, ctl, n1.Backend)
-					}
-				}
-			case n := <-n2.NotifyCh:
-				t.Logf("n2n: %v\n", n)
-				if n.State != nil {
-					s2 = *n.State
-					if s2 == NeedsMachineAuth {
-						authNode(t, ctl, n2.Backend)
-					}
-				}
-			case <-time.After(3 * time.Second):
-				t.Fatalf("\n\n\nFATAL: timed out waiting for notifications.\n\n\n")
-			}
-		}
-	})
-
-	n1addr := n1.Backend.NetMap().Addresses[0].IP
-	n2addr := n2.Backend.NetMap().Addresses[0].IP
-	t.Run("ping n2", func(t *testing.T) {
-		t.Skip("TODO(crawshaw): skipping ping test, it is flaky")
-		msg := tuntest.Ping(n2addr.IP(), n1addr.IP())
-		n1.ChannelTUN.Outbound <- msg
-		select {
-		case msgRecv := <-n2.ChannelTUN.Inbound:
-			if !bytes.Equal(msg, msgRecv) {
-				t.Error("bad ping")
-			}
-		case <-time.After(1 * time.Second):
-			t.Error("no ping seen")
-		}
-	})
-	t.Run("ping n1", func(t *testing.T) {
-		t.Skip("TODO(crawshaw): skipping ping test, it is flaky")
-		msg := tuntest.Ping(n1addr.IP(), n2addr.IP())
-		n2.ChannelTUN.Outbound <- msg
-		select {
-		case msgRecv := <-n1.ChannelTUN.Inbound:
-			if !bytes.Equal(msg, msgRecv) {
-				t.Error("bad ping")
-			}
-		case <-time.After(1 * time.Second):
-			t.Error("no ping seen")
-		}
-	})
-
-drain:
-	for {
-		select {
-		case <-n1.NotifyCh:
-		case <-n2.NotifyCh:
-		default:
-			break drain
-		}
-	}
-
-	n1.Backend.Logout()
-
-	t.Run("logout", func(t *testing.T) {
-		var s State
-		for {
-			select {
-			case n := <-n1.NotifyCh:
-				if n.State == nil {
-					continue
-				}
-				s = *n.State
-				t.Logf("n.State=%v", s)
-				if s == NeedsLogin {
-					return
-				}
-			case <-time.After(3 * time.Second):
-				t.Fatalf("timeout waiting for logout State=NeedsLogin, got State=%v", s)
-			}
-		}
-	})
-}
-
-type testNode struct {
-	Backend    *LocalBackend
-	ChannelTUN *tuntest.ChannelTUN
-	NotifyCh   <-chan Notify
-}
-
-// Create a new IPN node.
-func newNode(t *testing.T, prefix string, https *httptest.Server, weirdPrefs bool) testNode {
-	t.Helper()
-	logfe := func(fmt string, args ...interface{}) {
-		t.Logf(prefix+".e: "+fmt, args...)
-	}
-	logf := func(fmt string, args ...interface{}) {
-		t.Logf(prefix+": "+fmt, args...)
-	}
-
-	var err error
-	httpc := https.Client()
-	httpc.Jar, err = cookiejar.New(nil)
-	if err != nil {
-		t.Fatal(err)
-	}
-
-	tun := tuntest.NewChannelTUN()
-	e1, err := wgengine.NewUserspaceEngineAdvanced(logfe, tun.TUN(), router.NewFake, 0)
-	if err != nil {
-		t.Fatalf("NewFakeEngine: %v\n", err)
-	}
-	n, err := NewLocalBackend(logf, prefix, &MemoryStore{}, e1)
-	if err != nil {
-		t.Fatalf("NewLocalBackend: %v\n", err)
-	}
-	nch := make(chan Notify, 1000)
-	c := controlclient.Persist{
-		Provider:  "google",
-		LoginName: "test1@example.com",
-	}
-	prefs := NewPrefs()
-	prefs.ControlURL = https.URL
-	prefs.Persist = &c
-
-	if weirdPrefs {
-		// Let's test some nonempty extra prefs fields to make sure
-		// the server can handle them.
-		prefs.AdvertiseTags = []string{"tag:abc"}
-		cidr, err := wgcfg.ParseCIDR("1.2.3.4/24")
-		if err != nil {
-			t.Fatalf("ParseCIDR: %v", err)
-		}
-		prefs.AdvertiseRoutes = []wgcfg.CIDR{cidr}
-	}
-
-	n.Start(Options{
-		FrontendLogID: prefix + "-f",
-		Prefs:         prefs,
-		Notify: func(n Notify) {
-			// Automatically visit auth URLs
-			if n.BrowseToURL != nil {
-				t.Logf("BrowseToURL: %v", *n.BrowseToURL)
-
-				authURL := *n.BrowseToURL
-				i := strings.Index(authURL, "/a/")
-				if i == -1 {
-					panic("bad authURL: " + authURL)
-				}
-				authURL = authURL[:i] + "/login?refresh=true&next_url=" + url.PathEscape(authURL[i:])
-
-				form := url.Values{"user": []string{c.LoginName}}
-				req, err := http.NewRequest("POST", authURL, strings.NewReader(form.Encode()))
-				if err != nil {
-					t.Fatal(err)
-				}
-				req.Header.Add("Content-Type", "application/x-www-form-urlencoded")
-
-				if _, err := httpc.Do(req); err != nil {
-					t.Logf("BrowseToURL: %v\n", err)
-				}
-			}
-			nch <- n
-		},
-	})
-
-	return testNode{
-		Backend:    n,
-		ChannelTUN: tun,
-		NotifyCh:   nch,
-	}
-}
-
-// Tell the control server to authorize the given node.
-func authNode(t *testing.T, ctl *control.Server, n *LocalBackend) {
-	mk := n.prefs.Persist.PrivateMachineKey.Public()
-	nk := n.prefs.Persist.PrivateNodeKey.Public()
-	ctl.AuthorizeMachine(tailcfg.MachineKey(mk), tailcfg.NodeKey(nk))
-}

ipn/fake_test.go

@@ -8,6 +8,8 @@ import (
 	"log"
 	"time"
 
+	"golang.org/x/oauth2"
+	"tailscale.com/control/controlclient"
 	"tailscale.com/ipn/ipnstate"
 )
 
@@ -41,10 +43,18 @@ func (b *FakeBackend) newState(s State) {
 func (b *FakeBackend) StartLoginInteractive() {
 	u := b.serverURL + "/this/is/fake"
 	b.notify(Notify{BrowseToURL: &u})
+	b.login()
+}
+
+func (b *FakeBackend) Login(token *oauth2.Token) {
+	b.login()
+}
+
+func (b *FakeBackend) login() {
 	b.newState(NeedsMachineAuth)
 	b.newState(Stopped)
 	// TODO(apenwarr): Fill in a more interesting netmap here.
-	b.notify(Notify{NetMap: &NetworkMap{}})
+	b.notify(Notify{NetMap: &controlclient.NetworkMap{}})
 	b.newState(Starting)
 	// TODO(apenwarr): Fill in a more interesting status.
 	b.notify(Notify{Engine: &EngineStatus{}})
@@ -78,5 +88,5 @@ func (b *FakeBackend) RequestStatus() {
 }
 
 func (b *FakeBackend) FakeExpireAfter(x time.Duration) {
-	b.notify(Notify{NetMap: &NetworkMap{}})
+	b.notify(Notify{NetMap: &controlclient.NetworkMap{}})
 }

ipn/handle.go

@@ -9,6 +9,8 @@ import (
 	"time"
 
 	"github.com/tailscale/wireguard-go/wgcfg"
+	"golang.org/x/oauth2"
+	"tailscale.com/control/controlclient"
 	"tailscale.com/types/logger"
 )
 
@@ -20,7 +22,7 @@ type Handle struct {
 
 	// Mutex protects everything below
 	mu                sync.Mutex
-	netmapCache       *NetworkMap
+	netmapCache       *controlclient.NetworkMap
 	engineStatusCache EngineStatus
 	stateCache        State
 	prefsCache        *Prefs
@@ -127,7 +129,7 @@ func (h *Handle) LocalAddrs() []wgcfg.CIDR {
 	return []wgcfg.CIDR{}
 }
 
-func (h *Handle) NetMap() *NetworkMap {
+func (h *Handle) NetMap() *controlclient.NetworkMap {
 	h.mu.Lock()
 	defer h.mu.Unlock()
 
@@ -153,6 +155,10 @@ func (h *Handle) StartLoginInteractive() {
 	h.b.StartLoginInteractive()
 }
 
+func (h *Handle) Login(token *oauth2.Token) {
+	h.b.Login(token)
+}
+
 func (h *Handle) Logout() {
 	h.b.Logout()
 }

ipn/ipnserver/server.go

@@ -18,11 +18,11 @@ import (
 	"syscall"
 	"time"
 
-	"github.com/klauspost/compress/zstd"
 	"tailscale.com/control/controlclient"
 	"tailscale.com/ipn"
 	"tailscale.com/logtail/backoff"
 	"tailscale.com/safesocket"
+	"tailscale.com/smallzstd"
 	"tailscale.com/types/logger"
 	"tailscale.com/version"
 	"tailscale.com/wgengine"
@@ -33,15 +33,19 @@ type Options struct {
 	// SocketPath, on unix systems, is the unix socket path to listen
 	// on for frontend connections.
 	SocketPath string
+
 	// Port, on windows, is the localhost TCP port to listen on for
 	// frontend connections.
 	Port int
+
 	// StatePath is the path to the stored agent state.
 	StatePath string
+
 	// AutostartStateKey, if non-empty, immediately starts the agent
 	// using the given StateKey. If empty, the agent stays idle and
 	// waits for a frontend to start it.
 	AutostartStateKey ipn.StateKey
+
 	// LegacyConfigPath optionally specifies the old-style relaynode
 	// relay.conf location. If both LegacyConfigPath and
 	// AutostartStateKey are specified and the requested state doesn't
@@ -51,10 +55,15 @@ type Options struct {
 	// TODO(danderson): remove some time after the transition to
 	// tailscaled is done.
 	LegacyConfigPath string
+
 	// SurviveDisconnects specifies how the server reacts to its
 	// frontend disconnecting. If true, the server keeps running on
 	// its existing state, and accepts new frontend connections. If
 	// false, the server dumps its state and becomes idle.
+	//
+	// To support CLI connections (notably, "tailscale status"),
+	// the actual definition of "disconnect" is when the
+	// connection count transitions from 1 to 0.
 	SurviveDisconnects bool
 
 	// DebugMux, if non-nil, specifies an HTTP ServeMux in which
@@ -62,42 +71,146 @@ type Options struct {
 	DebugMux *http.ServeMux
 }
 
-func pump(logf logger.Logf, ctx context.Context, bs *ipn.BackendServer, s net.Conn) {
-	defer logf("Control connection done.")
+// server is an IPN backend and its set of 0 or more active connections
+// talking to an IPN backend.
+type server struct {
+	resetOnZero bool // call bs.Reset on transition from 1->0 connections
 
-	for ctx.Err() == nil && !bs.GotQuit {
-		msg, err := ipn.ReadMsg(s)
+	bsMu sync.Mutex // lock order: bsMu, then mu
+	bs   *ipn.BackendServer
+
+	mu      sync.Mutex
+	clients map[net.Conn]bool
+}
+
+func (s *server) serveConn(ctx context.Context, c net.Conn, logf logger.Logf) {
+	s.addConn(c)
+	logf("incoming control connection")
+	defer s.removeAndCloseConn(c)
+	for ctx.Err() == nil {
+		msg, err := ipn.ReadMsg(c)
 		if err != nil {
-			logf("ReadMsg: %v", err)
-			break
+			if ctx.Err() == nil {
+				logf("ReadMsg: %v", err)
+			}
+			return
 		}
-		err = bs.GotCommandMsg(msg)
-		if err != nil {
+		s.bsMu.Lock()
+		if err := s.bs.GotCommandMsg(msg); err != nil {
 			logf("GotCommandMsg: %v", err)
-			break
+		}
+		gotQuit := s.bs.GotQuit
+		s.bsMu.Unlock()
+		if gotQuit {
+			return
 		}
 	}
 }
 
-func Run(rctx context.Context, logf logger.Logf, logid string, opts Options, e wgengine.Engine) (err error) {
-	runDone := make(chan error, 1)
-	defer func() { runDone <- err }()
+func (s *server) addConn(c net.Conn) {
+	s.mu.Lock()
+	defer s.mu.Unlock()
+	if s.clients == nil {
+		s.clients = map[net.Conn]bool{}
+	}
+	s.clients[c] = true
+}
+
+func (s *server) removeAndCloseConn(c net.Conn) {
+	s.mu.Lock()
+	delete(s.clients, c)
+	remain := len(s.clients)
+	s.mu.Unlock()
+
+	if remain == 0 && s.resetOnZero {
+		s.bsMu.Lock()
+		s.bs.Reset()
+		s.bsMu.Unlock()
+	}
+	c.Close()
+}
+
+func (s *server) stopAll() {
+	s.mu.Lock()
+	defer s.mu.Unlock()
+	for c := range s.clients {
+		safesocket.ConnCloseRead(c)
+		safesocket.ConnCloseWrite(c)
+	}
+	s.clients = nil
+}
+
+func (s *server) writeToClients(b []byte) {
+	s.mu.Lock()
+	defer s.mu.Unlock()
+	for c := range s.clients {
+		ipn.WriteMsg(c, b)
+	}
+}
+
+// Run runs a Tailscale backend service.
+// The getEngine func is called repeatedly, once per connection, until it returns an engine successfully.
+func Run(ctx context.Context, logf logger.Logf, logid string, getEngine func() (wgengine.Engine, error), opts Options) error {
+	runDone := make(chan struct{})
+	defer close(runDone)
 
 	listen, _, err := safesocket.Listen(opts.SocketPath, uint16(opts.Port))
 	if err != nil {
 		return fmt.Errorf("safesocket.Listen: %v", err)
 	}
 
-	// Go listeners can't take a context, close it instead.
+	server := &server{
+		resetOnZero: !opts.SurviveDisconnects,
+	}
+
+	// When the context is closed or when we return, whichever is first, close our listner
+	// and all open connections.
 	go func() {
 		select {
-		case <-rctx.Done():
+		case <-ctx.Done():
 		case <-runDone:
 		}
+		server.stopAll()
 		listen.Close()
 	}()
 	logf("Listening on %v", listen.Addr())
 
+	bo := backoff.NewBackoff("ipnserver", logf, 30*time.Second)
+
+	var unservedConn net.Conn // if non-nil, accepted, but hasn't served yet
+
+	eng, err := getEngine()
+	if err != nil {
+		logf("Initial getEngine call: %v", err)
+		for i := 1; ctx.Err() == nil; i++ {
+			c, err := listen.Accept()
+			if err != nil {
+				logf("%d: Accept: %v", i, err)
+				bo.BackOff(ctx, err)
+				continue
+			}
+			logf("%d: trying getEngine again...", i)
+			eng, err = getEngine()
+			if err == nil {
+				logf("%d: GetEngine worked; exiting failure loop", i)
+				unservedConn = c
+				break
+			}
+			logf("%d: getEngine failed again: %v", i, err)
+			errMsg := err.Error()
+			go func() {
+				defer c.Close()
+				serverToClient := func(b []byte) { ipn.WriteMsg(c, b) }
+				bs := ipn.NewBackendServer(logf, nil, serverToClient)
+				bs.SendErrorMessage(errMsg)
+				time.Sleep(time.Second)
+			}()
+		}
+		if err := ctx.Err(); err != nil {
+			return err
+		}
+	}
+
 	var store ipn.StateStore
 	if opts.StatePath != "" {
 		store, err = ipn.NewFileStore(opts.StatePath)
@@ -108,12 +221,13 @@ func Run(rctx context.Context, logf logger.Logf, logid string, opts Options, e w
 		store = &ipn.MemoryStore{}
 	}
 
-	b, err := ipn.NewLocalBackend(logf, logid, store, e)
+	b, err := ipn.NewLocalBackend(logf, logid, store, eng)
 	if err != nil {
 		return fmt.Errorf("NewLocalBackend: %v", err)
 	}
+	defer b.Shutdown()
 	b.SetDecompressor(func() (controlclient.Decompressor, error) {
-		return zstd.NewReader(nil)
+		return smallzstd.NewDecoder(nil)
 	})
 
 	if opts.DebugMux != nil {
@@ -125,17 +239,10 @@ func Run(rctx context.Context, logf logger.Logf, logid string, opts Options, e w
 		})
 	}
 
-	var s net.Conn
-	serverToClient := func(b []byte) {
-		if s != nil { // TODO: racy access to s?
-			ipn.WriteMsg(s, b)
-		}
-	}
-
-	bs := ipn.NewBackendServer(logf, b, serverToClient)
+	server.bs = ipn.NewBackendServer(logf, b, server.writeToClients)
 
 	if opts.AutostartStateKey != "" {
-		bs.GotCommand(&ipn.Command{
+		server.bs.GotCommand(&ipn.Command{
 			Version: version.LONG,
 			Start: &ipn.StartArgs{
 				Opts: ipn.Options{
@@ -146,55 +253,25 @@ func Run(rctx context.Context, logf logger.Logf, logid string, opts Options, e w
 		})
 	}
 
-	var (
-		oldS   net.Conn
-		ctx    context.Context
-		cancel context.CancelFunc
-	)
-	stopAll := func() {
-		// Currently we only support one client connection at a time.
-		// Theoretically we could allow multiple clients, by passing
-		// notifications to all of them and accepting commands from
-		// any of them, but there doesn't seem to be much need for
-		// that right now.
-		if oldS != nil {
-			cancel()
-			safesocket.ConnCloseRead(oldS)
-			safesocket.ConnCloseWrite(oldS)
+	for i := 1; ctx.Err() == nil; i++ {
+		var c net.Conn
+		var err error
+		if unservedConn != nil {
+			c = unservedConn
+			unservedConn = nil
+		} else {
+			c, err = listen.Accept()
 		}
-	}
-
-	bo := backoff.Backoff{Name: "ipnserver"}
-
-	for i := 1; rctx.Err() == nil; i++ {
-		s, err = listen.Accept()
 		if err != nil {
-			logf("%d: Accept: %v", i, err)
-			bo.BackOff(rctx, err)
+			if ctx.Err() == nil {
+				logf("ipnserver: Accept: %v", err)
+				bo.BackOff(ctx, err)
+			}
 			continue
 		}
-		logf("%d: Incoming control connection.", i)
-		stopAll()
-
-		ctx, cancel = context.WithCancel(rctx)
-		oldS = s
-
-		go func(ctx context.Context, s net.Conn, i int) {
-			logf := logger.WithPrefix(logf, fmt.Sprintf("%d: ", i))
-			pump(logf, ctx, bs, s)
-			if !opts.SurviveDisconnects || bs.GotQuit {
-				bs.Reset()
-				s.Close()
-			}
-			// Quitting not allowed, just keep going.
-			bs.GotQuit = false
-		}(ctx, s, i)
-
-		bo.BackOff(ctx, nil)
+		go server.serveConn(ctx, c, logger.WithPrefix(logf, fmt.Sprintf("ipnserver: conn%d: ", i)))
 	}
-	stopAll()
-
-	return rctx.Err()
+	return ctx.Err()
 }
 
 func BabysitProc(ctx context.Context, args []string, logf logger.Logf) {
@@ -229,7 +306,7 @@ func BabysitProc(ctx context.Context, args []string, logf logger.Logf) {
 		proc.mu.Unlock()
 	}()
 
-	bo := backoff.Backoff{Name: "BabysitProc"}
+	bo := backoff.NewBackoff("BabysitProc", logf, 30*time.Second)
 
 	for {
 		startTime := time.Now()
@@ -312,3 +389,8 @@ func BabysitProc(ctx context.Context, args []string, logf logger.Logf) {
 		}
 	}
 }
+
+// FixedEngine returns a func that returns eng and a nil error.
+func FixedEngine(eng wgengine.Engine) func() (wgengine.Engine, error) {
+	return func() (wgengine.Engine, error) { return eng, nil }
+}

ipn/ipnserver/server_test.go

@@ -72,6 +72,6 @@ func TestRunMultipleAccepts(t *testing.T) {
 		SocketPath: socketPath,
 	}
 	t.Logf("pre-Run")
-	err = ipnserver.Run(ctx, logTriggerTestf, "dummy_logid", opts, eng)
+	err = ipnserver.Run(ctx, logTriggerTestf, "dummy_logid", ipnserver.FixedEngine(eng), opts)
 	t.Logf("ipnserver.Run = %v", err)
 }

ipn/ipnstate/ipnstate.go

@@ -18,6 +18,7 @@ import (
 	"sync"
 	"time"
 
+	"inet.af/netaddr"
 	"tailscale.com/tailcfg"
 	"tailscale.com/types/key"
 )
@@ -25,6 +26,7 @@ import (
 // Status represents the entire state of the IPN network.
 type Status struct {
 	BackendState string
+	TailscaleIPs []netaddr.IP // Tailscale IP(s) assigned to this node
 	Peer         map[key.Public]*PeerStatus
 	User         map[tailcfg.UserID]tailcfg.UserProfile
 }
@@ -49,10 +51,12 @@ type PeerStatus struct {
 	// Endpoints:
 	Addrs   []string
 	CurAddr string // one of Addrs, or unique if roaming
+	Relay   string // DERP region
 
 	RxBytes       int64
 	TxBytes       int64
 	Created       time.Time // time registered with tailcontrol
+	LastWrite     time.Time // time last packet sent
 	LastSeen      time.Time // last seen to tailcontrol
 	LastHandshake time.Time // with local wireguard
 	KeepAlive     bool
@@ -107,6 +111,18 @@ func (sb *StatusBuilder) AddUser(id tailcfg.UserID, up tailcfg.UserProfile) {
 	sb.st.User[id] = up
 }
 
+// AddIP adds a Tailscale IP address to the status.
+func (sb *StatusBuilder) AddTailscaleIP(ip netaddr.IP) {
+	sb.mu.Lock()
+	defer sb.mu.Unlock()
+	if sb.locked {
+		log.Printf("[unexpected] ipnstate: AddIP after Locked")
+		return
+	}
+
+	sb.st.TailscaleIPs = append(sb.st.TailscaleIPs, ip)
+}
+
 // AddPeer adds a peer node to the status.
 //
 // Its PeerStatus is mixed with any previous status already added.
@@ -135,6 +151,9 @@ func (sb *StatusBuilder) AddPeer(peer key.Public, st *PeerStatus) {
 	if v := st.HostName; v != "" {
 		e.HostName = v
 	}
+	if v := st.Relay; v != "" {
+		e.Relay = v
+	}
 	if v := st.UserID; v != 0 {
 		e.UserID = v
 	}
@@ -165,6 +184,9 @@ func (sb *StatusBuilder) AddPeer(peer key.Public, st *PeerStatus) {
 	if v := st.LastSeen; !v.IsZero() {
 		e.LastSeen = v
 	}
+	if v := st.LastWrite; !v.IsZero() {
+		e.LastWrite = v
+	}
 	if st.InNetworkMap {
 		e.InNetworkMap = true
 	}
@@ -186,35 +208,50 @@ type StatusUpdater interface {
 func (st *Status) WriteHTML(w io.Writer) {
 	f := func(format string, args ...interface{}) { fmt.Fprintf(w, format, args...) }
 
-	f(`<html><head><style>
-.owner { font-size: 80%%; color: #444; }
-.tailaddr { font-size: 80%%; font-family: monospace: }
-</style></head>`)
-	f("<body><h1>Tailscale State</h1>")
+	f(`<!DOCTYPE html>
+<html lang="en">
+<head>
+<title>Tailscale State</title>
+<style>
+body { font-family: monospace; }
+.owner { text-decoration: underline; }
+.tailaddr { font-style: italic; }
+.acenter { text-align: center; }
+.aright { text-align: right; }
+table, th, td { border: 1px solid black; border-spacing : 0; border-collapse : collapse; }
+thead { background-color: #FFA500; }
+th, td { padding: 5px; }
+td { vertical-align: top; }
+table tbody tr:nth-child(even) td { background-color: #f5f5f5; }
+</style>
+</head>
+<body>
+<h1>Tailscale State</h1>
+`)
+
 	//f("<p><b>logid:</b> %s</p>\n", logid)
 	//f("<p><b>opts:</b> <code>%s</code></p>\n", html.EscapeString(fmt.Sprintf("%+v", opts)))
 
-	f("<table border=1 cellpadding=5><tr><th>Peer</th><th>Node</th><th>Rx</th><th>Tx</th><th>Handshake</th><th>Endpoints</th></tr>")
+	ips := make([]string, 0, len(st.TailscaleIPs))
+	for _, ip := range st.TailscaleIPs {
+		ips = append(ips, ip.String())
+	}
+	f("<p>Tailscale IP: %s", strings.Join(ips, ", "))
+
+	f("<table>\n<thead>\n")
+	f("<tr><th>Peer</th><th>Node</th><th>Owner</th><th>Rx</th><th>Tx</th><th>Activity</th><th>Endpoints</th></tr>\n")
+	f("</thead>\n<tbody>\n")
 
 	now := time.Now()
 
-	// The tailcontrol server rounds LastSeen to 10 minutes. So we
-	// declare that a longAgo seen time of 15 minutes means
-	// they're not connected.
-	longAgo := now.Add(-15 * time.Minute)
-
 	for _, peer := range st.Peers() {
 		ps := st.Peer[peer]
-		var hsAgo string
-		if !ps.LastHandshake.IsZero() {
-			hsAgo = now.Sub(ps.LastHandshake).Round(time.Second).String() + " ago"
-		} else {
-			if ps.LastSeen.Before(longAgo) {
-				hsAgo = "<i>offline</i>"
-			} else if !ps.KeepAlive {
-				hsAgo = "on demand"
-			} else {
-				hsAgo = "<b>pending</b>"
+		var actAgo string
+		if !ps.LastWrite.IsZero() {
+			ago := now.Sub(ps.LastWrite)
+			actAgo = ago.Round(time.Second).String() + " ago"
+			if ago < 5*time.Minute {
+				actAgo = "<b>" + actAgo + "</b>"
 			}
 		}
 		var owner string
@@ -224,33 +261,46 @@ func (st *Status) WriteHTML(w io.Writer) {
 				owner = owner[:i]
 			}
 		}
-		f("<tr><td>%s</td><td>%s<div class=owner>%s</div><div class=tailaddr>%s</div></td><td>%v</td><td>%v</td><td>%v</td>",
+		f("<tr><td>%s</td><td>%s %s<br><span class=\"tailaddr\">%s</span></td><td class=\"acenter owner\">%s</td><td class=\"aright\">%v</td><td class=\"aright\">%v</td><td class=\"aright\">%v</td>",
 			peer.ShortString(),
-			osEmoji(ps.OS)+" "+html.EscapeString(ps.SimpleHostName()),
-			html.EscapeString(owner),
+			html.EscapeString(ps.SimpleHostName()),
+			osEmoji(ps.OS),
 			ps.TailAddr,
+			html.EscapeString(owner),
 			ps.RxBytes,
 			ps.TxBytes,
-			hsAgo,
+			actAgo,
 		)
-		f("<td>")
+		f("<td class=\"aright\">")
+		// TODO: let server report this active bool instead
+		active := !ps.LastWrite.IsZero() && time.Since(ps.LastWrite) < 2*time.Minute
+		relay := ps.Relay
+		if relay != "" {
+			if active && ps.CurAddr == "" {
+				f("🔗 <b>derp-%v</b><br>", html.EscapeString(relay))
+			} else {
+				f("derp-%v<br>", html.EscapeString(relay))
+			}
+		}
+
 		match := false
 		for _, addr := range ps.Addrs {
 			if addr == ps.CurAddr {
 				match = true
-				f("<b>%s</b> 🔗<br>\n", addr)
+				f("🔗 <b>%s</b><br>", addr)
 			} else {
-				f("%s<br>\n", addr)
+				f("%s<br>", addr)
 			}
 		}
 		if ps.CurAddr != "" && !match {
-			f("<b>%s</b> \xf0\x9f\xa7\xb3<br>\n", ps.CurAddr)
+			f("<b>%s</b> \xf0\x9f\xa7\xb3<br>", ps.CurAddr)
 		}
-		f("</tr>") // end Addrs
+		f("</td>") // end Addrs
 
 		f("</tr>\n")
 	}
-	f("</table>")
+	f("</tbody>\n</table>\n")
+	f("</body>\n</html>\n")
 }
 
 func osEmoji(os string) string {

ipn/message.go

@@ -13,6 +13,7 @@ import (
 	"log"
 	"time"
 
+	"golang.org/x/oauth2"
 	"tailscale.com/types/logger"
 	"tailscale.com/types/structs"
 	"tailscale.com/version"
@@ -35,13 +36,21 @@ type FakeExpireAfterArgs struct {
 // Command is a command message that is JSON encoded and sent by a
 // frontend to a backend.
 type Command struct {
-	_       structs.Incomparable
+	_ structs.Incomparable
+
+	// Version is the binary version of the frontend (the client).
 	Version string
 
+	// AllowVersionSkew controls whether it's permitted for the
+	// client and server to have a different version. The default
+	// (false) means to be strict.
+	AllowVersionSkew bool
+
 	// Exactly one of the following must be non-nil.
 	Quit                  *NoArgs
 	Start                 *StartArgs
 	StartLoginInteractive *NoArgs
+	Login                 *oauth2.Token
 	Logout                *NoArgs
 	SetPrefs              *SetPrefsArgs
 	RequestEngineStatus   *NoArgs
@@ -73,6 +82,10 @@ func (bs *BackendServer) send(n Notify) {
 	bs.sendNotifyMsg(b)
 }
 
+func (bs *BackendServer) SendErrorMessage(msg string) {
+	bs.send(Notify{ErrMessage: &msg})
+}
+
 // GotCommandMsg parses the incoming message b as a JSON Command and
 // calls GotCommand with it.
 func (bs *BackendServer) GotCommandMsg(b []byte) error {
@@ -92,7 +105,7 @@ func (bs *BackendServer) GotFakeCommand(cmd *Command) error {
 }
 
 func (bs *BackendServer) GotCommand(cmd *Command) error {
-	if cmd.Version != version.LONG {
+	if cmd.Version != version.LONG && !cmd.AllowVersionSkew {
 		vs := fmt.Sprintf("GotCommand: Version mismatch! frontend=%#v backend=%#v",
 			cmd.Version, version.LONG)
 		bs.logf("%s", vs)
@@ -117,6 +130,9 @@ func (bs *BackendServer) GotCommand(cmd *Command) error {
 	} else if c := cmd.StartLoginInteractive; c != nil {
 		bs.b.StartLoginInteractive()
 		return nil
+	} else if c := cmd.Login; c != nil {
+		bs.b.Login(c)
+		return nil
 	} else if c := cmd.Logout; c != nil {
 		bs.b.Logout()
 		return nil
@@ -147,6 +163,10 @@ type BackendClient struct {
 	logf           logger.Logf
 	sendCommandMsg func(jsonb []byte)
 	notify         func(Notify)
+
+	// AllowVersionSkew controls whether to allow mismatched
+	// frontend & backend versions.
+	AllowVersionSkew bool
 }
 
 func NewBackendClient(logf logger.Logf, sendCommandMsg func(jsonb []byte)) *BackendClient {
@@ -165,7 +185,7 @@ func (bc *BackendClient) GotNotifyMsg(b []byte) {
 	if err := json.Unmarshal(b, &n); err != nil {
 		log.Fatalf("BackendClient.Notify: cannot decode message (length=%d)\n%#v", len(b), string(b))
 	}
-	if n.Version != version.LONG {
+	if n.Version != version.LONG && !bc.AllowVersionSkew {
 		vs := fmt.Sprintf("GotNotify: Version mismatch! frontend=%#v backend=%#v",
 			version.LONG, n.Version)
 		bc.logf("%s", vs)
@@ -210,6 +230,10 @@ func (bc *BackendClient) StartLoginInteractive() {
 	bc.send(Command{StartLoginInteractive: &NoArgs{}})
 }
 
+func (bc *BackendClient) Login(token *oauth2.Token) {
+	bc.send(Command{Login: token})
+}
+
 func (bc *BackendClient) Logout() {
 	bc.send(Command{Logout: &NoArgs{}})
 }
@@ -223,7 +247,7 @@ func (bc *BackendClient) RequestEngineStatus() {
 }
 
 func (bc *BackendClient) RequestStatus() {
-	bc.send(Command{RequestStatus: &NoArgs{}})
+	bc.send(Command{AllowVersionSkew: true, RequestStatus: &NoArgs{}})
 }
 
 func (bc *BackendClient) FakeExpireAfter(x time.Duration) {
@@ -231,7 +255,7 @@ func (bc *BackendClient) FakeExpireAfter(x time.Duration) {
 }
 
 // MaxMessageSize is the maximum message size, in bytes.
-const MaxMessageSize = 1 << 20
+const MaxMessageSize = 10 << 20
 
 // TODO(apenwarr): incremental json decode?
 //  That would let us avoid storing the whole byte array uselessly in RAM.

ipn/message_test.go

@@ -9,12 +9,12 @@ import (
 	"testing"
 	"time"
 
+	"golang.org/x/oauth2"
 	"tailscale.com/tstest"
 )
 
 func TestReadWrite(t *testing.T) {
-	tstest.FixLogs(t)
-	defer tstest.UnfixLogs(t)
+	tstest.PanicOnLog()
 
 	rc := tstest.NewResourceCheck()
 	defer rc.Assert(t)
@@ -62,8 +62,7 @@ func TestReadWrite(t *testing.T) {
 }
 
 func TestClientServer(t *testing.T) {
-	tstest.FixLogs(t)
-	defer tstest.UnfixLogs(t)
+	tstest.PanicOnLog()
 
 	rc := tstest.NewResourceCheck()
 	defer rc.Assert(t)
@@ -179,4 +178,10 @@ func TestClientServer(t *testing.T) {
 
 	h.Logout()
 	flushUntil(NeedsLogin)
+
+	h.Login(&oauth2.Token{
+		AccessToken: "google_id_token",
+		TokenType:   GoogleIDTokenType,
+	})
+	flushUntil(Running)
 }

ipn/prefs.go

@@ -15,6 +15,7 @@ import (
 	"github.com/tailscale/wireguard-go/wgcfg"
 	"tailscale.com/atomicfile"
 	"tailscale.com/control/controlclient"
+	"tailscale.com/wgengine/router"
 )
 
 // Prefs are the user modifiable settings of the Tailscale node agent.
@@ -44,15 +45,19 @@ type Prefs struct {
 	// use the packet filter as provided. If true, we block incoming
 	// connections.
 	ShieldsUp bool
-	// AdvertiseRoutes specifies CIDR prefixes to advertise into the
-	// Tailscale network as reachable through the current node.
-	AdvertiseRoutes []wgcfg.CIDR
 	// AdvertiseTags specifies groups that this node wants to join, for
 	// purposes of ACL enforcement. These can be referenced from the ACL
 	// security policy. Note that advertising a tag doesn't guarantee that
 	// the control server will allow you to take on the rights for that
 	// tag.
 	AdvertiseTags []string
+	// Hostname is the hostname to use for identifying the node. If
+	// not set, os.Hostname is used.
+	Hostname string
+	// OSVersion overrides tailcfg.Hostinfo's OSVersion.
+	OSVersion string
+	// DeviceModel overrides tailcfg.Hostinfo's DeviceModel.
+	DeviceModel string
 
 	// NotepadURLs is a debugging setting that opens OAuth URLs in
 	// notepad.exe on Windows, rather than loading them in a browser.
@@ -65,6 +70,27 @@ type Prefs struct {
 	// DisableDERP prevents DERP from being used.
 	DisableDERP bool
 
+	// The following block of options only have an effect on Linux.
+
+	// AdvertiseRoutes specifies CIDR prefixes to advertise into the
+	// Tailscale network as reachable through the current
+	// node.
+	AdvertiseRoutes []wgcfg.CIDR
+	// NoSNAT specifies whether to source NAT traffic going to
+	// destinations in AdvertiseRoutes. The default is to apply source
+	// NAT, which makes the traffic appear to come from the router
+	// machine rather than the peer's Tailscale IP.
+	//
+	// Disabling SNAT requires additional manual configuration in your
+	// network to route Tailscale traffic back to the subnet relay
+	// machine.
+	//
+	// Linux-only.
+	NoSNAT bool
+	// NetfilterMode specifies how much to manage netfilter rules for
+	// Tailscale, if at all.
+	NetfilterMode router.NetfilterMode
+
 	// The Persist field is named 'Config' in the file for backward
 	// compatibility with earlier versions.
 	// TODO(apenwarr): We should move this out of here, it's not a pref.
@@ -83,9 +109,9 @@ func (p *Prefs) Pretty() string {
 	} else {
 		pp = "Persist=nil"
 	}
-	return fmt.Sprintf("Prefs{ra=%v mesh=%v dns=%v want=%v notepad=%v derp=%v shields=%v routes=%v %v}",
+	return fmt.Sprintf("Prefs{ra=%v mesh=%v dns=%v want=%v notepad=%v derp=%v shields=%v routes=%v snat=%v nf=%v %v}",
 		p.RouteAll, p.AllowSingleHosts, p.CorpDNS, p.WantRunning,
-		p.NotepadURLs, !p.DisableDERP, p.ShieldsUp, p.AdvertiseRoutes, pp)
+		p.NotepadURLs, !p.DisableDERP, p.ShieldsUp, p.AdvertiseRoutes, !p.NoSNAT, p.NetfilterMode, pp)
 }
 
 func (p *Prefs) ToBytes() []byte {
@@ -113,6 +139,11 @@ func (p *Prefs) Equals(p2 *Prefs) bool {
 		p.NotepadURLs == p2.NotepadURLs &&
 		p.DisableDERP == p2.DisableDERP &&
 		p.ShieldsUp == p2.ShieldsUp &&
+		p.NoSNAT == p2.NoSNAT &&
+		p.NetfilterMode == p2.NetfilterMode &&
+		p.Hostname == p2.Hostname &&
+		p.OSVersion == p2.OSVersion &&
+		p.DeviceModel == p2.DeviceModel &&
 		compareIPNets(p.AdvertiseRoutes, p2.AdvertiseRoutes) &&
 		compareStrings(p.AdvertiseTags, p2.AdvertiseTags) &&
 		p.Persist.Equals(p2.Persist)
@@ -152,6 +183,7 @@ func NewPrefs() *Prefs {
 		AllowSingleHosts: true,
 		CorpDNS:          true,
 		WantRunning:      true,
+		NetfilterMode:    router.NetfilterOn,
 	}
 }
 
@@ -191,10 +223,9 @@ func (p *Prefs) Clone() *Prefs {
 	return p2
 }
 
-// LoadLegacyPrefs loads a legacy relaynode config file into Prefs
-// with sensible migration defaults set. If enforceDefaults is true,
-// Prefs.RouteAll and Prefs.AllowSingleHosts are forced on.
-func LoadPrefs(filename string, enforceDefaults bool) (*Prefs, error) {
+// LoadPrefs loads a legacy relaynode config file into Prefs
+// with sensible migration defaults set.
+func LoadPrefs(filename string) (*Prefs, error) {
 	data, err := ioutil.ReadFile(filename)
 	if err != nil {
 		return nil, fmt.Errorf("loading prefs from %q: %v", filename, err)

ipn/prefs_test.go

@@ -10,6 +10,8 @@ import (
 
 	"github.com/tailscale/wireguard-go/wgcfg"
 	"tailscale.com/control/controlclient"
+	"tailscale.com/tstest"
+	"tailscale.com/wgengine/router"
 )
 
 func fieldsOf(t reflect.Type) (fields []string) {
@@ -20,7 +22,9 @@ func fieldsOf(t reflect.Type) (fields []string) {
 }
 
 func TestPrefsEqual(t *testing.T) {
-	prefsHandles := []string{"ControlURL", "RouteAll", "AllowSingleHosts", "CorpDNS", "WantRunning", "ShieldsUp", "AdvertiseRoutes", "AdvertiseTags", "NotepadURLs", "DisableDERP", "Persist"}
+	tstest.PanicOnLog()
+
+	prefsHandles := []string{"ControlURL", "RouteAll", "AllowSingleHosts", "CorpDNS", "WantRunning", "ShieldsUp", "AdvertiseTags", "Hostname", "OSVersion", "DeviceModel", "NotepadURLs", "DisableDERP", "AdvertiseRoutes", "NoSNAT", "NetfilterMode", "Persist"}
 	if have := fieldsOf(reflect.TypeOf(Prefs{})); !reflect.DeepEqual(have, prefsHandles) {
 		t.Errorf("Prefs.Equal check might be out of sync\nfields: %q\nhandled: %q\n",
 			have, prefsHandles)
@@ -111,6 +115,28 @@ func TestPrefsEqual(t *testing.T) {
 			true,
 		},
 
+		{
+			&Prefs{NoSNAT: true},
+			&Prefs{NoSNAT: false},
+			false,
+		},
+		{
+			&Prefs{NoSNAT: true},
+			&Prefs{NoSNAT: true},
+			true,
+		},
+
+		{
+			&Prefs{Hostname: "android-host01"},
+			&Prefs{Hostname: "android-host02"},
+			false,
+		},
+		{
+			&Prefs{Hostname: ""},
+			&Prefs{Hostname: ""},
+			true,
+		},
+
 		{
 			&Prefs{NotepadURLs: true},
 			&Prefs{NotepadURLs: false},
@@ -159,6 +185,17 @@ func TestPrefsEqual(t *testing.T) {
 			true,
 		},
 
+		{
+			&Prefs{NetfilterMode: router.NetfilterOff},
+			&Prefs{NetfilterMode: router.NetfilterOn},
+			false,
+		},
+		{
+			&Prefs{NetfilterMode: router.NetfilterOn},
+			&Prefs{NetfilterMode: router.NetfilterOn},
+			true,
+		},
+
 		{
 			&Prefs{Persist: &controlclient.Persist{}},
 			&Prefs{Persist: &controlclient.Persist{LoginName: "dave"}},
@@ -220,6 +257,8 @@ func checkPrefs(t *testing.T, p Prefs) {
 }
 
 func TestBasicPrefs(t *testing.T) {
+	tstest.PanicOnLog()
+
 	p := Prefs{
 		ControlURL: "https://login.tailscale.com",
 	}
@@ -227,6 +266,8 @@ func TestBasicPrefs(t *testing.T) {
 }
 
 func TestPrefsPersist(t *testing.T) {
+	tstest.PanicOnLog()
+
 	c := controlclient.Persist{
 		LoginName: "test@example.com",
 	}

ipn/store_test.go

@@ -8,6 +8,8 @@ import (
 	"io/ioutil"
 	"os"
 	"testing"
+
+	"tailscale.com/tstest"
 )
 
 func testStoreSemantics(t *testing.T, store StateStore) {
@@ -76,11 +78,15 @@ func testStoreSemantics(t *testing.T, store StateStore) {
 }
 
 func TestMemoryStore(t *testing.T) {
+	tstest.PanicOnLog()
+
 	store := &MemoryStore{}
 	testStoreSemantics(t, store)
 }
 
 func TestFileStore(t *testing.T) {
+	tstest.PanicOnLog()
+
 	f, err := ioutil.TempFile("", "test_ipn_store")
 	if err != nil {
 		t.Fatal(err)

log/logheap/logheap.go

@@ -0,0 +1,41 @@
+// Copyright (c) 2020 Tailscale Inc & AUTHORS All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+// Package logheap logs a heap pprof profile.
+package logheap
+
+import (
+	"bytes"
+	"context"
+	"log"
+	"net/http"
+	"runtime"
+	"runtime/pprof"
+	"time"
+)
+
+// LogHeap writes a JSON logtail record with the base64 heap pprof to
+// os.Stderr.
+func LogHeap(postURL string) {
+	if postURL == "" {
+		return
+	}
+	runtime.GC()
+	buf := new(bytes.Buffer)
+	pprof.WriteHeapProfile(buf)
+
+	ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
+	defer cancel()
+	req, err := http.NewRequestWithContext(ctx, "POST", postURL, buf)
+	if err != nil {
+		log.Printf("LogHeap: %v", err)
+		return
+	}
+	res, err := http.DefaultClient.Do(req)
+	if err != nil {
+		log.Printf("LogHeap: %v", err)
+		return
+	}
+	defer res.Body.Close()
+}

logpolicy/logpolicy.go

@@ -18,18 +18,21 @@ import (
 	"net"
 	"net/http"
 	"os"
+	"os/exec"
 	"path/filepath"
 	"runtime"
 	"strconv"
 	"strings"
 	"time"
 
-	"github.com/klauspost/compress/zstd"
 	"golang.org/x/crypto/ssh/terminal"
 	"tailscale.com/atomicfile"
 	"tailscale.com/logtail"
 	"tailscale.com/logtail/filch"
+	"tailscale.com/net/netns"
 	"tailscale.com/net/tlsdial"
+	"tailscale.com/smallzstd"
+	"tailscale.com/types/logger"
 	"tailscale.com/version"
 )
 
@@ -52,7 +55,7 @@ type Policy struct {
 func (c *Config) ToBytes() []byte {
 	data, err := json.MarshalIndent(c, "", "\t")
 	if err != nil {
-		log.Fatalf("logpolicy.Config marshal: %v\n", err)
+		log.Fatalf("logpolicy.Config marshal: %v", err)
 	}
 	return data
 }
@@ -99,21 +102,37 @@ func (l logWriter) Write(buf []byte) (int, error) {
 
 // logsDir returns the directory to use for log configuration and
 // buffer storage.
-func logsDir() string {
-	systemdCacheDir := os.Getenv("CACHE_DIRECTORY")
-	if systemdCacheDir != "" {
-		return systemdCacheDir
+func logsDir(logf logger.Logf) string {
+	systemdStateDir := os.Getenv("STATE_DIRECTORY")
+	if systemdStateDir != "" {
+		logf("logpolicy: using $STATE_DIRECTORY, %q", systemdStateDir)
+		return systemdStateDir
 	}
 
 	cacheDir, err := os.UserCacheDir()
 	if err == nil {
-		return filepath.Join(cacheDir, "Tailscale")
+		d := filepath.Join(cacheDir, "Tailscale")
+		logf("logpolicy: using UserCacheDir, %q", d)
+		return d
 	}
 
-	// No idea where to put stuff. This only happens when $HOME is
-	// unset, which os.UserCacheDir doesn't like. Use the current
-	// working directory and hope for the best.
-	return ""
+	// Use the current working directory, unless we're being run by a
+	// service manager that sets it to /.
+	wd, err := os.Getwd()
+	if err == nil && wd != "/" {
+		logf("logpolicy: using current directory, %q", wd)
+		return wd
+	}
+
+	// No idea where to put stuff. Try to create a temp dir. It'll
+	// mean we might lose some logs and rotate through log IDs, but
+	// it's something.
+	tmp, err := ioutil.TempDir("", "tailscaled-log-*")
+	if err != nil {
+		panic("no safe place found to store log state")
+	}
+	logf("logpolicy: using temp directory, %q", tmp)
+	return tmp
 }
 
 // runningUnderSystemd reports whether we're running under systemd.
@@ -125,6 +144,155 @@ func runningUnderSystemd() bool {
 	return false
 }
 
+// tryFixLogStateLocation is a temporary fixup for
+// https://github.com/tailscale/tailscale/issues/247 . We accidentally
+// wrote logging state files to /, and then later to $CACHE_DIRECTORY
+// (which is incorrect because the log ID is not reconstructible if
+// deleted - it's state, not cache data).
+//
+// If log state for cmdname exists in / or $CACHE_DIRECTORY, and no
+// log state for that command exists in dir, then the log state is
+// moved from whereever it does exist, into dir. Leftover logs state
+// in / and $CACHE_DIRECTORY is deleted.
+func tryFixLogStateLocation(dir, cmdname string) {
+	switch runtime.GOOS {
+	case "linux", "freebsd", "openbsd":
+		// These are the OSes where we might have written stuff into
+		// root. Others use different logic to find the logs storage
+		// dir.
+	default:
+		return
+	}
+	if cmdname == "" {
+		log.Printf("[unexpected] no cmdname given to tryFixLogStateLocation, please file a bug at https://github.com/tailscale/tailscale")
+		return
+	}
+	if dir == "/" {
+		// Trying to store things in / still. That's a bug, but don't
+		// abort hard.
+		log.Printf("[unexpected] storing logging config in /, please file a bug at https://github.com/tailscale/tailscale")
+		return
+	}
+	if os.Getuid() != 0 {
+		// Only root could have written log configs to weird places.
+		return
+	}
+
+	// We stored logs in 2 incorrect places: either /, or CACHE_DIR
+	// (aka /var/cache/tailscale). We want to move files into the
+	// provided dir, preferring those in CACHE_DIR over those in / if
+	// both exist. If files already exist in dir, don't
+	// overwrite. Finally, once we've maybe moved files around, we
+	// want to delete leftovers in / and CACHE_DIR, to clean up after
+	// our past selves.
+
+	files := []string{
+		fmt.Sprintf("%s.log.conf", cmdname),
+		fmt.Sprintf("%s.log1.txt", cmdname),
+		fmt.Sprintf("%s.log2.txt", cmdname),
+	}
+
+	// checks if any of the files above exist in d.
+	checkExists := func(d string) (bool, error) {
+		for _, file := range files {
+			p := filepath.Join(d, file)
+			_, err := os.Stat(p)
+			if os.IsNotExist(err) {
+				continue
+			} else if err != nil {
+				return false, fmt.Errorf("stat %q: %w", p, err)
+			}
+			return true, nil
+		}
+		return false, nil
+	}
+	// move files from d into dir, if they exist.
+	moveFiles := func(d string) error {
+		for _, file := range files {
+			src := filepath.Join(d, file)
+			_, err := os.Stat(src)
+			if os.IsNotExist(err) {
+				continue
+			} else if err != nil {
+				return fmt.Errorf("stat %q: %v", src, err)
+			}
+			dst := filepath.Join(dir, file)
+			bs, err := exec.Command("mv", src, dst).CombinedOutput()
+			if err != nil {
+				return fmt.Errorf("mv %q %q: %v (%s)", src, dst, err, bs)
+			}
+		}
+		return nil
+	}
+
+	existsInRoot, err := checkExists("/")
+	if err != nil {
+		log.Printf("checking for configs in /: %v", err)
+		return
+	}
+	existsInCache := false
+	cacheDir := os.Getenv("CACHE_DIRECTORY")
+	if cacheDir != "" {
+		existsInCache, err = checkExists("/var/cache/tailscale")
+		if err != nil {
+			log.Printf("checking for configs in %s: %v", cacheDir, err)
+		}
+	}
+	existsInDest, err := checkExists(dir)
+	if err != nil {
+		log.Printf("checking for configs in %s: %v", dir, err)
+		return
+	}
+
+	switch {
+	case !existsInRoot && !existsInCache:
+		// No leftover files, nothing to do.
+		return
+	case existsInDest:
+		// Already have "canonical" configs, just delete any remnants
+		// (below).
+	case existsInCache:
+		// CACHE_DIRECTORY takes precedence over /, move files from
+		// there.
+		if err := moveFiles(cacheDir); err != nil {
+			log.Print(err)
+			return
+		}
+	case existsInRoot:
+		// Files from root is better than nothing.
+		if err := moveFiles("/"); err != nil {
+			log.Print(err)
+			return
+		}
+	}
+
+	// If moving succeeded, or we didn't need to move files, try to
+	// delete any leftover files, but it's okay if we can't delete
+	// them for some reason.
+	dirs := []string{}
+	if existsInCache {
+		dirs = append(dirs, cacheDir)
+	}
+	if existsInRoot {
+		dirs = append(dirs, "/")
+	}
+	for _, d := range dirs {
+		for _, file := range files {
+			p := filepath.Join(d, file)
+			_, err := os.Stat(p)
+			if os.IsNotExist(err) {
+				continue
+			} else if err != nil {
+				log.Printf("stat %q: %v", p, err)
+				return
+			}
+			if err := os.Remove(p); err != nil {
+				log.Printf("rm %q: %v", p, err)
+			}
+		}
+	}
+}
+
 // New returns a new log policy (a logger and its instance ID) for a
 // given collection name.
 func New(collection string) *Policy {
@@ -141,18 +309,28 @@ func New(collection string) *Policy {
 	}
 	console := log.New(stderrWriter{}, "", lflags)
 
-	dir := logsDir()
-	cfgPath := filepath.Join(dir, fmt.Sprintf("%s.log.conf", version.CmdName()))
+	var earlyErrBuf bytes.Buffer
+	earlyLogf := func(format string, a ...interface{}) {
+		fmt.Fprintf(&earlyErrBuf, format, a...)
+		earlyErrBuf.WriteByte('\n')
+	}
+
+	dir := logsDir(earlyLogf)
+
+	cmdName := version.CmdName()
+	tryFixLogStateLocation(dir, cmdName)
+
+	cfgPath := filepath.Join(dir, fmt.Sprintf("%s.log.conf", cmdName))
 	var oldc *Config
 	data, err := ioutil.ReadFile(cfgPath)
 	if err != nil {
-		log.Printf("logpolicy.Read %v: %v\n", cfgPath, err)
+		earlyLogf("logpolicy.Read %v: %v", cfgPath, err)
 		oldc = &Config{}
 		oldc.Collection = collection
 	} else {
 		oldc, err = ConfigFromBytes(data)
 		if err != nil {
-			log.Printf("logpolicy.Config unmarshal: %v\n", err)
+			earlyLogf("logpolicy.Config unmarshal: %v", err)
 			oldc = &Config{}
 		}
 	}
@@ -174,7 +352,7 @@ func New(collection string) *Policy {
 	newc.PublicID = newc.PrivateID.Public()
 	if newc != *oldc {
 		if err := newc.save(cfgPath); err != nil {
-			log.Printf("logpolicy.Config.Save: %v\n", err)
+			earlyLogf("logpolicy.Config.Save: %v", err)
 		}
 	}
 
@@ -183,7 +361,7 @@ func New(collection string) *Policy {
 		PrivateID:  newc.PrivateID,
 		Stderr:     logWriter{console},
 		NewZstdEncoder: func() logtail.Encoder {
-			w, err := zstd.NewWriter(nil)
+			w, err := smallzstd.NewEncoder(nil)
 			if err != nil {
 				panic(err)
 			}
@@ -192,22 +370,25 @@ func New(collection string) *Policy {
 		HTTPC: &http.Client{Transport: newLogtailTransport(logtail.DefaultHost)},
 	}
 
-	filchBuf, filchErr := filch.New(filepath.Join(dir, version.CmdName()), filch.Options{})
+	filchBuf, filchErr := filch.New(filepath.Join(dir, cmdName), filch.Options{})
 	if filchBuf != nil {
 		c.Buffer = filchBuf
 	}
-	lw := logtail.Log(c)
+	lw := logtail.Log(c, log.Printf)
 	log.SetFlags(0) // other logflags are set on console, not here
 	log.SetOutput(lw)
 
-	log.Printf("Program starting: v%v, Go %v: %#v\n",
+	log.Printf("Program starting: v%v, Go %v: %#v",
 		version.LONG,
 		strings.TrimPrefix(runtime.Version(), "go"),
 		os.Args)
-	log.Printf("LogID: %v\n", newc.PublicID)
+	log.Printf("LogID: %v", newc.PublicID)
 	if filchErr != nil {
 		log.Printf("filch failed: %v", filchErr)
 	}
+	if earlyErrBuf.Len() != 0 {
+		log.Printf("%s", earlyErrBuf.Bytes())
+	}
 
 	return &Policy{
 		Logtail:  lw,
@@ -226,7 +407,7 @@ func (p *Policy) Close() {
 // log upload if it can be done before ctx is canceled.
 func (p *Policy) Shutdown(ctx context.Context) error {
 	if p.Logtail != nil {
-		log.Printf("flushing log.\n")
+		log.Printf("flushing log.")
 		return p.Logtail.Shutdown(ctx)
 	}
 	return nil
@@ -245,11 +426,10 @@ func newLogtailTransport(host string) *http.Transport {
 
 	// Log whenever we dial:
 	tr.DialContext = func(ctx context.Context, netw, addr string) (net.Conn, error) {
-		nd := &net.Dialer{
+		nd := netns.FromDialer(&net.Dialer{
 			Timeout:   30 * time.Second,
 			KeepAlive: 30 * time.Second,
-			DualStack: true,
-		}
+		})
 		t0 := time.Now()
 		c, err := nd.DialContext(ctx, netw, addr)
 		d := time.Since(t0).Round(time.Millisecond)

logtail/backoff/backoff.go

@@ -2,57 +2,81 @@
 // Use of this source code is governed by a BSD-style
 // license that can be found in the LICENSE file.
 
+// Package backoff provides a back-off timer type.
 package backoff
 
 import (
 	"context"
-	"log"
 	"math/rand"
 	"time"
+
+	"tailscale.com/types/logger"
 )
 
-const MAX_BACKOFF_MSEC = 30000
-
+// Backoff tracks state the history of consecutive failures and sleeps
+// an increasing amount of time, up to a provided limit.
 type Backoff struct {
-	n int
+	n          int // number of consecutive failures
+	maxBackoff time.Duration
+
 	// Name is the name of this backoff timer, for logging purposes.
-	Name string
-	// NewTimer is the function that acts like time.NewTimer().
-	// You can override this in unit tests.
-	NewTimer func(d time.Duration) *time.Timer
+	name string
+	// logf is the function used for log messages when backing off.
+	logf logger.Logf
+
+	// NewTimer is the function that acts like time.NewTimer.
+	// It's for use in unit tests.
+	NewTimer func(time.Duration) *time.Timer
+
 	// LogLongerThan sets the minimum time of a single backoff interval
 	// before we mention it in the log.
 	LogLongerThan time.Duration
 }
 
-func (b *Backoff) BackOff(ctx context.Context, err error) {
-	if ctx.Err() == nil && err != nil {
-		b.n++
-		// n^2 backoff timer is a little smoother than the
-		// common choice of 2^n.
-		msec := b.n * b.n * 10
-		if msec > MAX_BACKOFF_MSEC {
-			msec = MAX_BACKOFF_MSEC
-		}
-		// Randomize the delay between 0.5-1.5 x msec, in order
-		// to prevent accidental "thundering herd" problems.
-		msec = rand.Intn(msec) + msec/2
-		dur := time.Duration(msec) * time.Millisecond
-		if dur >= b.LogLongerThan {
-			log.Printf("%s: backoff: %d msec\n", b.Name, msec)
-		}
-		newTimer := b.NewTimer
-		if newTimer == nil {
-			newTimer = time.NewTimer
-		}
-		t := newTimer(dur)
-		select {
-		case <-ctx.Done():
-			t.Stop()
-		case <-t.C:
-		}
-	} else {
-		// not a regular error
-		b.n = 0
+// NewBackoff returns a new Backoff timer with the provided name (for logging), logger,
+// and max backoff time. By default, all failures (calls to BackOff with a non-nil err)
+// are logged unless the returned Backoff.LogLongerThan is adjusted.
+func NewBackoff(name string, logf logger.Logf, maxBackoff time.Duration) *Backoff {
+	return &Backoff{
+		name:       name,
+		logf:       logf,
+		maxBackoff: maxBackoff,
+		NewTimer:   time.NewTimer,
+	}
+}
+
+// Backoff sleeps an increasing amount of time if err is non-nil.
+// and the context is not a
+// It resets the backoff schedule once err is nil.
+func (b *Backoff) BackOff(ctx context.Context, err error) {
+	if err == nil {
+		// No error. Reset number of consecutive failures.
+		b.n = 0
+		return
+	}
+	if ctx.Err() != nil {
+		// Fast path.
+		return
+	}
+
+	b.n++
+	// n^2 backoff timer is a little smoother than the
+	// common choice of 2^n.
+	d := time.Duration(b.n*b.n) * 10 * time.Millisecond
+	if d > b.maxBackoff {
+		d = b.maxBackoff
+	}
+	// Randomize the delay between 0.5-1.5 x msec, in order
+	// to prevent accidental "thundering herd" problems.
+	d = time.Duration(float64(d) * (rand.Float64() + 0.5))
+
+	if d >= b.LogLongerThan {
+		b.logf("%s: backoff: %d msec", b.name, d.Milliseconds())
+	}
+	t := b.NewTimer(d)
+	select {
+	case <-ctx.Done():
+		t.Stop()
+	case <-t.C:
 	}
 }

logtail/example/logtail/logtail.go

@@ -34,7 +34,7 @@ func main() {
 	logger := logtail.Log(logtail.Config{
 		Collection: *collection,
 		PrivateID:  id,
-	})
+	}, log.Printf)
 	log.SetOutput(io.MultiWriter(logger, os.Stdout))
 	defer logger.Flush()
 	defer log.Printf("logtail exited")

logtail/filch/filch_test.go

@@ -56,18 +56,18 @@ func (f *filchTest) close(t *testing.T) {
 	}
 }
 
-func genFilePrefix(t *testing.T) string {
+func genFilePrefix(t *testing.T) (dir, prefix string) {
 	t.Helper()
-	filePrefix, err := ioutil.TempDir("", "filch")
+	dir, err := ioutil.TempDir("", "filch")
 	if err != nil {
 		t.Fatal(err)
 	}
-	return filepath.Join(filePrefix, "ringbuffer-")
+	return dir, filepath.Join(dir, "ringbuffer-")
 }
 
 func TestQueue(t *testing.T) {
-	filePrefix := genFilePrefix(t)
-	defer os.RemoveAll(filepath.Dir(filePrefix))
+	td, filePrefix := genFilePrefix(t)
+	defer os.RemoveAll(td)
 
 	f := newFilchTest(t, filePrefix, Options{ReplaceStderr: false})
 
@@ -90,8 +90,8 @@ func TestQueue(t *testing.T) {
 
 func TestRecover(t *testing.T) {
 	t.Run("empty", func(t *testing.T) {
-		filePrefix := genFilePrefix(t)
-		defer os.RemoveAll(filepath.Dir(filePrefix))
+		td, filePrefix := genFilePrefix(t)
+		defer os.RemoveAll(td)
 		f := newFilchTest(t, filePrefix, Options{ReplaceStderr: false})
 		f.write(t, "hello")
 		f.read(t, "hello")
@@ -104,8 +104,8 @@ func TestRecover(t *testing.T) {
 	})
 
 	t.Run("cur", func(t *testing.T) {
-		filePrefix := genFilePrefix(t)
-		defer os.RemoveAll(filepath.Dir(filePrefix))
+		td, filePrefix := genFilePrefix(t)
+		defer os.RemoveAll(td)
 		f := newFilchTest(t, filePrefix, Options{ReplaceStderr: false})
 		f.write(t, "hello")
 		f.close(t)
@@ -123,8 +123,8 @@ func TestRecover(t *testing.T) {
 		filch_test.go:129: r.ReadLine()="hello", want "world"
 		*/
 
-		filePrefix := genFilePrefix(t)
-		defer os.RemoveAll(filepath.Dir(filePrefix))
+		td, filePrefix := genFilePrefix(t)
+		defer os.RemoveAll(td)
 		f := newFilchTest(t, filePrefix, Options{ReplaceStderr: false})
 		f.write(t, "hello")
 		f.read(t, "hello")
@@ -155,8 +155,8 @@ func TestFilchStderr(t *testing.T) {
 		stderrFD = 2
 	}()
 
-	filePrefix := genFilePrefix(t)
-	defer os.RemoveAll(filepath.Dir(filePrefix))
+	td, filePrefix := genFilePrefix(t)
+	defer os.RemoveAll(td)
 	f := newFilchTest(t, filePrefix, Options{ReplaceStderr: true})
 	f.write(t, "hello")
 	if _, err := fmt.Fprintf(pipeW, "filch\n"); err != nil {

logtail/logtail.go

@@ -17,6 +17,7 @@ import (
 	"time"
 
 	"tailscale.com/logtail/backoff"
+	tslogger "tailscale.com/types/logger"
 )
 
 // DefaultHost is the default host name to upload logs to when
@@ -73,7 +74,7 @@ type Config struct {
 	DrainLogs <-chan struct{}
 }
 
-func Log(cfg Config) Logger {
+func Log(cfg Config, logf tslogger.Logf) Logger {
 	if cfg.BaseURL == "" {
 		cfg.BaseURL = "https://" + DefaultHost
 	}
@@ -104,9 +105,7 @@ func Log(cfg Config) Logger {
 		sentinel:       make(chan int32, 16),
 		drainLogs:      cfg.DrainLogs,
 		timeNow:        cfg.TimeNow,
-		bo: backoff.Backoff{
-			Name: "logtail",
-		},
+		bo:             backoff.NewBackoff("logtail", logf, 30*time.Second),
 
 		shutdownStart: make(chan struct{}),
 		shutdownDone:  make(chan struct{}),
@@ -134,7 +133,7 @@ type logger struct {
 	drainLogs      <-chan struct{} // if non-nil, external signal to attempt a drain
 	sentinel       chan int32
 	timeNow        func() time.Time
-	bo             backoff.Backoff
+	bo             *backoff.Backoff
 	zstdEncoder    Encoder
 	uploadCancel   func()
 
@@ -274,10 +273,10 @@ func (l *logger) uploading(ctx context.Context) {
 			if err != nil {
 				fmt.Fprintf(l.stderr, "logtail: upload: %v\n", err)
 			}
+			l.bo.BackOff(ctx, err)
 			if uploaded {
 				break
 			}
-			l.bo.BackOff(ctx, err)
 		}
 
 		select {
@@ -463,5 +462,6 @@ func (l *logger) Write(buf []byte) (int, error) {
 		}
 	}
 	b := l.encode(buf)
-	return l.send(b)
+	_, err := l.send(b)
+	return len(buf), err
 }

logtail/logtail_test.go

@@ -16,7 +16,7 @@ func TestFastShutdown(t *testing.T) {
 
 	l := Log(Config{
 		BaseURL: "http://localhost:1234",
-	})
+	}, t.Logf)
 	l.Shutdown(ctx)
 }
 
@@ -32,3 +32,18 @@ func TestLoggerEncodeTextAllocs(t *testing.T) {
 		t.Logf("allocs = %d; want 1", int(n))
 	}
 }
+
+func TestLoggerWriteLength(t *testing.T) {
+	lg := &logger{
+		timeNow: time.Now,
+		buffer:  NewMemoryBuffer(1024),
+	}
+	inBuf := []byte("some text to encode")
+	n, err := lg.Write(inBuf)
+	if err != nil {
+		t.Error(err)
+	}
+	if n != len(inBuf) {
+		t.Errorf("logger.Write wrote %d bytes, expected %d", n, len(inBuf))
+	}
+}

metrics/metrics.go

@@ -40,3 +40,10 @@ func (m *LabelMap) Get(key string) *expvar.Int {
 	m.Add(key, 0)
 	return m.Map.Get(key).(*expvar.Int)
 }
+
+// GetFloat returns a direct pointer to the expvar.Float for key, creating it
+// if necessary.
+func (m *LabelMap) GetFloat(key string) *expvar.Float {
+	m.AddFloat(key, 0.0)
+	return m.Map.Get(key).(*expvar.Float)
+}

net/interfaces/interfaces.go

@@ -10,6 +10,9 @@ import (
 	"net"
 	"reflect"
 	"strings"
+
+	"inet.af/netaddr"
+	"tailscale.com/net/tsaddr"
 )
 
 // Tailscale returns the current machine's Tailscale interface, if any.
@@ -37,39 +40,6 @@ func Tailscale() (net.IP, *net.Interface, error) {
 	return nil, nil, nil
 }
 
-// HaveIPv6GlobalAddress reports whether the machine appears to have a
-// global scope unicast IPv6 address.
-//
-// It only returns an error if there's a problem querying the system
-// interfaces.
-func HaveIPv6GlobalAddress() (bool, error) {
-	ifs, err := net.Interfaces()
-	if err != nil {
-		return false, err
-	}
-	for i := range ifs {
-		iface := &ifs[i]
-		if !isUp(iface) || isLoopback(iface) {
-			continue
-		}
-		addrs, err := iface.Addrs()
-		if err != nil {
-			continue
-		}
-		for _, a := range addrs {
-			ipnet, ok := a.(*net.IPNet)
-			if !ok {
-				continue
-			}
-			if ipnet.IP.To4() != nil || !ipnet.IP.IsGlobalUnicast() {
-				continue
-			}
-			return true, nil
-		}
-	}
-	return false, nil
-}
-
 // maybeTailscaleInterfaceName reports whether s is an interface
 // name that might be used by Tailscale.
 func maybeTailscaleInterfaceName(s string) bool {
@@ -82,7 +52,8 @@ func maybeTailscaleInterfaceName(s string) bool {
 // IsTailscaleIP reports whether ip is an IP in a range used by
 // Tailscale virtual network interfaces.
 func IsTailscaleIP(ip net.IP) bool {
-	return cgNAT.Contains(ip)
+	nip, _ := netaddr.FromStdIP(ip) // TODO: push this up to caller, change func signature
+	return tsaddr.IsTailscaleIP(nip)
 }
 
 func isUp(nif *net.Interface) bool       { return nif.Flags&net.FlagUp != 0 }
@@ -111,10 +82,13 @@ func LocalAddresses() (regular, loopback []string, err error) {
 		for _, a := range addrs {
 			switch v := a.(type) {
 			case *net.IPNet:
-				// TODO(crawshaw): IPv6 support.
-				// Easy to do here, but we need good endpoint ordering logic.
-				ip := v.IP.To4()
-				if ip == nil {
+				ip, ok := netaddr.FromStdIP(v.IP)
+				if !ok {
+					continue
+				}
+				if ip.Is6() {
+					// TODO(crawshaw): IPv6 support.
+					// Easy to do here, but we need good endpoint ordering logic.
 					continue
 				}
 				// TODO(apenwarr): don't special case cgNAT.
@@ -122,7 +96,7 @@ func LocalAddresses() (regular, loopback []string, err error) {
 				// very well be something we can route to
 				// directly, because both nodes are
 				// behind the same CGNAT router.
-				if cgNAT.Contains(ip) {
+				if tsaddr.IsTailscaleIP(ip) {
 					continue
 				}
 				if linkLocalIPv4.Contains(ip) {
@@ -148,7 +122,7 @@ func (i Interface) IsLoopback() bool { return isLoopback(i.Interface) }
 func (i Interface) IsUp() bool       { return isUp(i.Interface) }
 
 // ForeachInterfaceAddress calls fn for each interface's address on the machine.
-func ForeachInterfaceAddress(fn func(Interface, net.IP)) error {
+func ForeachInterfaceAddress(fn func(Interface, netaddr.IP)) error {
 	ifaces, err := net.Interfaces()
 	if err != nil {
 		return err
@@ -162,7 +136,9 @@ func ForeachInterfaceAddress(fn func(Interface, net.IP)) error {
 		for _, a := range addrs {
 			switch v := a.(type) {
 			case *net.IPNet:
-				fn(Interface{iface}, v.IP)
+				if ip, ok := netaddr.FromStdIP(v.IP); ok {
+					fn(Interface{iface}, ip)
+				}
 			}
 		}
 	}
@@ -173,9 +149,16 @@ func ForeachInterfaceAddress(fn func(Interface, net.IP)) error {
 // routing table, and other network configuration.
 // For now it's pretty basic.
 type State struct {
-	InterfaceIPs map[string][]net.IP
+	InterfaceIPs map[string][]netaddr.IP
 	InterfaceUp  map[string]bool
 
+	// HaveV6Global is whether this machine has an IPv6 global address
+	// on some interface.
+	HaveV6Global bool
+
+	// HaveV4 is whether the machine has some non-localhost IPv4 address.
+	HaveV4 bool
+
 	// IsExpensive is whether the current network interface is
 	// considered "expensive", which currently means LTE/etc
 	// instead of Wifi. This field is not populated by GetState.
@@ -204,12 +187,14 @@ func (s *State) RemoveTailscaleInterfaces() {
 // It does not set the returned State.IsExpensive. The caller can populate that.
 func GetState() (*State, error) {
 	s := &State{
-		InterfaceIPs: make(map[string][]net.IP),
+		InterfaceIPs: make(map[string][]netaddr.IP),
 		InterfaceUp:  make(map[string]bool),
 	}
-	if err := ForeachInterfaceAddress(func(ni Interface, ip net.IP) {
+	if err := ForeachInterfaceAddress(func(ni Interface, ip netaddr.IP) {
 		s.InterfaceIPs[ni.Name] = append(s.InterfaceIPs[ni.Name], ip)
 		s.InterfaceUp[ni.Name] = ni.IsUp()
+		s.HaveV6Global = s.HaveV6Global || isGlobalV6(ip)
+		s.HaveV4 = s.HaveV4 || (ip.Is4() && !ip.IsLoopback())
 	}); err != nil {
 		return nil, err
 	}
@@ -227,7 +212,7 @@ func HTTPOfListener(ln net.Listener) string {
 
 	var goodIP string
 	var privateIP string
-	ForeachInterfaceAddress(func(i Interface, ip net.IP) {
+	ForeachInterfaceAddress(func(i Interface, ip netaddr.IP) {
 		if isPrivateIP(ip) {
 			if privateIP == "" {
 				privateIP = ip.String()
@@ -246,22 +231,59 @@ func HTTPOfListener(ln net.Listener) string {
 
 }
 
-func isPrivateIP(ip net.IP) bool {
+var likelyHomeRouterIP func() (netaddr.IP, bool)
+
+// LikelyHomeRouterIP returns the likely IP of the residential router,
+// which will always be an IPv4 private address, if found.
+// In addition, it returns the IP address of the current machine on
+// the LAN using that gateway.
+// This is used as the destination for UPnP, NAT-PMP, PCP, etc queries.
+func LikelyHomeRouterIP() (gateway, myIP netaddr.IP, ok bool) {
+	if likelyHomeRouterIP != nil {
+		gateway, ok = likelyHomeRouterIP()
+		if !ok {
+			return
+		}
+	}
+	if !ok {
+		return
+	}
+	ForeachInterfaceAddress(func(i Interface, ip netaddr.IP) {
+		if !i.IsUp() || ip.IsZero() || !myIP.IsZero() {
+			return
+		}
+		for _, prefix := range privatev4s {
+			if prefix.Contains(gateway) && prefix.Contains(ip) {
+				myIP = ip
+				ok = true
+				return
+			}
+		}
+	})
+	return gateway, myIP, !myIP.IsZero()
+}
+
+func isPrivateIP(ip netaddr.IP) bool {
 	return private1.Contains(ip) || private2.Contains(ip) || private3.Contains(ip)
 }
 
-func mustCIDR(s string) *net.IPNet {
-	_, ipNet, err := net.ParseCIDR(s)
+func isGlobalV6(ip netaddr.IP) bool {
+	return v6Global1.Contains(ip)
+}
+
+func mustCIDR(s string) netaddr.IPPrefix {
+	prefix, err := netaddr.ParseIPPrefix(s)
 	if err != nil {
 		panic(err)
 	}
-	return ipNet
+	return prefix
 }
 
 var (
 	private1      = mustCIDR("10.0.0.0/8")
 	private2      = mustCIDR("172.16.0.0/12")
 	private3      = mustCIDR("192.168.0.0/16")
-	cgNAT         = mustCIDR("100.64.0.0/10")
+	privatev4s    = []netaddr.IPPrefix{private1, private2, private3}
 	linkLocalIPv4 = mustCIDR("169.254.0.0/16")
+	v6Global1     = mustCIDR("2000::/3")
 )

net/interfaces/interfaces_darwin.go

@@ -0,0 +1,69 @@
+// Copyright (c) 2020 Tailscale Inc & AUTHORS All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package interfaces
+
+import (
+	"os/exec"
+
+	"go4.org/mem"
+	"inet.af/netaddr"
+	"tailscale.com/util/lineread"
+	"tailscale.com/version"
+)
+
+/*
+Parse out 10.0.0.1 from:
+
+$ netstat -r -n -f inet
+Routing tables
+
+Internet:
+Destination        Gateway            Flags        Netif Expire
+default            10.0.0.1           UGSc           en0
+default            link#14            UCSI         utun2
+10/16              link#4             UCS            en0      !
+10.0.0.1/32        link#4             UCS            en0      !
+...
+
+*/
+func likelyHomeRouterIPDarwinExec() (ret netaddr.IP, ok bool) {
+	if version.IsMobile() {
+		// Don't try to do subprocesses on iOS. Ends up with log spam like:
+		// kernel: "Sandbox: IPNExtension(86580) deny(1) process-fork"
+		// This is why we have likelyHomeRouterIPDarwinSyscall.
+		return ret, false
+	}
+	cmd := exec.Command("/usr/sbin/netstat", "-r", "-n", "-f", "inet")
+	stdout, err := cmd.StdoutPipe()
+	if err != nil {
+		return
+	}
+	if err := cmd.Start(); err != nil {
+		return
+	}
+	defer cmd.Wait()
+
+	var f []mem.RO
+	lineread.Reader(stdout, func(lineb []byte) error {
+		line := mem.B(lineb)
+		if !mem.Contains(line, mem.S("default")) {
+			return nil
+		}
+		f = mem.AppendFields(f[:0], line)
+		if len(f) < 3 || !f[0].EqualString("default") {
+			return nil
+		}
+		ipm, flagsm := f[1], f[2]
+		if !mem.Contains(flagsm, mem.S("G")) {
+			return nil
+		}
+		ip, err := netaddr.ParseIP(string(mem.Append(nil, ipm)))
+		if err == nil && isPrivateIP(ip) {
+			ret = ip
+		}
+		return nil
+	})
+	return ret, !ret.IsZero()
+}

net/interfaces/interfaces_darwin_cgo.go

@@ -0,0 +1,127 @@
+// Copyright (c) 2020 Tailscale Inc & AUTHORS All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+// +build darwin,cgo
+
+package interfaces
+
+/*
+#import "route.h"
+#import <netinet/in.h>
+#import <sys/sysctl.h>
+#import <stdlib.h>
+#import <stdio.h>
+
+// privateGatewayIPFromRoute returns the private gateway ip address from rtm, if it exists.
+// Otherwise, it returns 0.
+int privateGatewayIPFromRoute(struct rt_msghdr2 *rtm)
+{
+    // sockaddrs are after the message header
+    struct sockaddr* dst_sa = (struct sockaddr *)(rtm + 1);
+
+    if((rtm->rtm_addrs & (RTA_DST|RTA_GATEWAY)) != (RTA_DST|RTA_GATEWAY))
+        return 0; // missing dst or gateway addr
+    if (dst_sa->sa_family != AF_INET)
+        return 0; // dst not IPv4
+    if ((rtm->rtm_flags & RTF_GATEWAY) == 0)
+        return 0; // gateway flag not set
+
+    struct sockaddr_in* dst_si = (struct sockaddr_in *)dst_sa;
+    if (dst_si->sin_addr.s_addr != INADDR_ANY)
+        return 0; // not default route
+
+    #define ROUNDUP(a) ((a) > 0 ? (1 + (((a) - 1) | (sizeof(long) - 1))) : sizeof(long))
+
+    struct sockaddr* gateway_sa = (struct sockaddr *)((char *)dst_sa + ROUNDUP(dst_sa->sa_len));
+    if (gateway_sa->sa_family != AF_INET)
+        return 0; // gateway not IPv4
+
+    struct sockaddr_in* gateway_si= (struct sockaddr_in *)gateway_sa;
+    int ip;
+    ip = gateway_si->sin_addr.s_addr;
+
+    unsigned char a, b;
+    a = (ip >> 0) & 0xff;
+    b = (ip >> 8) & 0xff;
+
+    // Check whether ip is private, that is, whether it is
+    // in one of 10.0.0.0/8, 172.16.0.0/12, or 192.168.0.0/16.
+    if (a == 10)
+        return ip; // matches 10.0.0.0/8
+    if (a == 172 && (b >> 4) == 1)
+        return ip; // matches 172.16.0.0/12
+    if (a == 192 && b == 168)
+        return ip; // matches 192.168.0.0/16
+
+    // Not a private IP.
+    return 0;
+}
+
+// privateGatewayIP returns the private gateway IP address, if it exists.
+// If no private gateway IP address was found, it returns 0.
+// On an error, it returns an error code in (0, 255].
+// Any private gateway IP address is > 255.
+int privateGatewayIP()
+{
+    size_t needed;
+    int mib[6];
+    char *buf;
+
+    mib[0] = CTL_NET;
+    mib[1] = PF_ROUTE;
+    mib[2] = 0;
+    mib[3] = 0;
+    mib[4] = NET_RT_DUMP2;
+    mib[5] = 0;
+
+    if (sysctl(mib, 6, NULL, &needed, NULL, 0) < 0)
+        return 1; // route dump size estimation failed
+    if ((buf = malloc(needed)) == 0)
+        return 2; // malloc failed
+    if (sysctl(mib, 6, buf, &needed, NULL, 0) < 0) {
+        free(buf);
+        return 3; // route dump failed
+    }
+
+    // Loop over all routes.
+    char *next, *lim;
+    lim = buf + needed;
+	struct rt_msghdr2 *rtm;
+    for (next = buf; next < lim; next += rtm->rtm_msglen) {
+        rtm = (struct rt_msghdr2 *)next;
+		int ip;
+        ip = privateGatewayIPFromRoute(rtm);
+        if (ip) {
+            free(buf);
+            return ip;
+        }
+    }
+    free(buf);
+    return 0; // no gateway found
+}
+*/
+import "C"
+
+import (
+	"encoding/binary"
+	"fmt"
+	"os"
+
+	"inet.af/netaddr"
+)
+
+func init() {
+	likelyHomeRouterIP = likelyHomeRouterIPDarwinSyscall
+}
+
+func likelyHomeRouterIPDarwinSyscall() (ret netaddr.IP, ok bool) {
+	ip := C.privateGatewayIP()
+	fmt.Fprintln(os.Stderr, "likelyHomeRouterIPDarwinSyscall", ip)
+	if ip < 255 {
+		return netaddr.IP{}, false
+	}
+	var q [4]byte
+	binary.LittleEndian.PutUint32(q[:], uint32(ip))
+	return netaddr.IPv4(q[0], q[1], q[2], q[3]), true
+}

net/interfaces/interfaces_darwin_cgo_test.go

@@ -0,0 +1,20 @@
+// Copyright (c) 2020 Tailscale Inc & AUTHORS All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+// +build cgo,darwin
+
+package interfaces
+
+import "testing"
+
+func TestLikelyHomeRouterIPSyscallExec(t *testing.T) {
+	syscallIP, syscallOK := likelyHomeRouterIPDarwinSyscall()
+	netstatIP, netstatOK := likelyHomeRouterIPDarwinExec()
+	if syscallOK != netstatOK || syscallIP != netstatIP {
+		t.Errorf("syscall() = %v, %v, netstat = %v, %v",
+			syscallIP, syscallOK,
+			netstatIP, netstatOK,
+		)
+	}
+}

net/interfaces/interfaces_darwin_nocgo.go

@@ -0,0 +1,11 @@
+// Copyright (c) 2020 Tailscale Inc & AUTHORS All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+// +build darwin,!cgo
+
+package interfaces
+
+func init() {
+	likelyHomeRouterIP = likelyHomeRouterIPDarwinExec
+}

net/interfaces/interfaces_linux.go

@@ -0,0 +1,210 @@
+// Copyright (c) 2020 Tailscale Inc & AUTHORS All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package interfaces
+
+import (
+	"bufio"
+	"bytes"
+	"errors"
+	"io"
+	"log"
+	"os"
+	"os/exec"
+	"runtime"
+	"strings"
+
+	"go4.org/mem"
+	"inet.af/netaddr"
+	"tailscale.com/syncs"
+	"tailscale.com/util/lineread"
+)
+
+func init() {
+	likelyHomeRouterIP = likelyHomeRouterIPLinux
+}
+
+var procNetRouteErr syncs.AtomicBool
+
+/*
+Parse 10.0.0.1 out of:
+
+$ cat /proc/net/route
+Iface   Destination     Gateway         Flags   RefCnt  Use     Metric  Mask            MTU     Window  IRTT
+ens18   00000000        0100000A        0003    0       0       0       00000000        0       0       0
+ens18   0000000A        00000000        0001    0       0       0       0000FFFF        0       0       0
+*/
+func likelyHomeRouterIPLinux() (ret netaddr.IP, ok bool) {
+	if procNetRouteErr.Get() {
+		// If we failed to read /proc/net/route previously, don't keep trying.
+		// But if we're on Android, go into the Android path.
+		if runtime.GOOS == "android" {
+			return likelyHomeRouterIPAndroid()
+		}
+		return ret, false
+	}
+	lineNum := 0
+	var f []mem.RO
+	err := lineread.File("/proc/net/route", func(line []byte) error {
+		lineNum++
+		if lineNum == 1 {
+			// Skip header line.
+			return nil
+		}
+		f = mem.AppendFields(f[:0], mem.B(line))
+		if len(f) < 4 {
+			return nil
+		}
+		gwHex, flagsHex := f[2], f[3]
+		flags, err := mem.ParseUint(flagsHex, 16, 16)
+		if err != nil {
+			return nil // ignore error, skip line and keep going
+		}
+		const RTF_UP = 0x0001
+		const RTF_GATEWAY = 0x0002
+		if flags&(RTF_UP|RTF_GATEWAY) != RTF_UP|RTF_GATEWAY {
+			return nil
+		}
+		ipu32, err := mem.ParseUint(gwHex, 16, 32)
+		if err != nil {
+			return nil // ignore error, skip line and keep going
+		}
+		ip := netaddr.IPv4(byte(ipu32), byte(ipu32>>8), byte(ipu32>>16), byte(ipu32>>24))
+		if isPrivateIP(ip) {
+			ret = ip
+		}
+		return nil
+	})
+	if err != nil {
+		procNetRouteErr.Set(true)
+		if runtime.GOOS == "android" {
+			return likelyHomeRouterIPAndroid()
+		}
+		log.Printf("interfaces: failed to read /proc/net/route: %v", err)
+	}
+	return ret, !ret.IsZero()
+}
+
+// Android apps don't have permission to read /proc/net/route, at
+// least on Google devices and the Android emulator.
+func likelyHomeRouterIPAndroid() (ret netaddr.IP, ok bool) {
+	cmd := exec.Command("/system/bin/ip", "route", "show", "table", "0")
+	out, err := cmd.StdoutPipe()
+	if err != nil {
+		return
+	}
+	if err := cmd.Start(); err != nil {
+		log.Printf("interfaces: running /system/bin/ip: %v", err)
+		return
+	}
+	// Search for line like "default via 10.0.2.2 dev radio0 table 1016 proto static mtu 1500 "
+	lineread.Reader(out, func(line []byte) error {
+		const pfx = "default via "
+		if !mem.HasPrefix(mem.B(line), mem.S(pfx)) {
+			return nil
+		}
+		line = line[len(pfx):]
+		sp := bytes.IndexByte(line, ' ')
+		if sp == -1 {
+			return nil
+		}
+		ipb := line[:sp]
+		if ip, err := netaddr.ParseIP(string(ipb)); err == nil && ip.Is4() {
+			ret = ip
+			log.Printf("interfaces: found Android default route %v", ip)
+		}
+		return nil
+	})
+	cmd.Process.Kill()
+	cmd.Wait()
+	return ret, !ret.IsZero()
+}
+
+// DefaultRouteInterface returns the name of the network interface that owns
+// the default route, not including any tailscale interfaces.
+func DefaultRouteInterface() (string, error) {
+	v, err := defaultRouteInterfaceProcNet()
+	if err == nil {
+		return v, nil
+	}
+	if runtime.GOOS == "android" {
+		return defaultRouteInterfaceAndroidIPRoute()
+	}
+	return v, err
+}
+
+var zeroRouteBytes = []byte("00000000")
+
+func defaultRouteInterfaceProcNet() (string, error) {
+	f, err := os.Open("/proc/net/route")
+	if err != nil {
+		return "", err
+	}
+	defer f.Close()
+	br := bufio.NewReaderSize(f, 128)
+	for {
+		line, err := br.ReadSlice('\n')
+		if err == io.EOF {
+			break
+		}
+		if err != nil {
+			return "", err
+		}
+		if !bytes.Contains(line, zeroRouteBytes) {
+			continue
+		}
+		fields := strings.Fields(string(line))
+		ifc := fields[0]
+		ip := fields[1]
+		netmask := fields[7]
+
+		if strings.HasPrefix(ifc, "tailscale") ||
+			strings.HasPrefix(ifc, "wg") {
+			continue
+		}
+		if ip == "00000000" && netmask == "00000000" {
+			// default route
+			return ifc, nil // interface name
+		}
+	}
+
+	return "", errors.New("no default routes found")
+
+}
+
+// defaultRouteInterfaceAndroidIPRoute tries to find the machine's default route interface name
+// by parsing the "ip route" command output. We use this on Android where /proc/net/route
+// can be missing entries or have locked-down permissions.
+// See also comments in https://github.com/tailscale/tailscale/pull/666.
+func defaultRouteInterfaceAndroidIPRoute() (ifname string, err error) {
+	cmd := exec.Command("/system/bin/ip", "route", "show", "table", "0")
+	out, err := cmd.StdoutPipe()
+	if err != nil {
+		return "", err
+	}
+	if err := cmd.Start(); err != nil {
+		log.Printf("interfaces: running /system/bin/ip: %v", err)
+		return "", err
+	}
+	// Search for line like "default via 10.0.2.2 dev radio0 table 1016 proto static mtu 1500 "
+	lineread.Reader(out, func(line []byte) error {
+		const pfx = "default via "
+		if !mem.HasPrefix(mem.B(line), mem.S(pfx)) {
+			return nil
+		}
+		ff := strings.Fields(string(line))
+		for i, v := range ff {
+			if i > 0 && ff[i-1] == "dev" && ifname == "" {
+				ifname = v
+			}
+		}
+		return nil
+	})
+	cmd.Process.Kill()
+	cmd.Wait()
+	if ifname == "" {
+		return "", errors.New("no default routes found")
+	}
+	return ifname, nil
+}

net/interfaces/interfaces_linux_test.go

@@ -0,0 +1,16 @@
+// Copyright (c) 2020 Tailscale Inc & AUTHORS All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package interfaces
+
+import "testing"
+
+func BenchmarkDefaultRouteInterface(b *testing.B) {
+	b.ReportAllocs()
+	for i := 0; i < b.N; i++ {
+		if _, err := DefaultRouteInterface(); err != nil {
+			b.Fatal(err)
+		}
+	}
+}

net/interfaces/interfaces_test.go

@@ -47,3 +47,12 @@ func TestGetState(t *testing.T) {
 		t.Fatal("two States back-to-back were not equal")
 	}
 }
+
+func TestLikelyHomeRouterIP(t *testing.T) {
+	gw, my, ok := LikelyHomeRouterIP()
+	if !ok {
+		t.Logf("no result")
+		return
+	}
+	t.Logf("myIP = %v; gw = %v", my, gw)
+}

net/interfaces/interfaces_windows.go

@@ -0,0 +1,73 @@
+// Copyright (c) 2020 Tailscale Inc & AUTHORS All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package interfaces
+
+import (
+	"os/exec"
+	"syscall"
+
+	"go4.org/mem"
+	"inet.af/netaddr"
+	"tailscale.com/util/lineread"
+)
+
+func init() {
+	likelyHomeRouterIP = likelyHomeRouterIPWindows
+}
+
+/*
+Parse out 10.0.0.1 from:
+
+Z:\>route print -4
+===========================================================================
+Interface List
+ 15...aa 15 48 ff 1c 72 ......Red Hat VirtIO Ethernet Adapter
+  5...........................Tailscale Tunnel
+  1...........................Software Loopback Interface 1
+===========================================================================
+
+IPv4 Route Table
+===========================================================================
+Active Routes:
+Network Destination        Netmask          Gateway       Interface  Metric
+          0.0.0.0          0.0.0.0         10.0.0.1       10.0.28.63      5
+         10.0.0.0      255.255.0.0         On-link        10.0.28.63    261
+       10.0.28.63  255.255.255.255         On-link        10.0.28.63    261
+        10.0.42.0    255.255.255.0   100.103.42.106   100.103.42.106      5
+     10.0.255.255  255.255.255.255         On-link        10.0.28.63    261
+   34.193.248.174  255.255.255.255   100.103.42.106   100.103.42.106      5
+
+*/
+func likelyHomeRouterIPWindows() (ret netaddr.IP, ok bool) {
+	cmd := exec.Command("route", "print", "-4")
+	cmd.SysProcAttr = &syscall.SysProcAttr{HideWindow: true}
+	stdout, err := cmd.StdoutPipe()
+	if err != nil {
+		return
+	}
+	if err := cmd.Start(); err != nil {
+		return
+	}
+	defer cmd.Wait()
+
+	var f []mem.RO
+	lineread.Reader(stdout, func(lineb []byte) error {
+		line := mem.B(lineb)
+		if !mem.Contains(line, mem.S("0.0.0.0")) {
+			return nil
+		}
+		f = mem.AppendFields(f[:0], line)
+		if len(f) < 3 || !f[0].EqualString("0.0.0.0") || !f[1].EqualString("0.0.0.0") {
+			return nil
+		}
+		ipm := f[2]
+		ip, err := netaddr.ParseIP(string(mem.Append(nil, ipm)))
+		if err == nil && isPrivateIP(ip) {
+			ret = ip
+		}
+		return nil
+	})
+	return ret, !ret.IsZero()
+}

net/interfaces/route.h

@@ -0,0 +1,257 @@
+/*
+ * Copyright (c) 2000-2017 Apple Inc. All rights reserved.
+ *
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
+ *
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
+ *
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ *
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
+ * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
+ *
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
+ */
+/*
+ * Copyright (c) 1980, 1986, 1993
+ *	The Regents of the University of California.  All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ * 3. All advertising materials mentioning features or use of this software
+ *    must display the following acknowledgement:
+ *	This product includes software developed by the University of
+ *	California, Berkeley and its contributors.
+ * 4. Neither the name of the University nor the names of its contributors
+ *    may be used to endorse or promote products derived from this software
+ *    without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED.  IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ *
+ *	@(#)route.h	8.3 (Berkeley) 4/19/94
+ * $FreeBSD: src/sys/net/route.h,v 1.36.2.1 2000/08/16 06:14:23 jayanth Exp $
+ */
+
+#ifndef _NET_ROUTE_H_
+#define _NET_ROUTE_H_
+#include <sys/appleapiopts.h>
+#include <stdint.h>
+#include <sys/types.h>
+#include <sys/socket.h>
+
+/*
+ * These numbers are used by reliable protocols for determining
+ * retransmission behavior and are included in the routing structure.
+ */
+struct rt_metrics {
+	u_int32_t       rmx_locks;      /* Kernel leaves these values alone */
+	u_int32_t       rmx_mtu;        /* MTU for this path */
+	u_int32_t       rmx_hopcount;   /* max hops expected */
+	int32_t         rmx_expire;     /* lifetime for route, e.g. redirect */
+	u_int32_t       rmx_recvpipe;   /* inbound delay-bandwidth product */
+	u_int32_t       rmx_sendpipe;   /* outbound delay-bandwidth product */
+	u_int32_t       rmx_ssthresh;   /* outbound gateway buffer limit */
+	u_int32_t       rmx_rtt;        /* estimated round trip time */
+	u_int32_t       rmx_rttvar;     /* estimated rtt variance */
+	u_int32_t       rmx_pksent;     /* packets sent using this route */
+	u_int32_t       rmx_state;      /* route state */
+	u_int32_t       rmx_filler[3];  /* will be used for T/TCP later */
+};
+
+/*
+ * rmx_rtt and rmx_rttvar are stored as microseconds;
+ */
+#define RTM_RTTUNIT     1000000 /* units for rtt, rttvar, as units per sec */
+
+
+
+#define RTF_UP          0x1             /* route usable */
+#define RTF_GATEWAY     0x2             /* destination is a gateway */
+#define RTF_HOST        0x4             /* host entry (net otherwise) */
+#define RTF_REJECT      0x8             /* host or net unreachable */
+#define RTF_DYNAMIC     0x10            /* created dynamically (by redirect) */
+#define RTF_MODIFIED    0x20            /* modified dynamically (by redirect) */
+#define RTF_DONE        0x40            /* message confirmed */
+#define RTF_DELCLONE    0x80            /* delete cloned route */
+#define RTF_CLONING     0x100           /* generate new routes on use */
+#define RTF_XRESOLVE    0x200           /* external daemon resolves name */
+#define RTF_LLINFO      0x400           /* DEPRECATED - exists ONLY for backward
+	                                 *  compatibility */
+#define RTF_LLDATA      0x400           /* used by apps to add/del L2 entries */
+#define RTF_STATIC      0x800           /* manually added */
+#define RTF_BLACKHOLE   0x1000          /* just discard pkts (during updates) */
+#define RTF_NOIFREF     0x2000          /* not eligible for RTF_IFREF */
+#define RTF_PROTO2      0x4000          /* protocol specific routing flag */
+#define RTF_PROTO1      0x8000          /* protocol specific routing flag */
+
+#define RTF_PRCLONING   0x10000         /* protocol requires cloning */
+#define RTF_WASCLONED   0x20000         /* route generated through cloning */
+#define RTF_PROTO3      0x40000         /* protocol specific routing flag */
+                                        /* 0x80000 unused */
+#define RTF_PINNED      0x100000        /* future use */
+#define RTF_LOCAL       0x200000        /* route represents a local address */
+#define RTF_BROADCAST   0x400000        /* route represents a bcast address */
+#define RTF_MULTICAST   0x800000        /* route represents a mcast address */
+#define RTF_IFSCOPE     0x1000000       /* has valid interface scope */
+#define RTF_CONDEMNED   0x2000000       /* defunct; no longer modifiable */
+#define RTF_IFREF       0x4000000       /* route holds a ref to interface */
+#define RTF_PROXY       0x8000000       /* proxying, no interface scope */
+#define RTF_ROUTER      0x10000000      /* host is a router */
+#define RTF_DEAD        0x20000000      /* Route entry is being freed */
+                                        /* 0x40000000 and up unassigned */
+
+#define RTPRF_OURS      RTF_PROTO3      /* set on routes we manage */
+#define RTF_BITS \
+	"\020\1UP\2GATEWAY\3HOST\4REJECT\5DYNAMIC\6MODIFIED\7DONE" \
+	"\10DELCLONE\11CLONING\12XRESOLVE\13LLINFO\14STATIC\15BLACKHOLE" \
+	"\16NOIFREF\17PROTO2\20PROTO1\21PRCLONING\22WASCLONED\23PROTO3" \
+	"\25PINNED\26LOCAL\27BROADCAST\30MULTICAST\31IFSCOPE\32CONDEMNED" \
+	"\33IFREF\34PROXY\35ROUTER"
+
+#define IS_DIRECT_HOSTROUTE(rt) \
+	(((rt)->rt_flags & (RTF_HOST | RTF_GATEWAY)) == RTF_HOST)
+/*
+ * Routing statistics.
+ */
+struct  rtstat {
+	short   rts_badredirect;        /* bogus redirect calls */
+	short   rts_dynamic;            /* routes created by redirects */
+	short   rts_newgateway;         /* routes modified by redirects */
+	short   rts_unreach;            /* lookups which failed */
+	short   rts_wildcard;           /* lookups satisfied by a wildcard */
+	short   rts_badrtgwroute;       /* route to gateway is not direct */
+};
+
+/*
+ * Structures for routing messages.
+ */
+struct rt_msghdr {
+	u_short rtm_msglen;     /* to skip over non-understood messages */
+	u_char  rtm_version;    /* future binary compatibility */
+	u_char  rtm_type;       /* message type */
+	u_short rtm_index;      /* index for associated ifp */
+	int     rtm_flags;      /* flags, incl. kern & message, e.g. DONE */
+	int     rtm_addrs;      /* bitmask identifying sockaddrs in msg */
+	pid_t   rtm_pid;        /* identify sender */
+	int     rtm_seq;        /* for sender to identify action */
+	int     rtm_errno;      /* why failed */
+	int     rtm_use;        /* from rtentry */
+	u_int32_t rtm_inits;    /* which metrics we are initializing */
+	struct rt_metrics rtm_rmx; /* metrics themselves */
+};
+
+struct rt_msghdr2 {
+	u_short rtm_msglen;     /* to skip over non-understood messages */
+	u_char  rtm_version;    /* future binary compatibility */
+	u_char  rtm_type;       /* message type */
+	u_short rtm_index;      /* index for associated ifp */
+	int     rtm_flags;      /* flags, incl. kern & message, e.g. DONE */
+	int     rtm_addrs;      /* bitmask identifying sockaddrs in msg */
+	int32_t rtm_refcnt;     /* reference count */
+	int     rtm_parentflags; /* flags of the parent route */
+	int     rtm_reserved;   /* reserved field set to 0 */
+	int     rtm_use;        /* from rtentry */
+	u_int32_t rtm_inits;    /* which metrics we are initializing */
+	struct rt_metrics rtm_rmx; /* metrics themselves */
+};
+
+
+#define RTM_VERSION     5       /* Up the ante and ignore older versions */
+
+/*
+ * Message types.
+ */
+#define RTM_ADD         0x1     /* Add Route */
+#define RTM_DELETE      0x2     /* Delete Route */
+#define RTM_CHANGE      0x3     /* Change Metrics or flags */
+#define RTM_GET         0x4     /* Report Metrics */
+#define RTM_LOSING      0x5     /* RTM_LOSING is no longer generated by xnu
+	                         *  and is deprecated */
+#define RTM_REDIRECT    0x6     /* Told to use different route */
+#define RTM_MISS        0x7     /* Lookup failed on this address */
+#define RTM_LOCK        0x8     /* fix specified metrics */
+#define RTM_OLDADD      0x9     /* caused by SIOCADDRT */
+#define RTM_OLDDEL      0xa     /* caused by SIOCDELRT */
+#define RTM_RESOLVE     0xb     /* req to resolve dst to LL addr */
+#define RTM_NEWADDR     0xc     /* address being added to iface */
+#define RTM_DELADDR     0xd     /* address being removed from iface */
+#define RTM_IFINFO      0xe     /* iface going up/down etc. */
+#define RTM_NEWMADDR    0xf     /* mcast group membership being added to if */
+#define RTM_DELMADDR    0x10    /* mcast group membership being deleted */
+#define RTM_IFINFO2     0x12    /* */
+#define RTM_NEWMADDR2   0x13    /* */
+#define RTM_GET2        0x14    /* */
+
+/*
+ * Bitmask values for rtm_inits and rmx_locks.
+ */
+#define RTV_MTU         0x1     /* init or lock _mtu */
+#define RTV_HOPCOUNT    0x2     /* init or lock _hopcount */
+#define RTV_EXPIRE      0x4     /* init or lock _expire */
+#define RTV_RPIPE       0x8     /* init or lock _recvpipe */
+#define RTV_SPIPE       0x10    /* init or lock _sendpipe */
+#define RTV_SSTHRESH    0x20    /* init or lock _ssthresh */
+#define RTV_RTT         0x40    /* init or lock _rtt */
+#define RTV_RTTVAR      0x80    /* init or lock _rttvar */
+
+/*
+ * Bitmask values for rtm_addrs.
+ */
+#define RTA_DST         0x1     /* destination sockaddr present */
+#define RTA_GATEWAY     0x2     /* gateway sockaddr present */
+#define RTA_NETMASK     0x4     /* netmask sockaddr present */
+#define RTA_GENMASK     0x8     /* cloning mask sockaddr present */
+#define RTA_IFP         0x10    /* interface name sockaddr present */
+#define RTA_IFA         0x20    /* interface addr sockaddr present */
+#define RTA_AUTHOR      0x40    /* sockaddr for author of redirect */
+#define RTA_BRD         0x80    /* for NEWADDR, broadcast or p-p dest addr */
+
+/*
+ * Index offsets for sockaddr array for alternate internal encoding.
+ */
+#define RTAX_DST        0       /* destination sockaddr present */
+#define RTAX_GATEWAY    1       /* gateway sockaddr present */
+#define RTAX_NETMASK    2       /* netmask sockaddr present */
+#define RTAX_GENMASK    3       /* cloning mask sockaddr present */
+#define RTAX_IFP        4       /* interface name sockaddr present */
+#define RTAX_IFA        5       /* interface addr sockaddr present */
+#define RTAX_AUTHOR     6       /* sockaddr for author of redirect */
+#define RTAX_BRD        7       /* for NEWADDR, broadcast or p-p dest addr */
+#define RTAX_MAX        8       /* size of array to allocate */
+
+struct rt_addrinfo {
+	int     rti_addrs;
+	struct  sockaddr *rti_info[RTAX_MAX];
+};
+
+
+#endif /* _NET_ROUTE_H_ */

net/netcheck/netcheck_test.go

@@ -0,0 +1,561 @@
+// Copyright (c) 2020 Tailscale Inc & AUTHORS All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package netcheck
+
+import (
+	"bytes"
+	"context"
+	"fmt"
+	"net"
+	"reflect"
+	"sort"
+	"strconv"
+	"strings"
+	"testing"
+	"time"
+
+	"inet.af/netaddr"
+	"tailscale.com/net/interfaces"
+	"tailscale.com/net/stun"
+	"tailscale.com/net/stun/stuntest"
+	"tailscale.com/tailcfg"
+)
+
+func TestHairpinSTUN(t *testing.T) {
+	tx := stun.NewTxID()
+	c := &Client{
+		curState: &reportState{
+			hairTX:      tx,
+			gotHairSTUN: make(chan netaddr.IPPort, 1),
+		},
+	}
+	req := stun.Request(tx)
+	if !stun.Is(req) {
+		t.Fatal("expected STUN message")
+	}
+	if !c.handleHairSTUNLocked(req, netaddr.IPPort{}) {
+		t.Fatal("expected true")
+	}
+	select {
+	case <-c.curState.gotHairSTUN:
+	default:
+		t.Fatal("expected value")
+	}
+}
+
+func TestBasic(t *testing.T) {
+	stunAddr, cleanup := stuntest.Serve(t)
+	defer cleanup()
+
+	c := &Client{
+		Logf: t.Logf,
+	}
+
+	ctx, cancel := context.WithTimeout(context.Background(), 1*time.Second)
+	defer cancel()
+
+	r, err := c.GetReport(ctx, stuntest.DERPMapOf(stunAddr.String()))
+	if err != nil {
+		t.Fatal(err)
+	}
+	if !r.UDP {
+		t.Error("want UDP")
+	}
+	if len(r.RegionLatency) != 1 {
+		t.Errorf("expected 1 key in DERPLatency; got %+v", r.RegionLatency)
+	}
+	if _, ok := r.RegionLatency[1]; !ok {
+		t.Errorf("expected key 1 in DERPLatency; got %+v", r.RegionLatency)
+	}
+	if r.GlobalV4 == "" {
+		t.Error("expected GlobalV4 set")
+	}
+	if r.PreferredDERP != 1 {
+		t.Errorf("PreferredDERP = %v; want 1", r.PreferredDERP)
+	}
+}
+
+func TestWorksWhenUDPBlocked(t *testing.T) {
+	blackhole, err := net.ListenPacket("udp4", "127.0.0.1:0")
+	if err != nil {
+		t.Fatalf("failed to open blackhole STUN listener: %v", err)
+	}
+	defer blackhole.Close()
+
+	stunAddr := blackhole.LocalAddr().String()
+
+	dm := stuntest.DERPMapOf(stunAddr)
+	dm.Regions[1].Nodes[0].STUNOnly = true
+
+	c := &Client{
+		Logf: t.Logf,
+	}
+	ctx, cancel := context.WithTimeout(context.Background(), 250*time.Millisecond)
+	defer cancel()
+
+	r, err := c.GetReport(ctx, dm)
+	if err != nil {
+		t.Fatal(err)
+	}
+	want := newReport()
+	r.UPnP = ""
+	r.PMP = ""
+	r.PCP = ""
+
+	if !reflect.DeepEqual(r, want) {
+		t.Errorf("mismatch\n got: %+v\nwant: %+v\n", r, want)
+	}
+}
+
+func TestAddReportHistoryAndSetPreferredDERP(t *testing.T) {
+	// report returns a *Report from (DERP host, time.Duration)+ pairs.
+	report := func(a ...interface{}) *Report {
+		r := &Report{RegionLatency: map[int]time.Duration{}}
+		for i := 0; i < len(a); i += 2 {
+			s := a[i].(string)
+			if !strings.HasPrefix(s, "d") {
+				t.Fatalf("invalid derp server key %q", s)
+			}
+			regionID, err := strconv.Atoi(s[1:])
+			if err != nil {
+				t.Fatalf("invalid derp server key %q", s)
+			}
+
+			switch v := a[i+1].(type) {
+			case time.Duration:
+				r.RegionLatency[regionID] = v
+			case int:
+				r.RegionLatency[regionID] = time.Second * time.Duration(v)
+			default:
+				panic(fmt.Sprintf("unexpected type %T", v))
+			}
+		}
+		return r
+	}
+	type step struct {
+		after time.Duration
+		r     *Report
+	}
+	tests := []struct {
+		name        string
+		steps       []step
+		wantDERP    int // want PreferredDERP on final step
+		wantPrevLen int // wanted len(c.prev)
+	}{
+		{
+			name: "first_reading",
+			steps: []step{
+				{0, report("d1", 2, "d2", 3)},
+			},
+			wantPrevLen: 1,
+			wantDERP:    1,
+		},
+		{
+			name: "with_two",
+			steps: []step{
+				{0, report("d1", 2, "d2", 3)},
+				{1 * time.Second, report("d1", 4, "d2", 3)},
+			},
+			wantPrevLen: 2,
+			wantDERP:    1, // t0's d1 of 2 is still best
+		},
+		{
+			name: "but_now_d1_gone",
+			steps: []step{
+				{0, report("d1", 2, "d2", 3)},
+				{1 * time.Second, report("d1", 4, "d2", 3)},
+				{2 * time.Second, report("d2", 3)},
+			},
+			wantPrevLen: 3,
+			wantDERP:    2, // only option
+		},
+		{
+			name: "d1_is_back",
+			steps: []step{
+				{0, report("d1", 2, "d2", 3)},
+				{1 * time.Second, report("d1", 4, "d2", 3)},
+				{2 * time.Second, report("d2", 3)},
+				{3 * time.Second, report("d1", 4, "d2", 3)}, // same as 2 seconds ago
+			},
+			wantPrevLen: 4,
+			wantDERP:    1, // t0's d1 of 2 is still best
+		},
+		{
+			name: "things_clean_up",
+			steps: []step{
+				{0, report("d1", 1, "d2", 2)},
+				{1 * time.Second, report("d1", 1, "d2", 2)},
+				{2 * time.Second, report("d1", 1, "d2", 2)},
+				{3 * time.Second, report("d1", 1, "d2", 2)},
+				{10 * time.Minute, report("d3", 3)},
+			},
+			wantPrevLen: 1, // t=[0123]s all gone. (too old, older than 10 min)
+			wantDERP:    3, // only option
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			fakeTime := time.Unix(123, 0)
+			c := &Client{
+				TimeNow: func() time.Time { return fakeTime },
+			}
+			for _, s := range tt.steps {
+				fakeTime = fakeTime.Add(s.after)
+				c.addReportHistoryAndSetPreferredDERP(s.r)
+			}
+			lastReport := tt.steps[len(tt.steps)-1].r
+			if got, want := len(c.prev), tt.wantPrevLen; got != want {
+				t.Errorf("len(prev) = %v; want %v", got, want)
+			}
+			if got, want := lastReport.PreferredDERP, tt.wantDERP; got != want {
+				t.Errorf("PreferredDERP = %v; want %v", got, want)
+			}
+		})
+	}
+}
+
+func TestMakeProbePlan(t *testing.T) {
+	// basicMap has 5 regions. each region has a number of nodes
+	// equal to the region number (1 has 1a, 2 has 2a and 2b, etc.)
+	basicMap := &tailcfg.DERPMap{
+		Regions: map[int]*tailcfg.DERPRegion{},
+	}
+	for rid := 1; rid <= 5; rid++ {
+		var nodes []*tailcfg.DERPNode
+		for nid := 0; nid < rid; nid++ {
+			nodes = append(nodes, &tailcfg.DERPNode{
+				Name:     fmt.Sprintf("%d%c", rid, 'a'+rune(nid)),
+				RegionID: rid,
+				HostName: fmt.Sprintf("derp%d-%d", rid, nid),
+				IPv4:     fmt.Sprintf("%d.0.0.%d", rid, nid),
+				IPv6:     fmt.Sprintf("%d::%d", rid, nid),
+			})
+		}
+		basicMap.Regions[rid] = &tailcfg.DERPRegion{
+			RegionID: rid,
+			Nodes:    nodes,
+		}
+	}
+
+	const ms = time.Millisecond
+	p := func(name string, c rune, d ...time.Duration) probe {
+		var proto probeProto
+		switch c {
+		case 4:
+			proto = probeIPv4
+		case 6:
+			proto = probeIPv6
+		case 'h':
+			proto = probeHTTPS
+		}
+		pr := probe{node: name, proto: proto}
+		if len(d) == 1 {
+			pr.delay = d[0]
+		} else if len(d) > 1 {
+			panic("too many args")
+		}
+		return pr
+	}
+	tests := []struct {
+		name    string
+		dm      *tailcfg.DERPMap
+		have6if bool
+		no4     bool // no IPv4
+		last    *Report
+		want    probePlan
+	}{
+		{
+			name:    "initial_v6",
+			dm:      basicMap,
+			have6if: true,
+			last:    nil, // initial
+			want: probePlan{
+				"region-1-v4": []probe{p("1a", 4), p("1a", 4, 100*ms), p("1a", 4, 200*ms)}, // all a
+				"region-1-v6": []probe{p("1a", 6), p("1a", 6, 100*ms), p("1a", 6, 200*ms)},
+				"region-2-v4": []probe{p("2a", 4), p("2b", 4, 100*ms), p("2a", 4, 200*ms)}, // a -> b -> a
+				"region-2-v6": []probe{p("2a", 6), p("2b", 6, 100*ms), p("2a", 6, 200*ms)},
+				"region-3-v4": []probe{p("3a", 4), p("3b", 4, 100*ms), p("3c", 4, 200*ms)}, // a -> b -> c
+				"region-3-v6": []probe{p("3a", 6), p("3b", 6, 100*ms), p("3c", 6, 200*ms)},
+				"region-4-v4": []probe{p("4a", 4), p("4b", 4, 100*ms), p("4c", 4, 200*ms)},
+				"region-4-v6": []probe{p("4a", 6), p("4b", 6, 100*ms), p("4c", 6, 200*ms)},
+				"region-5-v4": []probe{p("5a", 4), p("5b", 4, 100*ms), p("5c", 4, 200*ms)},
+				"region-5-v6": []probe{p("5a", 6), p("5b", 6, 100*ms), p("5c", 6, 200*ms)},
+			},
+		},
+		{
+			name:    "initial_no_v6",
+			dm:      basicMap,
+			have6if: false,
+			last:    nil, // initial
+			want: probePlan{
+				"region-1-v4": []probe{p("1a", 4), p("1a", 4, 100*ms), p("1a", 4, 200*ms)}, // all a
+				"region-2-v4": []probe{p("2a", 4), p("2b", 4, 100*ms), p("2a", 4, 200*ms)}, // a -> b -> a
+				"region-3-v4": []probe{p("3a", 4), p("3b", 4, 100*ms), p("3c", 4, 200*ms)}, // a -> b -> c
+				"region-4-v4": []probe{p("4a", 4), p("4b", 4, 100*ms), p("4c", 4, 200*ms)},
+				"region-5-v4": []probe{p("5a", 4), p("5b", 4, 100*ms), p("5c", 4, 200*ms)},
+			},
+		},
+		{
+			name:    "second_v4_no_6if",
+			dm:      basicMap,
+			have6if: false,
+			last: &Report{
+				RegionLatency: map[int]time.Duration{
+					1: 10 * time.Millisecond,
+					2: 20 * time.Millisecond,
+					3: 30 * time.Millisecond,
+					4: 40 * time.Millisecond,
+					// Pretend 5 is missing
+				},
+				RegionV4Latency: map[int]time.Duration{
+					1: 10 * time.Millisecond,
+					2: 20 * time.Millisecond,
+					3: 30 * time.Millisecond,
+					4: 40 * time.Millisecond,
+				},
+			},
+			want: probePlan{
+				"region-1-v4": []probe{p("1a", 4), p("1a", 4, 12*ms)},
+				"region-2-v4": []probe{p("2a", 4), p("2b", 4, 24*ms)},
+				"region-3-v4": []probe{p("3a", 4)},
+			},
+		},
+		{
+			name:    "second_v4_only_with_6if",
+			dm:      basicMap,
+			have6if: true,
+			last: &Report{
+				RegionLatency: map[int]time.Duration{
+					1: 10 * time.Millisecond,
+					2: 20 * time.Millisecond,
+					3: 30 * time.Millisecond,
+					4: 40 * time.Millisecond,
+					// Pretend 5 is missing
+				},
+				RegionV4Latency: map[int]time.Duration{
+					1: 10 * time.Millisecond,
+					2: 20 * time.Millisecond,
+					3: 30 * time.Millisecond,
+					4: 40 * time.Millisecond,
+				},
+			},
+			want: probePlan{
+				"region-1-v4": []probe{p("1a", 4), p("1a", 4, 12*ms)},
+				"region-1-v6": []probe{p("1a", 6)},
+				"region-2-v4": []probe{p("2a", 4), p("2b", 4, 24*ms)},
+				"region-2-v6": []probe{p("2a", 6)},
+				"region-3-v4": []probe{p("3a", 4)},
+			},
+		},
+		{
+			name:    "second_mixed",
+			dm:      basicMap,
+			have6if: true,
+			last: &Report{
+				RegionLatency: map[int]time.Duration{
+					1: 10 * time.Millisecond,
+					2: 20 * time.Millisecond,
+					3: 30 * time.Millisecond,
+					4: 40 * time.Millisecond,
+					// Pretend 5 is missing
+				},
+				RegionV4Latency: map[int]time.Duration{
+					1: 10 * time.Millisecond,
+					2: 20 * time.Millisecond,
+				},
+				RegionV6Latency: map[int]time.Duration{
+					3: 30 * time.Millisecond,
+					4: 40 * time.Millisecond,
+				},
+			},
+			want: probePlan{
+				"region-1-v4": []probe{p("1a", 4), p("1a", 4, 12*ms)},
+				"region-1-v6": []probe{p("1a", 6), p("1a", 6, 12*ms)},
+				"region-2-v4": []probe{p("2a", 4), p("2b", 4, 24*ms)},
+				"region-2-v6": []probe{p("2a", 6), p("2b", 6, 24*ms)},
+				"region-3-v4": []probe{p("3a", 4)},
+			},
+		},
+		{
+			name:    "only_v6_initial",
+			have6if: true,
+			no4:     true,
+			dm:      basicMap,
+			want: probePlan{
+				"region-1-v6": []probe{p("1a", 6), p("1a", 6, 100*ms), p("1a", 6, 200*ms)},
+				"region-2-v6": []probe{p("2a", 6), p("2b", 6, 100*ms), p("2a", 6, 200*ms)},
+				"region-3-v6": []probe{p("3a", 6), p("3b", 6, 100*ms), p("3c", 6, 200*ms)},
+				"region-4-v6": []probe{p("4a", 6), p("4b", 6, 100*ms), p("4c", 6, 200*ms)},
+				"region-5-v6": []probe{p("5a", 6), p("5b", 6, 100*ms), p("5c", 6, 200*ms)},
+			},
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			ifState := &interfaces.State{
+				HaveV6Global: tt.have6if,
+				HaveV4:       !tt.no4,
+			}
+			got := makeProbePlan(tt.dm, ifState, tt.last)
+			if !reflect.DeepEqual(got, tt.want) {
+				t.Errorf("unexpected plan; got:\n%v\nwant:\n%v\n", got, tt.want)
+			}
+		})
+	}
+}
+
+func (plan probePlan) String() string {
+	var sb strings.Builder
+	keys := []string{}
+	for k := range plan {
+		keys = append(keys, k)
+	}
+	sort.Strings(keys)
+
+	for _, key := range keys {
+		fmt.Fprintf(&sb, "[%s]", key)
+		pv := plan[key]
+		for _, p := range pv {
+			fmt.Fprintf(&sb, " %v", p)
+		}
+		sb.WriteByte('\n')
+	}
+	return sb.String()
+}
+
+func (p probe) String() string {
+	wait := ""
+	if p.wait > 0 {
+		wait = "+" + p.wait.String()
+	}
+	delay := ""
+	if p.delay > 0 {
+		delay = "@" + p.delay.String()
+	}
+	return fmt.Sprintf("%s-%s%s%s", p.node, p.proto, delay, wait)
+}
+
+func (p probeProto) String() string {
+	switch p {
+	case probeIPv4:
+		return "v4"
+	case probeIPv6:
+		return "v4"
+	case probeHTTPS:
+		return "https"
+	}
+	return "?"
+}
+
+func TestLogConciseReport(t *testing.T) {
+	dm := &tailcfg.DERPMap{
+		Regions: map[int]*tailcfg.DERPRegion{
+			1: nil,
+			2: nil,
+			3: nil,
+		},
+	}
+	const ms = time.Millisecond
+	tests := []struct {
+		name string
+		r    *Report
+		want string
+	}{
+		{
+			name: "no_udp",
+			r:    &Report{},
+			want: "udp=false v4=false v6=false mapvarydest= hair= portmap=? derp=0",
+		},
+		{
+			name: "ipv4_one_region",
+			r: &Report{
+				UDP:           true,
+				IPv4:          true,
+				PreferredDERP: 1,
+				RegionLatency: map[int]time.Duration{
+					1: 10 * ms,
+				},
+				RegionV4Latency: map[int]time.Duration{
+					1: 10 * ms,
+				},
+			},
+			want: "udp=true v6=false mapvarydest= hair= portmap=? derp=1 derpdist=1v4:10ms",
+		},
+		{
+			name: "ipv4_all_region",
+			r: &Report{
+				UDP:           true,
+				IPv4:          true,
+				PreferredDERP: 1,
+				RegionLatency: map[int]time.Duration{
+					1: 10 * ms,
+					2: 20 * ms,
+					3: 30 * ms,
+				},
+				RegionV4Latency: map[int]time.Duration{
+					1: 10 * ms,
+					2: 20 * ms,
+					3: 30 * ms,
+				},
+			},
+			want: "udp=true v6=false mapvarydest= hair= portmap=? derp=1 derpdist=1v4:10ms,2v4:20ms,3v4:30ms",
+		},
+		{
+			name: "ipboth_all_region",
+			r: &Report{
+				UDP:           true,
+				IPv4:          true,
+				IPv6:          true,
+				PreferredDERP: 1,
+				RegionLatency: map[int]time.Duration{
+					1: 10 * ms,
+					2: 20 * ms,
+					3: 30 * ms,
+				},
+				RegionV4Latency: map[int]time.Duration{
+					1: 10 * ms,
+					2: 20 * ms,
+					3: 30 * ms,
+				},
+				RegionV6Latency: map[int]time.Duration{
+					1: 10 * ms,
+					2: 20 * ms,
+					3: 30 * ms,
+				},
+			},
+			want: "udp=true v6=true mapvarydest= hair= portmap=? derp=1 derpdist=1v4:10ms,1v6:10ms,2v4:20ms,2v6:20ms,3v4:30ms,3v6:30ms",
+		},
+		{
+			name: "portmap_all",
+			r: &Report{
+				UDP:  true,
+				UPnP: "true",
+				PMP:  "true",
+				PCP:  "true",
+			},
+			want: "udp=true v4=false v6=false mapvarydest= hair= portmap=UMC derp=0",
+		},
+		{
+			name: "portmap_some",
+			r: &Report{
+				UDP:  true,
+				UPnP: "true",
+				PMP:  "false",
+				PCP:  "true",
+			},
+			want: "udp=true v4=false v6=false mapvarydest= hair= portmap=UC derp=0",
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			var buf bytes.Buffer
+			c := &Client{Logf: func(f string, a ...interface{}) { fmt.Fprintf(&buf, f, a...) }}
+			c.logConciseReport(tt.r, dm)
+			if got := buf.String(); got != tt.want {
+				t.Errorf("unexpected result.\n got: %#q\nwant: %#q\n", got, tt.want)
+			}
+		})
+	}
+}

net/netns/netns.go

@@ -0,0 +1,68 @@
+// Copyright (c) 2020 Tailscale Inc & AUTHORS All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+// Package netns contains the common code for using the Go net package
+// in a logical "network namespace" to avoid routing loops where
+// Tailscale-created packets would otherwise loop back through
+// Tailscale routes.
+//
+// Despite the name netns, the exact mechanism used differs by
+// operating system, and perhaps even by version of the OS.
+//
+// The netns package also handles connecting via SOCKS proxies when
+// configured by the environment.
+package netns
+
+import (
+	"context"
+	"net"
+)
+
+// Listener returns a new net.Listener with its Control hook func
+// initialized as necessary to run in logical network namespace that
+// doesn't route back into Tailscale.
+func Listener() *net.ListenConfig {
+	return &net.ListenConfig{Control: control}
+}
+
+// NewDialer returns a new Dialer using a net.Dialer with its Control
+// hook func initialized as necessary to run in a logical network
+// namespace that doesn't route back into Tailscale. It also handles
+// using a SOCKS if configured in the environment with ALL_PROXY.
+func NewDialer() Dialer {
+	return FromDialer(new(net.Dialer))
+}
+
+// FromDialer returns sets d.Control as necessary to run in a logical
+// network namespace that doesn't route back into Tailscale. It also
+// handles using a SOCKS if configured in the environment with
+// ALL_PROXY.
+func FromDialer(d *net.Dialer) Dialer {
+	d.Control = control
+	if wrapDialer != nil {
+		return wrapDialer(d)
+	}
+	return d
+}
+
+// IsSOCKSDialer reports whether d is SOCKS-proxying dialer as returned by
+// NewDialer or FromDialer.
+func IsSOCKSDialer(d Dialer) bool {
+	if d == nil {
+		return false
+	}
+	_, ok := d.(*net.Dialer)
+	return !ok
+}
+
+// wrapDialer, if non-nil, specifies a function to wrap a dialer in a
+// SOCKS-using dialer. It's set conditionally by socks.go.
+var wrapDialer func(Dialer) Dialer
+
+// Dialer is the interface for a dialer that can dial with or without a context.
+// It's the type implemented both by net.Dialer and the Go SOCKS dialer.
+type Dialer interface {
+	Dial(network, address string) (net.Conn, error)
+	DialContext(ctx context.Context, network, address string) (net.Conn, error)
+}

net/netns/netns_default.go

@@ -0,0 +1,14 @@
+// Copyright (c) 2020 Tailscale Inc & AUTHORS All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+// +build !linux
+
+package netns
+
+import "syscall"
+
+// control does nothing to c.
+func control(network, address string, c syscall.RawConn) error {
+	return nil
+}

net/netns/netns_linux.go

@@ -0,0 +1,103 @@
+// Copyright (c) 2020 Tailscale Inc & AUTHORS All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package netns
+
+import (
+	"flag"
+	"fmt"
+	"os"
+	"os/exec"
+	"sync"
+	"syscall"
+
+	"golang.org/x/sys/unix"
+	"tailscale.com/net/interfaces"
+)
+
+// tailscaleBypassMark is the mark indicating that packets originating
+// from a socket should bypass Tailscale-managed routes during routing
+// table lookups.
+//
+// Keep this in sync with tailscaleBypassMark in
+// wgengine/router/router_linux.go.
+const tailscaleBypassMark = 0x80000
+
+// ipRuleOnce is the sync.Once & cached value for ipRuleAvailable.
+var ipRuleOnce struct {
+	sync.Once
+	v bool
+}
+
+// ipRuleAvailable reports whether the 'ip rule' command works.
+// If it doesn't, we have to use SO_BINDTODEVICE on our sockets instead.
+func ipRuleAvailable() bool {
+	ipRuleOnce.Do(func() {
+		ipRuleOnce.v = exec.Command("ip", "rule").Run() == nil
+	})
+	return ipRuleOnce.v
+}
+
+// ignoreErrors returns true if we should ignore setsocketopt errors in
+// this instance.
+func ignoreErrors() bool {
+	// If we're in a test, ignore errors. Assume the test knows
+	// what it's doing and will do its own skips or permission
+	// checks if it's setting up a world that needs netns to work.
+	// But by default, assume that tests don't need netns and it's
+	// harmless to ignore the sockopts failing.
+	if flag.CommandLine.Lookup("test.v") != nil {
+		return true
+	}
+	if os.Getuid() != 0 {
+		// only root can manipulate these socket flags
+		return true
+	}
+	return false
+}
+
+// control marks c as necessary to dial in a separate network namespace.
+//
+// It's intentionally the same signature as net.Dialer.Control
+// and net.ListenConfig.Control.
+func control(network, address string, c syscall.RawConn) error {
+	var sockErr error
+	err := c.Control(func(fd uintptr) {
+		if ipRuleAvailable() {
+			sockErr = setBypassMark(fd)
+		} else {
+			sockErr = bindToDevice(fd)
+		}
+	})
+	if err != nil {
+		return fmt.Errorf("RawConn.Control on %T: %w", c, err)
+	}
+	if sockErr != nil && ignoreErrors() {
+		// TODO(bradfitz): maybe log once? probably too spammy for e.g. CLI tools like tailscale netcheck.
+		return nil
+	}
+	return sockErr
+}
+
+func setBypassMark(fd uintptr) error {
+	if err := unix.SetsockoptInt(int(fd), unix.SOL_SOCKET, unix.SO_MARK, tailscaleBypassMark); err != nil {
+		return fmt.Errorf("setting SO_MARK bypass: %w", err)
+	}
+	return nil
+}
+
+func bindToDevice(fd uintptr) error {
+	ifc, err := interfaces.DefaultRouteInterface()
+	if err != nil {
+		// Make sure we bind to *some* interface,
+		// or we could get a routing loop.
+		// "lo" is always wrong, but if we don't have
+		// a default route anyway, it doesn't matter.
+		ifc = "lo"
+	}
+	if err := unix.SetsockoptString(int(fd), unix.SOL_SOCKET, unix.SO_BINDTODEVICE, ifc); err != nil {
+		return fmt.Errorf("setting SO_BINDTODEVICE: %w", err)
+	}
+	return nil
+}

net/netns/netns_linux_test.go

@@ -0,0 +1,51 @@
+// Copyright (c) 2020 Tailscale Inc & AUTHORS All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package netns
+
+import (
+	"fmt"
+	"go/ast"
+	"go/parser"
+	"go/token"
+	"testing"
+)
+
+// verifies tailscaleBypassMark is in sync with wgengine.
+func TestBypassMarkInSync(t *testing.T) {
+	want := fmt.Sprintf("%q", fmt.Sprintf("0x%x", tailscaleBypassMark))
+	fset := token.NewFileSet()
+	f, err := parser.ParseFile(fset, "../../wgengine/router/router_linux.go", nil, 0)
+	if err != nil {
+		t.Fatal(err)
+	}
+	for _, decl := range f.Decls {
+		gd, ok := decl.(*ast.GenDecl)
+		if !ok || gd.Tok != token.CONST {
+			continue
+		}
+		for _, spec := range gd.Specs {
+			vs, ok := spec.(*ast.ValueSpec)
+			if !ok {
+				continue
+			}
+			for i, ident := range vs.Names {
+				if ident.Name != "tailscaleBypassMark" {
+					continue
+				}
+				valExpr := vs.Values[i]
+				lit, ok := valExpr.(*ast.BasicLit)
+				if !ok {
+					t.Errorf("tailscaleBypassMark = %T, expected *ast.BasicLit", valExpr)
+				}
+				if lit.Value == want {
+					// Pass.
+					return
+				}
+				t.Fatalf("router_linux.go's tailscaleBypassMark = %s; not in sync with netns's %s", lit.Value, want)
+			}
+		}
+	}
+	t.Errorf("tailscaleBypassMark not found in router_linux.go")
+}

net/netns/socks.go

@@ -0,0 +1,20 @@
+// Copyright (c) 2020 Tailscale Inc & AUTHORS All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+// +build !ios
+
+package netns
+
+import "golang.org/x/net/proxy"
+
+func init() {
+	wrapDialer = wrapSocks
+}
+
+func wrapSocks(d Dialer) Dialer {
+	if cd, ok := proxy.FromEnvironmentUsing(d).(Dialer); ok {
+		return cd
+	}
+	return d
+}

net/stun/stun.go

@@ -29,8 +29,6 @@ const (
 	bindingRequest = "\x00\x01"
 	magicCookie    = "\x21\x12\xa4\x42"
 	lenFingerprint = 8 // 2+byte header + 2-byte length + 4-byte crc32
-	ipv4Len        = 4
-	ipv6Len        = 16
 	headerLen      = 20
 )
 
@@ -135,7 +133,6 @@ var (
 func foreachAttr(b []byte, fn func(attrType uint16, a []byte) error) error {
 	for len(b) > 0 {
 		if len(b) < 4 {
-			return errors.New("effed-f1")
 			return ErrMalformedAttrs
 		}
 		attrType := binary.BigEndian.Uint16(b[:2])
@@ -143,7 +140,6 @@ func foreachAttr(b []byte, fn func(attrType uint16, a []byte) error) error {
 		attrLenPad := attrLen % 4
 		b = b[4:]
 		if attrLen+attrLenPad > len(b) {
-			return errors.New("effed-f2")
 			return ErrMalformedAttrs
 		}
 		if err := fn(attrType, b[:attrLen]); err != nil {
@@ -161,9 +157,9 @@ func Response(txID TxID, ip net.IP, port uint16) []byte {
 	}
 	var fam byte
 	switch len(ip) {
-	case 4:
+	case net.IPv4len:
 		fam = 1
-	case 16:
+	case net.IPv6len:
 		fam = 2
 	default:
 		return nil
@@ -194,8 +190,6 @@ func Response(txID TxID, ip net.IP, port uint16) []byte {
 	return b
 }
 
-func beu16(b []byte) uint16 { return binary.BigEndian.Uint16(b) }
-
 // ParseResponse parses a successful binding response STUN packet.
 // The IP address is extracted from the XOR-MAPPED-ADDRESS attribute.
 // The returned addr slice is owned by the caller and does not alias b.
@@ -207,7 +201,7 @@ func ParseResponse(b []byte) (tID TxID, addr []byte, port uint16, err error) {
 	if b[0] != 0x01 || b[1] != 0x01 {
 		return tID, nil, 0, ErrNotSuccessResponse
 	}
-	attrsLen := int(beu16(b[2:4]))
+	attrsLen := int(binary.BigEndian.Uint16(b[2:4]))
 	b = b[headerLen:] // remove STUN header
 	if attrsLen > len(b) {
 		return tID, nil, 0, ErrMalformedAttrs
@@ -272,7 +266,7 @@ func xorMappedAddress(tID TxID, b []byte) (addr []byte, port uint16, err error)
 	if len(b) < 4 {
 		return nil, 0, ErrMalformedAttrs
 	}
-	xorPort := beu16(b[2:4])
+	xorPort := binary.BigEndian.Uint16(b[2:4])
 	addrField := b[4:]
 	port = xorPort ^ 0x2112 // first half of magicCookie
 
@@ -298,9 +292,9 @@ func xorMappedAddress(tID TxID, b []byte) (addr []byte, port uint16, err error)
 func familyAddrLen(fam byte) int {
 	switch fam {
 	case 0x01: // IPv4
-		return ipv4Len
+		return net.IPv4len
 	case 0x02: // IPv6
-		return ipv6Len
+		return net.IPv6len
 	default:
 		return 0
 	}

net/stun/stun_test.go

@@ -10,7 +10,7 @@ import (
 	"net"
 	"testing"
 
-	"tailscale.com/stun"
+	"tailscale.com/net/stun"
 )
 
 // TODO(bradfitz): fuzz this.

net/stun/stuntest/stuntest.go

@@ -0,0 +1,135 @@
+// Copyright (c) 2020 Tailscale Inc & AUTHORS All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+// Package stuntest provides a STUN test server.
+package stuntest
+
+import (
+	"context"
+	"fmt"
+	"net"
+	"strconv"
+	"strings"
+	"sync"
+	"testing"
+
+	"inet.af/netaddr"
+	"tailscale.com/net/stun"
+	"tailscale.com/tailcfg"
+	"tailscale.com/types/nettype"
+)
+
+type stunStats struct {
+	mu       sync.Mutex
+	readIPv4 int
+	readIPv6 int
+}
+
+func Serve(t *testing.T) (addr *net.UDPAddr, cleanupFn func()) {
+	return ServeWithPacketListener(t, nettype.Std{})
+}
+
+func ServeWithPacketListener(t *testing.T, ln nettype.PacketListener) (addr *net.UDPAddr, cleanupFn func()) {
+	t.Helper()
+
+	// TODO(crawshaw): use stats to test re-STUN logic
+	var stats stunStats
+
+	pc, err := ln.ListenPacket(context.Background(), "udp4", ":0")
+	if err != nil {
+		t.Fatalf("failed to open STUN listener: %v", err)
+	}
+	addr = pc.LocalAddr().(*net.UDPAddr)
+	if len(addr.IP) == 0 || addr.IP.IsUnspecified() {
+		addr.IP = net.ParseIP("127.0.0.1")
+	}
+	doneCh := make(chan struct{})
+	go runSTUN(t, pc, &stats, doneCh)
+	return addr, func() {
+		pc.Close()
+		<-doneCh
+	}
+}
+
+func runSTUN(t *testing.T, pc net.PacketConn, stats *stunStats, done chan<- struct{}) {
+	defer close(done)
+
+	var buf [64 << 10]byte
+	for {
+		n, addr, err := pc.ReadFrom(buf[:])
+		if err != nil {
+			if strings.Contains(err.Error(), "closed network connection") {
+				t.Logf("STUN server shutdown")
+				return
+			}
+			continue
+		}
+		ua := addr.(*net.UDPAddr)
+		pkt := buf[:n]
+		if !stun.Is(pkt) {
+			continue
+		}
+		txid, err := stun.ParseBindingRequest(pkt)
+		if err != nil {
+			continue
+		}
+
+		stats.mu.Lock()
+		if ua.IP.To4() != nil {
+			stats.readIPv4++
+		} else {
+			stats.readIPv6++
+		}
+		stats.mu.Unlock()
+
+		res := stun.Response(txid, ua.IP, uint16(ua.Port))
+		if _, err := pc.WriteTo(res, addr); err != nil {
+			t.Logf("STUN server write failed: %v", err)
+		}
+	}
+}
+
+func DERPMapOf(stun ...string) *tailcfg.DERPMap {
+	m := &tailcfg.DERPMap{
+		Regions: map[int]*tailcfg.DERPRegion{},
+	}
+	for i, hostPortStr := range stun {
+		regionID := i + 1
+		host, portStr, err := net.SplitHostPort(hostPortStr)
+		if err != nil {
+			panic(fmt.Sprintf("bogus STUN hostport: %q", hostPortStr))
+		}
+		port, err := strconv.Atoi(portStr)
+		if err != nil {
+			panic(fmt.Sprintf("bogus port %q in %q", portStr, hostPortStr))
+		}
+		var ipv4, ipv6 string
+		ip, err := netaddr.ParseIP(host)
+		if err != nil {
+			panic(fmt.Sprintf("bogus non-IP STUN host %q in %q", host, hostPortStr))
+		}
+		if ip.Is4() {
+			ipv4 = host
+			ipv6 = "none"
+		}
+		if ip.Is6() {
+			ipv6 = host
+			ipv4 = "none"
+		}
+		node := &tailcfg.DERPNode{
+			Name:     fmt.Sprint(regionID) + "a",
+			RegionID: regionID,
+			HostName: fmt.Sprintf("d%d.invalid", regionID),
+			IPv4:     ipv4,
+			IPv6:     ipv6,
+			STUNPort: port,
+			STUNOnly: true,
+		}
+		m.Regions[regionID] = &tailcfg.DERPRegion{
+			RegionID: regionID,
+			Nodes:    []*tailcfg.DERPNode{node},
+		}
+	}
+	return m
+}

net/tlsdial/tlsdial.go

@@ -11,9 +11,14 @@
 // control, DERP).
 package tlsdial
 
-import "crypto/tls"
+import (
+	"crypto/tls"
+	"crypto/x509"
+	"errors"
+	"time"
+)
 
-// Config returns a tls.Config for dialing the given host.
+// Config returns a tls.Config for connecting to a server.
 // If base is non-nil, it's cloned as the base config before
 // being configured and returned.
 func Config(host string, base *tls.Config) *tls.Config {
@@ -27,3 +32,45 @@ func Config(host string, base *tls.Config) *tls.Config {
 
 	return conf
 }
+
+// SetConfigExpectedCert modifies c to expect and verify that the server returns
+// a certificate for the provided certDNSName.
+func SetConfigExpectedCert(c *tls.Config, certDNSName string) {
+	if c.ServerName == certDNSName {
+		return
+	}
+	if c.ServerName == "" {
+		c.ServerName = certDNSName
+		return
+	}
+	if c.VerifyPeerCertificate != nil {
+		panic("refusing to override tls.Config.VerifyPeerCertificate")
+	}
+	// Set InsecureSkipVerify to prevent crypto/tls from doing its
+	// own cert verification, but do the same work that it'd do
+	// (but using certDNSName) in the VerifyPeerCertificate hook.
+	c.InsecureSkipVerify = true
+	c.VerifyPeerCertificate = func(rawCerts [][]byte, _ [][]*x509.Certificate) error {
+		if len(rawCerts) == 0 {
+			return errors.New("no certs presented")
+		}
+		certs := make([]*x509.Certificate, len(rawCerts))
+		for i, asn1Data := range rawCerts {
+			cert, err := x509.ParseCertificate(asn1Data)
+			if err != nil {
+				return err
+			}
+			certs[i] = cert
+		}
+		opts := x509.VerifyOptions{
+			CurrentTime:   time.Now(),
+			DNSName:       certDNSName,
+			Intermediates: x509.NewCertPool(),
+		}
+		for _, cert := range certs[1:] {
+			opts.Intermediates.AddCert(cert)
+		}
+		_, err := certs[0].Verify(opts)
+		return err
+	}
+}

net/tsaddr/tsaddr.go

@@ -0,0 +1,74 @@
+// Copyright (c) 2020 Tailscale Inc & AUTHORS All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+// Package tsaddr handles Tailscale-specific IPs and ranges.
+package tsaddr
+
+import (
+	"sync"
+
+	"inet.af/netaddr"
+)
+
+// ChromeOSVMRange returns the subset of the CGNAT IPv4 range used by
+// ChromeOS to interconnect the host OS to containers and VMs. We
+// avoid allocating Tailscale IPs from it, to avoid conflicts.
+func ChromeOSVMRange() netaddr.IPPrefix {
+	chromeOSRange.Do(func() { mustPrefix(&chromeOSRange.v, "100.115.92.0/23") })
+	return chromeOSRange.v
+}
+
+var chromeOSRange oncePrefix
+
+// CGNATRange returns the Carrier Grade NAT address range that
+// is the superset range that Tailscale assigns out of.
+// See https://tailscale.com/kb/1015/100.x-addresses.
+// Note that Tailscale does not assign out of the ChromeOSVMRange.
+func CGNATRange() netaddr.IPPrefix {
+	cgnatRange.Do(func() { mustPrefix(&cgnatRange.v, "100.64.0.0/10") })
+	return cgnatRange.v
+}
+
+var cgnatRange oncePrefix
+
+// TailscaleServiceIP returns the listen address of services
+// provided by Tailscale itself such as the Magic DNS proxy.
+func TailscaleServiceIP() netaddr.IP {
+	serviceIP.Do(func() { mustIP(&serviceIP.v, "100.100.100.100") })
+	return serviceIP.v
+}
+
+var serviceIP onceIP
+
+// IsTailscaleIP reports whether ip is an IP address in a range that
+// Tailscale assigns from.
+func IsTailscaleIP(ip netaddr.IP) bool {
+	return CGNATRange().Contains(ip) && !ChromeOSVMRange().Contains(ip)
+}
+
+func mustPrefix(v *netaddr.IPPrefix, prefix string) {
+	var err error
+	*v, err = netaddr.ParseIPPrefix(prefix)
+	if err != nil {
+		panic(err)
+	}
+}
+
+type oncePrefix struct {
+	sync.Once
+	v netaddr.IPPrefix
+}
+
+func mustIP(v *netaddr.IP, ip string) {
+	var err error
+	*v, err = netaddr.ParseIP(ip)
+	if err != nil {
+		panic(err)
+	}
+}
+
+type onceIP struct {
+	sync.Once
+	v netaddr.IP
+}

net/tsaddr/tsaddr_test.go

@@ -0,0 +1,19 @@
+// Copyright (c) 2020 Tailscale Inc & AUTHORS All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package tsaddr
+
+import "testing"
+
+func TestChromeOSVMRange(t *testing.T) {
+	if got, want := ChromeOSVMRange().String(), "100.115.92.0/23"; got != want {
+		t.Errorf("got %q; want %q", got, want)
+	}
+}
+
+func TestCGNATRange(t *testing.T) {
+	if got, want := CGNATRange().String(), "100.64.0.0/10"; got != want {
+		t.Errorf("got %q; want %q", got, want)
+	}
+}

netcheck/netcheck.go

@@ -1,564 +0,0 @@
-// Copyright (c) 2020 Tailscale Inc & AUTHORS All rights reserved.
-// Use of this source code is governed by a BSD-style
-// license that can be found in the LICENSE file.
-
-// Package netcheck checks the network conditions from the current host.
-package netcheck
-
-import (
-	"bytes"
-	"context"
-	"errors"
-	"fmt"
-	"io"
-	"log"
-	"net"
-	"sort"
-	"sync"
-	"time"
-
-	"golang.org/x/sync/errgroup"
-	"tailscale.com/derp/derpmap"
-	"tailscale.com/net/dnscache"
-	"tailscale.com/net/interfaces"
-	"tailscale.com/stun"
-	"tailscale.com/stunner"
-	"tailscale.com/types/logger"
-	"tailscale.com/types/opt"
-)
-
-type Report struct {
-	UDP                   bool                     // UDP works
-	IPv6                  bool                     // IPv6 works
-	MappingVariesByDestIP opt.Bool                 // for IPv4
-	HairPinning           opt.Bool                 // for IPv4
-	PreferredDERP         int                      // or 0 for unknown
-	DERPLatency           map[string]time.Duration // keyed by STUN host:port
-
-	GlobalV4 string // ip:port of global IPv4
-	GlobalV6 string // [ip]:port of global IPv6 // TODO
-
-	// TODO: update Clone when adding new fields
-}
-
-func (r *Report) Clone() *Report {
-	if r == nil {
-		return nil
-	}
-	r2 := *r
-	if r2.DERPLatency != nil {
-		r2.DERPLatency = map[string]time.Duration{}
-		for k, v := range r.DERPLatency {
-			r2.DERPLatency[k] = v
-		}
-	}
-	return &r2
-}
-
-// Client generates a netcheck Report.
-type Client struct {
-	// DERP is the DERP world to use.
-	DERP *derpmap.World
-
-	// DNSCache optionally specifies a DNSCache to use.
-	// If nil, a DNS cache is not used.
-	DNSCache *dnscache.Resolver
-
-	// Logf optionally specifies where to log to.
-	Logf logger.Logf
-
-	// TimeNow, if non-nil, is used instead of time.Now.
-	TimeNow func() time.Time
-
-	GetSTUNConn4 func() STUNConn
-	GetSTUNConn6 func() STUNConn
-
-	mu          sync.Mutex            // guards following
-	prev        map[time.Time]*Report // some previous reports
-	last        *Report               // most recent report
-	s4          *stunner.Stunner
-	s6          *stunner.Stunner
-	hairTX      stun.TxID
-	gotHairSTUN chan *net.UDPAddr // non-nil if we're in GetReport
-}
-
-// STUNConn is the interface required by the netcheck Client when
-// reusing an existing UDP connection.
-type STUNConn interface {
-	WriteTo([]byte, net.Addr) (int, error)
-	ReadFrom([]byte) (int, net.Addr, error)
-}
-
-func (c *Client) logf(format string, a ...interface{}) {
-	if c.Logf != nil {
-		c.Logf(format, a...)
-	} else {
-		log.Printf(format, a...)
-	}
-}
-
-// handleHairSTUN reports whether pkt (from src) was our magic hairpin
-// probe packet that we sent to ourselves.
-func (c *Client) handleHairSTUN(pkt []byte, src *net.UDPAddr) bool {
-	c.mu.Lock()
-	defer c.mu.Unlock()
-	return c.handleHairSTUNLocked(pkt, src)
-}
-
-func (c *Client) handleHairSTUNLocked(pkt []byte, src *net.UDPAddr) bool {
-	if tx, err := stun.ParseBindingRequest(pkt); err == nil && tx == c.hairTX {
-		select {
-		case c.gotHairSTUN <- src:
-		default:
-		}
-		return true
-	}
-	return false
-}
-
-func (c *Client) ReceiveSTUNPacket(pkt []byte, src *net.UDPAddr) {
-	if src == nil || src.IP == nil {
-		panic("bogus src")
-	}
-
-	c.mu.Lock()
-
-	if c.handleHairSTUNLocked(pkt, src) {
-		c.mu.Unlock()
-		return
-	}
-
-	var st *stunner.Stunner
-	if src.IP.To4() != nil {
-		st = c.s4
-	} else {
-		st = c.s6
-	}
-
-	c.mu.Unlock()
-
-	if st != nil {
-		st.Receive(pkt, src)
-	}
-}
-
-// pickSubset selects a subset of IPv4 and IPv6 STUN server addresses
-// to hit based on history.
-//
-// maxTries is the max number of tries per server.
-//
-// The caller owns the returned values.
-func (c *Client) pickSubset() (stuns4, stuns6 []string, maxTries map[string]int, err error) {
-	c.mu.Lock()
-	defer c.mu.Unlock()
-
-	const defaultMaxTries = 2
-	maxTries = map[string]int{}
-
-	var prev4, prev6 []string // sorted fastest to slowest
-	if c.last != nil {
-		condAppend := func(dst []string, server string) []string {
-			if server != "" && c.last.DERPLatency[server] != 0 {
-				return append(dst, server)
-			}
-			return dst
-		}
-		c.DERP.ForeachServer(func(s *derpmap.Server) {
-			prev4 = condAppend(prev4, s.STUN4)
-			prev6 = condAppend(prev6, s.STUN6)
-		})
-		sort.Slice(prev4, func(i, j int) bool { return c.last.DERPLatency[prev4[i]] < c.last.DERPLatency[prev4[j]] })
-		sort.Slice(prev6, func(i, j int) bool { return c.last.DERPLatency[prev6[i]] < c.last.DERPLatency[prev6[j]] })
-	}
-
-	c.DERP.ForeachServer(func(s *derpmap.Server) {
-		if s.STUN4 == "" {
-			return
-		}
-		// STUN against all DERP's IPv4 endpoints, but
-		// if the previous report had results from
-		// more than 2 servers, only do 1 try against
-		// all but the first two.
-		stuns4 = append(stuns4, s.STUN4)
-		tries := defaultMaxTries
-		if len(prev4) > 2 && !stringsContains(prev4[:2], s.STUN4) {
-			tries = 1
-		}
-		maxTries[s.STUN4] = tries
-		if s.STUN6 != "" && tries == defaultMaxTries {
-			// For IPv6, we mostly care whether the user has IPv6 at all,
-			// so we don't need to send to all servers. The IPv4 timing
-			// information is enough for now. (We don't yet support IPv6-only)
-			// So only add the two fastest ones, or all if this is a fresh one.
-			stuns6 = append(stuns6, s.STUN6)
-			maxTries[s.STUN6] = 1
-		}
-	})
-
-	if len(stuns4) == 0 {
-		// TODO: make this work? if we ever need it
-		// to. Requirement for self-hosted Tailscale might be
-		// to run a DERP+STUN server co-resident with the
-		// Control server.
-		return nil, nil, nil, errors.New("netcheck: GetReport: no STUN servers, no Report")
-	}
-	sort.Strings(stuns4)
-	sort.Strings(stuns6)
-	return stuns4, stuns6, maxTries, nil
-}
-
-// GetReport gets a report.
-//
-// It may not be called concurrently with itself.
-func (c *Client) GetReport(ctx context.Context) (*Report, error) {
-	// Mask user context with ours that we guarantee to cancel so
-	// we can depend on it being closed in goroutines later.
-	// (User ctx might be context.Background, etc)
-	ctx, cancel := context.WithTimeout(ctx, 3*time.Second)
-	defer cancel()
-
-	if c.DERP == nil {
-		return nil, errors.New("netcheck: GetReport: Client.DERP is nil")
-	}
-
-	c.mu.Lock()
-	if c.gotHairSTUN != nil {
-		c.mu.Unlock()
-		return nil, errors.New("invalid concurrent call to GetReport")
-	}
-	hairTX := stun.NewTxID() // random payload
-	c.hairTX = hairTX
-	gotHairSTUN := make(chan *net.UDPAddr, 1)
-	c.gotHairSTUN = gotHairSTUN
-	c.mu.Unlock()
-
-	defer func() {
-		c.mu.Lock()
-		defer c.mu.Unlock()
-		c.s4 = nil
-		c.s6 = nil
-		c.gotHairSTUN = nil
-	}()
-
-	stuns4, stuns6, maxTries, err := c.pickSubset()
-	if err != nil {
-		return nil, err
-	}
-
-	closeOnCtx := func(c io.Closer) {
-		<-ctx.Done()
-		c.Close()
-	}
-
-	v6iface, err := interfaces.HaveIPv6GlobalAddress()
-	if err != nil {
-		c.logf("interfaces: %v", err)
-	}
-
-	// Create a UDP4 socket used for sending to our discovered IPv4 address.
-	pc4Hair, err := net.ListenPacket("udp4", ":0")
-	if err != nil {
-		c.logf("udp4: %v", err)
-		return nil, err
-	}
-	defer pc4Hair.Close()
-	hairTimeout := make(chan bool, 1)
-	startHairCheck := func(dstEP string) {
-		if dst, err := net.ResolveUDPAddr("udp4", dstEP); err == nil {
-			pc4Hair.WriteTo(stun.Request(hairTX), dst)
-			time.AfterFunc(500*time.Millisecond, func() { hairTimeout <- true })
-		}
-	}
-
-	var (
-		mu  sync.Mutex
-		ret = &Report{
-			DERPLatency: map[string]time.Duration{},
-		}
-		gotEP  = map[string]string{} // server -> ipPort
-		gotEP4 string
-	)
-	anyV6 := func() bool {
-		mu.Lock()
-		defer mu.Unlock()
-		return ret.IPv6
-	}
-	anyV4 := func() bool {
-		mu.Lock()
-		defer mu.Unlock()
-		return gotEP4 != ""
-	}
-	add := func(server, ipPort string, d time.Duration) {
-		ua, err := net.ResolveUDPAddr("udp", ipPort)
-		if err != nil {
-			c.logf("[unexpected] STUN addr %q", ipPort)
-			return
-		}
-		isV6 := ua.IP.To4() == nil
-
-		mu.Lock()
-		defer mu.Unlock()
-		ret.UDP = true
-		ret.DERPLatency[server] = d
-		if isV6 {
-			ret.IPv6 = true
-			ret.GlobalV6 = ipPort
-			// TODO: track MappingVariesByDestIP for IPv6
-			// too? Would be sad if so, but who knows.
-		} else {
-			// IPv4
-			if gotEP4 == "" {
-				gotEP4 = ipPort
-				ret.GlobalV4 = ipPort
-				startHairCheck(ipPort)
-			} else {
-				if gotEP4 != ipPort {
-					ret.MappingVariesByDestIP.Set(true)
-				} else if ret.MappingVariesByDestIP == "" {
-					ret.MappingVariesByDestIP.Set(false)
-				}
-			}
-		}
-		gotEP[server] = ipPort
-	}
-
-	var pc4, pc6 STUNConn
-
-	if f := c.GetSTUNConn4; f != nil {
-		pc4 = f()
-	} else {
-		u4, err := net.ListenPacket("udp4", ":0")
-		if err != nil {
-			c.logf("udp4: %v", err)
-			return nil, err
-		}
-		pc4 = u4
-		go closeOnCtx(u4)
-	}
-
-	if v6iface {
-		if f := c.GetSTUNConn6; f != nil {
-			pc6 = f()
-		} else {
-			u6, err := net.ListenPacket("udp6", ":0")
-			if err != nil {
-				c.logf("udp6: %v", err)
-			} else {
-				pc6 = u6
-				go closeOnCtx(u6)
-			}
-		}
-	}
-
-	reader := func(s *stunner.Stunner, pc STUNConn) {
-		var buf [64 << 10]byte
-		for {
-			n, addr, err := pc.ReadFrom(buf[:])
-			if err != nil {
-				if ctx.Err() != nil {
-					return
-				}
-				c.logf("ReadFrom: %v", err)
-				return
-			}
-			ua, ok := addr.(*net.UDPAddr)
-			if !ok {
-				c.logf("ReadFrom: unexpected addr %T", addr)
-				continue
-			}
-			if c.handleHairSTUN(buf[:n], ua) {
-				continue
-			}
-			s.Receive(buf[:n], ua)
-		}
-
-	}
-
-	var grp errgroup.Group
-
-	s4 := &stunner.Stunner{
-		Send:     pc4.WriteTo,
-		Endpoint: add,
-		Servers:  stuns4,
-		Logf:     c.logf,
-		DNSCache: dnscache.Get(),
-		MaxTries: maxTries,
-	}
-
-	c.mu.Lock()
-	c.s4 = s4
-	c.mu.Unlock()
-
-	grp.Go(func() error {
-		err := s4.Run(ctx)
-		if errors.Is(err, context.DeadlineExceeded) {
-			if !anyV4() {
-				c.logf("netcheck: no IPv4 UDP STUN replies")
-			}
-			return nil
-		}
-		return err
-	})
-	if c.GetSTUNConn4 == nil {
-		go reader(s4, pc4)
-	}
-
-	if pc6 != nil && len(stuns6) > 0 {
-		s6 := &stunner.Stunner{
-			Endpoint: add,
-			Send:     pc6.WriteTo,
-			Servers:  stuns6,
-			Logf:     c.logf,
-			OnlyIPv6: true,
-			DNSCache: dnscache.Get(),
-			MaxTries: maxTries,
-		}
-
-		c.mu.Lock()
-		c.s6 = s6
-		c.mu.Unlock()
-
-		grp.Go(func() error {
-			err := s6.Run(ctx)
-			if errors.Is(err, context.DeadlineExceeded) {
-				if !anyV6() {
-					// IPv6 seemed like it was configured, but actually failed.
-					// Just log and return a nil error.
-					c.logf("IPv6 seemed configured, but no UDP STUN replies")
-				}
-				return nil
-			}
-			// Otherwise must be some invalid use of Stunner.
-			return err //
-		})
-		if c.GetSTUNConn6 == nil {
-			go reader(s6, pc6)
-		}
-	}
-
-	err = grp.Wait()
-	if err != nil {
-		return nil, err
-	}
-
-	mu.Lock()
-	defer mu.Unlock()
-
-	// Check hairpinning.
-	if ret.MappingVariesByDestIP == "false" && gotEP4 != "" {
-		select {
-		case <-gotHairSTUN:
-			ret.HairPinning.Set(true)
-		case <-hairTimeout:
-			ret.HairPinning.Set(false)
-		}
-	}
-
-	// TODO: if UDP is blocked, try to measure TCP connect times
-	// to DERP nodes instead? So UDP-blocked users still get a
-	// decent DERP node, rather than being randomly assigned to
-	// the other side of the planet? Or try ICMP? (likely also
-	// blocked?)
-
-	report := ret.Clone()
-
-	c.addReportHistoryAndSetPreferredDERP(report)
-	c.logConciseReport(report)
-
-	return report, nil
-}
-
-func (c *Client) logConciseReport(r *Report) {
-	buf := bytes.NewBuffer(make([]byte, 0, 256)) // empirically: 5 DERPs + IPv6 == ~233 bytes
-	fmt.Fprintf(buf, "udp=%v", r.UDP)
-	fmt.Fprintf(buf, " v6=%v", r.IPv6)
-	fmt.Fprintf(buf, " mapvarydest=%v", r.MappingVariesByDestIP)
-	fmt.Fprintf(buf, " hair=%v", r.HairPinning)
-	if r.GlobalV4 != "" {
-		fmt.Fprintf(buf, " v4a=%v", r.GlobalV4)
-	}
-	if r.GlobalV6 != "" {
-		fmt.Fprintf(buf, " v6a=%v", r.GlobalV6)
-	}
-	fmt.Fprintf(buf, " derp=%v", r.PreferredDERP)
-	if r.PreferredDERP != 0 {
-		fmt.Fprintf(buf, " derpdist=")
-		for i, id := range c.DERP.IDs() {
-			if i != 0 {
-				buf.WriteByte(',')
-			}
-			s := c.DERP.ServerByID(id)
-			needComma := false
-			if d := r.DERPLatency[s.STUN4]; d != 0 {
-				fmt.Fprintf(buf, "%dv4:%v", id, d.Round(time.Millisecond))
-				needComma = true
-			}
-			if d := r.DERPLatency[s.STUN6]; d != 0 {
-				if needComma {
-					buf.WriteByte(',')
-				}
-				fmt.Fprintf(buf, "%dv6:%v", id, d.Round(time.Millisecond))
-			}
-		}
-	}
-
-	c.logf("%s", buf.Bytes())
-}
-
-func (c *Client) timeNow() time.Time {
-	if c.TimeNow != nil {
-		return c.TimeNow()
-	}
-	return time.Now()
-}
-
-// addReportHistoryAndSetPreferredDERP adds r to the set of recent Reports
-// and mutates r.PreferredDERP to contain the best recent one.
-func (c *Client) addReportHistoryAndSetPreferredDERP(r *Report) {
-	c.mu.Lock()
-	defer c.mu.Unlock()
-
-	if c.prev == nil {
-		c.prev = map[time.Time]*Report{}
-	}
-	now := c.timeNow()
-	c.prev[now] = r
-	c.last = r
-
-	const maxAge = 5 * time.Minute
-
-	// STUN host:port => its best recent latency in last maxAge
-	bestRecent := map[string]time.Duration{}
-
-	for t, pr := range c.prev {
-		if now.Sub(t) > maxAge {
-			delete(c.prev, t)
-			continue
-		}
-		for hp, d := range pr.DERPLatency {
-			if bd, ok := bestRecent[hp]; !ok || d < bd {
-				bestRecent[hp] = d
-			}
-		}
-	}
-
-	// Then, pick which currently-alive DERP server from the
-	// current report has the best latency over the past maxAge.
-	var bestAny time.Duration
-	for hp := range r.DERPLatency {
-		best := bestRecent[hp]
-		if r.PreferredDERP == 0 || best < bestAny {
-			bestAny = best
-			r.PreferredDERP = c.DERP.NodeIDOfSTUNServer(hp)
-		}
-	}
-}
-
-func stringsContains(ss []string, s string) bool {
-	for _, v := range ss {
-		if s == v {
-			return true
-		}
-	}
-	return false
-}

netcheck/netcheck_test.go

@@ -1,292 +0,0 @@
-// Copyright (c) 2020 Tailscale Inc & AUTHORS All rights reserved.
-// Use of this source code is governed by a BSD-style
-// license that can be found in the LICENSE file.
-
-package netcheck
-
-import (
-	"context"
-	"fmt"
-	"net"
-	"reflect"
-	"testing"
-	"time"
-
-	"tailscale.com/derp/derpmap"
-	"tailscale.com/stun"
-	"tailscale.com/stun/stuntest"
-)
-
-func TestHairpinSTUN(t *testing.T) {
-	c := &Client{
-		hairTX:      stun.NewTxID(),
-		gotHairSTUN: make(chan *net.UDPAddr, 1),
-	}
-	req := stun.Request(c.hairTX)
-	if !stun.Is(req) {
-		t.Fatal("expected STUN message")
-	}
-	if !c.handleHairSTUN(req, nil) {
-		t.Fatal("expected true")
-	}
-	select {
-	case <-c.gotHairSTUN:
-	default:
-		t.Fatal("expected value")
-	}
-}
-
-func TestBasic(t *testing.T) {
-	stunAddr, cleanup := stuntest.Serve(t)
-	defer cleanup()
-
-	c := &Client{
-		DERP: derpmap.NewTestWorld(stunAddr),
-		Logf: t.Logf,
-	}
-
-	ctx, cancel := context.WithTimeout(context.Background(), 1*time.Second)
-	defer cancel()
-
-	r, err := c.GetReport(ctx)
-	if err != nil {
-		t.Fatal(err)
-	}
-	if !r.UDP {
-		t.Error("want UDP")
-	}
-	if len(r.DERPLatency) != 1 {
-		t.Errorf("expected 1 key in DERPLatency; got %+v", r.DERPLatency)
-	}
-	if _, ok := r.DERPLatency[stunAddr]; !ok {
-		t.Errorf("expected key %q in DERPLatency; got %+v", stunAddr, r.DERPLatency)
-	}
-	if r.GlobalV4 == "" {
-		t.Error("expected GlobalV4 set")
-	}
-	if r.PreferredDERP != 1 {
-		t.Errorf("PreferredDERP = %v; want 1", r.PreferredDERP)
-	}
-}
-
-func TestWorksWhenUDPBlocked(t *testing.T) {
-	blackhole, err := net.ListenPacket("udp4", "127.0.0.1:0")
-	if err != nil {
-		t.Fatalf("failed to open blackhole STUN listener: %v", err)
-	}
-	defer blackhole.Close()
-
-	stunAddr := blackhole.LocalAddr().String()
-
-	c := &Client{
-		DERP: derpmap.NewTestWorld(stunAddr),
-		Logf: t.Logf,
-	}
-	ctx, cancel := context.WithTimeout(context.Background(), 250*time.Millisecond)
-	defer cancel()
-
-	r, err := c.GetReport(ctx)
-	if err != nil {
-		t.Fatal(err)
-	}
-	want := &Report{
-		DERPLatency: map[string]time.Duration{},
-	}
-
-	if !reflect.DeepEqual(r, want) {
-		t.Errorf("mismatch\n got: %+v\nwant: %+v\n", r, want)
-	}
-}
-
-func TestAddReportHistoryAndSetPreferredDERP(t *testing.T) {
-	derps := derpmap.NewTestWorldWith(
-		&derpmap.Server{
-			ID:    1,
-			STUN4: "d1:1",
-		},
-		&derpmap.Server{
-			ID:    2,
-			STUN4: "d2:1",
-		},
-		&derpmap.Server{
-			ID:    3,
-			STUN4: "d3:1",
-		},
-	)
-	// report returns a *Report from (DERP host, time.Duration)+ pairs.
-	report := func(a ...interface{}) *Report {
-		r := &Report{DERPLatency: map[string]time.Duration{}}
-		for i := 0; i < len(a); i += 2 {
-			k := a[i].(string) + ":1"
-			switch v := a[i+1].(type) {
-			case time.Duration:
-				r.DERPLatency[k] = v
-			case int:
-				r.DERPLatency[k] = time.Second * time.Duration(v)
-			default:
-				panic(fmt.Sprintf("unexpected type %T", v))
-			}
-		}
-		return r
-	}
-	type step struct {
-		after time.Duration
-		r     *Report
-	}
-	tests := []struct {
-		name        string
-		steps       []step
-		wantDERP    int // want PreferredDERP on final step
-		wantPrevLen int // wanted len(c.prev)
-	}{
-		{
-			name: "first_reading",
-			steps: []step{
-				{0, report("d1", 2, "d2", 3)},
-			},
-			wantPrevLen: 1,
-			wantDERP:    1,
-		},
-		{
-			name: "with_two",
-			steps: []step{
-				{0, report("d1", 2, "d2", 3)},
-				{1 * time.Second, report("d1", 4, "d2", 3)},
-			},
-			wantPrevLen: 2,
-			wantDERP:    1, // t0's d1 of 2 is still best
-		},
-		{
-			name: "but_now_d1_gone",
-			steps: []step{
-				{0, report("d1", 2, "d2", 3)},
-				{1 * time.Second, report("d1", 4, "d2", 3)},
-				{2 * time.Second, report("d2", 3)},
-			},
-			wantPrevLen: 3,
-			wantDERP:    2, // only option
-		},
-		{
-			name: "d1_is_back",
-			steps: []step{
-				{0, report("d1", 2, "d2", 3)},
-				{1 * time.Second, report("d1", 4, "d2", 3)},
-				{2 * time.Second, report("d2", 3)},
-				{3 * time.Second, report("d1", 4, "d2", 3)}, // same as 2 seconds ago
-			},
-			wantPrevLen: 4,
-			wantDERP:    1, // t0's d1 of 2 is still best
-		},
-		{
-			name: "things_clean_up",
-			steps: []step{
-				{0, report("d1", 1, "d2", 2)},
-				{1 * time.Second, report("d1", 1, "d2", 2)},
-				{2 * time.Second, report("d1", 1, "d2", 2)},
-				{3 * time.Second, report("d1", 1, "d2", 2)},
-				{10 * time.Minute, report("d3", 3)},
-			},
-			wantPrevLen: 1, // t=[0123]s all gone. (too old, older than 10 min)
-			wantDERP:    3, // only option
-		},
-	}
-	for _, tt := range tests {
-		t.Run(tt.name, func(t *testing.T) {
-			fakeTime := time.Unix(123, 0)
-			c := &Client{
-				DERP:    derps,
-				TimeNow: func() time.Time { return fakeTime },
-			}
-			for _, s := range tt.steps {
-				fakeTime = fakeTime.Add(s.after)
-				c.addReportHistoryAndSetPreferredDERP(s.r)
-			}
-			lastReport := tt.steps[len(tt.steps)-1].r
-			if got, want := len(c.prev), tt.wantPrevLen; got != want {
-				t.Errorf("len(prev) = %v; want %v", got, want)
-			}
-			if got, want := lastReport.PreferredDERP, tt.wantDERP; got != want {
-				t.Errorf("PreferredDERP = %v; want %v", got, want)
-			}
-		})
-	}
-}
-
-func TestPickSubset(t *testing.T) {
-	derps := derpmap.NewTestWorldWith(
-		&derpmap.Server{
-			ID:    1,
-			STUN4: "d1:4",
-			STUN6: "d1:6",
-		},
-		&derpmap.Server{
-			ID:    2,
-			STUN4: "d2:4",
-			STUN6: "d2:6",
-		},
-		&derpmap.Server{
-			ID:    3,
-			STUN4: "d3:4",
-			STUN6: "d3:6",
-		},
-	)
-	tests := []struct {
-		name      string
-		last      *Report
-		want4     []string
-		want6     []string
-		wantTries map[string]int
-	}{
-		{
-			name:  "fresh",
-			last:  nil,
-			want4: []string{"d1:4", "d2:4", "d3:4"},
-			want6: []string{"d1:6", "d2:6", "d3:6"},
-			wantTries: map[string]int{
-				"d1:4": 2,
-				"d2:4": 2,
-				"d3:4": 2,
-				"d1:6": 1,
-				"d2:6": 1,
-				"d3:6": 1,
-			},
-		},
-		{
-			name: "1_and_3_closest",
-			last: &Report{
-				DERPLatency: map[string]time.Duration{
-					"d1:4": 15 * time.Millisecond,
-					"d2:4": 300 * time.Millisecond,
-					"d3:4": 25 * time.Millisecond,
-				},
-			},
-			want4: []string{"d1:4", "d2:4", "d3:4"},
-			want6: []string{"d1:6", "d3:6"},
-			wantTries: map[string]int{
-				"d1:4": 2,
-				"d3:4": 2,
-				"d2:4": 1,
-				"d1:6": 1,
-				"d3:6": 1,
-			},
-		},
-	}
-	for _, tt := range tests {
-		t.Run(tt.name, func(t *testing.T) {
-			c := &Client{DERP: derps, last: tt.last}
-			got4, got6, gotTries, err := c.pickSubset()
-			if err != nil {
-				t.Fatal(err)
-			}
-			if !reflect.DeepEqual(got4, tt.want4) {
-				t.Errorf("stuns4 = %q; want %q", got4, tt.want4)
-			}
-			if !reflect.DeepEqual(got6, tt.want6) {
-				t.Errorf("stuns6 = %q; want %q", got6, tt.want6)
-			}
-			if !reflect.DeepEqual(gotTries, tt.wantTries) {
-				t.Errorf("tries = %v; want %v", gotTries, tt.wantTries)
-			}
-		})
-	}
-}

paths/paths.go

@@ -11,9 +11,15 @@ import (
 	"runtime"
 )
 
-// LegacyConfigPath is the path used by the pre-tailscaled "relaynode"
-// daemon's config file.
-const LegacyConfigPath = "/var/lib/tailscale/relay.conf"
+// LegacyConfigPath returns the path used by the pre-tailscaled
+// "relaynode" daemon's config file. It returns the empty string for
+// platforms where relaynode never ran.
+func LegacyConfigPath() string {
+	if runtime.GOOS == "windows" {
+		return ""
+	}
+	return "/var/lib/tailscale/relay.conf"
+}
 
 // DefaultTailscaledSocket returns the path to the tailscaled Unix socket
 // or the empty string if there's no reasonable default.

portlist/netstat_exec.go

@@ -13,12 +13,22 @@ import (
 	exec "tailscale.com/tempfork/osexec"
 )
 
+var osHideWindow func(*exec.Cmd) // non-nil on Windows; see portlist_windows.go
+
+// hideWindow returns c. On Windows it first sets SysProcAttr.HideWindow.
+func hideWindow(c *exec.Cmd) *exec.Cmd {
+	if osHideWindow != nil {
+		osHideWindow(c)
+	}
+	return c
+}
+
 func listPortsNetstat(arg string) (List, error) {
 	exe, err := exec.LookPath("netstat")
 	if err != nil {
 		return nil, fmt.Errorf("netstat: lookup: %v", err)
 	}
-	output, err := exec.Command(exe, arg).Output()
+	output, err := hideWindow(exec.Command(exe, arg)).Output()
 	if err != nil {
 		xe, ok := err.(*exec.ExitError)
 		stderr := ""

portlist/portlist.go

@@ -21,8 +21,6 @@ type Port struct {
 // List is a list of Ports.
 type List []Port
 
-var protos = []string{"tcp", "udp"}
-
 func (a *Port) lessThan(b *Port) bool {
 	if a.Port < b.Port {
 		return true