net/netns, net/interfaces: move defaultRouteInterface, add Android fallback

Signed-off-by: Brad Fitzpatrick <bradfitz@tailscale.com>

.github/workflows/linux32.yml

@@ -0,0 +1,48 @@
+name: Linux 32-bit
+
+on:
+  push:
+    branches:
+      - main
+  pull_request:
+    branches:
+      - '*'
+
+jobs:
+  build:
+    runs-on: ubuntu-latest
+
+    if: "!contains(github.event.head_commit.message, '[ci skip]')"
+
+    steps:
+
+    - name: Set up Go
+      uses: actions/setup-go@v1
+      with:
+        go-version: 1.14
+      id: go
+
+    - name: Check out code into the Go module directory
+      uses: actions/checkout@v1
+
+    - name: Basic build
+      run: GOARCH=386 go build ./cmd/...
+
+    - name: Run tests on linux
+      run: GOARCH=386 go test ./...
+
+    - uses: k0kubun/action-slack@v2.0.0
+      with:
+        payload: |
+          {
+            "attachments": [{
+              "text": "${{ job.status }}: ${{ github.workflow }} <https://github.com/${{ github.repository }}/commit/${{ github.sha }}/checks|${{ env.COMMIT_DATE }} #${{ env.COMMIT_NUMBER_OF_DAY }}> " +
+                      "(<https://github.com/${{ github.repository }}/commit/${{ github.sha }}|" + "${{ github.sha }}".substring(0, 10) + ">) " +
+                      "of ${{ github.repository }}@" + "${{ github.ref }}".split('/').reverse()[0] + " by ${{ github.event.head_commit.committer.name }}",
+              "color": "danger"
+            }]
+          }
+      env:
+        SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK_URL }}
+      if: failure() && github.event_name == 'push'
+

.github/workflows/staticcheck.yml

@@ -21,6 +21,9 @@ jobs:
     - name: Check out code
       uses: actions/checkout@v1
 
+    - name: Run go vet
+      run: go vet ./...
+
     - name: Print staticcheck version
       run: go run honnef.co/go/tools/cmd/staticcheck -version
 

README.md

@@ -6,17 +6,24 @@ Private WireGuard® networks made easy
 
 ## Overview
 
-This repository contains all the open source Tailscale code.
-It currently includes the Linux client.
+This repository contains all the open source Tailscale client code and
+the `tailscaled` daemon and `tailscale` CLI tool. The `tailscaled`
+daemon runs primarily on Linux; it also works to varying degrees on
+FreeBSD, OpenBSD, Darwin, and Windows.
 
-The Linux client is currently `cmd/relaynode`, but will
-soon be replaced by `cmd/tailscaled`.
+The Android app is at https://github.com/tailscale/tailscale-android
 
 ## Using
 
 We serve packages for a variety of distros at
 https://pkgs.tailscale.com .
 
+## Other clients
+
+The [macOS, iOS, and Windows clients](https://tailscale.com/download)
+use the code in this repository but additionally include small GUI
+wrappers that are not open source.
+
 ## Building
 
 ```
@@ -35,10 +42,8 @@ Please file any issues about this code or the hosted service on
 
 ## Contributing
 
-`under_construction.gif`
-
-PRs welcome, but we are still working out our contribution process and
-tooling.
+PRs welcome! But please file bugs. Commit messages should [reference
+bugs](https://docs.github.com/en/github/writing-on-github/autolinked-references-and-urls).
 
 We require [Developer Certificate of
 Origin](https://en.wikipedia.org/wiki/Developer_Certificate_of_Origin)
@@ -46,7 +51,7 @@ Origin](https://en.wikipedia.org/wiki/Developer_Certificate_of_Origin)
 
 ## About Us
 
-We are apenwarr, bradfitz, crawshaw, danderson, dfcarney,
+We are apenwarr, bradfitz, crawshaw, danderson, dfcarney, josharian
 from Tailscale Inc.
 You can learn more about us from [our website](https://tailscale.com).
 

atomicfile/atomicfile.go

@@ -9,20 +9,39 @@
 package atomicfile // import "tailscale.com/atomicfile"
 
 import (
-	"fmt"
 	"io/ioutil"
 	"os"
+	"path/filepath"
+	"runtime"
 )
 
 // WriteFile writes data to filename+some suffix, then renames it
 // into filename.
-func WriteFile(filename string, data []byte, perm os.FileMode) error {
-	tmpname := filename + ".new.tmp"
-	if err := ioutil.WriteFile(tmpname, data, perm); err != nil {
-		return fmt.Errorf("%#v: %v", tmpname, err)
+func WriteFile(filename string, data []byte, perm os.FileMode) (err error) {
+	f, err := ioutil.TempFile(filepath.Dir(filename), filepath.Base(filename)+".tmp")
+	if err != nil {
+		return err
 	}
-	if err := os.Rename(tmpname, filename); err != nil {
-		return fmt.Errorf("%#v->%#v: %v", tmpname, filename, err)
+	tmpName := f.Name()
+	defer func() {
+		if err != nil {
+			f.Close()
+			os.Remove(tmpName)
+		}
+	}()
+	if _, err := f.Write(data); err != nil {
+		return err
 	}
-	return nil
+	if runtime.GOOS != "windows" {
+		if err := f.Chmod(perm); err != nil {
+			return err
+		}
+	}
+	if err := f.Sync(); err != nil {
+		return err
+	}
+	if err := f.Close(); err != nil {
+		return err
+	}
+	return os.Rename(tmpName, filename)
 }

cmd/cloner/cloner.go

@@ -0,0 +1,264 @@
+// Copyright (c) 2020 Tailscale Inc & AUTHORS All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+// Cloner is a tool to automate the creation of a Clone method.
+//
+// The result of the Clone method aliases no memory that can be edited
+// with the original.
+//
+// This tool makes lots of implicit assumptions about the types you feed it.
+// In particular, it can only write relatively "shallow" Clone methods.
+// That is, if a type contains another named struct type, cloner assumes that
+// named type will also have a Clone method.
+package main
+
+import (
+	"bytes"
+	"flag"
+	"fmt"
+	"go/ast"
+	"go/format"
+	"go/token"
+	"go/types"
+	"io/ioutil"
+	"log"
+	"os"
+	"strings"
+
+	"golang.org/x/tools/go/packages"
+)
+
+var (
+	flagTypes     = flag.String("type", "", "comma-separated list of types; required")
+	flagOutput    = flag.String("output", "", "output file; required")
+	flagBuildTags = flag.String("tags", "", "compiler build tags to apply")
+)
+
+func main() {
+	log.SetFlags(0)
+	log.SetPrefix("cloner: ")
+	flag.Parse()
+	if len(*flagTypes) == 0 {
+		flag.Usage()
+		os.Exit(2)
+	}
+	typeNames := strings.Split(*flagTypes, ",")
+
+	cfg := &packages.Config{
+		Mode:  packages.NeedTypes | packages.NeedTypesInfo | packages.NeedSyntax | packages.NeedName,
+		Tests: false,
+	}
+	if *flagBuildTags != "" {
+		cfg.BuildFlags = []string{"-tags=" + *flagBuildTags}
+	}
+	pkgs, err := packages.Load(cfg, ".")
+	if err != nil {
+		log.Fatal(err)
+	}
+	if len(pkgs) != 1 {
+		log.Fatalf("wrong number of packages: %d", len(pkgs))
+	}
+	pkg := pkgs[0]
+	buf := new(bytes.Buffer)
+	imports := make(map[string]struct{})
+	for _, typeName := range typeNames {
+		found := false
+		for _, file := range pkg.Syntax {
+			//var fbuf bytes.Buffer
+			//ast.Fprint(&fbuf, pkg.Fset, file, nil)
+			//fmt.Println(fbuf.String())
+
+			for _, d := range file.Decls {
+				decl, ok := d.(*ast.GenDecl)
+				if !ok || decl.Tok != token.TYPE {
+					continue
+				}
+				for _, s := range decl.Specs {
+					spec, ok := s.(*ast.TypeSpec)
+					if !ok || spec.Name.Name != typeName {
+						continue
+					}
+					typeNameObj := pkg.TypesInfo.Defs[spec.Name]
+					typ, ok := typeNameObj.Type().(*types.Named)
+					if !ok {
+						continue
+					}
+					pkg := typeNameObj.Pkg()
+					gen(buf, imports, typeName, typ, pkg)
+				}
+				found = true
+			}
+		}
+		if !found {
+			log.Fatalf("could not find type %s", typeName)
+		}
+	}
+
+	contents := new(bytes.Buffer)
+	fmt.Fprintf(contents, header, *flagTypes, pkg.Name)
+	fmt.Fprintf(contents, "import (\n")
+	for s := range imports {
+		fmt.Fprintf(contents, "\t%q\n", s)
+	}
+	fmt.Fprintf(contents, ")\n\n")
+	contents.Write(buf.Bytes())
+
+	out, err := format.Source(contents.Bytes())
+	if err != nil {
+		log.Fatalf("%s, in source:\n%s", err, contents.Bytes())
+	}
+
+	output := *flagOutput
+	if output == "" {
+		flag.Usage()
+		os.Exit(2)
+	}
+	if err := ioutil.WriteFile(output, out, 0666); err != nil {
+		log.Fatal(err)
+	}
+}
+
+const header = `// Copyright (c) 2020 Tailscale Inc & AUTHORS All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+// Code generated by tailscale.com/cmd/cloner -type %s; DO NOT EDIT.
+
+package %s
+
+`
+
+func gen(buf *bytes.Buffer, imports map[string]struct{}, name string, typ *types.Named, thisPkg *types.Package) {
+	pkgQual := func(pkg *types.Package) string {
+		if thisPkg == pkg {
+			return ""
+		}
+		imports[pkg.Path()] = struct{}{}
+		return pkg.Name()
+	}
+	importedName := func(t types.Type) string {
+		return types.TypeString(t, pkgQual)
+	}
+
+	switch t := typ.Underlying().(type) {
+	case *types.Struct:
+		_ = t
+		name := typ.Obj().Name()
+		fmt.Fprintf(buf, "// Clone makes a deep copy of %s.\n", name)
+		fmt.Fprintf(buf, "// The result aliases no memory with the original.\n")
+		fmt.Fprintf(buf, "func (src *%s) Clone() *%s {\n", name, name)
+		writef := func(format string, args ...interface{}) {
+			fmt.Fprintf(buf, "\t"+format+"\n", args...)
+		}
+		writef("if src == nil {")
+		writef("\treturn nil")
+		writef("}")
+		writef("dst := new(%s)", name)
+		writef("*dst = *src")
+		for i := 0; i < t.NumFields(); i++ {
+			fname := t.Field(i).Name()
+			ft := t.Field(i).Type()
+			if !containsPointers(ft) {
+				continue
+			}
+			if named, _ := ft.(*types.Named); named != nil && !hasBasicUnderlying(ft) {
+				writef("dst.%s = *src.%s.Clone()", fname, fname)
+				continue
+			}
+			switch ft := ft.Underlying().(type) {
+			case *types.Slice:
+				if containsPointers(ft.Elem()) {
+					n := importedName(ft.Elem())
+					writef("dst.%s = make([]%s, len(src.%s))", fname, n, fname)
+					writef("for i := range dst.%s {", fname)
+					if _, isPtr := ft.Elem().(*types.Pointer); isPtr {
+						writef("\tdst.%s[i] = src.%s[i].Clone()", fname, fname)
+					} else {
+						writef("\tdst.%s[i] = *src.%s[i].Clone()", fname, fname)
+					}
+					writef("}")
+				} else {
+					writef("dst.%s = append(src.%s[:0:0], src.%s...)", fname, fname, fname)
+				}
+			case *types.Pointer:
+				if named, _ := ft.Elem().(*types.Named); named != nil && containsPointers(ft.Elem()) {
+					writef("dst.%s = src.%s.Clone()", fname, fname)
+					continue
+				}
+				n := importedName(ft.Elem())
+				writef("if dst.%s != nil {", fname)
+				writef("\tdst.%s = new(%s)", fname, n)
+				writef("\t*dst.%s = *src.%s", fname, fname)
+				if containsPointers(ft.Elem()) {
+					writef("\t" + `panic("TODO pointers in pointers")`)
+				}
+				writef("}")
+			case *types.Map:
+				writef("if dst.%s != nil {", fname)
+				writef("\tdst.%s = map[%s]%s{}", fname, importedName(ft.Key()), importedName(ft.Elem()))
+				if sliceType, isSlice := ft.Elem().(*types.Slice); isSlice {
+					n := importedName(sliceType.Elem())
+					writef("\tfor k := range src.%s {", fname)
+					// use zero-length slice instead of nil to ensure
+					// the key is always copied.
+					writef("\t\tdst.%s[k] = append([]%s{}, src.%s[k]...)", fname, n, fname)
+					writef("\t}")
+				} else if containsPointers(ft.Elem()) {
+					writef("\t\t" + `panic("TODO map value pointers")`)
+				} else {
+					writef("\tfor k, v := range src.%s {", fname)
+					writef("\t\tdst.%s[k] = v", fname)
+					writef("\t}")
+				}
+				writef("}")
+			case *types.Struct:
+				writef(`panic("TODO struct %s")`, fname)
+			default:
+				writef(`panic(fmt.Sprintf("TODO: %T", ft))`)
+			}
+		}
+		writef("return dst")
+		fmt.Fprintf(buf, "}\n\n")
+	}
+}
+
+func hasBasicUnderlying(typ types.Type) bool {
+	switch typ.Underlying().(type) {
+	case *types.Slice, *types.Map:
+		return true
+	default:
+		return false
+	}
+}
+
+func containsPointers(typ types.Type) bool {
+	switch typ.String() {
+	case "time.Time":
+		// time.Time contains a pointer that does not need copying
+		return false
+	case "inet.af/netaddr.IP":
+		return false
+	}
+	switch ft := typ.Underlying().(type) {
+	case *types.Array:
+		return containsPointers(ft.Elem())
+	case *types.Chan:
+		return true
+	case *types.Interface:
+		return true // a little too broad
+	case *types.Map:
+		return true
+	case *types.Pointer:
+		return true
+	case *types.Slice:
+		return true
+	case *types.Struct:
+		for i := 0; i < ft.NumFields(); i++ {
+			if containsPointers(ft.Field(i).Type()) {
+				return true
+			}
+		}
+	}
+	return false
+}

cmd/derper/derper.go

@@ -33,6 +33,7 @@ import (
 	"tailscale.com/net/stun"
 	"tailscale.com/tsweb"
 	"tailscale.com/types/key"
+	"tailscale.com/version"
 )
 
 var (
@@ -185,7 +186,7 @@ func main() {
 		}
 		httpsrv.TLSConfig = certManager.TLSConfig()
 		go func() {
-			err := http.ListenAndServe(":80", certManager.HTTPHandler(tsweb.Port80Handler{mux}))
+			err := http.ListenAndServe(":80", certManager.HTTPHandler(tsweb.Port80Handler{Main: mux}))
 			if err != nil {
 				if err != http.ErrServerClosed {
 					log.Fatal(err)
@@ -221,6 +222,7 @@ func debugHandler(s *derp.Server) http.Handler {
 		f("<li><b>Hostname:</b> %v</li>\n", *hostname)
 		f("<li><b>Uptime:</b> %v</li>\n", tsweb.Uptime())
 		f("<li><b>Mesh Key:</b> %v</li>\n", s.HasMeshKey())
+		f("<li><b>Version:</b> %v</li>\n", version.LONG)
 
 		f(`<li><a href="/debug/vars">/debug/vars</a> (Go)</li>
    <li><a href="/debug/varz">/debug/varz</a> (Prometheus)</li>

cmd/relaynode/relaynode.od

@@ -1 +0,0 @@
-# placeholder to work around redo bug

cmd/tailscale/cli/cli.go

@@ -0,0 +1,128 @@
+// Copyright (c) 2020 Tailscale Inc & AUTHORS All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+// Package cli contains the cmd/tailscale CLI code in a package that can be included
+// in other wrapper binaries such as the Mac and Windows clients.
+package cli
+
+import (
+	"context"
+	"flag"
+	"log"
+	"net"
+	"os"
+	"os/signal"
+	"runtime"
+	"strings"
+	"syscall"
+
+	"github.com/peterbourgon/ff/v2/ffcli"
+	"tailscale.com/ipn"
+	"tailscale.com/paths"
+	"tailscale.com/safesocket"
+)
+
+// ActLikeCLI reports whether a GUI application should act like the
+// CLI based on os.Args, GOOS, the context the process is running in
+// (pty, parent PID), etc.
+func ActLikeCLI() bool {
+	if len(os.Args) < 2 {
+		return false
+	}
+	switch os.Args[1] {
+	case "up", "status", "netcheck", "version",
+		"-V", "--version", "-h", "--help":
+		return true
+	}
+	return false
+}
+
+// Run runs the CLI. The args do not include the binary name.
+func Run(args []string) error {
+	if len(args) == 1 && (args[0] == "-V" || args[0] == "--version") {
+		args = []string{"version"}
+	}
+
+	rootfs := flag.NewFlagSet("tailscale", flag.ExitOnError)
+	rootfs.StringVar(&rootArgs.socket, "socket", paths.DefaultTailscaledSocket(), "path to tailscaled's unix socket")
+
+	rootCmd := &ffcli.Command{
+		Name:       "tailscale",
+		ShortUsage: "tailscale subcommand [flags]",
+		ShortHelp:  "The easiest, most secure way to use WireGuard.",
+		LongHelp: strings.TrimSpace(`
+This CLI is still under active development. Commands and flags will
+change in the future.
+`),
+		Subcommands: []*ffcli.Command{
+			upCmd,
+			netcheckCmd,
+			statusCmd,
+			versionCmd,
+		},
+		FlagSet: rootfs,
+		Exec:    func(context.Context, []string) error { return flag.ErrHelp },
+	}
+
+	if err := rootCmd.Parse(args); err != nil {
+		return err
+	}
+
+	err := rootCmd.Run(context.Background())
+	if err == flag.ErrHelp {
+		return nil
+	}
+	return err
+}
+
+func fatalf(format string, a ...interface{}) {
+	log.SetFlags(0)
+	log.Fatalf(format, a...)
+}
+
+var rootArgs struct {
+	socket string
+}
+
+func connect(ctx context.Context) (net.Conn, *ipn.BackendClient, context.Context, context.CancelFunc) {
+	c, err := safesocket.Connect(rootArgs.socket, 41112)
+	if err != nil {
+		if runtime.GOOS != "windows" && rootArgs.socket == "" {
+			fatalf("--socket cannot be empty")
+		}
+		fatalf("Failed to connect to connect to tailscaled. (safesocket.Connect: %v)\n", err)
+	}
+	clientToServer := func(b []byte) {
+		ipn.WriteMsg(c, b)
+	}
+
+	ctx, cancel := context.WithCancel(ctx)
+
+	go func() {
+		interrupt := make(chan os.Signal, 1)
+		signal.Notify(interrupt, syscall.SIGINT, syscall.SIGTERM)
+		<-interrupt
+		c.Close()
+		cancel()
+	}()
+
+	bc := ipn.NewBackendClient(log.Printf, clientToServer)
+	return c, bc, ctx, cancel
+}
+
+// pump receives backend messages on conn and pushes them into bc.
+func pump(ctx context.Context, bc *ipn.BackendClient, conn net.Conn) {
+	defer conn.Close()
+	for ctx.Err() == nil {
+		msg, err := ipn.ReadMsg(conn)
+		if err != nil {
+			if ctx.Err() != nil {
+				return
+			}
+			log.Printf("ReadMsg: %v\n", err)
+			break
+		}
+		bc.GotNotifyMsg(msg)
+	}
+}

cmd/tailscale/cli/netcheck.go

@@ -2,7 +2,7 @@
 // Use of this source code is governed by a BSD-style
 // license that can be found in the LICENSE file.
 
-package main
+package cli
 
 import (
 	"context"
@@ -117,6 +117,7 @@ func printReport(dm *tailcfg.DERPMap, report *netcheck.Report) error {
 	}
 	fmt.Printf("\t* MappingVariesByDestIP: %v\n", report.MappingVariesByDestIP)
 	fmt.Printf("\t* HairPinning: %v\n", report.HairPinning)
+	fmt.Printf("\t* PortMapping: %v\n", portMapping(report))
 
 	// When DERP latency checking failed,
 	// magicsock will try to pick the DERP server that
@@ -142,3 +143,20 @@ func printReport(dm *tailcfg.DERPMap, report *netcheck.Report) error {
 	}
 	return nil
 }
+
+func portMapping(r *netcheck.Report) string {
+	if !r.AnyPortMappingChecked() {
+		return "not checked"
+	}
+	var got []string
+	if r.UPnP.EqualBool(true) {
+		got = append(got, "UPnP")
+	}
+	if r.PMP.EqualBool(true) {
+		got = append(got, "NAT-PMP")
+	}
+	if r.PCP.EqualBool(true) {
+		got = append(got, "PCP")
+	}
+	return strings.Join(got, ", ")
+}

cmd/tailscale/cli/status.go

@@ -2,7 +2,7 @@
 // Use of this source code is governed by a BSD-style
 // license that can be found in the LICENSE file.
 
-package main
+package cli
 
 import (
 	"bytes"
@@ -25,13 +25,14 @@ import (
 
 var statusCmd = &ffcli.Command{
 	Name:       "status",
-	ShortUsage: "status [-web] [-json]",
+	ShortUsage: "status [-active] [-web] [-json]",
 	ShortHelp:  "Show state of tailscaled and its connections",
 	Exec:       runStatus,
 	FlagSet: (func() *flag.FlagSet {
 		fs := flag.NewFlagSet("status", flag.ExitOnError)
 		fs.BoolVar(&statusArgs.json, "json", false, "output in JSON format (WARNING: format subject to change)")
 		fs.BoolVar(&statusArgs.web, "web", false, "run webserver with HTML showing status")
+		fs.BoolVar(&statusArgs.active, "active", false, "filter output to only peers with active sessions (not applicable to web mode)")
 		fs.StringVar(&statusArgs.listen, "listen", "127.0.0.1:8384", "listen address; use port 0 for automatic")
 		fs.BoolVar(&statusArgs.browser, "browser", true, "Open a browser in web mode")
 		return fs
@@ -43,6 +44,7 @@ var statusArgs struct {
 	web     bool   // run webserver
 	listen  string // in web mode, webserver address to listen on, empty means auto
 	browser bool   // in web mode, whether to open browser
+	active  bool   // in CLI mode, filter output to only peers with active sessions
 }
 
 func runStatus(ctx context.Context, args []string) error {
@@ -76,6 +78,13 @@ func runStatus(ctx context.Context, args []string) error {
 		return err
 	}
 	if statusArgs.json {
+		if statusArgs.active {
+			for peer, ps := range st.Peer {
+				if !peerActive(ps) {
+					delete(st.Peer, peer)
+				}
+			}
+		}
 		j, err := json.MarshalIndent(st, "", "  ")
 		if err != nil {
 			return err
@@ -120,6 +129,10 @@ func runStatus(ctx context.Context, args []string) error {
 	f := func(format string, a ...interface{}) { fmt.Fprintf(&buf, format, a...) }
 	for _, peer := range st.Peers() {
 		ps := st.Peer[peer]
+		active := peerActive(ps)
+		if statusArgs.active && !active {
+			continue
+		}
 		f("%s %-7s %-15s %-18s tx=%8d rx=%8d ",
 			peer.ShortString(),
 			ps.OS,
@@ -128,8 +141,6 @@ func runStatus(ctx context.Context, args []string) error {
 			ps.TxBytes,
 			ps.RxBytes,
 		)
-		// TODO: let server report this active bool instead
-		active := !ps.LastWrite.IsZero() && time.Since(ps.LastWrite) < 2*time.Minute
 		relay := ps.Relay
 		if active && relay != "" && ps.CurAddr == "" {
 			relay = "*" + relay + "*"
@@ -152,3 +163,10 @@ func runStatus(ctx context.Context, args []string) error {
 	os.Stdout.Write(buf.Bytes())
 	return nil
 }
+
+// peerActive reports whether ps has recent activity.
+//
+// TODO: have the server report this bool instead.
+func peerActive(ps *ipnstate.PeerStatus) bool {
+	return !ps.LastWrite.IsZero() && time.Since(ps.LastWrite) < 2*time.Minute
+}

cmd/tailscale/cli/up.go

@@ -0,0 +1,258 @@
+// Copyright (c) 2020 Tailscale Inc & AUTHORS All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package cli
+
+import (
+	"bytes"
+	"context"
+	"flag"
+	"fmt"
+	"log"
+	"os"
+	"os/exec"
+	"runtime"
+	"strconv"
+	"strings"
+
+	"github.com/peterbourgon/ff/v2/ffcli"
+	"github.com/tailscale/wireguard-go/wgcfg"
+	"inet.af/netaddr"
+	"tailscale.com/ipn"
+	"tailscale.com/tailcfg"
+	"tailscale.com/version"
+	"tailscale.com/wgengine/router"
+)
+
+// globalStateKey is the ipn.StateKey that tailscaled loads on
+// startup.
+//
+// We have to support multiple state keys for other OSes (Windows in
+// particular), but right now Unix daemons run with a single
+// node-global state. To keep open the option of having per-user state
+// later, the global state key doesn't look like a username.
+const globalStateKey = "_daemon"
+
+var upCmd = &ffcli.Command{
+	Name:       "up",
+	ShortUsage: "up [flags]",
+	ShortHelp:  "Connect to your Tailscale network",
+
+	LongHelp: strings.TrimSpace(`
+"tailscale up" connects this machine to your Tailscale network,
+triggering authentication if necessary.
+
+The flags passed to this command are specific to this machine. If you don't
+specify any flags, options are reset to their default.
+`),
+	FlagSet: (func() *flag.FlagSet {
+		upf := flag.NewFlagSet("up", flag.ExitOnError)
+		upf.StringVar(&upArgs.server, "login-server", "https://login.tailscale.com", "base URL of control server")
+		upf.BoolVar(&upArgs.acceptRoutes, "accept-routes", false, "accept routes advertised by other Tailscale nodes")
+		upf.BoolVar(&upArgs.acceptDNS, "accept-dns", true, "accept DNS configuration from the admin panel")
+		upf.BoolVar(&upArgs.singleRoutes, "host-routes", true, "install host routes to other Tailscale nodes")
+		upf.BoolVar(&upArgs.shieldsUp, "shields-up", false, "don't allow incoming connections")
+		upf.StringVar(&upArgs.advertiseTags, "advertise-tags", "", "ACL tags to request (comma-separated, e.g. eng,montreal,ssh)")
+		upf.StringVar(&upArgs.authKey, "authkey", "", "node authorization key")
+		upf.StringVar(&upArgs.hostname, "hostname", "", "hostname to use instead of the one provided by the OS")
+		upf.BoolVar(&upArgs.enableDERP, "enable-derp", true, "enable the use of DERP servers")
+		if runtime.GOOS == "linux" || isBSD(runtime.GOOS) || version.OS() == "macOS" {
+			upf.StringVar(&upArgs.advertiseRoutes, "advertise-routes", "", "routes to advertise to other nodes (comma-separated, e.g. 10.0.0.0/8,192.168.0.0/24)")
+		}
+		if runtime.GOOS == "linux" {
+			upf.BoolVar(&upArgs.snat, "snat-subnet-routes", true, "source NAT traffic to local routes advertised with -advertise-routes")
+			upf.StringVar(&upArgs.netfilterMode, "netfilter-mode", "on", "netfilter mode (one of on, nodivert, off)")
+		}
+		return upf
+	})(),
+	Exec: runUp,
+}
+
+var upArgs struct {
+	server          string
+	acceptRoutes    bool
+	acceptDNS       bool
+	singleRoutes    bool
+	shieldsUp       bool
+	advertiseRoutes string
+	advertiseTags   string
+	enableDERP      bool
+	snat            bool
+	netfilterMode   string
+	authKey         string
+	hostname        string
+}
+
+// parseIPOrCIDR parses an IP address or a CIDR prefix. If the input
+// is an IP address, it is returned in CIDR form with a /32 mask for
+// IPv4 or a /128 mask for IPv6.
+func parseIPOrCIDR(s string) (wgcfg.CIDR, bool) {
+	if strings.Contains(s, "/") {
+		ret, err := wgcfg.ParseCIDR(s)
+		if err != nil {
+			return wgcfg.CIDR{}, false
+		}
+		return ret, true
+	}
+
+	ip, ok := wgcfg.ParseIP(s)
+	if !ok {
+		return wgcfg.CIDR{}, false
+	}
+	if ip.Is4() {
+		return wgcfg.CIDR{IP: ip, Mask: 32}, true
+	} else {
+		return wgcfg.CIDR{IP: ip, Mask: 128}, true
+	}
+}
+
+func isBSD(s string) bool {
+	return s == "dragonfly" || s == "freebsd" || s == "netbsd" || s == "openbsd"
+}
+
+func warnf(format string, args ...interface{}) {
+	fmt.Printf("Warning: "+format+"\n", args...)
+}
+
+// checkIPForwarding prints warnings if IP forwarding is not
+// enabled, or if we were unable to verify the state of IP forwarding.
+func checkIPForwarding() {
+	var key string
+
+	if runtime.GOOS == "linux" {
+		key = "net.ipv4.ip_forward"
+	} else if isBSD(runtime.GOOS) || version.OS() == "macOS" {
+		key = "net.inet.ip.forwarding"
+	} else {
+		return
+	}
+
+	bs, err := exec.Command("sysctl", "-n", key).Output()
+	if err != nil {
+		warnf("couldn't check %s (%v).\nSubnet routes won't work without IP forwarding.", key, err)
+		return
+	}
+	on, err := strconv.ParseBool(string(bytes.TrimSpace(bs)))
+	if err != nil {
+		warnf("couldn't parse %s (%v).\nSubnet routes won't work without IP forwarding.", key, err)
+		return
+	}
+	if !on {
+		warnf("%s is disabled. Subnet routes won't work.", key)
+	}
+}
+
+func runUp(ctx context.Context, args []string) error {
+	if len(args) > 0 {
+		log.Fatalf("too many non-flag arguments: %q", args)
+	}
+
+	var routes []wgcfg.CIDR
+	if upArgs.advertiseRoutes != "" {
+		advroutes := strings.Split(upArgs.advertiseRoutes, ",")
+		for _, s := range advroutes {
+			cidr, ok := parseIPOrCIDR(s)
+			ipp, err := netaddr.ParseIPPrefix(s) // parse it with other pawith both packages
+			if !ok || err != nil {
+				fatalf("%q is not a valid IP address or CIDR prefix", s)
+			}
+			if ipp != ipp.Masked() {
+				fatalf("%s has non-address bits set; expected %s", ipp, ipp.Masked())
+			}
+			routes = append(routes, cidr)
+		}
+		checkIPForwarding()
+	}
+
+	var tags []string
+	if upArgs.advertiseTags != "" {
+		tags = strings.Split(upArgs.advertiseTags, ",")
+		for _, tag := range tags {
+			err := tailcfg.CheckTag(tag)
+			if err != nil {
+				fatalf("tag: %q: %s", tag, err)
+			}
+		}
+	}
+
+	if len(upArgs.hostname) > 256 {
+		fatalf("hostname too long: %d bytes (max 256)", len(upArgs.hostname))
+	}
+
+	// TODO(apenwarr): fix different semantics between prefs and uflags
+	// TODO(apenwarr): allow setting/using CorpDNS
+	prefs := ipn.NewPrefs()
+	prefs.ControlURL = upArgs.server
+	prefs.WantRunning = true
+	prefs.RouteAll = upArgs.acceptRoutes
+	prefs.CorpDNS = upArgs.acceptDNS
+	prefs.AllowSingleHosts = upArgs.singleRoutes
+	prefs.ShieldsUp = upArgs.shieldsUp
+	prefs.AdvertiseRoutes = routes
+	prefs.AdvertiseTags = tags
+	prefs.NoSNAT = !upArgs.snat
+	prefs.DisableDERP = !upArgs.enableDERP
+	prefs.Hostname = upArgs.hostname
+	if runtime.GOOS == "linux" {
+		switch upArgs.netfilterMode {
+		case "on":
+			prefs.NetfilterMode = router.NetfilterOn
+		case "nodivert":
+			prefs.NetfilterMode = router.NetfilterNoDivert
+			warnf("netfilter=nodivert; add iptables calls to ts-* chains manually.")
+		case "off":
+			prefs.NetfilterMode = router.NetfilterOff
+			warnf("netfilter=off; configure iptables yourself.")
+		default:
+			fatalf("invalid value --netfilter-mode: %q", upArgs.netfilterMode)
+		}
+	}
+
+	c, bc, ctx, cancel := connect(ctx)
+	defer cancel()
+
+	var printed bool
+
+	bc.SetPrefs(prefs)
+	opts := ipn.Options{
+		StateKey: globalStateKey,
+		AuthKey:  upArgs.authKey,
+		Notify: func(n ipn.Notify) {
+			if n.ErrMessage != nil {
+				fatalf("backend error: %v\n", *n.ErrMessage)
+			}
+			if s := n.State; s != nil {
+				switch *s {
+				case ipn.NeedsLogin:
+					printed = true
+					bc.StartLoginInteractive()
+				case ipn.NeedsMachineAuth:
+					printed = true
+					fmt.Fprintf(os.Stderr, "\nTo authorize your machine, visit (as admin):\n\n\t%s/admin/machines\n\n", upArgs.server)
+				case ipn.Starting, ipn.Running:
+					// Done full authentication process
+					if printed {
+						// Only need to print an update if we printed the "please click" message earlier.
+						fmt.Fprintf(os.Stderr, "Success.\n")
+					}
+					cancel()
+				}
+			}
+			if url := n.BrowseToURL; url != nil {
+				fmt.Fprintf(os.Stderr, "\nTo authenticate, visit:\n\n\t%s\n\n", *url)
+			}
+		},
+	}
+	// We still have to Start right now because it's the only way to
+	// set up notifications and whatnot. This causes a bunch of churn
+	// every time the CLI touches anything.
+	//
+	// TODO(danderson): redo the frontend/backend API to assume
+	// ephemeral frontends that read/modify/write state, once
+	// Windows/Mac state is moved into backend.
+	bc.Start(opts)
+	pump(ctx, bc, c)
+
+	return nil
+}

cmd/tailscale/cli/version.go

@@ -0,0 +1,69 @@
+// Copyright (c) 2020 Tailscale Inc & AUTHORS All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package cli
+
+import (
+	"context"
+	"flag"
+	"fmt"
+	"log"
+
+	"github.com/peterbourgon/ff/v2/ffcli"
+	"tailscale.com/ipn"
+	"tailscale.com/version"
+)
+
+var versionCmd = &ffcli.Command{
+	Name:       "version",
+	ShortUsage: "version [flags]",
+	ShortHelp:  "Print Tailscale version",
+	FlagSet: (func() *flag.FlagSet {
+		fs := flag.NewFlagSet("version", flag.ExitOnError)
+		fs.BoolVar(&versionArgs.daemon, "daemon", false, "also print local node's daemon version")
+		return fs
+	})(),
+	Exec: runVersion,
+}
+
+var versionArgs struct {
+	daemon bool // also check local node's daemon version
+}
+
+func runVersion(ctx context.Context, args []string) error {
+	if len(args) > 0 {
+		log.Fatalf("too many non-flag arguments: %q", args)
+	}
+	if !versionArgs.daemon {
+		fmt.Println(version.LONG)
+		return nil
+	}
+	fmt.Printf("Client: %s\n", version.LONG)
+
+	c, bc, ctx, cancel := connect(ctx)
+	defer cancel()
+
+	bc.AllowVersionSkew = true
+
+	done := make(chan struct{})
+
+	bc.SetNotifyCallback(func(n ipn.Notify) {
+		if n.ErrMessage != nil {
+			log.Fatal(*n.ErrMessage)
+		}
+		if n.Status != nil {
+			fmt.Printf("Daemon: %s\n", n.Version)
+			close(done)
+		}
+	})
+	go pump(ctx, bc, c)
+
+	bc.RequestStatus()
+	select {
+	case <-done:
+		return nil
+	case <-ctx.Done():
+		return ctx.Err()
+	}
+}

cmd/tailscale/tailscale.go

@@ -7,319 +7,22 @@
 package main // import "tailscale.com/cmd/tailscale"
 
 import (
-	"bytes"
-	"context"
-	"flag"
 	"fmt"
 	"log"
-	"net"
 	"os"
-	"os/exec"
-	"os/signal"
-	"runtime"
-	"strconv"
-	"strings"
-	"syscall"
 
 	"github.com/apenwarr/fixconsole"
-	"github.com/peterbourgon/ff/v2/ffcli"
-	"github.com/tailscale/wireguard-go/wgcfg"
-	"tailscale.com/ipn"
-	"tailscale.com/paths"
-	"tailscale.com/safesocket"
-	"tailscale.com/tailcfg"
-	"tailscale.com/wgengine/router"
+	"tailscale.com/cmd/tailscale/cli"
 )
 
-// globalStateKey is the ipn.StateKey that tailscaled loads on
-// startup.
-//
-// We have to support multiple state keys for other OSes (Windows in
-// particular), but right now Unix daemons run with a single
-// node-global state. To keep open the option of having per-user state
-// later, the global state key doesn't look like a username.
-const globalStateKey = "_daemon"
-
-var rootArgs struct {
-	socket string
-}
-
 func main() {
 	err := fixconsole.FixConsoleIfNeeded()
 	if err != nil {
 		log.Printf("fixConsoleOutput: %v\n", err)
 	}
 
-	upf := flag.NewFlagSet("up", flag.ExitOnError)
-	upf.StringVar(&upArgs.server, "login-server", "https://login.tailscale.com", "base URL of control server")
-	upf.BoolVar(&upArgs.acceptRoutes, "accept-routes", false, "accept routes advertised by other Tailscale nodes")
-	upf.BoolVar(&upArgs.singleRoutes, "host-routes", true, "install host routes to other Tailscale nodes")
-	upf.BoolVar(&upArgs.shieldsUp, "shields-up", false, "don't allow incoming connections")
-	upf.StringVar(&upArgs.advertiseTags, "advertise-tags", "", "ACL tags to request (comma-separated, e.g. eng,montreal,ssh)")
-	upf.StringVar(&upArgs.authKey, "authkey", "", "node authorization key")
-	upf.BoolVar(&upArgs.enableDERP, "enable-derp", true, "enable the use of DERP servers")
-	if runtime.GOOS == "linux" || isBSD(runtime.GOOS) {
-		upf.StringVar(&upArgs.advertiseRoutes, "advertise-routes", "", "routes to advertise to other nodes (comma-separated, e.g. 10.0.0.0/8,192.168.0.0/24)")
-	}
-	if runtime.GOOS == "linux" {
-		upf.BoolVar(&upArgs.snat, "snat-subnet-routes", true, "source NAT traffic to local routes advertised with -advertise-routes")
-		upf.StringVar(&upArgs.netfilterMode, "netfilter-mode", "on", "netfilter mode (one of on, nodivert, off)")
-	}
-	upCmd := &ffcli.Command{
-		Name:       "up",
-		ShortUsage: "up [flags]",
-		ShortHelp:  "Connect to your Tailscale network",
-
-		LongHelp: strings.TrimSpace(`
-"tailscale up" connects this machine to your Tailscale network,
-triggering authentication if necessary.
-
-The flags passed to this command are specific to this machine. If you don't
-specify any flags, options are reset to their default.
-`),
-		FlagSet: upf,
-		Exec:    runUp,
-	}
-
-	rootfs := flag.NewFlagSet("tailscale", flag.ExitOnError)
-	rootfs.StringVar(&rootArgs.socket, "socket", paths.DefaultTailscaledSocket(), "path to tailscaled's unix socket")
-
-	rootCmd := &ffcli.Command{
-		Name:       "tailscale",
-		ShortUsage: "tailscale subcommand [flags]",
-		ShortHelp:  "The easiest, most secure way to use WireGuard.",
-		LongHelp: strings.TrimSpace(`
-This CLI is still under active development. Commands and flags will
-change in the future.
-`),
-		Subcommands: []*ffcli.Command{
-			upCmd,
-			netcheckCmd,
-			statusCmd,
-		},
-		FlagSet: rootfs,
-		Exec:    func(context.Context, []string) error { return flag.ErrHelp },
-	}
-
-	if err := rootCmd.ParseAndRun(context.Background(), os.Args[1:]); err != nil && err != flag.ErrHelp {
-		log.Fatal(err)
-	}
-}
-
-var upArgs struct {
-	server          string
-	acceptRoutes    bool
-	singleRoutes    bool
-	shieldsUp       bool
-	advertiseRoutes string
-	advertiseTags   string
-	enableDERP      bool
-	snat            bool
-	netfilterMode   string
-	authKey         string
-}
-
-// parseIPOrCIDR parses an IP address or a CIDR prefix. If the input
-// is an IP address, it is returned in CIDR form with a /32 mask for
-// IPv4 or a /128 mask for IPv6.
-func parseIPOrCIDR(s string) (wgcfg.CIDR, bool) {
-	if strings.Contains(s, "/") {
-		ret, err := wgcfg.ParseCIDR(s)
-		if err != nil {
-			return wgcfg.CIDR{}, false
-		}
-		return ret, true
-	}
-
-	ip, ok := wgcfg.ParseIP(s)
-	if !ok {
-		return wgcfg.CIDR{}, false
-	}
-	if ip.Is4() {
-		return wgcfg.CIDR{ip, 32}, true
-	} else {
-		return wgcfg.CIDR{ip, 128}, true
-	}
-}
-
-func isBSD(s string) bool {
-	return s == "dragonfly" || s == "freebsd" || s == "netbsd" || s == "openbsd"
-}
-
-func warning(format string, args ...interface{}) {
-	fmt.Printf("Warning: "+format+"\n", args...)
-}
-
-// checkIPForwarding prints warnings on linux if IP forwarding is not
-// enabled, or if we were unable to verify the state of IP forwarding.
-func checkIPForwarding() {
-	var key string
-
-	if runtime.GOOS == "linux" {
-		key = "net.ipv4.ip_forward"
-	} else if isBSD(runtime.GOOS) {
-		key = "net.inet.ip.forwarding"
-	} else {
-		return
-	}
-
-	bs, err := exec.Command("sysctl", "-n", key).Output()
-	if err != nil {
-		warning("couldn't check %s (%v).\nSubnet routes won't work without IP forwarding.", key, err)
-		return
-	}
-	on, err := strconv.ParseBool(string(bytes.TrimSpace(bs)))
-	if err != nil {
-		warning("couldn't parse %s (%v).\nSubnet routes won't work without IP forwarding.", key, err)
-		return
-	}
-	if !on {
-		warning("%s is disabled. Subnet routes won't work.", key)
-	}
-}
-
-func runUp(ctx context.Context, args []string) error {
-	if len(args) > 0 {
-		log.Fatalf("too many non-flag arguments: %q", args)
-	}
-
-	var routes []wgcfg.CIDR
-	if upArgs.advertiseRoutes != "" {
-		checkIPForwarding()
-		advroutes := strings.Split(upArgs.advertiseRoutes, ",")
-		for _, s := range advroutes {
-			cidr, ok := parseIPOrCIDR(s)
-			if !ok {
-				log.Fatalf("%q is not a valid IP address or CIDR prefix", s)
-			}
-			routes = append(routes, cidr)
-		}
-	}
-
-	var tags []string
-	if upArgs.advertiseTags != "" {
-		tags = strings.Split(upArgs.advertiseTags, ",")
-		for _, tag := range tags {
-			err := tailcfg.CheckTag(tag)
-			if err != nil {
-				log.Fatalf("tag: %q: %s", tag, err)
-			}
-		}
-	}
-
-	// TODO(apenwarr): fix different semantics between prefs and uflags
-	// TODO(apenwarr): allow setting/using CorpDNS
-	prefs := ipn.NewPrefs()
-	prefs.ControlURL = upArgs.server
-	prefs.WantRunning = true
-	prefs.RouteAll = upArgs.acceptRoutes
-	prefs.AllowSingleHosts = upArgs.singleRoutes
-	prefs.ShieldsUp = upArgs.shieldsUp
-	prefs.AdvertiseRoutes = routes
-	prefs.AdvertiseTags = tags
-	prefs.NoSNAT = !upArgs.snat
-	prefs.DisableDERP = !upArgs.enableDERP
-	if runtime.GOOS == "linux" {
-		switch upArgs.netfilterMode {
-		case "on":
-			prefs.NetfilterMode = router.NetfilterOn
-		case "nodivert":
-			prefs.NetfilterMode = router.NetfilterNoDivert
-			warning("netfilter=nodivert; add iptables calls to ts-* chains manually.")
-		case "off":
-			prefs.NetfilterMode = router.NetfilterOff
-			warning("netfilter=off; configure iptables yourself.")
-		default:
-			log.Fatalf("invalid value --netfilter-mode: %q", upArgs.netfilterMode)
-		}
-	}
-
-	c, bc, ctx, cancel := connect(ctx)
-	defer cancel()
-
-	var printed bool
-
-	bc.SetPrefs(prefs)
-	opts := ipn.Options{
-		StateKey: globalStateKey,
-		AuthKey:  upArgs.authKey,
-		Notify: func(n ipn.Notify) {
-			if n.ErrMessage != nil {
-				log.Fatalf("backend error: %v\n", *n.ErrMessage)
-			}
-			if s := n.State; s != nil {
-				switch *s {
-				case ipn.NeedsLogin:
-					printed = true
-					bc.StartLoginInteractive()
-				case ipn.NeedsMachineAuth:
-					printed = true
-					fmt.Fprintf(os.Stderr, "\nTo authorize your machine, visit (as admin):\n\n\t%s/admin/machines\n\n", upArgs.server)
-				case ipn.Starting, ipn.Running:
-					// Done full authentication process
-					if printed {
-						// Only need to print an update if we printed the "please click" message earlier.
-						fmt.Fprintf(os.Stderr, "Success.\n")
-					}
-					cancel()
-				}
-			}
-			if url := n.BrowseToURL; url != nil {
-				fmt.Fprintf(os.Stderr, "\nTo authenticate, visit:\n\n\t%s\n\n", *url)
-			}
-		},
-	}
-	// We still have to Start right now because it's the only way to
-	// set up notifications and whatnot. This causes a bunch of churn
-	// every time the CLI touches anything.
-	//
-	// TODO(danderson): redo the frontend/backend API to assume
-	// ephemeral frontends that read/modify/write state, once
-	// Windows/Mac state is moved into backend.
-	bc.Start(opts)
-	pump(ctx, bc, c)
-
-	return nil
-}
-
-func connect(ctx context.Context) (net.Conn, *ipn.BackendClient, context.Context, context.CancelFunc) {
-	c, err := safesocket.Connect(rootArgs.socket, 41112)
-	if err != nil {
-		if runtime.GOOS != "windows" && rootArgs.socket == "" {
-			log.Fatalf("--socket cannot be empty")
-		}
-		log.Fatalf("Failed to connect to connect to tailscaled. (safesocket.Connect: %v)\n", err)
-	}
-	clientToServer := func(b []byte) {
-		ipn.WriteMsg(c, b)
-	}
-
-	ctx, cancel := context.WithCancel(ctx)
-
-	go func() {
-		interrupt := make(chan os.Signal, 1)
-		signal.Notify(interrupt, syscall.SIGINT, syscall.SIGTERM)
-		<-interrupt
-		c.Close()
-		cancel()
-	}()
-
-	bc := ipn.NewBackendClient(log.Printf, clientToServer)
-	return c, bc, ctx, cancel
-}
-
-// pump receives backend messages on conn and pushes them into bc.
-func pump(ctx context.Context, bc *ipn.BackendClient, conn net.Conn) {
-	defer conn.Close()
-	for ctx.Err() == nil {
-		msg, err := ipn.ReadMsg(conn)
-		if err != nil {
-			if ctx.Err() != nil {
-				return
-			}
-			log.Printf("ReadMsg: %v\n", err)
-			break
-		}
-		bc.GotNotifyMsg(msg)
+	if err := cli.Run(os.Args[1:]); err != nil {
+		fmt.Fprintln(os.Stderr, err)
+		os.Exit(1)
 	}
 }

cmd/tailscaled/tailscaled.go

@@ -15,8 +15,10 @@ import (
 	"net/http"
 	"net/http/pprof"
 	"os"
+	"os/signal"
 	"runtime"
 	"runtime/debug"
+	"syscall"
 	"time"
 
 	"github.com/apenwarr/fixconsole"
@@ -27,6 +29,7 @@ import (
 	"tailscale.com/types/logger"
 	"tailscale.com/wgengine"
 	"tailscale.com/wgengine/magicsock"
+	"tailscale.com/wgengine/router"
 )
 
 // globalStateKey is the ipn.StateKey that tailscaled loads on
@@ -38,6 +41,27 @@ import (
 // later, the global state key doesn't look like a username.
 const globalStateKey = "_daemon"
 
+// defaultTunName returns the default tun device name for the platform.
+func defaultTunName() string {
+	switch runtime.GOOS {
+	case "openbsd":
+		return "tun"
+	case "windows":
+		return "Tailscale"
+	}
+	return "tailscale0"
+}
+
+var args struct {
+	cleanup    bool
+	fake       bool
+	debug      string
+	tunname    string
+	port       uint16
+	statepath  string
+	socketpath string
+}
+
 func main() {
 	// We aren't very performance sensitive, and the parts that are
 	// performance sensitive (wireguard) try hard not to do any memory
@@ -47,77 +71,117 @@ func main() {
 		debug.SetGCPercent(10)
 	}
 
-	defaultTunName := "tailscale0"
-	if runtime.GOOS == "openbsd" {
-		defaultTunName = "tun"
-	}
+	// Set default values for getopt.
+	args.tunname = defaultTunName()
+	args.port = magicsock.DefaultPort
+	args.statepath = paths.DefaultTailscaledStateFile()
+	args.socketpath = paths.DefaultTailscaledSocket()
 
-	fake := getopt.BoolLong("fake", 0, "fake tunnel+routing instead of tuntap")
-	debug := getopt.StringLong("debug", 0, "", "Address of debug server")
-	tunname := getopt.StringLong("tun", 0, defaultTunName, "tunnel interface name")
-	listenport := getopt.Uint16Long("port", 'p', magicsock.DefaultPort, "WireGuard port (0=autoselect)")
-	statepath := getopt.StringLong("state", 0, paths.DefaultTailscaledStateFile(), "Path of state file")
-	socketpath := getopt.StringLong("socket", 's', paths.DefaultTailscaledSocket(), "Path of the service unix socket")
-
-	logf := wgengine.RusagePrefixLog(log.Printf)
-	logf = logger.RateLimitedFn(logf, 5*time.Second, 5, 100)
+	getopt.FlagLong(&args.cleanup, "cleanup", 0, "clean up system state and exit")
+	getopt.FlagLong(&args.fake, "fake", 0, "fake tunnel+routing instead of tuntap")
+	getopt.FlagLong(&args.debug, "debug", 0, "address of debug server")
+	getopt.FlagLong(&args.tunname, "tun", 0, "tunnel interface name")
+	getopt.FlagLong(&args.port, "port", 'p', "WireGuard port (0=autoselect)")
+	getopt.FlagLong(&args.statepath, "state", 0, "path of state file")
+	getopt.FlagLong(&args.socketpath, "socket", 's', "path of the service unix socket")
 
 	err := fixconsole.FixConsoleIfNeeded()
 	if err != nil {
-		logf("fixConsoleOutput: %v", err)
+		log.Fatalf("fixConsoleOutput: %v", err)
 	}
-	pol := logpolicy.New("tailnode.log.tailscale.io")
 
 	getopt.Parse()
 	if len(getopt.Args()) > 0 {
 		log.Fatalf("too many non-flag arguments: %#v", getopt.Args()[0])
 	}
 
-	if *statepath == "" {
+	if args.statepath == "" {
 		log.Fatalf("--state is required")
 	}
 
-	if *socketpath == "" {
+	if args.socketpath == "" && runtime.GOOS != "windows" {
 		log.Fatalf("--socket is required")
 	}
 
+	if err := run(); err != nil {
+		// No need to log; the func already did
+		os.Exit(1)
+	}
+}
+
+func run() error {
+	var err error
+
+	pol := logpolicy.New("tailnode.log.tailscale.io")
+	defer func() {
+		// Finish uploading logs after closing everything else.
+		ctx, cancel := context.WithTimeout(context.Background(), time.Second)
+		defer cancel()
+		pol.Shutdown(ctx)
+	}()
+
+	logf := wgengine.RusagePrefixLog(log.Printf)
+	logf = logger.RateLimitedFn(logf, 5*time.Second, 5, 100)
+
+	if args.cleanup {
+		router.Cleanup(logf, args.tunname)
+		return nil
+	}
+
 	var debugMux *http.ServeMux
-	if *debug != "" {
+	if args.debug != "" {
 		debugMux = newDebugMux()
-		go runDebugServer(debugMux, *debug)
+		go runDebugServer(debugMux, args.debug)
 	}
 
 	var e wgengine.Engine
-	if *fake {
+	if args.fake {
 		e, err = wgengine.NewFakeUserspaceEngine(logf, 0)
 	} else {
-		e, err = wgengine.NewUserspaceEngine(logf, *tunname, *listenport)
+		e, err = wgengine.NewUserspaceEngine(logf, args.tunname, args.port)
 	}
 	if err != nil {
-		log.Fatalf("wgengine.New: %v", err)
+		logf("wgengine.New: %v", err)
+		return err
 	}
 	e = wgengine.NewWatchdog(e)
 
+	ctx, cancel := context.WithCancel(context.Background())
+	// Exit gracefully by cancelling the ipnserver context in most common cases:
+	// interrupted from the TTY or killed by a service manager.
+	interrupt := make(chan os.Signal, 1)
+	signal.Notify(interrupt, syscall.SIGINT, syscall.SIGTERM)
+	// SIGPIPE sometimes gets generated when CLIs disconnect from
+	// tailscaled. The default action is to terminate the process, we
+	// want to keep running.
+	signal.Ignore(syscall.SIGPIPE)
+	go func() {
+		select {
+		case s := <-interrupt:
+			logf("tailscaled got signal %v; shutting down", s)
+			cancel()
+		case <-ctx.Done():
+			// continue
+		}
+	}()
+
 	opts := ipnserver.Options{
-		SocketPath:         *socketpath,
+		SocketPath:         args.socketpath,
 		Port:               41112,
-		StatePath:          *statepath,
+		StatePath:          args.statepath,
 		AutostartStateKey:  globalStateKey,
-		LegacyConfigPath:   paths.LegacyConfigPath,
+		LegacyConfigPath:   paths.LegacyConfigPath(),
 		SurviveDisconnects: true,
 		DebugMux:           debugMux,
 	}
-	err = ipnserver.Run(context.Background(), logf, pol.PublicID.String(), opts, e)
-	if err != nil {
-		log.Fatalf("tailscaled: %v", err)
+	err = ipnserver.Run(ctx, logf, pol.PublicID.String(), ipnserver.FixedEngine(e), opts)
+	// Cancelation is not an error: it is the only way to stop ipnserver.
+	if err != nil && err != context.Canceled {
+		logf("ipnserver.Run: %v", err)
+		return err
 	}
 
-	// TODO(crawshaw): It would be nice to start a timeout context the moment a signal
-	// is received and use that timeout to give us a moment to finish uploading logs
-	// here. But the signal is handled inside ipnserver.Run, so some plumbing is needed.
-	ctx, cancel := context.WithCancel(context.Background())
-	cancel()
-	pol.Shutdown(ctx)
+	return nil
 }
 
 func newDebugMux() *http.ServeMux {

cmd/tailscaled/tailscaled.service

@@ -3,12 +3,11 @@ Description=Tailscale node agent
 Documentation=https://tailscale.com/kb/
 Wants=network-pre.target
 After=network-pre.target
-StartLimitIntervalSec=0
-StartLimitBurst=0
 
 [Service]
 EnvironmentFile=/etc/default/tailscaled
 ExecStart=/usr/sbin/tailscaled --state=/var/lib/tailscale/tailscaled.state --socket=/run/tailscale/tailscaled.sock --port $PORT $FLAGS
+ExecStopPost=/usr/sbin/tailscaled --cleanup
 
 Restart=on-failure
 

control/controlclient/auto.go

@@ -117,13 +117,15 @@ type Client struct {
 	mu         sync.Mutex   // mutex guards the following fields
 	statusFunc func(Status) // called to update Client status
 
-	loggedIn     bool       // true if currently logged in
-	loginGoal    *LoginGoal // non-nil if some login activity is desired
-	synced       bool       // true if our netmap is up-to-date
-	hostinfo     *tailcfg.Hostinfo
-	inPollNetMap bool // true if currently running a PollNetMap
-	inSendStatus int  // number of sendStatus calls currently in progress
-	state        State
+	paused         bool // whether we should stop making HTTP requests
+	unpauseWaiters []chan struct{}
+	loggedIn       bool       // true if currently logged in
+	loginGoal      *LoginGoal // non-nil if some login activity is desired
+	synced         bool       // true if our netmap is up-to-date
+	hostinfo       *tailcfg.Hostinfo
+	inPollNetMap   bool // true if currently running a PollNetMap
+	inSendStatus   int  // number of sendStatus calls currently in progress
+	state          State
 
 	authCtx    context.Context // context used for auth requests
 	mapCtx     context.Context // context used for netmap requests
@@ -169,6 +171,27 @@ func NewNoStart(opts Options) (*Client, error) {
 	return c, nil
 }
 
+// SetPaused controls whether HTTP activity should be paused.
+//
+// The client can be paused and unpaused repeatedly, unlike Start and Shutdown, which can only be used once.
+func (c *Client) SetPaused(paused bool) {
+	c.mu.Lock()
+	defer c.mu.Unlock()
+	if paused == c.paused {
+		return
+	}
+	c.paused = paused
+	if paused {
+		// Just cancel the map routine. The auth routine isn't expensive.
+		c.cancelMapLocked()
+	} else {
+		for _, ch := range c.unpauseWaiters {
+			close(ch)
+		}
+		c.unpauseWaiters = nil
+	}
+}
+
 // Start starts the client's goroutines.
 //
 // It should only be called for clients created by NewNoStart.
@@ -241,7 +264,7 @@ func (c *Client) cancelMapSafely() {
 
 func (c *Client) authRoutine() {
 	defer close(c.authDone)
-	bo := backoff.NewBackoff("authRoutine", c.logf)
+	bo := backoff.NewBackoff("authRoutine", c.logf, 30*time.Second)
 
 	for {
 		c.mu.Lock()
@@ -272,6 +295,7 @@ func (c *Client) authRoutine() {
 		if goal == nil {
 			// Wait for something interesting to happen
 			var exp <-chan time.Time
+			var expTimer *time.Timer
 			if expiry != nil && !expiry.IsZero() {
 				// if expiry is in the future, don't delay
 				// past that time.
@@ -284,11 +308,15 @@ func (c *Client) authRoutine() {
 					if delay > 5*time.Second {
 						delay = time.Second
 					}
-					exp = time.After(delay)
+					expTimer = time.NewTimer(delay)
+					exp = expTimer.C
 				}
 			}
 			select {
 			case <-ctx.Done():
+				if expTimer != nil {
+					expTimer.Stop()
+				}
 				c.logf("authRoutine: context done.")
 			case <-exp:
 				// Unfortunately the key expiry isn't provided
@@ -310,7 +338,7 @@ func (c *Client) authRoutine() {
 				}
 			}
 		} else if !goal.wantLoggedIn {
-			err := c.direct.TryLogout(c.authCtx)
+			err := c.direct.TryLogout(ctx)
 			if err != nil {
 				report(err, "TryLogout")
 				bo.BackOff(ctx, err)
@@ -355,12 +383,13 @@ func (c *Client) authRoutine() {
 					err = fmt.Errorf("weird: server required a new url?")
 					report(err, "WaitLoginURL")
 				}
-				goal.url = url
-				goal.token = nil
-				goal.flags = LoginDefault
 
 				c.mu.Lock()
-				c.loginGoal = goal
+				c.loginGoal = &LoginGoal{
+					wantLoggedIn: true,
+					flags:        LoginDefault,
+					url:          url,
+				}
 				c.state = StateURLVisitRequired
 				c.synced = false
 				c.mu.Unlock()
@@ -398,12 +427,35 @@ func (c *Client) Direct() *Direct {
 	return c.direct
 }
 
+// unpausedChanLocked returns a new channel that is closed when the
+// current Client pause is unpaused.
+//
+// c.mu must be held
+func (c *Client) unpausedChanLocked() <-chan struct{} {
+	unpaused := make(chan struct{})
+	c.unpauseWaiters = append(c.unpauseWaiters, unpaused)
+	return unpaused
+}
+
 func (c *Client) mapRoutine() {
 	defer close(c.mapDone)
-	bo := backoff.NewBackoff("mapRoutine", c.logf)
+	bo := backoff.NewBackoff("mapRoutine", c.logf, 30*time.Second)
 
 	for {
 		c.mu.Lock()
+		if c.paused {
+			unpaused := c.unpausedChanLocked()
+			c.mu.Unlock()
+			c.logf("mapRoutine: awaiting unpause")
+			select {
+			case <-unpaused:
+				c.logf("mapRoutine: unpaused")
+			case <-c.quit:
+				c.logf("mapRoutine: quit")
+				return
+			}
+			continue
+		}
 		c.logf("mapRoutine: %s", c.state)
 		loggedIn := c.loggedIn
 		ctx := c.mapCtx
@@ -486,8 +538,14 @@ func (c *Client) mapRoutine() {
 			if c.state == StateSynchronized {
 				c.state = StateAuthenticated
 			}
+			paused := c.paused
 			c.mu.Unlock()
 
+			if paused {
+				c.logf("mapRoutine: paused")
+				continue
+			}
+
 			if err != nil {
 				report(err, "PollNetMap")
 				bo.BackOff(ctx, err)
@@ -516,7 +574,7 @@ func (c *Client) SetHostinfo(hi *tailcfg.Hostinfo) {
 		panic("nil Hostinfo")
 	}
 	if !c.direct.SetHostinfo(hi) {
-		c.logf("[unexpected] duplicate Hostinfo: %v", hi)
+		// No changes. Don't log.
 		return
 	}
 	c.logf("Hostinfo: %v", hi)

control/controlclient/controlclient_test.go

@@ -70,3 +70,10 @@ func TestStatusEqual(t *testing.T) {
 		}
 	}
 }
+
+func TestOSVersion(t *testing.T) {
+	if osVersion == nil {
+		t.Skip("not available for OS")
+	}
+	t.Logf("Got: %#q", osVersion())
+}

control/controlclient/direct.go

@@ -4,6 +4,8 @@
 
 package controlclient
 
+//go:generate go run tailscale.com/cmd/cloner -type=Persist -output=direct_clone.go
+
 import (
 	"bytes"
 	"context"
@@ -19,6 +21,8 @@ import (
 	"net/url"
 	"os"
 	"reflect"
+	"runtime"
+	"sort"
 	"strconv"
 	"strings"
 	"sync"
@@ -27,6 +31,7 @@ import (
 	"github.com/tailscale/wireguard-go/wgcfg"
 	"golang.org/x/crypto/nacl/box"
 	"golang.org/x/oauth2"
+	"inet.af/netaddr"
 	"tailscale.com/log/logheap"
 	"tailscale.com/net/netns"
 	"tailscale.com/net/tlsdial"
@@ -165,12 +170,20 @@ func NewDirect(opts Options) (*Direct, error) {
 	return c, nil
 }
 
+var osVersion func() string // non-nil on some platforms
+
 func NewHostinfo() *tailcfg.Hostinfo {
 	hostname, _ := os.Hostname()
+	var osv string
+	if osVersion != nil {
+		osv = osVersion()
+	}
 	return &tailcfg.Hostinfo{
 		IPNVersion: version.LONG,
 		Hostname:   hostname,
 		OS:         version.OS(),
+		OSVersion:  osv,
+		GoArch:     runtime.GOARCH,
 	}
 }
 
@@ -266,7 +279,7 @@ func (c *Direct) doLogin(ctx context.Context, t *oauth2.Token, flags LoginFlags,
 	tryingNewKey := c.tryingNewKey
 	serverKey := c.serverKey
 	authKey := c.authKey
-	hostinfo := c.hostinfo
+	hostinfo := c.hostinfo.Clone()
 	backendLogID := hostinfo.BackendLogID
 	expired := c.expiry != nil && !c.expiry.IsZero() && c.expiry.Before(c.timeNow())
 	c.mu.Unlock()
@@ -456,7 +469,7 @@ func (c *Direct) PollNetMap(ctx context.Context, maxPolls int, cb func(*NetworkM
 	persist := c.persist
 	serverURL := c.serverURL
 	serverKey := c.serverKey
-	hostinfo := c.hostinfo
+	hostinfo := c.hostinfo.Clone()
 	backendLogID := hostinfo.BackendLogID
 	localPort := c.localPort
 	ep := append([]string(nil), c.endpoints...)
@@ -477,6 +490,7 @@ func (c *Direct) PollNetMap(ctx context.Context, maxPolls int, cb func(*NetworkM
 	request := tailcfg.MapRequest{
 		Version:         4,
 		IncludeIPv6:     true,
+		DeltaPeers:      true,
 		KeepAlive:       c.keepAlive,
 		NodeKey:         tailcfg.NodeKey(persist.PrivateNodeKey.Public()),
 		DiscoKey:        c.discoPubKey,
@@ -561,6 +575,7 @@ func (c *Direct) PollNetMap(ctx context.Context, maxPolls int, cb func(*NetworkM
 	// the same format before just closing the connection.
 	// We can use this same read loop either way.
 	var msg []byte
+	var previousPeers []*tailcfg.Node // for delta-purposes
 	for i := 0; i < maxPolls || maxPolls < 0; i++ {
 		vlogf("netmap: starting size read after %v (poll %v)", time.Since(t0).Round(time.Millisecond), i)
 		var siz [4]byte
@@ -582,18 +597,25 @@ func (c *Direct) PollNetMap(ctx context.Context, maxPolls int, cb func(*NetworkM
 			vlogf("netmap: decode error: %v")
 			return err
 		}
+
 		if resp.KeepAlive {
 			vlogf("netmap: got keep-alive")
-			select {
-			case timeoutReset <- struct{}{}:
-				vlogf("netmap: sent keep-alive timer reset")
-			case <-ctx.Done():
-				c.logf("netmap: not resetting timer for keep-alive due to: %v", ctx.Err())
-				return ctx.Err()
-			}
+		} else {
+			vlogf("netmap: got new map")
+		}
+		select {
+		case timeoutReset <- struct{}{}:
+			vlogf("netmap: sent timer reset")
+		case <-ctx.Done():
+			c.logf("netmap: not resetting timer; context done: %v", ctx.Err())
+			return ctx.Err()
+		}
+		if resp.KeepAlive {
 			continue
 		}
-		vlogf("netmap: got new map")
+
+		undeltaPeers(&resp, previousPeers)
+		previousPeers = cloneNodes(resp.Peers) // defensive/lazy clone, since this escapes to who knows where
 
 		if resp.DERPMap != nil {
 			vlogf("netmap: new map contains DERP map")
@@ -619,6 +641,7 @@ func (c *Direct) PollNetMap(ctx context.Context, maxPolls int, cb func(*NetworkM
 			NodeKey:      tailcfg.NodeKey(persist.PrivateNodeKey.Public()),
 			PrivateKey:   persist.PrivateNodeKey,
 			Expiry:       resp.Node.KeyExpiry,
+			Name:         resp.Node.Name,
 			Addresses:    resp.Node.Addresses,
 			Peers:        resp.Peers,
 			LocalPort:    localPort,
@@ -626,8 +649,7 @@ func (c *Direct) PollNetMap(ctx context.Context, maxPolls int, cb func(*NetworkM
 			UserProfiles: make(map[tailcfg.UserID]tailcfg.UserProfile),
 			Domain:       resp.Domain,
 			Roles:        resp.Roles,
-			DNS:          resp.DNS,
-			DNSDomains:   resp.SearchPaths,
+			DNS:          resp.DNSConfig,
 			Hostinfo:     resp.Node.Hostinfo,
 			PacketFilter: c.parsePacketFilter(resp.PacketFilter),
 			DERPMap:      lastDERPMap,
@@ -641,6 +663,15 @@ func (c *Direct) PollNetMap(ctx context.Context, maxPolls int, cb func(*NetworkM
 		} else {
 			nm.MachineStatus = tailcfg.MachineUnauthorized
 		}
+		if len(resp.DNS) > 0 {
+			nm.DNS.Nameservers = wgIPToNetaddr(resp.DNS)
+		}
+		if len(resp.SearchPaths) > 0 {
+			nm.DNS.Domains = resp.SearchPaths
+		}
+		if Debug.ProxyDNS {
+			nm.DNS.Proxied = true
+		}
 
 		// Printing the netmap can be extremely verbose, but is very
 		// handy for debugging. Let's limit how often we do it.
@@ -678,8 +709,10 @@ func decode(res *http.Response, v interface{}, serverKey *wgcfg.Key, mkey *wgcfg
 }
 
 func (c *Direct) decodeMsg(msg []byte, v interface{}) error {
+	c.mu.Lock()
 	mkey := c.persist.PrivateMachineKey
 	serverKey := c.serverKey
+	c.mu.Unlock()
 
 	decrypted, err := decryptMsg(msg, &serverKey, &mkey)
 	if err != nil {
@@ -778,12 +811,24 @@ func loadServerKey(ctx context.Context, httpc *http.Client, serverURL string) (w
 	return key, nil
 }
 
+func wgIPToNetaddr(ips []wgcfg.IP) (ret []netaddr.IP) {
+	for _, ip := range ips {
+		nip, ok := netaddr.FromStdIP(ip.IP())
+		if !ok {
+			panic(fmt.Sprintf("conversion of %s from wgcfg to netaddr IP failed", ip))
+		}
+		ret = append(ret, nip.Unmap())
+	}
+	return ret
+}
+
 // Debug contains temporary internal-only debug knobs.
 // They're unexported to not draw attention to them.
 var Debug = initDebug()
 
 type debug struct {
 	NetMap     bool
+	ProxyDNS   bool
 	OnlyDisco  bool
 	Disco      bool
 	ForceDisco bool // ask control server to not filter out our disco key
@@ -792,6 +837,7 @@ type debug struct {
 func initDebug() debug {
 	d := debug{
 		NetMap:     envBool("TS_DEBUG_NETMAP"),
+		ProxyDNS:   envBool("TS_DEBUG_PROXY_DNS"),
 		OnlyDisco:  os.Getenv("TS_DEBUG_USE_DISCO") == "only",
 		ForceDisco: os.Getenv("TS_DEBUG_USE_DISCO") == "only" || envBool("TS_DEBUG_USE_DISCO"),
 	}
@@ -813,3 +859,97 @@ func envBool(k string) bool {
 	}
 	return v
 }
+
+// undeltaPeers updates mapRes.Peers to be complete based on the provided previous peer list
+// and the PeersRemoved and PeersChanged fields in mapRes.
+// It then also nils out the delta fields.
+func undeltaPeers(mapRes *tailcfg.MapResponse, prev []*tailcfg.Node) {
+	if len(mapRes.Peers) > 0 {
+		// Not delta encoded.
+		if !nodesSorted(mapRes.Peers) {
+			log.Printf("netmap: undeltaPeers: MapResponse.Peers not sorted; sorting")
+			sortNodes(mapRes.Peers)
+		}
+		return
+	}
+
+	var removed map[tailcfg.NodeID]bool
+	if pr := mapRes.PeersRemoved; len(pr) > 0 {
+		removed = make(map[tailcfg.NodeID]bool, len(pr))
+		for _, id := range pr {
+			removed[id] = true
+		}
+	}
+	changed := mapRes.PeersChanged
+
+	if len(removed) == 0 && len(changed) == 0 {
+		// No changes fast path.
+		mapRes.Peers = prev
+		return
+	}
+
+	if !nodesSorted(changed) {
+		log.Printf("netmap: undeltaPeers: MapResponse.PeersChanged not sorted; sorting")
+		sortNodes(changed)
+	}
+	if !nodesSorted(prev) {
+		// Internal error (unrelated to the network) if we get here.
+		log.Printf("netmap: undeltaPeers: [unexpected] prev not sorted; sorting")
+		sortNodes(prev)
+	}
+
+	newFull := make([]*tailcfg.Node, 0, len(prev)-len(removed))
+	for len(prev) > 0 && len(changed) > 0 {
+		pID := prev[0].ID
+		cID := changed[0].ID
+		if removed[pID] {
+			prev = prev[1:]
+			continue
+		}
+		switch {
+		case pID < cID:
+			newFull = append(newFull, prev[0])
+			prev = prev[1:]
+		case pID == cID:
+			newFull = append(newFull, changed[0])
+			prev, changed = prev[1:], changed[1:]
+		case cID < pID:
+			newFull = append(newFull, changed[0])
+			changed = changed[1:]
+		}
+	}
+	newFull = append(newFull, changed...)
+	for _, n := range prev {
+		if !removed[n.ID] {
+			newFull = append(newFull, n)
+		}
+	}
+	sortNodes(newFull)
+	mapRes.Peers = newFull
+	mapRes.PeersChanged = nil
+	mapRes.PeersRemoved = nil
+}
+
+func nodesSorted(v []*tailcfg.Node) bool {
+	for i, n := range v {
+		if i > 0 && n.ID <= v[i-1].ID {
+			return false
+		}
+	}
+	return true
+}
+
+func sortNodes(v []*tailcfg.Node) {
+	sort.Slice(v, func(i, j int) bool { return v[i].ID < v[j].ID })
+}
+
+func cloneNodes(v1 []*tailcfg.Node) []*tailcfg.Node {
+	if v1 == nil {
+		return nil
+	}
+	v2 := make([]*tailcfg.Node, len(v1))
+	for i, n := range v1 {
+		v2[i] = n.Clone()
+	}
+	return v2
+}

control/controlclient/direct_clone.go

@@ -0,0 +1,20 @@
+// Copyright (c) 2020 Tailscale Inc & AUTHORS All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+// Code generated by tailscale.com/cmd/cloner -type Persist; DO NOT EDIT.
+
+package controlclient
+
+import ()
+
+// Clone makes a deep copy of Persist.
+// The result aliases no memory with the original.
+func (src *Persist) Clone() *Persist {
+	if src == nil {
+		return nil
+	}
+	dst := new(Persist)
+	*dst = *src
+	return dst
+}

control/controlclient/direct_test.go

@@ -0,0 +1,93 @@
+// Copyright (c) 2020 Tailscale Inc & AUTHORS All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package controlclient
+
+import (
+	"fmt"
+	"reflect"
+	"strings"
+	"testing"
+
+	"tailscale.com/tailcfg"
+)
+
+func TestUndeltaPeers(t *testing.T) {
+	n := func(id tailcfg.NodeID, name string) *tailcfg.Node {
+		return &tailcfg.Node{ID: id, Name: name}
+	}
+	peers := func(nv ...*tailcfg.Node) []*tailcfg.Node { return nv }
+	tests := []struct {
+		name   string
+		mapRes *tailcfg.MapResponse
+		prev   []*tailcfg.Node
+		want   []*tailcfg.Node
+	}{
+		{
+			name: "full_peers",
+			mapRes: &tailcfg.MapResponse{
+				Peers: peers(n(1, "foo"), n(2, "bar")),
+			},
+			want: peers(n(1, "foo"), n(2, "bar")),
+		},
+		{
+			name: "full_peers_ignores_deltas",
+			mapRes: &tailcfg.MapResponse{
+				Peers:        peers(n(1, "foo"), n(2, "bar")),
+				PeersRemoved: []tailcfg.NodeID{2},
+			},
+			want: peers(n(1, "foo"), n(2, "bar")),
+		},
+		{
+			name: "add_and_update",
+			prev: peers(n(1, "foo"), n(2, "bar")),
+			mapRes: &tailcfg.MapResponse{
+				PeersChanged: peers(n(0, "zero"), n(2, "bar2"), n(3, "three")),
+			},
+			want: peers(n(0, "zero"), n(1, "foo"), n(2, "bar2"), n(3, "three")),
+		},
+		{
+			name: "remove",
+			prev: peers(n(1, "foo"), n(2, "bar")),
+			mapRes: &tailcfg.MapResponse{
+				PeersRemoved: []tailcfg.NodeID{1},
+			},
+			want: peers(n(2, "bar")),
+		},
+		{
+			name: "add_and_remove",
+			prev: peers(n(1, "foo"), n(2, "bar")),
+			mapRes: &tailcfg.MapResponse{
+				PeersChanged: peers(n(1, "foo2")),
+				PeersRemoved: []tailcfg.NodeID{2},
+			},
+			want: peers(n(1, "foo2")),
+		},
+		{
+			name:   "unchanged",
+			prev:   peers(n(1, "foo"), n(2, "bar")),
+			mapRes: &tailcfg.MapResponse{},
+			want:   peers(n(1, "foo"), n(2, "bar")),
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			undeltaPeers(tt.mapRes, tt.prev)
+			if !reflect.DeepEqual(tt.mapRes.Peers, tt.want) {
+				t.Errorf("wrong results\n got: %s\nwant: %s", formatNodes(tt.mapRes.Peers), formatNodes(tt.want))
+			}
+		})
+	}
+}
+
+func formatNodes(nodes []*tailcfg.Node) string {
+	var sb strings.Builder
+	for i, n := range nodes {
+		if i > 0 {
+			sb.WriteString(", ")
+		}
+		fmt.Fprintf(&sb, "(%d, %q)", n.ID, n.Name)
+	}
+	return sb.String()
+}

control/controlclient/hostinfo_linux.go

@@ -0,0 +1,87 @@
+// Copyright (c) 2020 Tailscale Inc & AUTHORS All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+// +build linux,!android
+
+package controlclient
+
+import (
+	"bytes"
+	"fmt"
+	"io"
+	"io/ioutil"
+	"strings"
+	"syscall"
+
+	"go4.org/mem"
+	"tailscale.com/util/lineread"
+)
+
+func init() {
+	osVersion = osVersionLinux
+}
+
+func osVersionLinux() string {
+	m := map[string]string{}
+	lineread.File("/etc/os-release", func(line []byte) error {
+		eq := bytes.IndexByte(line, '=')
+		if eq == -1 {
+			return nil
+		}
+		k, v := string(line[:eq]), strings.Trim(string(line[eq+1:]), `"`)
+		m[k] = v
+		return nil
+	})
+
+	var un syscall.Utsname
+	syscall.Uname(&un)
+
+	var attrBuf strings.Builder
+	attrBuf.WriteString("; kernel=")
+	for _, b := range un.Release {
+		if b == 0 {
+			break
+		}
+		attrBuf.WriteByte(byte(b))
+	}
+	if inContainer() {
+		attrBuf.WriteString("; container")
+	}
+	attr := attrBuf.String()
+
+	id := m["ID"]
+
+	switch id {
+	case "debian":
+		slurp, _ := ioutil.ReadFile("/etc/debian_version")
+		return fmt.Sprintf("Debian %s (%s)%s", bytes.TrimSpace(slurp), m["VERSION_CODENAME"], attr)
+	case "ubuntu":
+		return fmt.Sprintf("Ubuntu %s%s", m["VERSION"], attr)
+	case "", "centos": // CentOS 6 has no /etc/os-release, so its id is ""
+		if cr, _ := ioutil.ReadFile("/etc/centos-release"); len(cr) > 0 { // "CentOS release 6.10 (Final)
+			return fmt.Sprintf("%s%s", bytes.TrimSpace(cr), attr)
+		}
+		fallthrough
+	case "fedora", "rhel", "alpine":
+		// Their PRETTY_NAME is fine as-is for all versions I tested.
+		fallthrough
+	default:
+		if v := m["PRETTY_NAME"]; v != "" {
+			return fmt.Sprintf("%s%s", v, attr)
+		}
+	}
+	return fmt.Sprintf("Other%s", attr)
+}
+
+func inContainer() (ret bool) {
+	lineread.File("/proc/1/cgroup", func(line []byte) error {
+		if mem.Contains(mem.B(line), mem.S("/docker/")) ||
+			mem.Contains(mem.B(line), mem.S("/lxc/")) {
+			ret = true
+			return io.EOF // arbitrary non-nil error to stop loop
+		}
+		return nil
+	})
+	return
+}

control/controlclient/hostinfo_windows.go

@@ -0,0 +1,26 @@
+// Copyright (c) 2020 Tailscale Inc & AUTHORS All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package controlclient
+
+import (
+	"os/exec"
+	"strings"
+	"syscall"
+)
+
+func init() {
+	osVersion = osVersionWindows
+}
+
+func osVersionWindows() string {
+	cmd := exec.Command("cmd", "/c", "ver")
+	cmd.SysProcAttr = &syscall.SysProcAttr{HideWindow: true}
+	out, _ := cmd.Output() // "\nMicrosoft Windows [Version 10.0.19041.388]\n\n"
+	s := strings.TrimSpace(string(out))
+	s = strings.TrimPrefix(s, "Microsoft Windows [")
+	s = strings.TrimSuffix(s, "]")
+	s = strings.TrimPrefix(s, "Version ") // is this localized? do it last in case.
+	return s                              // "10.0.19041.388", ideally
+}

control/controlclient/netmap.go

@@ -5,12 +5,11 @@
 package controlclient
 
 import (
-	"bytes"
-	"encoding/base64"
 	"encoding/json"
 	"fmt"
-	"log"
+	"net"
 	"reflect"
+	"strconv"
 	"strings"
 	"time"
 
@@ -23,15 +22,16 @@ import (
 type NetworkMap struct {
 	// Core networking
 
-	NodeKey       tailcfg.NodeKey
-	PrivateKey    wgcfg.PrivateKey
-	Expiry        time.Time
+	NodeKey    tailcfg.NodeKey
+	PrivateKey wgcfg.PrivateKey
+	Expiry     time.Time
+	// Name is the DNS name assigned to this node.
+	Name          string
 	Addresses     []wgcfg.CIDR
 	LocalPort     uint16 // used for debugging
 	MachineStatus tailcfg.MachineStatus
-	Peers         []*tailcfg.Node
-	DNS           []wgcfg.IP
-	DNSDomains    []string
+	Peers         []*tailcfg.Node // sorted by Node.ID
+	DNS           tailcfg.DNSConfig
 	Hostinfo      tailcfg.Hostinfo
 	PacketFilter  filter.Matches
 
@@ -54,25 +54,25 @@ type NetworkMap struct {
 	// TODO(crawshaw): Capabilities []tailcfg.Capability
 }
 
-func (n *NetworkMap) Equal(n2 *NetworkMap) bool {
-	// TODO(crawshaw): this is crude, but is an easy way to avoid bugs.
-	b, err := json.Marshal(n)
-	if err != nil {
-		panic(err)
-	}
-	b2, err := json.Marshal(n2)
-	if err != nil {
-		panic(err)
-	}
-	return bytes.Equal(b, b2)
-}
-
 func (nm NetworkMap) String() string {
 	return nm.Concise()
 }
 
 func (nm *NetworkMap) Concise() string {
 	buf := new(strings.Builder)
+
+	nm.printConciseHeader(buf)
+	for _, p := range nm.Peers {
+		printPeerConcise(buf, p)
+	}
+	return buf.String()
+}
+
+// printConciseHeader prints a concise header line representing nm to buf.
+//
+// If this function is changed to access different fields of nm, keep
+// in equalConciseHeader in sync.
+func (nm *NetworkMap) printConciseHeader(buf *strings.Builder) {
 	fmt.Fprintf(buf, "netmap: self: %v auth=%v",
 		nm.NodeKey.ShortString(), nm.MachineStatus)
 	if nm.LocalPort != 0 {
@@ -84,72 +84,123 @@ func (nm *NetworkMap) Concise() string {
 	}
 	fmt.Fprintf(buf, " %v", nm.Addresses)
 	buf.WriteByte('\n')
-	for _, p := range nm.Peers {
-		aip := make([]string, len(p.AllowedIPs))
-		for i, a := range p.AllowedIPs {
-			s := strings.TrimSuffix(fmt.Sprint(a), "/32")
-			aip[i] = s
-		}
+}
 
-		ep := make([]string, len(p.Endpoints))
-		for i, e := range p.Endpoints {
-			// Align vertically on the ':' between IP and port
-			colon := strings.IndexByte(e, ':')
-			spaces := 0
-			for colon > 0 && len(e)+spaces-colon < 6 {
-				spaces++
-				colon--
-			}
-			ep[i] = fmt.Sprintf("%21v", e+strings.Repeat(" ", spaces))
-		}
-
-		derp := p.DERP
-		const derpPrefix = "127.3.3.40:"
-		if strings.HasPrefix(derp, derpPrefix) {
-			derp = "D" + derp[len(derpPrefix):]
-		}
-
-		// Most of the time, aip is just one element, so format the
-		// table to look good in that case. This will also make multi-
-		// subnet nodes stand out visually.
-		fmt.Fprintf(buf, " %v %-2v %-15v : %v\n",
-			p.Key.ShortString(), derp,
-			strings.Join(aip, " "),
-			strings.Join(ep, " "))
+// equalConciseHeader reports whether a and b are equal for the fields
+// used by printConciseHeader.
+func (a *NetworkMap) equalConciseHeader(b *NetworkMap) bool {
+	if a.NodeKey != b.NodeKey ||
+		a.MachineStatus != b.MachineStatus ||
+		a.LocalPort != b.LocalPort ||
+		len(a.Addresses) != len(b.Addresses) {
+		return false
 	}
-	return buf.String()
+	for i, a := range a.Addresses {
+		if b.Addresses[i] != a {
+			return false
+		}
+	}
+	return (a.Debug == nil && b.Debug == nil) || reflect.DeepEqual(a.Debug, b.Debug)
+}
+
+// printPeerConcise appends to buf a line repsenting the peer p.
+//
+// If this function is changed to access different fields of p, keep
+// in nodeConciseEqual in sync.
+func printPeerConcise(buf *strings.Builder, p *tailcfg.Node) {
+	aip := make([]string, len(p.AllowedIPs))
+	for i, a := range p.AllowedIPs {
+		s := strings.TrimSuffix(fmt.Sprint(a), "/32")
+		aip[i] = s
+	}
+
+	ep := make([]string, len(p.Endpoints))
+	for i, e := range p.Endpoints {
+		// Align vertically on the ':' between IP and port
+		colon := strings.IndexByte(e, ':')
+		spaces := 0
+		for colon > 0 && len(e)+spaces-colon < 6 {
+			spaces++
+			colon--
+		}
+		ep[i] = fmt.Sprintf("%21v", e+strings.Repeat(" ", spaces))
+	}
+
+	derp := p.DERP
+	const derpPrefix = "127.3.3.40:"
+	if strings.HasPrefix(derp, derpPrefix) {
+		derp = "D" + derp[len(derpPrefix):]
+	}
+	var discoShort string
+	if !p.DiscoKey.IsZero() {
+		discoShort = p.DiscoKey.ShortString() + " "
+	}
+
+	// Most of the time, aip is just one element, so format the
+	// table to look good in that case. This will also make multi-
+	// subnet nodes stand out visually.
+	fmt.Fprintf(buf, " %v %s%-2v %-15v : %v\n",
+		p.Key.ShortString(),
+		discoShort,
+		derp,
+		strings.Join(aip, " "),
+		strings.Join(ep, " "))
+}
+
+// nodeConciseEqual reports whether a and b are equal for the fields accessed by printPeerConcise.
+func nodeConciseEqual(a, b *tailcfg.Node) bool {
+	return a.Key == b.Key &&
+		a.DERP == b.DERP &&
+		a.DiscoKey == b.DiscoKey &&
+		eqCIDRsIgnoreNil(a.AllowedIPs, b.AllowedIPs) &&
+		eqStringsIgnoreNil(a.Endpoints, b.Endpoints)
 }
 
 func (b *NetworkMap) ConciseDiffFrom(a *NetworkMap) string {
-	if reflect.DeepEqual(a, b) {
-		// Fast path that only does one allocation.
-		return ""
-	}
-	out := []string{}
-	ra := strings.Split(a.Concise(), "\n")
-	rb := strings.Split(b.Concise(), "\n")
+	var diff strings.Builder
 
-	ma := map[string]struct{}{}
-	for _, s := range ra {
-		ma[s] = struct{}{}
+	// See if header (non-peers, "bare") part of the network map changed.
+	// If so, print its diff lines first.
+	if !a.equalConciseHeader(b) {
+		diff.WriteByte('-')
+		a.printConciseHeader(&diff)
+		diff.WriteByte('+')
+		b.printConciseHeader(&diff)
 	}
 
-	mb := map[string]struct{}{}
-	for _, s := range rb {
-		mb[s] = struct{}{}
-	}
-
-	for _, s := range ra {
-		if _, ok := mb[s]; !ok {
-			out = append(out, "-"+s)
+	aps, bps := a.Peers, b.Peers
+	for len(aps) > 0 && len(bps) > 0 {
+		pa, pb := aps[0], bps[0]
+		switch {
+		case pa.ID == pb.ID:
+			if !nodeConciseEqual(pa, pb) {
+				diff.WriteByte('-')
+				printPeerConcise(&diff, pa)
+				diff.WriteByte('+')
+				printPeerConcise(&diff, pb)
+			}
+			aps, bps = aps[1:], bps[1:]
+		case pa.ID > pb.ID:
+			// New peer in b.
+			diff.WriteByte('+')
+			printPeerConcise(&diff, pb)
+			bps = bps[1:]
+		case pb.ID > pa.ID:
+			// Deleted peer in b.
+			diff.WriteByte('-')
+			printPeerConcise(&diff, pa)
+			aps = aps[1:]
 		}
 	}
-	for _, s := range rb {
-		if _, ok := ma[s]; !ok {
-			out = append(out, "+"+s)
-		}
+	for _, pa := range aps {
+		diff.WriteByte('-')
+		printPeerConcise(&diff, pa)
 	}
-	return strings.Join(out, "\n")
+	for _, pb := range bps {
+		diff.WriteByte('+')
+		printPeerConcise(&diff, pb)
+	}
+	return diff.String()
 }
 
 func (nm *NetworkMap) JSON() string {
@@ -160,141 +211,126 @@ func (nm *NetworkMap) JSON() string {
 	return string(b)
 }
 
+// WGConfigFlags is a bitmask of flags to control the behavior of the
+// wireguard configuration generation done by NetMap.WGCfg.
+type WGConfigFlags int
+
 const (
-	UAllowSingleHosts = 1 << iota
-	UAllowSubnetRoutes
-	UAllowDefaultRoute
-	UHackDefaultRoute
-
-	UDefault = 0
+	AllowSingleHosts WGConfigFlags = 1 << iota
+	AllowSubnetRoutes
+	AllowDefaultRoute
+	HackDefaultRoute
 )
 
-// Several programs need to parse these arguments into uflags, so let's
-// centralize it here.
-func UFlagsHelper(uroutes, rroutes, droutes bool) int {
-	uflags := 0
-	if uroutes {
-		uflags |= UAllowSingleHosts
-	}
-	if rroutes {
-		uflags |= UAllowSubnetRoutes
-	}
-	if droutes {
-		uflags |= UAllowDefaultRoute
-	}
-	return uflags
-}
-
-// TODO(bradfitz): UAPI seems to only be used by the old confnode and
-// pingnode; delete this when those are deleted/rewritten?
-func (nm *NetworkMap) UAPI(uflags int, dnsOverride []wgcfg.IP) string {
-	wgcfg, err := nm.WGCfg(log.Printf, uflags, dnsOverride)
-	if err != nil {
-		log.Fatalf("WGCfg() failed unexpectedly: %v\n", err)
-	}
-	s, err := wgcfg.ToUAPI()
-	if err != nil {
-		log.Fatalf("ToUAPI() failed unexpectedly: %v\n", err)
-	}
-	return s
-}
-
-func (nm *NetworkMap) WGCfg(logf logger.Logf, uflags int, dnsOverride []wgcfg.IP) (*wgcfg.Config, error) {
-	s := nm._WireGuardConfig(logf, uflags, dnsOverride, true)
-	return wgcfg.FromWgQuick(s, "tailscale")
-}
-
 // EndpointDiscoSuffix is appended to the hex representation of a peer's discovery key
 // and is then the sole wireguard endpoint for peers with a non-zero discovery key.
 // This form is then recognize by magicsock's CreateEndpoint.
 const EndpointDiscoSuffix = ".disco.tailscale:12345"
 
-func (nm *NetworkMap) _WireGuardConfig(logf logger.Logf, uflags int, dnsOverride []wgcfg.IP, allEndpoints bool) string {
-	buf := new(strings.Builder)
-	fmt.Fprintf(buf, "[Interface]\n")
-	fmt.Fprintf(buf, "PrivateKey = %s\n", base64.StdEncoding.EncodeToString(nm.PrivateKey[:]))
-	if len(nm.Addresses) > 0 {
-		fmt.Fprintf(buf, "Address = ")
-		for i, cidr := range nm.Addresses {
-			if i > 0 {
-				fmt.Fprintf(buf, ", ")
-			}
-			fmt.Fprintf(buf, "%s", cidr)
-		}
-		fmt.Fprintf(buf, "\n")
+// WGCfg returns the NetworkMaps's Wireguard configuration.
+func (nm *NetworkMap) WGCfg(logf logger.Logf, flags WGConfigFlags) (*wgcfg.Config, error) {
+	cfg := &wgcfg.Config{
+		Name:       "tailscale",
+		PrivateKey: nm.PrivateKey,
+		Addresses:  nm.Addresses,
+		ListenPort: nm.LocalPort,
+		Peers:      make([]wgcfg.Peer, 0, len(nm.Peers)),
 	}
-	fmt.Fprintf(buf, "ListenPort = %d\n", nm.LocalPort)
-	if len(dnsOverride) > 0 {
-		dnss := []string{}
-		for _, ip := range dnsOverride {
-			dnss = append(dnss, ip.String())
-		}
-		fmt.Fprintf(buf, "DNS = %s\n", strings.Join(dnss, ","))
-	}
-	fmt.Fprintf(buf, "\n")
 
-	for i, peer := range nm.Peers {
+	for _, peer := range nm.Peers {
 		if Debug.OnlyDisco && peer.DiscoKey.IsZero() {
 			continue
 		}
-		if (uflags&UAllowSingleHosts) == 0 && len(peer.AllowedIPs) < 2 {
-			logf("wgcfg: %v skipping a single-host peer.\n", peer.Key.ShortString())
+		if (flags&AllowSingleHosts) == 0 && len(peer.AllowedIPs) < 2 {
+			logf("wgcfg: %v skipping a single-host peer.", peer.Key.ShortString())
 			continue
 		}
-		if i > 0 {
-			fmt.Fprintf(buf, "\n")
+		cfg.Peers = append(cfg.Peers, wgcfg.Peer{
+			PublicKey: wgcfg.Key(peer.Key),
+		})
+		cpeer := &cfg.Peers[len(cfg.Peers)-1]
+		if peer.KeepAlive {
+			cpeer.PersistentKeepalive = 25 // seconds
 		}
-		fmt.Fprintf(buf, "[Peer]\n")
-		fmt.Fprintf(buf, "PublicKey = %s\n", base64.StdEncoding.EncodeToString(peer.Key[:]))
-		var endpoints []string
+
 		if !peer.DiscoKey.IsZero() {
-			fmt.Fprintf(buf, "Endpoint = %x%s\n", peer.DiscoKey[:], EndpointDiscoSuffix)
-		} else {
-			if peer.DERP != "" {
-				endpoints = append(endpoints, peer.DERP)
+			if err := appendEndpoint(cpeer, fmt.Sprintf("%x%s", peer.DiscoKey[:], EndpointDiscoSuffix)); err != nil {
+				return nil, err
 			}
-			endpoints = append(endpoints, peer.Endpoints...)
-			if len(endpoints) > 0 {
-				if len(endpoints) == 1 {
-					fmt.Fprintf(buf, "Endpoint = %s", endpoints[0])
-				} else if allEndpoints {
-					// TODO(apenwarr): This mode is incompatible.
-					// Normal wireguard clients don't know how to
-					// parse it (yet?)
-					fmt.Fprintf(buf, "Endpoint = %s", strings.Join(endpoints, ","))
-				} else {
-					fmt.Fprintf(buf, "Endpoint = %s # other endpoints: %s",
-						endpoints[0],
-						strings.Join(endpoints[1:], ", "))
+			cpeer.Endpoints = []wgcfg.Endpoint{{Host: fmt.Sprintf("%x.disco.tailscale", peer.DiscoKey[:]), Port: 12345}}
+		} else {
+			if err := appendEndpoint(cpeer, peer.DERP); err != nil {
+				return nil, err
+			}
+			for _, ep := range peer.Endpoints {
+				if err := appendEndpoint(cpeer, ep); err != nil {
+					return nil, err
 				}
-				buf.WriteByte('\n')
 			}
 		}
-		var aips []string
 		for _, allowedIP := range peer.AllowedIPs {
-			aip := allowedIP.String()
 			if allowedIP.Mask == 0 {
-				if (uflags & UAllowDefaultRoute) == 0 {
-					logf("wgcfg: %v skipping default route\n", peer.Key.ShortString())
+				if (flags & AllowDefaultRoute) == 0 {
+					logf("wgcfg: %v skipping default route", peer.Key.ShortString())
 					continue
 				}
-				if (uflags & UHackDefaultRoute) != 0 {
-					aip = "10.0.0.0/8"
-					logf("wgcfg: %v converting default route => %v\n", peer.Key.ShortString(), aip)
+				if (flags & HackDefaultRoute) != 0 {
+					allowedIP = wgcfg.CIDR{IP: wgcfg.IPv4(10, 0, 0, 0), Mask: 8}
+					logf("wgcfg: %v converting default route => %v", peer.Key.ShortString(), allowedIP.String())
 				}
 			} else if allowedIP.Mask < 32 {
-				if (uflags & UAllowSubnetRoutes) == 0 {
-					logf("wgcfg: %v skipping subnet route\n", peer.Key.ShortString())
+				if (flags & AllowSubnetRoutes) == 0 {
+					logf("wgcfg: %v skipping subnet route", peer.Key.ShortString())
 					continue
 				}
 			}
-			aips = append(aips, aip)
-		}
-		fmt.Fprintf(buf, "AllowedIPs = %s\n", strings.Join(aips, ", "))
-		if peer.KeepAlive {
-			fmt.Fprintf(buf, "PersistentKeepalive = 25\n")
+			cpeer.AllowedIPs = append(cpeer.AllowedIPs, allowedIP)
 		}
 	}
 
-	return buf.String()
+	return cfg, nil
+}
+
+func appendEndpoint(peer *wgcfg.Peer, epStr string) error {
+	if epStr == "" {
+		return nil
+	}
+	host, port, err := net.SplitHostPort(epStr)
+	if err != nil {
+		return fmt.Errorf("malformed endpoint %q for peer %v", epStr, peer.PublicKey.ShortString())
+	}
+	port16, err := strconv.ParseUint(port, 10, 16)
+	if err != nil {
+		return fmt.Errorf("invalid port in endpoint %q for peer %v", epStr, peer.PublicKey.ShortString())
+	}
+	peer.Endpoints = append(peer.Endpoints, wgcfg.Endpoint{Host: host, Port: uint16(port16)})
+	return nil
+}
+
+// eqStringsIgnoreNil reports whether a and b have the same length and
+// contents, but ignore whether a or b are nil.
+func eqStringsIgnoreNil(a, b []string) bool {
+	if len(a) != len(b) {
+		return false
+	}
+	for i, v := range a {
+		if v != b[i] {
+			return false
+		}
+	}
+	return true
+}
+
+// eqCIDRsIgnoreNil reports whether a and b have the same length and
+// contents, but ignore whether a or b are nil.
+func eqCIDRsIgnoreNil(a, b []wgcfg.CIDR) bool {
+	if len(a) != len(b) {
+		return false
+	}
+	for i, v := range a {
+		if v != b[i] {
+			return false
+		}
+	}
+	return true
 }

control/controlclient/netmap_test.go

@@ -5,18 +5,30 @@
 package controlclient
 
 import (
+	"encoding/hex"
 	"testing"
 
+	"github.com/tailscale/wireguard-go/wgcfg"
 	"tailscale.com/tailcfg"
 )
 
-func TestNetworkMapConcise(t *testing.T) {
-	nodekey := func(b byte) (ret tailcfg.NodeKey) {
-		for i := range ret {
-			ret[i] = b
-		}
-		return
+func testNodeKey(b byte) (ret tailcfg.NodeKey) {
+	for i := range ret {
+		ret[i] = b
 	}
+	return
+}
+
+func testDiscoKey(hexPrefix string) (ret tailcfg.DiscoKey) {
+	b, err := hex.DecodeString(hexPrefix)
+	if err != nil {
+		panic(err)
+	}
+	copy(ret[:], b)
+	return
+}
+
+func TestNetworkMapConcise(t *testing.T) {
 	for _, tt := range []struct {
 		name string
 		nm   *NetworkMap
@@ -25,15 +37,15 @@ func TestNetworkMapConcise(t *testing.T) {
 		{
 			name: "basic",
 			nm: &NetworkMap{
-				NodeKey: nodekey(1),
+				NodeKey: testNodeKey(1),
 				Peers: []*tailcfg.Node{
 					{
-						Key:       nodekey(2),
+						Key:       testNodeKey(2),
 						DERP:      "127.3.3.40:2",
 						Endpoints: []string{"192.168.0.100:12", "192.168.0.100:12354"},
 					},
 					{
-						Key:       nodekey(3),
+						Key:       testNodeKey(3),
 						DERP:      "127.3.3.40:4",
 						Endpoints: []string{"10.2.0.100:12", "10.1.0.100:12345"},
 					},
@@ -44,7 +56,7 @@ func TestNetworkMapConcise(t *testing.T) {
 		{
 			name: "debug_non_nil",
 			nm: &NetworkMap{
-				NodeKey: nodekey(1),
+				NodeKey: testNodeKey(1),
 				Debug:   &tailcfg.Debug{},
 			},
 			want: "netmap: self: [AQEBA] auth=machine-unknown debug={} []\n",
@@ -52,7 +64,7 @@ func TestNetworkMapConcise(t *testing.T) {
 		{
 			name: "debug_values",
 			nm: &NetworkMap{
-				NodeKey: nodekey(1),
+				NodeKey: testNodeKey(1),
 				Debug:   &tailcfg.Debug{LogHeapPprof: true},
 			},
 			want: "netmap: self: [AQEBA] auth=machine-unknown debug={\"LogHeapPprof\":true} []\n",
@@ -70,3 +82,203 @@ func TestNetworkMapConcise(t *testing.T) {
 		})
 	}
 }
+
+func TestConciseDiffFrom(t *testing.T) {
+	for _, tt := range []struct {
+		name string
+		a, b *NetworkMap
+		want string
+	}{
+		{
+			name: "no_change",
+			a: &NetworkMap{
+				NodeKey: testNodeKey(1),
+				Peers: []*tailcfg.Node{
+					{
+						Key:       testNodeKey(2),
+						DERP:      "127.3.3.40:2",
+						Endpoints: []string{"192.168.0.100:12", "192.168.0.100:12354"},
+					},
+				},
+			},
+			b: &NetworkMap{
+				NodeKey: testNodeKey(1),
+				Peers: []*tailcfg.Node{
+					{
+						Key:       testNodeKey(2),
+						DERP:      "127.3.3.40:2",
+						Endpoints: []string{"192.168.0.100:12", "192.168.0.100:12354"},
+					},
+				},
+			},
+			want: "",
+		},
+		{
+			name: "header_change",
+			a: &NetworkMap{
+				NodeKey: testNodeKey(1),
+				Peers: []*tailcfg.Node{
+					{
+						Key:       testNodeKey(2),
+						DERP:      "127.3.3.40:2",
+						Endpoints: []string{"192.168.0.100:12", "192.168.0.100:12354"},
+					},
+				},
+			},
+			b: &NetworkMap{
+				NodeKey: testNodeKey(2),
+				Peers: []*tailcfg.Node{
+					{
+						Key:       testNodeKey(2),
+						DERP:      "127.3.3.40:2",
+						Endpoints: []string{"192.168.0.100:12", "192.168.0.100:12354"},
+					},
+				},
+			},
+			want: "-netmap: self: [AQEBA] auth=machine-unknown []\n+netmap: self: [AgICA] auth=machine-unknown []\n",
+		},
+		{
+			name: "peer_add",
+			a: &NetworkMap{
+				NodeKey: testNodeKey(1),
+				Peers: []*tailcfg.Node{
+					{
+						ID:        2,
+						Key:       testNodeKey(2),
+						DERP:      "127.3.3.40:2",
+						Endpoints: []string{"192.168.0.100:12", "192.168.0.100:12354"},
+					},
+				},
+			},
+			b: &NetworkMap{
+				NodeKey: testNodeKey(1),
+				Peers: []*tailcfg.Node{
+					{
+						ID:        1,
+						Key:       testNodeKey(1),
+						DERP:      "127.3.3.40:1",
+						Endpoints: []string{"192.168.0.100:12", "192.168.0.100:12354"},
+					},
+					{
+						ID:        2,
+						Key:       testNodeKey(2),
+						DERP:      "127.3.3.40:2",
+						Endpoints: []string{"192.168.0.100:12", "192.168.0.100:12354"},
+					},
+					{
+						ID:        3,
+						Key:       testNodeKey(3),
+						DERP:      "127.3.3.40:3",
+						Endpoints: []string{"192.168.0.100:12", "192.168.0.100:12354"},
+					},
+				},
+			},
+			want: "+ [AQEBA] D1                 :    192.168.0.100:12     192.168.0.100:12354\n+ [AwMDA] D3                 :    192.168.0.100:12     192.168.0.100:12354\n",
+		},
+		{
+			name: "peer_remove",
+			a: &NetworkMap{
+				NodeKey: testNodeKey(1),
+				Peers: []*tailcfg.Node{
+					{
+						ID:        1,
+						Key:       testNodeKey(1),
+						DERP:      "127.3.3.40:1",
+						Endpoints: []string{"192.168.0.100:12", "192.168.0.100:12354"},
+					},
+					{
+						ID:        2,
+						Key:       testNodeKey(2),
+						DERP:      "127.3.3.40:2",
+						Endpoints: []string{"192.168.0.100:12", "192.168.0.100:12354"},
+					},
+					{
+						ID:        3,
+						Key:       testNodeKey(3),
+						DERP:      "127.3.3.40:3",
+						Endpoints: []string{"192.168.0.100:12", "192.168.0.100:12354"},
+					},
+				},
+			},
+			b: &NetworkMap{
+				NodeKey: testNodeKey(1),
+				Peers: []*tailcfg.Node{
+					{
+						ID:        2,
+						Key:       testNodeKey(2),
+						DERP:      "127.3.3.40:2",
+						Endpoints: []string{"192.168.0.100:12", "192.168.0.100:12354"},
+					},
+				},
+			},
+			want: "- [AQEBA] D1                 :    192.168.0.100:12     192.168.0.100:12354\n- [AwMDA] D3                 :    192.168.0.100:12     192.168.0.100:12354\n",
+		},
+		{
+			name: "peer_port_change",
+			a: &NetworkMap{
+				NodeKey: testNodeKey(1),
+				Peers: []*tailcfg.Node{
+					{
+						ID:        2,
+						Key:       testNodeKey(2),
+						DERP:      "127.3.3.40:2",
+						Endpoints: []string{"192.168.0.100:12", "1.1.1.1:1"},
+					},
+				},
+			},
+			b: &NetworkMap{
+				NodeKey: testNodeKey(1),
+				Peers: []*tailcfg.Node{
+					{
+						ID:        2,
+						Key:       testNodeKey(2),
+						DERP:      "127.3.3.40:2",
+						Endpoints: []string{"192.168.0.100:12", "1.1.1.1:2"},
+					},
+				},
+			},
+			want: "- [AgICA] D2                 :    192.168.0.100:12             1.1.1.1:1  \n+ [AgICA] D2                 :    192.168.0.100:12             1.1.1.1:2  \n",
+		},
+		{
+			name: "disco_key_only_change",
+			a: &NetworkMap{
+				NodeKey: testNodeKey(1),
+				Peers: []*tailcfg.Node{
+					{
+						ID:         2,
+						Key:        testNodeKey(2),
+						DERP:       "127.3.3.40:2",
+						Endpoints:  []string{"192.168.0.100:41641", "1.1.1.1:41641"},
+						DiscoKey:   testDiscoKey("f00f00f00f"),
+						AllowedIPs: []wgcfg.CIDR{{IP: wgcfg.IPv4(100, 102, 103, 104), Mask: 32}},
+					},
+				},
+			},
+			b: &NetworkMap{
+				NodeKey: testNodeKey(1),
+				Peers: []*tailcfg.Node{
+					{
+						ID:         2,
+						Key:        testNodeKey(2),
+						DERP:       "127.3.3.40:2",
+						Endpoints:  []string{"192.168.0.100:41641", "1.1.1.1:41641"},
+						DiscoKey:   testDiscoKey("ba4ba4ba4b"),
+						AllowedIPs: []wgcfg.CIDR{{IP: wgcfg.IPv4(100, 102, 103, 104), Mask: 32}},
+					},
+				},
+			},
+			want: "- [AgICA] d:f00f00f00f000000 D2 100.102.103.104 :   192.168.0.100:41641         1.1.1.1:41641\n+ [AgICA] d:ba4ba4ba4b000000 D2 100.102.103.104 :   192.168.0.100:41641         1.1.1.1:41641\n",
+		},
+	} {
+		t.Run(tt.name, func(t *testing.T) {
+			var got string
+			n := int(testing.AllocsPerRun(50, func() {
+				got = tt.b.ConciseDiffFrom(tt.a)
+			}))
+			t.Logf("Allocs = %d", n)
+			if got != tt.want {
+				t.Errorf("Wrong output\n Got: %q\nWant: %q\n## Got (unescaped):\n%s\n## Want (unescaped):\n%s\n", got, tt.want, got, tt.want)
+			}
+		})
+	}
+}

derp/derp_server.go

@@ -29,6 +29,7 @@ import (
 	"tailscale.com/metrics"
 	"tailscale.com/types/key"
 	"tailscale.com/types/logger"
+	"tailscale.com/version"
 )
 
 var debug, _ = strconv.ParseBool(os.Getenv("DERP_DEBUG_LOGS"))
@@ -38,6 +39,13 @@ const (
 	writeTimeout            = 2 * time.Second
 )
 
+const host64bit = (^uint(0) >> 32) & 1 // 1 on 64-bit, 0 on 32-bit
+
+// pad32bit is 4 on 32-bit machines and 0 on 64-bit.
+// It exists so the Server struct's atomic fields can be aligned to 8
+// byte boundaries. (As tested by GOARCH=386 go test, etc)
+const pad32bit = 4 - host64bit*4 // 0 on 64-bit, 4 on 32-bit
+
 // Server is a DERP server.
 type Server struct {
 	// WriteTimeout, if non-zero, specifies how long to wait
@@ -51,6 +59,7 @@ type Server struct {
 	meshKey    string
 
 	// Counters:
+	_                        [pad32bit]byte
 	packetsSent, bytesSent   expvar.Int
 	packetsRecv, bytesRecv   expvar.Int
 	packetsDropped           expvar.Int
@@ -61,6 +70,7 @@ type Server struct {
 	packetsDroppedQueueHead  *expvar.Int // queue full, drop head packet
 	packetsDroppedQueueTail  *expvar.Int // queue full, drop tail packet
 	packetsDroppedWrite      *expvar.Int // error writing to dst conn
+	_                        [pad32bit]byte
 	packetsForwardedOut      expvar.Int
 	packetsForwardedIn       expvar.Int
 	peerGoneFrames           expvar.Int // number of peer gone frames sent
@@ -1182,6 +1192,9 @@ func (s *Server) ExpVar() expvar.Var {
 	m.Set("multiforwarder_created", &s.multiForwarderCreated)
 	m.Set("multiforwarder_deleted", &s.multiForwarderDeleted)
 	m.Set("packet_forwarder_delete_other_value", &s.removePktForwardOther)
+	var expvarVersion expvar.String
+	expvarVersion.Set(version.LONG)
+	m.Set("version", &expvarVersion)
 	return m
 }
 

derp/derp_test.go

@@ -23,10 +23,10 @@ import (
 	"tailscale.com/types/logger"
 )
 
-func newPrivateKey(t *testing.T) (k key.Private) {
-	t.Helper()
+func newPrivateKey(tb testing.TB) (k key.Private) {
+	tb.Helper()
 	if _, err := crand.Read(k[:]); err != nil {
-		t.Fatal(err)
+		tb.Fatal(err)
 	}
 	return
 }
@@ -742,3 +742,64 @@ func TestForwarderRegistration(t *testing.T) {
 		u1: testFwd(3),
 	})
 }
+
+func BenchmarkSendRecv(b *testing.B) {
+	for _, size := range []int{10, 100, 1000, 10000} {
+		b.Run(fmt.Sprintf("msgsize=%d", size), func(b *testing.B) { benchmarkSendRecvSize(b, size) })
+	}
+}
+
+func benchmarkSendRecvSize(b *testing.B, packetSize int) {
+	serverPrivateKey := newPrivateKey(b)
+	s := NewServer(serverPrivateKey, logger.Discard)
+	defer s.Close()
+
+	key := newPrivateKey(b)
+	clientKey := key.Public()
+
+	ln, err := net.Listen("tcp", "127.0.0.1:0")
+	if err != nil {
+		b.Fatal(err)
+	}
+	defer ln.Close()
+
+	connOut, err := net.Dial("tcp", ln.Addr().String())
+	if err != nil {
+		b.Fatal(err)
+	}
+	defer connOut.Close()
+
+	connIn, err := ln.Accept()
+	if err != nil {
+		b.Fatal(err)
+	}
+	defer connIn.Close()
+
+	brwServer := bufio.NewReadWriter(bufio.NewReader(connIn), bufio.NewWriter(connIn))
+	go s.Accept(connIn, brwServer, "test-client")
+
+	brw := bufio.NewReadWriter(bufio.NewReader(connOut), bufio.NewWriter(connOut))
+	client, err := NewClient(key, connOut, brw, logger.Discard)
+	if err != nil {
+		b.Fatalf("client: %v", err)
+	}
+
+	go func() {
+		for {
+			_, err := client.Recv()
+			if err != nil {
+				return
+			}
+		}
+	}()
+
+	msg := make([]byte, packetSize)
+	b.SetBytes(int64(len(msg)))
+	b.ReportAllocs()
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		if err := client.Send(clientKey, msg); err != nil {
+			b.Fatal(err)
+		}
+	}
+}

derp/derphttp/derphttp_client.go

@@ -338,6 +338,9 @@ func (c *Client) dialRegion(ctx context.Context, reg *tailcfg.DERPRegion) (net.C
 	var firstErr error
 	for _, n := range reg.Nodes {
 		if n.STUNOnly {
+			if firstErr == nil {
+				firstErr = fmt.Errorf("no non-STUNOnly nodes for %s", c.targetString(reg))
+			}
 			continue
 		}
 		c, err := c.dialNode(ctx, n)

disco/disco.go

@@ -31,6 +31,8 @@ import (
 // Magic is the 6 byte header of all discovery messages.
 const Magic = "TS💬" // 6 bytes: 0x54 53 f0 9f 92 ac
 
+const keyLen = 32
+
 // NonceLen is the length of the nonces used by nacl secretboxes.
 const NonceLen = 24
 
@@ -46,6 +48,15 @@ const v0 = byte(0)
 
 var errShort = errors.New("short message")
 
+// LooksLikeDiscoWrapper reports whether p looks like it's a packet
+// containing an encrypted disco message.
+func LooksLikeDiscoWrapper(p []byte) bool {
+	if len(p) < len(Magic)+keyLen+NonceLen {
+		return false
+	}
+	return string(p[:len(Magic)]) == Magic
+}
+
 // Parse parses the encrypted part of the message from inside the
 // nacl secretbox.
 func Parse(p []byte) (Message, error) {

go.mod

@@ -9,26 +9,30 @@ require (
 	github.com/flynn/go-shlex v0.0.0-20150515145356-3f9db97f8568 // indirect
 	github.com/gliderlabs/ssh v0.2.2
 	github.com/go-ole/go-ole v1.2.4
+	github.com/godbus/dbus/v5 v5.0.3
 	github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e
 	github.com/google/go-cmp v0.4.0
 	github.com/goreleaser/nfpm v1.1.10
+	github.com/jsimonetti/rtnetlink v0.0.0-20200117123717-f846d4f6c1f4
 	github.com/klauspost/compress v1.10.10
 	github.com/kr/pty v1.1.1
 	github.com/mdlayher/netlink v1.1.0
+	github.com/miekg/dns v1.1.30
 	github.com/pborman/getopt v0.0.0-20190409184431-ee0cd42419d3
 	github.com/peterbourgon/ff/v2 v2.0.0
 	github.com/tailscale/winipcfg-go v0.0.0-20200413171540-609dcf2df55f
-	github.com/tailscale/wireguard-go v0.0.0-20200624060658-de1f1af1f35f
+	github.com/tailscale/wireguard-go v0.0.0-20200806235025-91988cfbaa3a
 	github.com/tcnksm/go-httpstat v0.2.0
 	github.com/toqueteos/webbrowser v1.2.0
-	go4.org/mem v0.0.0-20200601023850-d8ee1dfa5518
-	golang.org/x/crypto v0.0.0-20200317142112-1b76d66859c6
-	golang.org/x/net v0.0.0-20200301022130-244492dfa37a
+	go4.org/mem v0.0.0-20200706164138-185c595c3ecc
+	golang.org/x/crypto v0.0.0-20200429183012-4b2356b1ed79
+	golang.org/x/net v0.0.0-20200501053045-e0ff5e5a1de5
 	golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d
 	golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e
-	golang.org/x/sys v0.0.0-20200501052902-10377860bb8e
+	golang.org/x/sys v0.0.0-20200501145240-bc7a7d42d5c3
 	golang.org/x/time v0.0.0-20191024005414-555d28b269f0
+	golang.org/x/tools v0.0.0-20191216052735-49a3e744a425
 	honnef.co/go/tools v0.0.1-2020.1.4
-	inet.af/netaddr v0.0.0-20200702150737-4591d218f82c
+	inet.af/netaddr v0.0.0-20200810144936-56928fe48a98
 	rsc.io/goversion v1.2.0
 )

go.sum

@@ -30,6 +30,8 @@ github.com/gliderlabs/ssh v0.2.2 h1:6zsha5zo/TWhRhwqCD3+EarCAgZ2yN28ipRnGPnwkI0=
 github.com/gliderlabs/ssh v0.2.2/go.mod h1:U7qILu1NlMHj9FlMhZLlkCdDnU1DBEAqr0aevW3Awn0=
 github.com/go-ole/go-ole v1.2.4 h1:nNBDSCOigTSiarFpYE9J/KtEA1IOW4CNeqT9TQDqCxI=
 github.com/go-ole/go-ole v1.2.4/go.mod h1:XCwSNxSkXRo4vlyPy93sltvi/qJq0jqQhjqQNIwKuxM=
+github.com/godbus/dbus/v5 v5.0.3 h1:ZqHaoEF7TBzh4jzPmqVhE/5A1z9of6orkAe5uHoAeME=
+github.com/godbus/dbus/v5 v5.0.3/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA=
 github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e h1:1r7pUrabqp18hOBcwBwiTsbnFeTZHV9eER/QT5JVZxY=
 github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
 github.com/golang/protobuf v1.2.0 h1:P3YflyNX/ehuJFLhxviNdFxQPkGK5cDcApsge1SqnvM=
@@ -63,6 +65,8 @@ github.com/mdlayher/netlink v0.0.0-20190409211403-11939a169225/go.mod h1:eQB3mZE
 github.com/mdlayher/netlink v1.0.0/go.mod h1:KxeJAFOFLG6AjpyDkQ/iIhxygIUKD+vcwqcnu43w/+M=
 github.com/mdlayher/netlink v1.1.0 h1:mpdLgm+brq10nI9zM1BpX1kpDbh3NLl3RSnVq6ZSkfg=
 github.com/mdlayher/netlink v1.1.0/go.mod h1:H4WCitaheIsdF9yOYu8CFmCgQthAPIWZmcKp9uZHgmY=
+github.com/miekg/dns v1.1.30 h1:Qww6FseFn8PRfw07jueqIXqodm0JKiiKuK0DeXSqfyo=
+github.com/miekg/dns v1.1.30/go.mod h1:KNUDUusw/aVsxyTYZM1oqvCicbwhgbNgztCETuNZ7xM=
 github.com/op/go-logging v0.0.0-20160315200505-970db520ece7 h1:lDH9UUVJtmYCjyT0CI4q8xvlXPxeZ0gYCVvWbmPlp88=
 github.com/op/go-logging v0.0.0-20160315200505-970db520ece7/go.mod h1:HzydrMdWErDVzsI23lYNej1Htcns9BCg93Dk0bBINWk=
 github.com/pborman/getopt v0.0.0-20190409184431-ee0cd42419d3 h1:YtFkrqsMEj7YqpIhRteVxJxCeC3jJBieuLr0d4C4rSA=
@@ -82,8 +86,8 @@ github.com/stretchr/testify v1.4.0 h1:2E4SXV/wtOkTonXsotYi4li6zVWxYlZuYNCXe9XRJy
 github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4=
 github.com/tailscale/winipcfg-go v0.0.0-20200413171540-609dcf2df55f h1:uFj5bslHsMzxIM8UTjAhq4VXeo6GfNW91rpoh/WMJaY=
 github.com/tailscale/winipcfg-go v0.0.0-20200413171540-609dcf2df55f/go.mod h1:x880GWw5fvrl2DVTQ04ttXQD4DuppTt1Yz6wLibbjNE=
-github.com/tailscale/wireguard-go v0.0.0-20200615180905-687c10194779 h1:zg0rgvhBZGA4nvh17nDKcqkEXw6Nbc/Ma2VBvLaW7LU=
-github.com/tailscale/wireguard-go v0.0.0-20200615180905-687c10194779/go.mod h1:JPm5cTfu1K+qDFRbiHy0sOlHUylYQbpl356sdYFD8V4=
+github.com/tailscale/wireguard-go v0.0.0-20200806235025-91988cfbaa3a h1:dQEgNpoOJf+8MswlvXJicb8ZDQqZAGe8f/WfzbDMvtE=
+github.com/tailscale/wireguard-go v0.0.0-20200806235025-91988cfbaa3a/go.mod h1:WXq+IkSOJGIgfF1XW+4z4oW+LX/TXzU9DcKlT5EZLi4=
 github.com/tcnksm/go-httpstat v0.2.0 h1:rP7T5e5U2HfmOBmZzGgGZjBQ5/GluWUylujl0tJ04I0=
 github.com/tcnksm/go-httpstat v0.2.0/go.mod h1:s3JVJFtQxtBEBC9dwcdTTXS9xFnM3SXAZwPG41aurT8=
 github.com/toqueteos/webbrowser v1.2.0 h1:tVP/gpK69Fx+qMJKsLE7TD8LuGWPnEV71wBN9rrstGQ=
@@ -92,26 +96,28 @@ github.com/ulikunitz/xz v0.5.6 h1:jGHAfXawEGZQ3blwU5wnWKQJvAraT7Ftq9EXjnXYgt8=
 github.com/ulikunitz/xz v0.5.6/go.mod h1:2bypXElzHzzJZwzH67Y6wb67pO62Rzfn7BSiF4ABRW8=
 github.com/xi2/xz v0.0.0-20171230120015-48954b6210f8 h1:nIPpBwaJSVYIxUFsDv3M8ofmx9yWTog9BfvIu0q41lo=
 github.com/xi2/xz v0.0.0-20171230120015-48954b6210f8/go.mod h1:HUYIGzjTL3rfEspMxjDjgmT5uz5wzYJKVo23qUhYTos=
-go4.org/mem v0.0.0-20200601023850-d8ee1dfa5518 h1:AA3bSGklCgkrqIGnvL4894oa/2K9ltE0RejXh8CgyvA=
-go4.org/mem v0.0.0-20200601023850-d8ee1dfa5518/go.mod h1:NEYvpHWemiG/E5UWfaN5QAIGZeT1sa0Z2UNk6oeMb/k=
+go4.org/mem v0.0.0-20200706164138-185c595c3ecc h1:paujszgN6SpsO/UsXC7xax3gQAKz/XQKCYZLQdU34Tw=
+go4.org/mem v0.0.0-20200706164138-185c595c3ecc/go.mod h1:NEYvpHWemiG/E5UWfaN5QAIGZeT1sa0Z2UNk6oeMb/k=
 golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
 golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
 golang.org/x/crypto v0.0.0-20191002192127-34f69633bfdc/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
-golang.org/x/crypto v0.0.0-20200317142112-1b76d66859c6 h1:TjszyFsQsyZNHwdVdZ5m7bjmreu0znc2kRYsEml9/Ww=
-golang.org/x/crypto v0.0.0-20200317142112-1b76d66859c6/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
+golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
+golang.org/x/crypto v0.0.0-20200429183012-4b2356b1ed79 h1:IaQbIIB2X/Mp/DKctl6ROxz1KyMlKp4uyvL6+kQ7C88=
+golang.org/x/crypto v0.0.0-20200429183012-4b2356b1ed79/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
 golang.org/x/mod v0.0.0-20190513183733-4bf6d317e70e/go.mod h1:mXi4GBBbnImb6dmsKGUJ2LatrhH/nqhxcFungHvyanc=
+golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg=
 golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/net v0.0.0-20190108225652-1e06a53dbb7e/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
 golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
 golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
 golang.org/x/net v0.0.0-20190827160401-ba9fcec4b297/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
-golang.org/x/net v0.0.0-20191003171128-d98b1b443823/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
+golang.org/x/net v0.0.0-20190923162816-aa69164e4478/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
 golang.org/x/net v0.0.0-20191007182048-72f939374954/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
 golang.org/x/net v0.0.0-20200202094626-16171245cfb2 h1:CCH4IOTTfewWjGOlSp+zGcjutRKlBEZQ6wTn8ozI/nI=
 golang.org/x/net v0.0.0-20200202094626-16171245cfb2/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
-golang.org/x/net v0.0.0-20200301022130-244492dfa37a h1:GuSPYbZzB5/dcLNCwLQLsg3obCJtX9IJhpXkvY7kzk0=
-golang.org/x/net v0.0.0-20200301022130-244492dfa37a/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
+golang.org/x/net v0.0.0-20200501053045-e0ff5e5a1de5 h1:WQ8q63x+f/zpC8Ac1s9wLElVoHhm32p6tudrU72n1QA=
+golang.org/x/net v0.0.0-20200501053045-e0ff5e5a1de5/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A=
 golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d h1:TzXSXBo42m9gQenoE3b9BGiEpg5IG2JkU5FkPIawgtw=
 golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
 golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4 h1:YUO/7uOKsKeq9UokNS62b8FYywz3ker1l1vDZRCRefw=
@@ -126,12 +132,13 @@ golang.org/x/sys v0.0.0-20190405154228-4b34438f7a67/go.mod h1:h1NjWce9XRLGQEsW7w
 golang.org/x/sys v0.0.0-20190411185658-b44545bcd369/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20190826190057-c7b8b68b1456/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20191003212358-c178f38b412c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20190924154521-2837fb4f24fe/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20191008105621-543471e840be/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20200202164722-d101bd2416d5 h1:LfCXLvNmTYH9kEmVgqbnsWfruoXZIrh4YBgqVHtDvw0=
 golang.org/x/sys v0.0.0-20200202164722-d101bd2416d5/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20200501052902-10377860bb8e h1:hq86ru83GdWTlfQFZGO4nZJTU4Bs2wfHl8oFHRaXsfc=
-golang.org/x/sys v0.0.0-20200501052902-10377860bb8e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200501145240-bc7a7d42d5c3 h1:5B6i6EAiSYyejWfvc5Rc9BbI3rzIsrrXfAQBWnYfn+w=
+golang.org/x/sys v0.0.0-20200501145240-bc7a7d42d5c3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
 golang.org/x/text v0.3.2 h1:tW2bmiBqwgJj/UpqtC8EpXEZVYOwU0yG4iWbprSVAcs=
 golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk=
@@ -140,7 +147,10 @@ golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxb
 golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
 golang.org/x/tools v0.0.0-20191130070609-6e064ea0cf2d h1:/iIZNFGxc/a7C3yWjGcnboV+Tkc7mxr+p6fDztwoxuM=
 golang.org/x/tools v0.0.0-20191130070609-6e064ea0cf2d/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
+golang.org/x/tools v0.0.0-20191216052735-49a3e744a425 h1:VvQyQJN0tSuecqgcIxMWnnfG5kSmgy9KZR9sW3W5QeA=
+golang.org/x/tools v0.0.0-20191216052735-49a3e744a425/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=
 golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
+golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
 golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543 h1:E7g+9GITq07hpfrRu66IVDexMakfv52eLZ2CXBWiKr4=
 golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
 google.golang.org/appengine v1.4.0 h1:/wp5JvzpHIxhs/dumFmF7BXTf3Z+dd4uXta4kVyO508=
@@ -156,7 +166,9 @@ gopkg.in/yaml.v2 v2.2.7 h1:VUgggvou5XRW9mHwD/yXxIYSMtY0zoKQf/v226p2nyo=
 gopkg.in/yaml.v2 v2.2.7/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
 honnef.co/go/tools v0.0.1-2020.1.4 h1:UoveltGrhghAA7ePc+e+QYDHXrBps2PqFZiHkGR/xK8=
 honnef.co/go/tools v0.0.1-2020.1.4/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k=
-inet.af/netaddr v0.0.0-20200702150737-4591d218f82c h1:j3Z4HL4KcLBDU1kmRpXTD5fikKBqIkE+7vFKS5mCz3Y=
-inet.af/netaddr v0.0.0-20200702150737-4591d218f82c/go.mod h1:qqYzz/2whtrbWJvt+DNWQyvekNN4ePQZcg2xc2/Yjww=
+inet.af/netaddr v0.0.0-20200718043157-99321d6ad24c h1:si3Owrfem175Ry6gKqnh59eOXxDojyBTIHxUKuvK/Eo=
+inet.af/netaddr v0.0.0-20200718043157-99321d6ad24c/go.mod h1:qqYzz/2whtrbWJvt+DNWQyvekNN4ePQZcg2xc2/Yjww=
+inet.af/netaddr v0.0.0-20200810144936-56928fe48a98 h1:bWyWDZP0l6VnQ1TDKf6yNwuiEDV6Q3q1Mv34m+lzT1I=
+inet.af/netaddr v0.0.0-20200810144936-56928fe48a98/go.mod h1:qqYzz/2whtrbWJvt+DNWQyvekNN4ePQZcg2xc2/Yjww=
 rsc.io/goversion v1.2.0 h1:SPn+NLTiAG7w30IRK/DKp1BjvpWabYgxlLp/+kx5J8w=
 rsc.io/goversion v1.2.0/go.mod h1:Eih9y/uIBS3ulggl7KNJ09xGSLcuNaLgmvvqa07sgfo=

internal/deepprint/deepprint.go

@@ -18,13 +18,23 @@ import (
 	"reflect"
 )
 
-func Hash(v interface{}) string {
+func Hash(v ...interface{}) string {
 	h := sha256.New()
 	Print(h, v)
 	return fmt.Sprintf("%x", h.Sum(nil))
 }
 
-func Print(w io.Writer, v interface{}) {
+// UpdateHash sets last to the hash of v and reports whether its value changed.
+func UpdateHash(last *string, v ...interface{}) (changed bool) {
+	sig := Hash(v)
+	if *last != sig {
+		*last = sig
+		return true
+	}
+	return false
+}
+
+func Print(w io.Writer, v ...interface{}) {
 	print(w, reflect.ValueOf(v), make(map[uintptr]bool))
 }
 

internal/deepprint/deepprint_test.go

@@ -11,6 +11,7 @@ import (
 	"github.com/tailscale/wireguard-go/wgcfg"
 	"inet.af/netaddr"
 	"tailscale.com/wgengine/router"
+	"tailscale.com/wgengine/router/dns"
 )
 
 func TestDeepPrint(t *testing.T) {
@@ -50,7 +51,10 @@ func getVal() []interface{} {
 			},
 		},
 		&router.Config{
-			DNS: []netaddr.IP{netaddr.IPv4(8, 8, 8, 8)},
+			DNS: dns.Config{
+				Nameservers: []netaddr.IP{netaddr.IPv4(8, 8, 8, 8)},
+				Domains:     []string{"tailscale.net"},
+			},
 		},
 		map[string]string{
 			"key1": "val1",

ipn/backend.go

@@ -8,6 +8,7 @@ import (
 	"net/http"
 	"time"
 
+	"golang.org/x/oauth2"
 	"tailscale.com/control/controlclient"
 	"tailscale.com/ipn/ipnstate"
 	"tailscale.com/tailcfg"
@@ -27,6 +28,10 @@ const (
 	Running
 )
 
+// GoogleIDToken Type is the oauth2.Token.TokenType for the Google
+// ID tokens used by the Android client.
+const GoogleIDTokenType = "ts_android_google_login"
+
 func (s State) String() string {
 	return [...]string{"NoState", "NeedsLogin", "NeedsMachineAuth",
 		"Stopped", "Starting", "Running"}[s]
@@ -58,6 +63,12 @@ type Notify struct {
 	BrowseToURL   *string                   // UI should open a browser right now
 	BackendLogID  *string                   // public logtail id used by backend
 
+	// LocalTCPPort, if non-nil, informs the UI frontend which
+	// (non-zero) localhost TCP port it's listening on.
+	// This is currently only used by Tailscale when run in the
+	// macOS Network Extension.
+	LocalTCPPort *uint16 `json:",omitempty"`
+
 	// type is mirrored in xcode/Shared/IPN.swift
 }
 
@@ -123,6 +134,8 @@ type Backend interface {
 	// flow. This should trigger a new BrowseToURL notification
 	// eventually.
 	StartLoginInteractive()
+	// Login logs in with an OAuth2 token.
+	Login(token *oauth2.Token)
 	// Logout terminates the current login session and stops the
 	// wireguard engine.
 	Logout()

ipn/fake_test.go

@@ -8,6 +8,7 @@ import (
 	"log"
 	"time"
 
+	"golang.org/x/oauth2"
 	"tailscale.com/control/controlclient"
 	"tailscale.com/ipn/ipnstate"
 )
@@ -42,6 +43,14 @@ func (b *FakeBackend) newState(s State) {
 func (b *FakeBackend) StartLoginInteractive() {
 	u := b.serverURL + "/this/is/fake"
 	b.notify(Notify{BrowseToURL: &u})
+	b.login()
+}
+
+func (b *FakeBackend) Login(token *oauth2.Token) {
+	b.login()
+}
+
+func (b *FakeBackend) login() {
 	b.newState(NeedsMachineAuth)
 	b.newState(Stopped)
 	// TODO(apenwarr): Fill in a more interesting netmap here.

ipn/handle.go

@@ -9,6 +9,7 @@ import (
 	"time"
 
 	"github.com/tailscale/wireguard-go/wgcfg"
+	"golang.org/x/oauth2"
 	"tailscale.com/control/controlclient"
 	"tailscale.com/types/logger"
 )
@@ -154,6 +155,10 @@ func (h *Handle) StartLoginInteractive() {
 	h.b.StartLoginInteractive()
 }
 
+func (h *Handle) Login(token *oauth2.Token) {
+	h.b.Login(token)
+}
+
 func (h *Handle) Logout() {
 	h.b.Logout()
 }

ipn/ipnserver/server.go

@@ -33,15 +33,19 @@ type Options struct {
 	// SocketPath, on unix systems, is the unix socket path to listen
 	// on for frontend connections.
 	SocketPath string
+
 	// Port, on windows, is the localhost TCP port to listen on for
 	// frontend connections.
 	Port int
+
 	// StatePath is the path to the stored agent state.
 	StatePath string
+
 	// AutostartStateKey, if non-empty, immediately starts the agent
 	// using the given StateKey. If empty, the agent stays idle and
 	// waits for a frontend to start it.
 	AutostartStateKey ipn.StateKey
+
 	// LegacyConfigPath optionally specifies the old-style relaynode
 	// relay.conf location. If both LegacyConfigPath and
 	// AutostartStateKey are specified and the requested state doesn't
@@ -51,10 +55,15 @@ type Options struct {
 	// TODO(danderson): remove some time after the transition to
 	// tailscaled is done.
 	LegacyConfigPath string
+
 	// SurviveDisconnects specifies how the server reacts to its
 	// frontend disconnecting. If true, the server keeps running on
 	// its existing state, and accepts new frontend connections. If
 	// false, the server dumps its state and becomes idle.
+	//
+	// To support CLI connections (notably, "tailscale status"),
+	// the actual definition of "disconnect" is when the
+	// connection count transitions from 1 to 0.
 	SurviveDisconnects bool
 
 	// DebugMux, if non-nil, specifies an HTTP ServeMux in which
@@ -62,42 +71,146 @@ type Options struct {
 	DebugMux *http.ServeMux
 }
 
-func pump(logf logger.Logf, ctx context.Context, bs *ipn.BackendServer, s net.Conn) {
-	defer logf("Control connection done.")
+// server is an IPN backend and its set of 0 or more active connections
+// talking to an IPN backend.
+type server struct {
+	resetOnZero bool // call bs.Reset on transition from 1->0 connections
 
-	for ctx.Err() == nil && !bs.GotQuit {
-		msg, err := ipn.ReadMsg(s)
+	bsMu sync.Mutex // lock order: bsMu, then mu
+	bs   *ipn.BackendServer
+
+	mu      sync.Mutex
+	clients map[net.Conn]bool
+}
+
+func (s *server) serveConn(ctx context.Context, c net.Conn, logf logger.Logf) {
+	s.addConn(c)
+	logf("incoming control connection")
+	defer s.removeAndCloseConn(c)
+	for ctx.Err() == nil {
+		msg, err := ipn.ReadMsg(c)
 		if err != nil {
-			logf("ReadMsg: %v", err)
-			break
+			if ctx.Err() == nil {
+				logf("ReadMsg: %v", err)
+			}
+			return
 		}
-		err = bs.GotCommandMsg(msg)
-		if err != nil {
+		s.bsMu.Lock()
+		if err := s.bs.GotCommandMsg(msg); err != nil {
 			logf("GotCommandMsg: %v", err)
-			break
+		}
+		gotQuit := s.bs.GotQuit
+		s.bsMu.Unlock()
+		if gotQuit {
+			return
 		}
 	}
 }
 
-func Run(rctx context.Context, logf logger.Logf, logid string, opts Options, e wgengine.Engine) (err error) {
-	runDone := make(chan error, 1)
-	defer func() { runDone <- err }()
+func (s *server) addConn(c net.Conn) {
+	s.mu.Lock()
+	defer s.mu.Unlock()
+	if s.clients == nil {
+		s.clients = map[net.Conn]bool{}
+	}
+	s.clients[c] = true
+}
+
+func (s *server) removeAndCloseConn(c net.Conn) {
+	s.mu.Lock()
+	delete(s.clients, c)
+	remain := len(s.clients)
+	s.mu.Unlock()
+
+	if remain == 0 && s.resetOnZero {
+		s.bsMu.Lock()
+		s.bs.Reset()
+		s.bsMu.Unlock()
+	}
+	c.Close()
+}
+
+func (s *server) stopAll() {
+	s.mu.Lock()
+	defer s.mu.Unlock()
+	for c := range s.clients {
+		safesocket.ConnCloseRead(c)
+		safesocket.ConnCloseWrite(c)
+	}
+	s.clients = nil
+}
+
+func (s *server) writeToClients(b []byte) {
+	s.mu.Lock()
+	defer s.mu.Unlock()
+	for c := range s.clients {
+		ipn.WriteMsg(c, b)
+	}
+}
+
+// Run runs a Tailscale backend service.
+// The getEngine func is called repeatedly, once per connection, until it returns an engine successfully.
+func Run(ctx context.Context, logf logger.Logf, logid string, getEngine func() (wgengine.Engine, error), opts Options) error {
+	runDone := make(chan struct{})
+	defer close(runDone)
 
 	listen, _, err := safesocket.Listen(opts.SocketPath, uint16(opts.Port))
 	if err != nil {
 		return fmt.Errorf("safesocket.Listen: %v", err)
 	}
 
-	// Go listeners can't take a context, close it instead.
+	server := &server{
+		resetOnZero: !opts.SurviveDisconnects,
+	}
+
+	// When the context is closed or when we return, whichever is first, close our listner
+	// and all open connections.
 	go func() {
 		select {
-		case <-rctx.Done():
+		case <-ctx.Done():
 		case <-runDone:
 		}
+		server.stopAll()
 		listen.Close()
 	}()
 	logf("Listening on %v", listen.Addr())
 
+	bo := backoff.NewBackoff("ipnserver", logf, 30*time.Second)
+
+	var unservedConn net.Conn // if non-nil, accepted, but hasn't served yet
+
+	eng, err := getEngine()
+	if err != nil {
+		logf("Initial getEngine call: %v", err)
+		for i := 1; ctx.Err() == nil; i++ {
+			c, err := listen.Accept()
+			if err != nil {
+				logf("%d: Accept: %v", i, err)
+				bo.BackOff(ctx, err)
+				continue
+			}
+			logf("%d: trying getEngine again...", i)
+			eng, err = getEngine()
+			if err == nil {
+				logf("%d: GetEngine worked; exiting failure loop", i)
+				unservedConn = c
+				break
+			}
+			logf("%d: getEngine failed again: %v", i, err)
+			errMsg := err.Error()
+			go func() {
+				defer c.Close()
+				serverToClient := func(b []byte) { ipn.WriteMsg(c, b) }
+				bs := ipn.NewBackendServer(logf, nil, serverToClient)
+				bs.SendErrorMessage(errMsg)
+				time.Sleep(time.Second)
+			}()
+		}
+		if err := ctx.Err(); err != nil {
+			return err
+		}
+	}
+
 	var store ipn.StateStore
 	if opts.StatePath != "" {
 		store, err = ipn.NewFileStore(opts.StatePath)
@@ -108,10 +221,11 @@ func Run(rctx context.Context, logf logger.Logf, logid string, opts Options, e w
 		store = &ipn.MemoryStore{}
 	}
 
-	b, err := ipn.NewLocalBackend(logf, logid, store, e)
+	b, err := ipn.NewLocalBackend(logf, logid, store, eng)
 	if err != nil {
 		return fmt.Errorf("NewLocalBackend: %v", err)
 	}
+	defer b.Shutdown()
 	b.SetDecompressor(func() (controlclient.Decompressor, error) {
 		return smallzstd.NewDecoder(nil)
 	})
@@ -125,17 +239,10 @@ func Run(rctx context.Context, logf logger.Logf, logid string, opts Options, e w
 		})
 	}
 
-	var s net.Conn
-	serverToClient := func(b []byte) {
-		if s != nil { // TODO: racy access to s?
-			ipn.WriteMsg(s, b)
-		}
-	}
-
-	bs := ipn.NewBackendServer(logf, b, serverToClient)
+	server.bs = ipn.NewBackendServer(logf, b, server.writeToClients)
 
 	if opts.AutostartStateKey != "" {
-		bs.GotCommand(&ipn.Command{
+		server.bs.GotCommand(&ipn.Command{
 			Version: version.LONG,
 			Start: &ipn.StartArgs{
 				Opts: ipn.Options{
@@ -146,55 +253,25 @@ func Run(rctx context.Context, logf logger.Logf, logid string, opts Options, e w
 		})
 	}
 
-	var (
-		oldS   net.Conn
-		ctx    context.Context
-		cancel context.CancelFunc
-	)
-	stopAll := func() {
-		// Currently we only support one client connection at a time.
-		// Theoretically we could allow multiple clients, by passing
-		// notifications to all of them and accepting commands from
-		// any of them, but there doesn't seem to be much need for
-		// that right now.
-		if oldS != nil {
-			cancel()
-			safesocket.ConnCloseRead(oldS)
-			safesocket.ConnCloseWrite(oldS)
+	for i := 1; ctx.Err() == nil; i++ {
+		var c net.Conn
+		var err error
+		if unservedConn != nil {
+			c = unservedConn
+			unservedConn = nil
+		} else {
+			c, err = listen.Accept()
 		}
-	}
-
-	bo := backoff.NewBackoff("ipnserver", logf)
-
-	for i := 1; rctx.Err() == nil; i++ {
-		s, err = listen.Accept()
 		if err != nil {
-			logf("%d: Accept: %v", i, err)
-			bo.BackOff(rctx, err)
+			if ctx.Err() == nil {
+				logf("ipnserver: Accept: %v", err)
+				bo.BackOff(ctx, err)
+			}
 			continue
 		}
-		logf("%d: Incoming control connection.", i)
-		stopAll()
-
-		ctx, cancel = context.WithCancel(rctx)
-		oldS = s
-
-		go func(ctx context.Context, s net.Conn, i int) {
-			logf := logger.WithPrefix(logf, fmt.Sprintf("%d: ", i))
-			pump(logf, ctx, bs, s)
-			if !opts.SurviveDisconnects || bs.GotQuit {
-				bs.Reset()
-				s.Close()
-			}
-			// Quitting not allowed, just keep going.
-			bs.GotQuit = false
-		}(ctx, s, i)
-
-		bo.BackOff(ctx, nil)
+		go server.serveConn(ctx, c, logger.WithPrefix(logf, fmt.Sprintf("ipnserver: conn%d: ", i)))
 	}
-	stopAll()
-
-	return rctx.Err()
+	return ctx.Err()
 }
 
 func BabysitProc(ctx context.Context, args []string, logf logger.Logf) {
@@ -229,7 +306,7 @@ func BabysitProc(ctx context.Context, args []string, logf logger.Logf) {
 		proc.mu.Unlock()
 	}()
 
-	bo := backoff.NewBackoff("BabysitProc", logf)
+	bo := backoff.NewBackoff("BabysitProc", logf, 30*time.Second)
 
 	for {
 		startTime := time.Now()
@@ -312,3 +389,8 @@ func BabysitProc(ctx context.Context, args []string, logf logger.Logf) {
 		}
 	}
 }
+
+// FixedEngine returns a func that returns eng and a nil error.
+func FixedEngine(eng wgengine.Engine) func() (wgengine.Engine, error) {
+	return func() (wgengine.Engine, error) { return eng, nil }
+}

ipn/ipnserver/server_test.go

@@ -72,6 +72,6 @@ func TestRunMultipleAccepts(t *testing.T) {
 		SocketPath: socketPath,
 	}
 	t.Logf("pre-Run")
-	err = ipnserver.Run(ctx, logTriggerTestf, "dummy_logid", opts, eng)
+	err = ipnserver.Run(ctx, logTriggerTestf, "dummy_logid", ipnserver.FixedEngine(eng), opts)
 	t.Logf("ipnserver.Run = %v", err)
 }

ipn/ipnstate/ipnstate.go

@@ -18,6 +18,7 @@ import (
 	"sync"
 	"time"
 
+	"inet.af/netaddr"
 	"tailscale.com/tailcfg"
 	"tailscale.com/types/key"
 )
@@ -25,6 +26,7 @@ import (
 // Status represents the entire state of the IPN network.
 type Status struct {
 	BackendState string
+	TailscaleIPs []netaddr.IP // Tailscale IP(s) assigned to this node
 	Peer         map[key.Public]*PeerStatus
 	User         map[tailcfg.UserID]tailcfg.UserProfile
 }
@@ -109,6 +111,18 @@ func (sb *StatusBuilder) AddUser(id tailcfg.UserID, up tailcfg.UserProfile) {
 	sb.st.User[id] = up
 }
 
+// AddIP adds a Tailscale IP address to the status.
+func (sb *StatusBuilder) AddTailscaleIP(ip netaddr.IP) {
+	sb.mu.Lock()
+	defer sb.mu.Unlock()
+	if sb.locked {
+		log.Printf("[unexpected] ipnstate: AddIP after Locked")
+		return
+	}
+
+	sb.st.TailscaleIPs = append(sb.st.TailscaleIPs, ip)
+}
+
 // AddPeer adds a peer node to the status.
 //
 // Its PeerStatus is mixed with any previous status already added.
@@ -218,6 +232,12 @@ table tbody tr:nth-child(even) td { background-color: #f5f5f5; }
 	//f("<p><b>logid:</b> %s</p>\n", logid)
 	//f("<p><b>opts:</b> <code>%s</code></p>\n", html.EscapeString(fmt.Sprintf("%+v", opts)))
 
+	ips := make([]string, 0, len(st.TailscaleIPs))
+	for _, ip := range st.TailscaleIPs {
+		ips = append(ips, ip.String())
+	}
+	f("<p>Tailscale IP: %s", strings.Join(ips, ", "))
+
 	f("<table>\n<thead>\n")
 	f("<tr><th>Peer</th><th>Node</th><th>Owner</th><th>Rx</th><th>Tx</th><th>Activity</th><th>Endpoints</th></tr>\n")
 	f("</thead>\n<tbody>\n")

ipn/local.go

@@ -13,10 +13,13 @@ import (
 	"time"
 
 	"github.com/tailscale/wireguard-go/wgcfg"
+	"golang.org/x/oauth2"
 	"inet.af/netaddr"
 	"tailscale.com/control/controlclient"
+	"tailscale.com/internal/deepprint"
 	"tailscale.com/ipn/ipnstate"
 	"tailscale.com/ipn/policy"
+	"tailscale.com/net/tsaddr"
 	"tailscale.com/portlist"
 	"tailscale.com/tailcfg"
 	"tailscale.com/types/empty"
@@ -26,6 +29,7 @@ import (
 	"tailscale.com/wgengine"
 	"tailscale.com/wgengine/filter"
 	"tailscale.com/wgengine/router"
+	"tailscale.com/wgengine/router/dns"
 	"tailscale.com/wgengine/tsdns"
 )
 
@@ -49,12 +53,11 @@ type LocalBackend struct {
 	store           StateStore
 	backendLogID    string
 	portpoll        *portlist.Poller // may be nil
+	portpollOnce    sync.Once
+	serverURL       string // tailcontrol URL
 	newDecompressor func() (controlclient.Decompressor, error)
 
-	// TODO: these fields are accessed unsafely by concurrent
-	// goroutines. They need to be protected.
-	serverURL       string // tailcontrol URL
-	lastFilterPrint time.Time
+	filterHash string
 
 	// The mutex protects the following elements.
 	mu       sync.Mutex
@@ -184,21 +187,56 @@ func (b *LocalBackend) SetDecompressor(fn func() (controlclient.Decompressor, er
 // setClientStatus is the callback invoked by the control client whenever it posts a new status.
 // Among other things, this is where we update the netmap, packet filters, DNS and DERP maps.
 func (b *LocalBackend) setClientStatus(st controlclient.Status) {
+	// The following do not depend on any data for which we need to lock b.
+	if st.Err != "" {
+		// TODO(crawshaw): display in the UI.
+		b.logf("Received error: %v", st.Err)
+		return
+	}
 	if st.LoginFinished != nil {
 		// Auth completed, unblock the engine
 		b.blockEngineUpdates(false)
 		b.authReconfig()
 		b.send(Notify{LoginFinished: &empty.Message{}})
 	}
+
+	prefsChanged := false
+
+	// Lock b once and do only the things that require locking.
+	b.mu.Lock()
+
+	prefs := b.prefs
+	stateKey := b.stateKey
+	netMap := b.netMap
+	interact := b.interact
+
 	if st.Persist != nil {
-		persist := *st.Persist // copy
+		if !b.prefs.Persist.Equals(st.Persist) {
+			prefsChanged = true
+			b.prefs.Persist = st.Persist.Clone()
+		}
+	}
+	if st.NetMap != nil {
+		b.netMap = st.NetMap
+	}
+	if st.URL != "" {
+		b.authURL = st.URL
+	}
+	if b.state == NeedsLogin {
+		if !b.prefs.WantRunning {
+			prefsChanged = true
+		}
+		b.prefs.WantRunning = true
+	}
+	// Prefs will be written out; this is not safe unless locked or cloned.
+	if prefsChanged {
+		prefs = b.prefs.Clone()
+	}
 
-		b.mu.Lock()
-		b.prefs.Persist = &persist
-		prefs := b.prefs.Clone()
-		stateKey := b.stateKey
-		b.mu.Unlock()
+	b.mu.Unlock()
 
+	// Now complete the lock-free parts of what we started while locked.
+	if prefsChanged {
 		if stateKey != "" {
 			if err := b.store.WriteState(stateKey, prefs.ToBytes()); err != nil {
 				b.logf("Failed to save new controlclient state: %v", err)
@@ -207,63 +245,41 @@ func (b *LocalBackend) setClientStatus(st controlclient.Status) {
 		b.send(Notify{Prefs: prefs})
 	}
 	if st.NetMap != nil {
-		// Netmap is unchanged only when the diff is empty.
-		changed := true
-		b.mu.Lock()
-		if b.netMap != nil {
-			diff := st.NetMap.ConciseDiffFrom(b.netMap)
+		if netMap != nil {
+			diff := st.NetMap.ConciseDiffFrom(netMap)
 			if strings.TrimSpace(diff) == "" {
-				changed = false
 				b.logf("netmap diff: (none)")
 			} else {
 				b.logf("netmap diff:\n%v", diff)
 			}
 		}
-		disableDERP := b.prefs != nil && b.prefs.DisableDERP
-		b.netMap = st.NetMap
-		b.mu.Unlock()
 
-		b.send(Notify{NetMap: st.NetMap})
-		// There is nothing to update if the map hasn't changed.
-		if changed {
-			b.updateFilter(st.NetMap)
+		b.updateFilter(st.NetMap, prefs)
+		b.e.SetNetworkMap(st.NetMap)
+
+		if !dnsMapsEqual(st.NetMap, netMap) {
 			b.updateDNSMap(st.NetMap)
-			b.e.SetNetworkMap(st.NetMap)
 		}
+
+		disableDERP := prefs != nil && prefs.DisableDERP
 		if disableDERP {
 			b.e.SetDERPMap(nil)
 		} else {
 			b.e.SetDERPMap(st.NetMap.DERPMap)
 		}
+
+		b.send(Notify{NetMap: st.NetMap})
 	}
 	if st.URL != "" {
 		b.logf("Received auth URL: %.20v...", st.URL)
-
-		b.mu.Lock()
-		interact := b.interact
-		b.authURL = st.URL
-		b.mu.Unlock()
-
 		if interact > 0 {
 			b.popBrowserAuthNow()
 		}
 	}
-	if st.Err != "" {
-		// TODO(crawshaw): display in the UI.
-		b.logf("Received error: %v", st.Err)
-		return
-	}
-	if st.NetMap != nil {
-		b.mu.Lock()
-		if b.state == NeedsLogin {
-			b.prefs.WantRunning = true
-		}
-		prefs := b.prefs
-		b.mu.Unlock()
-
-		b.SetPrefs(prefs)
-	}
 	b.stateMachine()
+	// This is currently (2020-07-28) necessary; conditionally disabling it is fragile!
+	// This is where netmap information gets propagated to router and magicsock.
+	b.authReconfig()
 }
 
 // setWgengineStatus is the callback by the wireguard engine whenever it posts a new status.
@@ -351,19 +367,18 @@ func (b *LocalBackend) Start(opts Options) error {
 	b.serverURL = b.prefs.ControlURL
 	hostinfo.RoutableIPs = append(hostinfo.RoutableIPs, b.prefs.AdvertiseRoutes...)
 	hostinfo.RequestTags = append(hostinfo.RequestTags, b.prefs.AdvertiseTags...)
+	applyPrefsToHostinfo(hostinfo, b.prefs)
 
 	b.notify = opts.Notify
 	b.netMap = nil
 	persist := b.prefs.Persist
 	b.mu.Unlock()
 
-	b.updateFilter(nil)
+	b.updateFilter(nil, nil)
 
 	var discoPublic tailcfg.DiscoKey
 	if controlclient.Debug.Disco {
-		discoPrivate := key.NewPrivate()
-		b.e.SetDiscoPrivateKey(discoPrivate)
-		discoPublic = tailcfg.DiscoKey(discoPrivate.Public())
+		discoPublic = b.e.DiscoPublicKey()
 	}
 
 	var err error
@@ -389,8 +404,10 @@ func (b *LocalBackend) Start(opts Options) error {
 	// At this point, we have finished using hostinfo without synchronization,
 	// so it is safe to start readPoller which concurrently writes to it.
 	if b.portpoll != nil {
-		go b.portpoll.Run(b.ctx)
-		go b.readPoller()
+		b.portpollOnce.Do(func() {
+			go b.portpoll.Run(b.ctx)
+			go b.readPoller()
+		})
 	}
 
 	b.mu.Lock()
@@ -421,58 +438,121 @@ func (b *LocalBackend) Start(opts Options) error {
 
 // updateFilter updates the packet filter in wgengine based on the
 // given netMap and user preferences.
-func (b *LocalBackend) updateFilter(netMap *controlclient.NetworkMap) {
-	if netMap == nil {
-		// Not configured yet, block everything
+func (b *LocalBackend) updateFilter(netMap *controlclient.NetworkMap, prefs *Prefs) {
+	// NOTE(danderson): keep change detection as the first thing in
+	// this function. Don't try to optimize by returning early, more
+	// likely than not you'll just end up breaking the change
+	// detection and end up with the wrong filter installed. This is
+	// quite hard to debug, so save yourself the trouble.
+	var (
+		haveNetmap   = netMap != nil
+		addrs        []wgcfg.CIDR
+		packetFilter filter.Matches
+		advRoutes    []wgcfg.CIDR
+		shieldsUp    = prefs == nil || prefs.ShieldsUp // Be conservative when not ready
+	)
+	if haveNetmap {
+		addrs = netMap.Addresses
+		packetFilter = netMap.PacketFilter
+	}
+	if prefs != nil {
+		advRoutes = prefs.AdvertiseRoutes
+	}
+
+	changed := deepprint.UpdateHash(&b.filterHash, haveNetmap, addrs, packetFilter, advRoutes, shieldsUp)
+	if !changed {
+		return
+	}
+
+	if !haveNetmap {
 		b.logf("netmap packet filter: (not ready yet)")
 		b.e.SetFilter(filter.NewAllowNone(b.logf))
 		return
 	}
 
-	b.mu.Lock()
-	advRoutes := b.prefs.AdvertiseRoutes
-	b.mu.Unlock()
 	localNets := wgCIDRsToFilter(netMap.Addresses, advRoutes)
 
-	if b.shieldsAreUp() {
-		// Shields up, block everything
+	if shieldsUp {
 		b.logf("netmap packet filter: (shields up)")
 		var prevFilter *filter.Filter // don't reuse old filter state
 		b.e.SetFilter(filter.New(filter.Matches{}, localNets, prevFilter, b.logf))
-		return
+	} else {
+		b.logf("netmap packet filter: %v", packetFilter)
+		b.e.SetFilter(filter.New(packetFilter, localNets, b.e.GetFilter(), b.logf))
+	}
+}
+
+// dnsCIDRsEqual determines whether two CIDR lists are equal
+// for DNS map construction purposes (that is, only the first entry counts).
+func dnsCIDRsEqual(newAddr, oldAddr []wgcfg.CIDR) bool {
+	if len(newAddr) != len(oldAddr) {
+		return false
+	}
+	if len(newAddr) == 0 || newAddr[0] == oldAddr[0] {
+		return true
+	}
+	return false
+}
+
+// dnsMapsEqual determines whether the new and the old network map
+// induce the same DNS map. It does so without allocating memory,
+// at the expense of giving false negatives if peers are reordered.
+func dnsMapsEqual(new, old *controlclient.NetworkMap) bool {
+	if (old == nil) != (new == nil) {
+		return false
+	}
+	if old == nil && new == nil {
+		return true
 	}
 
-	// TODO(apenwarr): don't replace filter at all if unchanged.
-	// TODO(apenwarr): print a diff instead of full filter.
-	now := time.Now()
-	if now.Sub(b.lastFilterPrint) > 1*time.Minute {
-		b.logf("netmap packet filter: %v", netMap.PacketFilter)
-		b.lastFilterPrint = now
-	} else {
-		b.logf("netmap packet filter: (length %d)", len(netMap.PacketFilter))
+	if len(new.Peers) != len(old.Peers) {
+		return false
 	}
-	b.e.SetFilter(filter.New(netMap.PacketFilter, localNets, b.e.GetFilter(), b.logf))
+
+	if new.Name != old.Name {
+		return false
+	}
+	if !dnsCIDRsEqual(new.Addresses, old.Addresses) {
+		return false
+	}
+
+	for i, newPeer := range new.Peers {
+		oldPeer := old.Peers[i]
+		if newPeer.Name != oldPeer.Name {
+			return false
+		}
+		if !dnsCIDRsEqual(newPeer.Addresses, oldPeer.Addresses) {
+			return false
+		}
+	}
+
+	return true
 }
 
 // updateDNSMap updates the domain map in the DNS resolver in wgengine
 // based on the given netMap and user preferences.
 func (b *LocalBackend) updateDNSMap(netMap *controlclient.NetworkMap) {
 	if netMap == nil {
+		b.logf("dns map: (not ready)")
 		return
 	}
-	domainToIP := make(map[string]netaddr.IP)
-	for _, peer := range netMap.Peers {
-		if len(peer.Addresses) == 0 {
-			continue
+
+	nameToIP := make(map[string]netaddr.IP)
+	set := func(name string, addrs []wgcfg.CIDR) {
+		if len(addrs) == 0 {
+			return
 		}
-		domain := peer.Hostinfo.Hostname
-		// Like PeerStatus.SimpleHostName()
-		domain = strings.TrimSuffix(domain, ".local")
-		domain = strings.TrimSuffix(domain, ".localdomain")
-		domain = domain + ".ipn.dev"
-		domainToIP[domain] = netaddr.IPFrom16(peer.Addresses[0].IP.Addr)
+		nameToIP[name] = netaddr.IPFrom16(addrs[0].IP.Addr)
 	}
-	b.e.SetDNSMap(tsdns.NewMap(domainToIP))
+
+	for _, peer := range netMap.Peers {
+		set(peer.Name, peer.Addresses)
+	}
+	set(netMap.Name, netMap.Addresses)
+
+	dnsMap := tsdns.NewMap(nameToIP)
+	// map diff will be logged in tsdns.Resolver.SetMap.
+	b.e.SetDNSMap(dnsMap)
 }
 
 // readPoller is a goroutine that receives service lists from
@@ -517,6 +597,8 @@ func (b *LocalBackend) send(n Notify) {
 	if notify != nil {
 		n.Version = version.LONG
 		notify(n)
+	} else {
+		b.logf("nil notify callback; dropping %+v", n)
 	}
 }
 
@@ -569,7 +651,7 @@ func (b *LocalBackend) loadStateLocked(key StateKey, prefs *Prefs, legacyPath st
 	if err != nil {
 		if errors.Is(err, ErrStateNotExist) {
 			if legacyPath != "" {
-				b.prefs, err = LoadPrefs(legacyPath, true)
+				b.prefs, err = LoadPrefs(legacyPath)
 				if err != nil {
 					b.logf("Failed to load legacy prefs: %v", err)
 					b.prefs = NewPrefs()
@@ -611,6 +693,16 @@ func (b *LocalBackend) getEngineStatus() EngineStatus {
 	return b.engineStatus
 }
 
+// Login implements Backend.
+func (b *LocalBackend) Login(token *oauth2.Token) {
+	b.mu.Lock()
+	b.assertClientLocked()
+	c := b.c
+	b.mu.Unlock()
+
+	c.Login(token, controlclient.LoginInteractive)
+}
+
 // StartLoginInteractive implements Backend. It requests a new
 // interactive login from controlclient, unless such a flow is already
 // in progress, in which case StartLoginInteractive attempts to pick
@@ -699,38 +791,45 @@ func (b *LocalBackend) SetPrefs(new *Prefs) {
 	}
 
 	b.mu.Lock()
+
+	netMap := b.netMap
+	stateKey := b.stateKey
+
 	old := b.prefs
 	new.Persist = old.Persist // caller isn't allowed to override this
 	b.prefs = new
-	if b.stateKey != "" {
-		if err := b.store.WriteState(b.stateKey, b.prefs.ToBytes()); err != nil {
-			b.logf("Failed to save new controlclient state: %v", err)
-		}
-	}
+	// We do this to avoid holding the lock while doing everything else.
+	new = b.prefs.Clone()
+
 	oldHi := b.hostinfo
 	newHi := oldHi.Clone()
 	newHi.RoutableIPs = append([]wgcfg.CIDR(nil), b.prefs.AdvertiseRoutes...)
-	if h := new.Hostname; h != "" {
-		newHi.Hostname = h
-	}
+	applyPrefsToHostinfo(newHi, new)
 	b.hostinfo = newHi
+	hostInfoChanged := !oldHi.Equal(newHi)
+
 	b.mu.Unlock()
 
+	if stateKey != "" {
+		if err := b.store.WriteState(stateKey, new.ToBytes()); err != nil {
+			b.logf("Failed to save new controlclient state: %v", err)
+		}
+	}
+
 	b.logf("SetPrefs: %v", new.Pretty())
 
-	if old.ShieldsUp != new.ShieldsUp || !oldHi.Equal(newHi) {
+	if old.ShieldsUp != new.ShieldsUp || hostInfoChanged {
 		b.doSetHostinfoFilterServices(newHi)
 	}
 
-	b.updateFilter(b.netMap)
-	// TODO(dmytro): when Prefs gain an EnableTailscaleDNS toggle, updateDNSMap here.
+	b.updateFilter(netMap, new)
 
 	turnDERPOff := new.DisableDERP && !old.DisableDERP
 	turnDERPOn := !new.DisableDERP && old.DisableDERP
 	if turnDERPOff {
 		b.e.SetDERPMap(nil)
-	} else if turnDERPOn && b.netMap != nil {
-		b.e.SetDERPMap(b.netMap.DERPMap)
+	} else if turnDERPOn && netMap != nil {
+		b.e.SetDERPMap(netMap.DERPMap)
 	}
 
 	if old.WantRunning != new.WantRunning {
@@ -807,44 +906,88 @@ func (b *LocalBackend) authReconfig() {
 		return
 	}
 
-	uflags := controlclient.UDefault
+	var flags controlclient.WGConfigFlags
 	if uc.RouteAll {
-		uflags |= controlclient.UAllowDefaultRoute
+		flags |= controlclient.AllowDefaultRoute
 		// TODO(apenwarr): Make subnet routes a different pref?
-		uflags |= controlclient.UAllowSubnetRoutes
+		flags |= controlclient.AllowSubnetRoutes
 		// TODO(apenwarr): Remove this once we sort out subnet routes.
 		//  Right now default routes are broken in Windows, but
 		//  controlclient doesn't properly send subnet routes. So
 		//  let's convert a default route into a subnet route in order
 		//  to allow experimentation.
-		uflags |= controlclient.UHackDefaultRoute
+		flags |= controlclient.HackDefaultRoute
 	}
 	if uc.AllowSingleHosts {
-		uflags |= controlclient.UAllowSingleHosts
+		flags |= controlclient.AllowSingleHosts
 	}
 
-	dns := nm.DNS
-	dom := nm.DNSDomains
-	if !uc.CorpDNS {
-		dns = []wgcfg.IP{}
-		dom = []string{}
-	}
-	cfg, err := nm.WGCfg(b.logf, uflags, dns)
+	cfg, err := nm.WGCfg(b.logf, flags)
 	if err != nil {
 		b.logf("wgcfg: %v", err)
 		return
 	}
 
-	err = b.e.Reconfig(cfg, routerConfig(cfg, uc, dom))
+	rcfg := routerConfig(cfg, uc)
+
+	// If CorpDNS is false, rcfg.DNS remains the zero value.
+	if uc.CorpDNS {
+		domains := nm.DNS.Domains
+		proxied := nm.DNS.Proxied
+		if proxied {
+			if len(nm.DNS.Nameservers) == 0 {
+				b.logf("[unexpected] dns proxied but no nameservers")
+				proxied = false
+			} else {
+				// Domains for proxying should come first to avoid leaking queries.
+				domains = append(domainsForProxying(nm), domains...)
+			}
+		}
+		rcfg.DNS = dns.Config{
+			Nameservers: nm.DNS.Nameservers,
+			Domains:     domains,
+			PerDomain:   nm.DNS.PerDomain,
+			Proxied:     proxied,
+		}
+	}
+
+	err = b.e.Reconfig(cfg, rcfg)
 	if err == wgengine.ErrNoChanges {
 		return
 	}
-	b.logf("authReconfig: ra=%v dns=%v 0x%02x: %v", uc.RouteAll, uc.CorpDNS, uflags, err)
+	b.logf("authReconfig: ra=%v dns=%v 0x%02x: %v", uc.RouteAll, uc.CorpDNS, flags, err)
 }
 
-// routerConfig produces a router.Config from a wireguard config,
-// IPN prefs, and the dnsDomains pulled from control's network map.
-func routerConfig(cfg *wgcfg.Config, prefs *Prefs, dnsDomains []string) *router.Config {
+// domainsForProxying produces a list of search domains for proxied DNS.
+func domainsForProxying(nm *controlclient.NetworkMap) []string {
+	var domains []string
+	if idx := strings.IndexByte(nm.Name, '.'); idx != -1 {
+		domains = append(domains, nm.Name[idx+1:])
+	}
+	for _, peer := range nm.Peers {
+		idx := strings.IndexByte(peer.Name, '.')
+		if idx == -1 {
+			continue
+		}
+		domain := peer.Name[idx+1:]
+		seen := false
+		// In theory this makes the function O(n^2) worst case,
+		// but in practice we expect domains to contain very few elements
+		// (only one until invitations are introduced).
+		for _, seenDomain := range domains {
+			if domain == seenDomain {
+				seen = true
+			}
+		}
+		if !seen {
+			domains = append(domains, domain)
+		}
+	}
+	return domains
+}
+
+// routerConfig produces a router.Config from a wireguard config and IPN prefs.
+func routerConfig(cfg *wgcfg.Config, prefs *Prefs) *router.Config {
 	var addrs []wgcfg.CIDR
 	for _, addr := range cfg.Addresses {
 		addrs = append(addrs, wgcfg.CIDR{
@@ -855,8 +998,6 @@ func routerConfig(cfg *wgcfg.Config, prefs *Prefs, dnsDomains []string) *router.
 
 	rs := &router.Config{
 		LocalAddrs:       wgCIDRToNetaddr(addrs),
-		DNS:              wgIPToNetaddr(cfg.DNS),
-		DNSDomains:       dnsDomains,
 		SubnetRoutes:     wgCIDRToNetaddr(prefs.AdvertiseRoutes),
 		SNATSubnetRoutes: !prefs.NoSNAT,
 		NetfilterMode:    prefs.NetfilterMode,
@@ -866,10 +1007,8 @@ func routerConfig(cfg *wgcfg.Config, prefs *Prefs, dnsDomains []string) *router.
 		rs.Routes = append(rs.Routes, wgCIDRToNetaddr(peer.AllowedIPs)...)
 	}
 
-	// The Tailscale DNS IP.
-	// TODO(dmytro): make this configurable.
 	rs.Routes = append(rs.Routes, netaddr.IPPrefix{
-		IP:   netaddr.IPv4(100, 100, 100, 100),
+		IP:   tsaddr.TailscaleServiceIP(),
 		Bits: 32,
 	})
 
@@ -893,17 +1032,6 @@ func wgCIDRsToFilter(cidrLists ...[]wgcfg.CIDR) (ret []filter.Net) {
 	return ret
 }
 
-func wgIPToNetaddr(ips []wgcfg.IP) (ret []netaddr.IP) {
-	for _, ip := range ips {
-		nip, ok := netaddr.FromStdIP(ip.IP())
-		if !ok {
-			panic(fmt.Sprintf("conversion of %s from wgcfg to netaddr IP failed", ip))
-		}
-		ret = append(ret, nip.Unmap())
-	}
-	return ret
-}
-
 func wgCIDRToNetaddr(cidrs []wgcfg.CIDR) (ret []netaddr.IPPrefix) {
 	for _, cidr := range cidrs {
 		ncidr, ok := netaddr.FromStdIPNet(cidr.IPNet())
@@ -916,6 +1044,18 @@ func wgCIDRToNetaddr(cidrs []wgcfg.CIDR) (ret []netaddr.IPPrefix) {
 	return ret
 }
 
+func applyPrefsToHostinfo(hi *tailcfg.Hostinfo, prefs *Prefs) {
+	if h := prefs.Hostname; h != "" {
+		hi.Hostname = h
+	}
+	if v := prefs.OSVersion; v != "" {
+		hi.OSVersion = v
+	}
+	if m := prefs.DeviceModel; m != "" {
+		hi.DeviceModel = m
+	}
+}
+
 // enterState transitions the backend into newState, updating internal
 // state and propagating events out as needed.
 //
@@ -929,6 +1069,7 @@ func (b *LocalBackend) enterState(newState State) {
 	b.state = newState
 	prefs := b.prefs
 	notify := b.notify
+	bc := b.c
 	b.mu.Unlock()
 
 	if state == newState {
@@ -940,6 +1081,10 @@ func (b *LocalBackend) enterState(newState State) {
 		b.send(Notify{State: &newState})
 	}
 
+	if bc != nil {
+		bc.SetPaused(newState == Stopped)
+	}
+
 	switch newState {
 	case NeedsLogin:
 		b.blockEngineUpdates(true)
@@ -1015,7 +1160,7 @@ func (b *LocalBackend) RequestEngineStatus() {
 // RequestStatus implements Backend.
 func (b *LocalBackend) RequestStatus() {
 	st := b.Status()
-	b.notify(Notify{Status: st})
+	b.send(Notify{Status: st})
 }
 
 // stateMachine updates the state machine state based on other things

ipn/message.go

@@ -13,6 +13,7 @@ import (
 	"log"
 	"time"
 
+	"golang.org/x/oauth2"
 	"tailscale.com/types/logger"
 	"tailscale.com/types/structs"
 	"tailscale.com/version"
@@ -49,6 +50,7 @@ type Command struct {
 	Quit                  *NoArgs
 	Start                 *StartArgs
 	StartLoginInteractive *NoArgs
+	Login                 *oauth2.Token
 	Logout                *NoArgs
 	SetPrefs              *SetPrefsArgs
 	RequestEngineStatus   *NoArgs
@@ -80,6 +82,10 @@ func (bs *BackendServer) send(n Notify) {
 	bs.sendNotifyMsg(b)
 }
 
+func (bs *BackendServer) SendErrorMessage(msg string) {
+	bs.send(Notify{ErrMessage: &msg})
+}
+
 // GotCommandMsg parses the incoming message b as a JSON Command and
 // calls GotCommand with it.
 func (bs *BackendServer) GotCommandMsg(b []byte) error {
@@ -124,6 +130,9 @@ func (bs *BackendServer) GotCommand(cmd *Command) error {
 	} else if c := cmd.StartLoginInteractive; c != nil {
 		bs.b.StartLoginInteractive()
 		return nil
+	} else if c := cmd.Login; c != nil {
+		bs.b.Login(c)
+		return nil
 	} else if c := cmd.Logout; c != nil {
 		bs.b.Logout()
 		return nil
@@ -221,6 +230,10 @@ func (bc *BackendClient) StartLoginInteractive() {
 	bc.send(Command{StartLoginInteractive: &NoArgs{}})
 }
 
+func (bc *BackendClient) Login(token *oauth2.Token) {
+	bc.send(Command{Login: token})
+}
+
 func (bc *BackendClient) Logout() {
 	bc.send(Command{Logout: &NoArgs{}})
 }
@@ -242,7 +255,7 @@ func (bc *BackendClient) FakeExpireAfter(x time.Duration) {
 }
 
 // MaxMessageSize is the maximum message size, in bytes.
-const MaxMessageSize = 1 << 20
+const MaxMessageSize = 10 << 20
 
 // TODO(apenwarr): incremental json decode?
 //  That would let us avoid storing the whole byte array uselessly in RAM.

ipn/message_test.go

@@ -9,6 +9,7 @@ import (
 	"testing"
 	"time"
 
+	"golang.org/x/oauth2"
 	"tailscale.com/tstest"
 )
 
@@ -177,4 +178,10 @@ func TestClientServer(t *testing.T) {
 
 	h.Logout()
 	flushUntil(NeedsLogin)
+
+	h.Login(&oauth2.Token{
+		AccessToken: "google_id_token",
+		TokenType:   GoogleIDTokenType,
+	})
+	flushUntil(Running)
 }

ipn/prefs.go

@@ -54,6 +54,10 @@ type Prefs struct {
 	// Hostname is the hostname to use for identifying the node. If
 	// not set, os.Hostname is used.
 	Hostname string
+	// OSVersion overrides tailcfg.Hostinfo's OSVersion.
+	OSVersion string
+	// DeviceModel overrides tailcfg.Hostinfo's DeviceModel.
+	DeviceModel string
 
 	// NotepadURLs is a debugging setting that opens OAuth URLs in
 	// notepad.exe on Windows, rather than loading them in a browser.
@@ -138,6 +142,8 @@ func (p *Prefs) Equals(p2 *Prefs) bool {
 		p.NoSNAT == p2.NoSNAT &&
 		p.NetfilterMode == p2.NetfilterMode &&
 		p.Hostname == p2.Hostname &&
+		p.OSVersion == p2.OSVersion &&
+		p.DeviceModel == p2.DeviceModel &&
 		compareIPNets(p.AdvertiseRoutes, p2.AdvertiseRoutes) &&
 		compareStrings(p.AdvertiseTags, p2.AdvertiseTags) &&
 		p.Persist.Equals(p2.Persist)
@@ -217,10 +223,9 @@ func (p *Prefs) Clone() *Prefs {
 	return p2
 }
 
-// LoadLegacyPrefs loads a legacy relaynode config file into Prefs
-// with sensible migration defaults set. If enforceDefaults is true,
-// Prefs.RouteAll and Prefs.AllowSingleHosts are forced on.
-func LoadPrefs(filename string, enforceDefaults bool) (*Prefs, error) {
+// LoadPrefs loads a legacy relaynode config file into Prefs
+// with sensible migration defaults set.
+func LoadPrefs(filename string) (*Prefs, error) {
 	data, err := ioutil.ReadFile(filename)
 	if err != nil {
 		return nil, fmt.Errorf("loading prefs from %q: %v", filename, err)

ipn/prefs_test.go

@@ -24,7 +24,7 @@ func fieldsOf(t reflect.Type) (fields []string) {
 func TestPrefsEqual(t *testing.T) {
 	tstest.PanicOnLog()
 
-	prefsHandles := []string{"ControlURL", "RouteAll", "AllowSingleHosts", "CorpDNS", "WantRunning", "ShieldsUp", "AdvertiseTags", "Hostname", "NotepadURLs", "DisableDERP", "AdvertiseRoutes", "NoSNAT", "NetfilterMode", "Persist"}
+	prefsHandles := []string{"ControlURL", "RouteAll", "AllowSingleHosts", "CorpDNS", "WantRunning", "ShieldsUp", "AdvertiseTags", "Hostname", "OSVersion", "DeviceModel", "NotepadURLs", "DisableDERP", "AdvertiseRoutes", "NoSNAT", "NetfilterMode", "Persist"}
 	if have := fieldsOf(reflect.TypeOf(Prefs{})); !reflect.DeepEqual(have, prefsHandles) {
 		t.Errorf("Prefs.Equal check might be out of sync\nfields: %q\nhandled: %q\n",
 			have, prefsHandles)

logpolicy/logpolicy.go

@@ -32,6 +32,7 @@ import (
 	"tailscale.com/net/netns"
 	"tailscale.com/net/tlsdial"
 	"tailscale.com/smallzstd"
+	"tailscale.com/types/logger"
 	"tailscale.com/version"
 )
 
@@ -54,7 +55,7 @@ type Policy struct {
 func (c *Config) ToBytes() []byte {
 	data, err := json.MarshalIndent(c, "", "\t")
 	if err != nil {
-		log.Fatalf("logpolicy.Config marshal: %v\n", err)
+		log.Fatalf("logpolicy.Config marshal: %v", err)
 	}
 	return data
 }
@@ -101,21 +102,25 @@ func (l logWriter) Write(buf []byte) (int, error) {
 
 // logsDir returns the directory to use for log configuration and
 // buffer storage.
-func logsDir() string {
+func logsDir(logf logger.Logf) string {
 	systemdStateDir := os.Getenv("STATE_DIRECTORY")
 	if systemdStateDir != "" {
+		logf("logpolicy: using $STATE_DIRECTORY, %q", systemdStateDir)
 		return systemdStateDir
 	}
 
 	cacheDir, err := os.UserCacheDir()
 	if err == nil {
-		return filepath.Join(cacheDir, "Tailscale")
+		d := filepath.Join(cacheDir, "Tailscale")
+		logf("logpolicy: using UserCacheDir, %q", d)
+		return d
 	}
 
 	// Use the current working directory, unless we're being run by a
 	// service manager that sets it to /.
 	wd, err := os.Getwd()
 	if err == nil && wd != "/" {
+		logf("logpolicy: using current directory, %q", wd)
 		return wd
 	}
 
@@ -126,6 +131,7 @@ func logsDir() string {
 	if err != nil {
 		panic("no safe place found to store log state")
 	}
+	logf("logpolicy: using temp directory, %q", tmp)
 	return tmp
 }
 
@@ -149,6 +155,14 @@ func runningUnderSystemd() bool {
 // moved from whereever it does exist, into dir. Leftover logs state
 // in / and $CACHE_DIRECTORY is deleted.
 func tryFixLogStateLocation(dir, cmdname string) {
+	switch runtime.GOOS {
+	case "linux", "freebsd", "openbsd":
+		// These are the OSes where we might have written stuff into
+		// root. Others use different logic to find the logs storage
+		// dir.
+	default:
+		return
+	}
 	if cmdname == "" {
 		log.Printf("[unexpected] no cmdname given to tryFixLogStateLocation, please file a bug at https://github.com/tailscale/tailscale")
 		return
@@ -163,14 +177,6 @@ func tryFixLogStateLocation(dir, cmdname string) {
 		// Only root could have written log configs to weird places.
 		return
 	}
-	switch runtime.GOOS {
-	case "linux", "freebsd", "openbsd":
-		// These are the OSes where we might have written stuff into
-		// root. Others use different logic to find the logs storage
-		// dir.
-	default:
-		return
-	}
 
 	// We stored logs in 2 incorrect places: either /, or CACHE_DIR
 	// (aka /var/cache/tailscale). We want to move files into the
@@ -303,23 +309,28 @@ func New(collection string) *Policy {
 	}
 	console := log.New(stderrWriter{}, "", lflags)
 
-	dir := logsDir()
-
-	if runtime.GOOS != "windows" { // version.CmdName call was blowing some Windows stack limit via goversion DLL loading
-		tryFixLogStateLocation(dir, version.CmdName())
+	var earlyErrBuf bytes.Buffer
+	earlyLogf := func(format string, a ...interface{}) {
+		fmt.Fprintf(&earlyErrBuf, format, a...)
+		earlyErrBuf.WriteByte('\n')
 	}
 
-	cfgPath := filepath.Join(dir, fmt.Sprintf("%s.log.conf", version.CmdName()))
+	dir := logsDir(earlyLogf)
+
+	cmdName := version.CmdName()
+	tryFixLogStateLocation(dir, cmdName)
+
+	cfgPath := filepath.Join(dir, fmt.Sprintf("%s.log.conf", cmdName))
 	var oldc *Config
 	data, err := ioutil.ReadFile(cfgPath)
 	if err != nil {
-		log.Printf("logpolicy.Read %v: %v\n", cfgPath, err)
+		earlyLogf("logpolicy.Read %v: %v", cfgPath, err)
 		oldc = &Config{}
 		oldc.Collection = collection
 	} else {
 		oldc, err = ConfigFromBytes(data)
 		if err != nil {
-			log.Printf("logpolicy.Config unmarshal: %v\n", err)
+			earlyLogf("logpolicy.Config unmarshal: %v", err)
 			oldc = &Config{}
 		}
 	}
@@ -341,7 +352,7 @@ func New(collection string) *Policy {
 	newc.PublicID = newc.PrivateID.Public()
 	if newc != *oldc {
 		if err := newc.save(cfgPath); err != nil {
-			log.Printf("logpolicy.Config.Save: %v\n", err)
+			earlyLogf("logpolicy.Config.Save: %v", err)
 		}
 	}
 
@@ -359,7 +370,7 @@ func New(collection string) *Policy {
 		HTTPC: &http.Client{Transport: newLogtailTransport(logtail.DefaultHost)},
 	}
 
-	filchBuf, filchErr := filch.New(filepath.Join(dir, version.CmdName()), filch.Options{})
+	filchBuf, filchErr := filch.New(filepath.Join(dir, cmdName), filch.Options{})
 	if filchBuf != nil {
 		c.Buffer = filchBuf
 	}
@@ -367,14 +378,17 @@ func New(collection string) *Policy {
 	log.SetFlags(0) // other logflags are set on console, not here
 	log.SetOutput(lw)
 
-	log.Printf("Program starting: v%v, Go %v: %#v\n",
+	log.Printf("Program starting: v%v, Go %v: %#v",
 		version.LONG,
 		strings.TrimPrefix(runtime.Version(), "go"),
 		os.Args)
-	log.Printf("LogID: %v\n", newc.PublicID)
+	log.Printf("LogID: %v", newc.PublicID)
 	if filchErr != nil {
 		log.Printf("filch failed: %v", filchErr)
 	}
+	if earlyErrBuf.Len() != 0 {
+		log.Printf("%s", earlyErrBuf.Bytes())
+	}
 
 	return &Policy{
 		Logtail:  lw,
@@ -393,7 +407,7 @@ func (p *Policy) Close() {
 // log upload if it can be done before ctx is canceled.
 func (p *Policy) Shutdown(ctx context.Context) error {
 	if p.Logtail != nil {
-		log.Printf("flushing log.\n")
+		log.Printf("flushing log.")
 		return p.Logtail.Shutdown(ctx)
 	}
 	return nil

logtail/backoff/backoff.go

@@ -2,6 +2,7 @@
 // Use of this source code is governed by a BSD-style
 // license that can be found in the LICENSE file.
 
+// Package backoff provides a back-off timer type.
 package backoff
 
 import (
@@ -12,54 +13,70 @@ import (
 	"tailscale.com/types/logger"
 )
 
-const MAX_BACKOFF_MSEC = 30000
-
+// Backoff tracks state the history of consecutive failures and sleeps
+// an increasing amount of time, up to a provided limit.
 type Backoff struct {
-	n int
+	n          int // number of consecutive failures
+	maxBackoff time.Duration
+
 	// Name is the name of this backoff timer, for logging purposes.
 	name string
 	// logf is the function used for log messages when backing off.
 	logf logger.Logf
-	// NewTimer is the function that acts like time.NewTimer().
-	// You can override this in unit tests.
-	NewTimer func(d time.Duration) *time.Timer
+
+	// NewTimer is the function that acts like time.NewTimer.
+	// It's for use in unit tests.
+	NewTimer func(time.Duration) *time.Timer
+
 	// LogLongerThan sets the minimum time of a single backoff interval
 	// before we mention it in the log.
 	LogLongerThan time.Duration
 }
 
-func NewBackoff(name string, logf logger.Logf) Backoff {
-	return Backoff{
-		name:     name,
-		logf:     logf,
-		NewTimer: time.NewTimer,
+// NewBackoff returns a new Backoff timer with the provided name (for logging), logger,
+// and max backoff time. By default, all failures (calls to BackOff with a non-nil err)
+// are logged unless the returned Backoff.LogLongerThan is adjusted.
+func NewBackoff(name string, logf logger.Logf, maxBackoff time.Duration) *Backoff {
+	return &Backoff{
+		name:       name,
+		logf:       logf,
+		maxBackoff: maxBackoff,
+		NewTimer:   time.NewTimer,
 	}
 }
 
+// Backoff sleeps an increasing amount of time if err is non-nil.
+// and the context is not a
+// It resets the backoff schedule once err is nil.
 func (b *Backoff) BackOff(ctx context.Context, err error) {
-	if ctx.Err() == nil && err != nil {
-		b.n++
-		// n^2 backoff timer is a little smoother than the
-		// common choice of 2^n.
-		msec := b.n * b.n * 10
-		if msec > MAX_BACKOFF_MSEC {
-			msec = MAX_BACKOFF_MSEC
-		}
-		// Randomize the delay between 0.5-1.5 x msec, in order
-		// to prevent accidental "thundering herd" problems.
-		msec = rand.Intn(msec) + msec/2
-		dur := time.Duration(msec) * time.Millisecond
-		if dur >= b.LogLongerThan {
-			b.logf("%s: backoff: %d msec\n", b.name, msec)
-		}
-		t := b.NewTimer(dur)
-		select {
-		case <-ctx.Done():
-			t.Stop()
-		case <-t.C:
-		}
-	} else {
-		// not a regular error
+	if err == nil {
+		// No error. Reset number of consecutive failures.
 		b.n = 0
+		return
+	}
+	if ctx.Err() != nil {
+		// Fast path.
+		return
+	}
+
+	b.n++
+	// n^2 backoff timer is a little smoother than the
+	// common choice of 2^n.
+	d := time.Duration(b.n*b.n) * 10 * time.Millisecond
+	if d > b.maxBackoff {
+		d = b.maxBackoff
+	}
+	// Randomize the delay between 0.5-1.5 x msec, in order
+	// to prevent accidental "thundering herd" problems.
+	d = time.Duration(float64(d) * (rand.Float64() + 0.5))
+
+	if d >= b.LogLongerThan {
+		b.logf("%s: backoff: %d msec", b.name, d.Milliseconds())
+	}
+	t := b.NewTimer(d)
+	select {
+	case <-ctx.Done():
+		t.Stop()
+	case <-t.C:
 	}
 }

logtail/logtail.go

@@ -105,7 +105,7 @@ func Log(cfg Config, logf tslogger.Logf) Logger {
 		sentinel:       make(chan int32, 16),
 		drainLogs:      cfg.DrainLogs,
 		timeNow:        cfg.TimeNow,
-		bo:             backoff.NewBackoff("logtail", logf),
+		bo:             backoff.NewBackoff("logtail", logf, 30*time.Second),
 
 		shutdownStart: make(chan struct{}),
 		shutdownDone:  make(chan struct{}),
@@ -133,7 +133,7 @@ type logger struct {
 	drainLogs      <-chan struct{} // if non-nil, external signal to attempt a drain
 	sentinel       chan int32
 	timeNow        func() time.Time
-	bo             backoff.Backoff
+	bo             *backoff.Backoff
 	zstdEncoder    Encoder
 	uploadCancel   func()
 
@@ -462,5 +462,6 @@ func (l *logger) Write(buf []byte) (int, error) {
 		}
 	}
 	b := l.encode(buf)
-	return l.send(b)
+	_, err := l.send(b)
+	return len(buf), err
 }

logtail/logtail_test.go

@@ -32,3 +32,18 @@ func TestLoggerEncodeTextAllocs(t *testing.T) {
 		t.Logf("allocs = %d; want 1", int(n))
 	}
 }
+
+func TestLoggerWriteLength(t *testing.T) {
+	lg := &logger{
+		timeNow: time.Now,
+		buffer:  NewMemoryBuffer(1024),
+	}
+	inBuf := []byte("some text to encode")
+	n, err := lg.Write(inBuf)
+	if err != nil {
+		t.Error(err)
+	}
+	if n != len(inBuf) {
+		t.Errorf("logger.Write wrote %d bytes, expected %d", n, len(inBuf))
+	}
+}

metrics/metrics.go

@@ -40,3 +40,10 @@ func (m *LabelMap) Get(key string) *expvar.Int {
 	m.Add(key, 0)
 	return m.Map.Get(key).(*expvar.Int)
 }
+
+// GetFloat returns a direct pointer to the expvar.Float for key, creating it
+// if necessary.
+func (m *LabelMap) GetFloat(key string) *expvar.Float {
+	m.AddFloat(key, 0.0)
+	return m.Map.Get(key).(*expvar.Float)
+}

net/interfaces/interfaces.go

@@ -12,6 +12,7 @@ import (
 	"strings"
 
 	"inet.af/netaddr"
+	"tailscale.com/net/tsaddr"
 )
 
 // Tailscale returns the current machine's Tailscale interface, if any.
@@ -52,7 +53,7 @@ func maybeTailscaleInterfaceName(s string) bool {
 // Tailscale virtual network interfaces.
 func IsTailscaleIP(ip net.IP) bool {
 	nip, _ := netaddr.FromStdIP(ip) // TODO: push this up to caller, change func signature
-	return cgNAT.Contains(nip)
+	return tsaddr.IsTailscaleIP(nip)
 }
 
 func isUp(nif *net.Interface) bool       { return nif.Flags&net.FlagUp != 0 }
@@ -95,7 +96,7 @@ func LocalAddresses() (regular, loopback []string, err error) {
 				// very well be something we can route to
 				// directly, because both nodes are
 				// behind the same CGNAT router.
-				if cgNAT.Contains(ip) {
+				if tsaddr.IsTailscaleIP(ip) {
 					continue
 				}
 				if linkLocalIPv4.Contains(ip) {
@@ -230,6 +231,38 @@ func HTTPOfListener(ln net.Listener) string {
 
 }
 
+var likelyHomeRouterIP func() (netaddr.IP, bool)
+
+// LikelyHomeRouterIP returns the likely IP of the residential router,
+// which will always be an IPv4 private address, if found.
+// In addition, it returns the IP address of the current machine on
+// the LAN using that gateway.
+// This is used as the destination for UPnP, NAT-PMP, PCP, etc queries.
+func LikelyHomeRouterIP() (gateway, myIP netaddr.IP, ok bool) {
+	if likelyHomeRouterIP != nil {
+		gateway, ok = likelyHomeRouterIP()
+		if !ok {
+			return
+		}
+	}
+	if !ok {
+		return
+	}
+	ForeachInterfaceAddress(func(i Interface, ip netaddr.IP) {
+		if !i.IsUp() || ip.IsZero() || !myIP.IsZero() {
+			return
+		}
+		for _, prefix := range privatev4s {
+			if prefix.Contains(gateway) && prefix.Contains(ip) {
+				myIP = ip
+				ok = true
+				return
+			}
+		}
+	})
+	return gateway, myIP, !myIP.IsZero()
+}
+
 func isPrivateIP(ip netaddr.IP) bool {
 	return private1.Contains(ip) || private2.Contains(ip) || private3.Contains(ip)
 }
@@ -250,7 +283,7 @@ var (
 	private1      = mustCIDR("10.0.0.0/8")
 	private2      = mustCIDR("172.16.0.0/12")
 	private3      = mustCIDR("192.168.0.0/16")
-	cgNAT         = mustCIDR("100.64.0.0/10")
+	privatev4s    = []netaddr.IPPrefix{private1, private2, private3}
 	linkLocalIPv4 = mustCIDR("169.254.0.0/16")
 	v6Global1     = mustCIDR("2000::/3")
 )

net/interfaces/interfaces_darwin.go

@@ -0,0 +1,69 @@
+// Copyright (c) 2020 Tailscale Inc & AUTHORS All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package interfaces
+
+import (
+	"os/exec"
+
+	"go4.org/mem"
+	"inet.af/netaddr"
+	"tailscale.com/util/lineread"
+	"tailscale.com/version"
+)
+
+/*
+Parse out 10.0.0.1 from:
+
+$ netstat -r -n -f inet
+Routing tables
+
+Internet:
+Destination        Gateway            Flags        Netif Expire
+default            10.0.0.1           UGSc           en0
+default            link#14            UCSI         utun2
+10/16              link#4             UCS            en0      !
+10.0.0.1/32        link#4             UCS            en0      !
+...
+
+*/
+func likelyHomeRouterIPDarwinExec() (ret netaddr.IP, ok bool) {
+	if version.IsMobile() {
+		// Don't try to do subprocesses on iOS. Ends up with log spam like:
+		// kernel: "Sandbox: IPNExtension(86580) deny(1) process-fork"
+		// This is why we have likelyHomeRouterIPDarwinSyscall.
+		return ret, false
+	}
+	cmd := exec.Command("/usr/sbin/netstat", "-r", "-n", "-f", "inet")
+	stdout, err := cmd.StdoutPipe()
+	if err != nil {
+		return
+	}
+	if err := cmd.Start(); err != nil {
+		return
+	}
+	defer cmd.Wait()
+
+	var f []mem.RO
+	lineread.Reader(stdout, func(lineb []byte) error {
+		line := mem.B(lineb)
+		if !mem.Contains(line, mem.S("default")) {
+			return nil
+		}
+		f = mem.AppendFields(f[:0], line)
+		if len(f) < 3 || !f[0].EqualString("default") {
+			return nil
+		}
+		ipm, flagsm := f[1], f[2]
+		if !mem.Contains(flagsm, mem.S("G")) {
+			return nil
+		}
+		ip, err := netaddr.ParseIP(string(mem.Append(nil, ipm)))
+		if err == nil && isPrivateIP(ip) {
+			ret = ip
+		}
+		return nil
+	})
+	return ret, !ret.IsZero()
+}

net/interfaces/interfaces_darwin_cgo.go

@@ -0,0 +1,127 @@
+// Copyright (c) 2020 Tailscale Inc & AUTHORS All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+// +build darwin,cgo
+
+package interfaces
+
+/*
+#import "route.h"
+#import <netinet/in.h>
+#import <sys/sysctl.h>
+#import <stdlib.h>
+#import <stdio.h>
+
+// privateGatewayIPFromRoute returns the private gateway ip address from rtm, if it exists.
+// Otherwise, it returns 0.
+int privateGatewayIPFromRoute(struct rt_msghdr2 *rtm)
+{
+    // sockaddrs are after the message header
+    struct sockaddr* dst_sa = (struct sockaddr *)(rtm + 1);
+
+    if((rtm->rtm_addrs & (RTA_DST|RTA_GATEWAY)) != (RTA_DST|RTA_GATEWAY))
+        return 0; // missing dst or gateway addr
+    if (dst_sa->sa_family != AF_INET)
+        return 0; // dst not IPv4
+    if ((rtm->rtm_flags & RTF_GATEWAY) == 0)
+        return 0; // gateway flag not set
+
+    struct sockaddr_in* dst_si = (struct sockaddr_in *)dst_sa;
+    if (dst_si->sin_addr.s_addr != INADDR_ANY)
+        return 0; // not default route
+
+    #define ROUNDUP(a) ((a) > 0 ? (1 + (((a) - 1) | (sizeof(long) - 1))) : sizeof(long))
+
+    struct sockaddr* gateway_sa = (struct sockaddr *)((char *)dst_sa + ROUNDUP(dst_sa->sa_len));
+    if (gateway_sa->sa_family != AF_INET)
+        return 0; // gateway not IPv4
+
+    struct sockaddr_in* gateway_si= (struct sockaddr_in *)gateway_sa;
+    int ip;
+    ip = gateway_si->sin_addr.s_addr;
+
+    unsigned char a, b;
+    a = (ip >> 0) & 0xff;
+    b = (ip >> 8) & 0xff;
+
+    // Check whether ip is private, that is, whether it is
+    // in one of 10.0.0.0/8, 172.16.0.0/12, or 192.168.0.0/16.
+    if (a == 10)
+        return ip; // matches 10.0.0.0/8
+    if (a == 172 && (b >> 4) == 1)
+        return ip; // matches 172.16.0.0/12
+    if (a == 192 && b == 168)
+        return ip; // matches 192.168.0.0/16
+
+    // Not a private IP.
+    return 0;
+}
+
+// privateGatewayIP returns the private gateway IP address, if it exists.
+// If no private gateway IP address was found, it returns 0.
+// On an error, it returns an error code in (0, 255].
+// Any private gateway IP address is > 255.
+int privateGatewayIP()
+{
+    size_t needed;
+    int mib[6];
+    char *buf;
+
+    mib[0] = CTL_NET;
+    mib[1] = PF_ROUTE;
+    mib[2] = 0;
+    mib[3] = 0;
+    mib[4] = NET_RT_DUMP2;
+    mib[5] = 0;
+
+    if (sysctl(mib, 6, NULL, &needed, NULL, 0) < 0)
+        return 1; // route dump size estimation failed
+    if ((buf = malloc(needed)) == 0)
+        return 2; // malloc failed
+    if (sysctl(mib, 6, buf, &needed, NULL, 0) < 0) {
+        free(buf);
+        return 3; // route dump failed
+    }
+
+    // Loop over all routes.
+    char *next, *lim;
+    lim = buf + needed;
+	struct rt_msghdr2 *rtm;
+    for (next = buf; next < lim; next += rtm->rtm_msglen) {
+        rtm = (struct rt_msghdr2 *)next;
+		int ip;
+        ip = privateGatewayIPFromRoute(rtm);
+        if (ip) {
+            free(buf);
+            return ip;
+        }
+    }
+    free(buf);
+    return 0; // no gateway found
+}
+*/
+import "C"
+
+import (
+	"encoding/binary"
+	"fmt"
+	"os"
+
+	"inet.af/netaddr"
+)
+
+func init() {
+	likelyHomeRouterIP = likelyHomeRouterIPDarwinSyscall
+}
+
+func likelyHomeRouterIPDarwinSyscall() (ret netaddr.IP, ok bool) {
+	ip := C.privateGatewayIP()
+	fmt.Fprintln(os.Stderr, "likelyHomeRouterIPDarwinSyscall", ip)
+	if ip < 255 {
+		return netaddr.IP{}, false
+	}
+	var q [4]byte
+	binary.LittleEndian.PutUint32(q[:], uint32(ip))
+	return netaddr.IPv4(q[0], q[1], q[2], q[3]), true
+}

net/interfaces/interfaces_darwin_cgo_test.go

@@ -0,0 +1,20 @@
+// Copyright (c) 2020 Tailscale Inc & AUTHORS All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+// +build cgo,darwin
+
+package interfaces
+
+import "testing"
+
+func TestLikelyHomeRouterIPSyscallExec(t *testing.T) {
+	syscallIP, syscallOK := likelyHomeRouterIPDarwinSyscall()
+	netstatIP, netstatOK := likelyHomeRouterIPDarwinExec()
+	if syscallOK != netstatOK || syscallIP != netstatIP {
+		t.Errorf("syscall() = %v, %v, netstat = %v, %v",
+			syscallIP, syscallOK,
+			netstatIP, netstatOK,
+		)
+	}
+}

net/interfaces/interfaces_darwin_nocgo.go

@@ -0,0 +1,11 @@
+// Copyright (c) 2020 Tailscale Inc & AUTHORS All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+// +build darwin,!cgo
+
+package interfaces
+
+func init() {
+	likelyHomeRouterIP = likelyHomeRouterIPDarwinExec
+}

net/interfaces/interfaces_linux.go

@@ -0,0 +1,210 @@
+// Copyright (c) 2020 Tailscale Inc & AUTHORS All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package interfaces
+
+import (
+	"bufio"
+	"bytes"
+	"errors"
+	"io"
+	"log"
+	"os"
+	"os/exec"
+	"runtime"
+	"strings"
+
+	"go4.org/mem"
+	"inet.af/netaddr"
+	"tailscale.com/syncs"
+	"tailscale.com/util/lineread"
+)
+
+func init() {
+	likelyHomeRouterIP = likelyHomeRouterIPLinux
+}
+
+var procNetRouteErr syncs.AtomicBool
+
+/*
+Parse 10.0.0.1 out of:
+
+$ cat /proc/net/route
+Iface   Destination     Gateway         Flags   RefCnt  Use     Metric  Mask            MTU     Window  IRTT
+ens18   00000000        0100000A        0003    0       0       0       00000000        0       0       0
+ens18   0000000A        00000000        0001    0       0       0       0000FFFF        0       0       0
+*/
+func likelyHomeRouterIPLinux() (ret netaddr.IP, ok bool) {
+	if procNetRouteErr.Get() {
+		// If we failed to read /proc/net/route previously, don't keep trying.
+		// But if we're on Android, go into the Android path.
+		if runtime.GOOS == "android" {
+			return likelyHomeRouterIPAndroid()
+		}
+		return ret, false
+	}
+	lineNum := 0
+	var f []mem.RO
+	err := lineread.File("/proc/net/route", func(line []byte) error {
+		lineNum++
+		if lineNum == 1 {
+			// Skip header line.
+			return nil
+		}
+		f = mem.AppendFields(f[:0], mem.B(line))
+		if len(f) < 4 {
+			return nil
+		}
+		gwHex, flagsHex := f[2], f[3]
+		flags, err := mem.ParseUint(flagsHex, 16, 16)
+		if err != nil {
+			return nil // ignore error, skip line and keep going
+		}
+		const RTF_UP = 0x0001
+		const RTF_GATEWAY = 0x0002
+		if flags&(RTF_UP|RTF_GATEWAY) != RTF_UP|RTF_GATEWAY {
+			return nil
+		}
+		ipu32, err := mem.ParseUint(gwHex, 16, 32)
+		if err != nil {
+			return nil // ignore error, skip line and keep going
+		}
+		ip := netaddr.IPv4(byte(ipu32), byte(ipu32>>8), byte(ipu32>>16), byte(ipu32>>24))
+		if isPrivateIP(ip) {
+			ret = ip
+		}
+		return nil
+	})
+	if err != nil {
+		procNetRouteErr.Set(true)
+		if runtime.GOOS == "android" {
+			return likelyHomeRouterIPAndroid()
+		}
+		log.Printf("interfaces: failed to read /proc/net/route: %v", err)
+	}
+	return ret, !ret.IsZero()
+}
+
+// Android apps don't have permission to read /proc/net/route, at
+// least on Google devices and the Android emulator.
+func likelyHomeRouterIPAndroid() (ret netaddr.IP, ok bool) {
+	cmd := exec.Command("/system/bin/ip", "route", "show", "table", "0")
+	out, err := cmd.StdoutPipe()
+	if err != nil {
+		return
+	}
+	if err := cmd.Start(); err != nil {
+		log.Printf("interfaces: running /system/bin/ip: %v", err)
+		return
+	}
+	// Search for line like "default via 10.0.2.2 dev radio0 table 1016 proto static mtu 1500 "
+	lineread.Reader(out, func(line []byte) error {
+		const pfx = "default via "
+		if !mem.HasPrefix(mem.B(line), mem.S(pfx)) {
+			return nil
+		}
+		line = line[len(pfx):]
+		sp := bytes.IndexByte(line, ' ')
+		if sp == -1 {
+			return nil
+		}
+		ipb := line[:sp]
+		if ip, err := netaddr.ParseIP(string(ipb)); err == nil && ip.Is4() {
+			ret = ip
+			log.Printf("interfaces: found Android default route %v", ip)
+		}
+		return nil
+	})
+	cmd.Process.Kill()
+	cmd.Wait()
+	return ret, !ret.IsZero()
+}
+
+// DefaultRouteInterface returns the name of the network interface that owns
+// the default route, not including any tailscale interfaces.
+func DefaultRouteInterface() (string, error) {
+	v, err := defaultRouteInterfaceProcNet()
+	if err == nil {
+		return v, nil
+	}
+	if runtime.GOOS == "android" {
+		return defaultRouteInterfaceAndroidIPRoute()
+	}
+	return v, err
+}
+
+var zeroRouteBytes = []byte("00000000")
+
+func defaultRouteInterfaceProcNet() (string, error) {
+	f, err := os.Open("/proc/net/route")
+	if err != nil {
+		return "", err
+	}
+	defer f.Close()
+	br := bufio.NewReaderSize(f, 128)
+	for {
+		line, err := br.ReadSlice('\n')
+		if err == io.EOF {
+			break
+		}
+		if err != nil {
+			return "", err
+		}
+		if !bytes.Contains(line, zeroRouteBytes) {
+			continue
+		}
+		fields := strings.Fields(string(line))
+		ifc := fields[0]
+		ip := fields[1]
+		netmask := fields[7]
+
+		if strings.HasPrefix(ifc, "tailscale") ||
+			strings.HasPrefix(ifc, "wg") {
+			continue
+		}
+		if ip == "00000000" && netmask == "00000000" {
+			// default route
+			return ifc, nil // interface name
+		}
+	}
+
+	return "", errors.New("no default routes found")
+
+}
+
+// defaultRouteInterfaceAndroidIPRoute tries to find the machine's default route interface name
+// by parsing the "ip route" command output. We use this on Android where /proc/net/route
+// can be missing entries or have locked-down permissions.
+// See also comments in https://github.com/tailscale/tailscale/pull/666.
+func defaultRouteInterfaceAndroidIPRoute() (ifname string, err error) {
+	cmd := exec.Command("/system/bin/ip", "route", "show", "table", "0")
+	out, err := cmd.StdoutPipe()
+	if err != nil {
+		return "", err
+	}
+	if err := cmd.Start(); err != nil {
+		log.Printf("interfaces: running /system/bin/ip: %v", err)
+		return "", err
+	}
+	// Search for line like "default via 10.0.2.2 dev radio0 table 1016 proto static mtu 1500 "
+	lineread.Reader(out, func(line []byte) error {
+		const pfx = "default via "
+		if !mem.HasPrefix(mem.B(line), mem.S(pfx)) {
+			return nil
+		}
+		ff := strings.Fields(string(line))
+		for i, v := range ff {
+			if i > 0 && ff[i-1] == "dev" && ifname == "" {
+				ifname = v
+			}
+		}
+		return nil
+	})
+	cmd.Process.Kill()
+	cmd.Wait()
+	if ifname == "" {
+		return "", errors.New("no default routes found")
+	}
+	return ifname, nil
+}

net/interfaces/interfaces_linux_test.go

@@ -0,0 +1,16 @@
+// Copyright (c) 2020 Tailscale Inc & AUTHORS All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package interfaces
+
+import "testing"
+
+func BenchmarkDefaultRouteInterface(b *testing.B) {
+	b.ReportAllocs()
+	for i := 0; i < b.N; i++ {
+		if _, err := DefaultRouteInterface(); err != nil {
+			b.Fatal(err)
+		}
+	}
+}

net/interfaces/interfaces_test.go

@@ -47,3 +47,12 @@ func TestGetState(t *testing.T) {
 		t.Fatal("two States back-to-back were not equal")
 	}
 }
+
+func TestLikelyHomeRouterIP(t *testing.T) {
+	gw, my, ok := LikelyHomeRouterIP()
+	if !ok {
+		t.Logf("no result")
+		return
+	}
+	t.Logf("myIP = %v; gw = %v", my, gw)
+}

net/interfaces/interfaces_windows.go

@@ -0,0 +1,73 @@
+// Copyright (c) 2020 Tailscale Inc & AUTHORS All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package interfaces
+
+import (
+	"os/exec"
+	"syscall"
+
+	"go4.org/mem"
+	"inet.af/netaddr"
+	"tailscale.com/util/lineread"
+)
+
+func init() {
+	likelyHomeRouterIP = likelyHomeRouterIPWindows
+}
+
+/*
+Parse out 10.0.0.1 from:
+
+Z:\>route print -4
+===========================================================================
+Interface List
+ 15...aa 15 48 ff 1c 72 ......Red Hat VirtIO Ethernet Adapter
+  5...........................Tailscale Tunnel
+  1...........................Software Loopback Interface 1
+===========================================================================
+
+IPv4 Route Table
+===========================================================================
+Active Routes:
+Network Destination        Netmask          Gateway       Interface  Metric
+          0.0.0.0          0.0.0.0         10.0.0.1       10.0.28.63      5
+         10.0.0.0      255.255.0.0         On-link        10.0.28.63    261
+       10.0.28.63  255.255.255.255         On-link        10.0.28.63    261
+        10.0.42.0    255.255.255.0   100.103.42.106   100.103.42.106      5
+     10.0.255.255  255.255.255.255         On-link        10.0.28.63    261
+   34.193.248.174  255.255.255.255   100.103.42.106   100.103.42.106      5
+
+*/
+func likelyHomeRouterIPWindows() (ret netaddr.IP, ok bool) {
+	cmd := exec.Command("route", "print", "-4")
+	cmd.SysProcAttr = &syscall.SysProcAttr{HideWindow: true}
+	stdout, err := cmd.StdoutPipe()
+	if err != nil {
+		return
+	}
+	if err := cmd.Start(); err != nil {
+		return
+	}
+	defer cmd.Wait()
+
+	var f []mem.RO
+	lineread.Reader(stdout, func(lineb []byte) error {
+		line := mem.B(lineb)
+		if !mem.Contains(line, mem.S("0.0.0.0")) {
+			return nil
+		}
+		f = mem.AppendFields(f[:0], line)
+		if len(f) < 3 || !f[0].EqualString("0.0.0.0") || !f[1].EqualString("0.0.0.0") {
+			return nil
+		}
+		ipm := f[2]
+		ip, err := netaddr.ParseIP(string(mem.Append(nil, ipm)))
+		if err == nil && isPrivateIP(ip) {
+			ret = ip
+		}
+		return nil
+	})
+	return ret, !ret.IsZero()
+}

net/interfaces/route.h

@@ -0,0 +1,257 @@
+/*
+ * Copyright (c) 2000-2017 Apple Inc. All rights reserved.
+ *
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
+ *
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
+ *
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ *
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
+ * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
+ *
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
+ */
+/*
+ * Copyright (c) 1980, 1986, 1993
+ *	The Regents of the University of California.  All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ * 3. All advertising materials mentioning features or use of this software
+ *    must display the following acknowledgement:
+ *	This product includes software developed by the University of
+ *	California, Berkeley and its contributors.
+ * 4. Neither the name of the University nor the names of its contributors
+ *    may be used to endorse or promote products derived from this software
+ *    without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED.  IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ *
+ *	@(#)route.h	8.3 (Berkeley) 4/19/94
+ * $FreeBSD: src/sys/net/route.h,v 1.36.2.1 2000/08/16 06:14:23 jayanth Exp $
+ */
+
+#ifndef _NET_ROUTE_H_
+#define _NET_ROUTE_H_
+#include <sys/appleapiopts.h>
+#include <stdint.h>
+#include <sys/types.h>
+#include <sys/socket.h>
+
+/*
+ * These numbers are used by reliable protocols for determining
+ * retransmission behavior and are included in the routing structure.
+ */
+struct rt_metrics {
+	u_int32_t       rmx_locks;      /* Kernel leaves these values alone */
+	u_int32_t       rmx_mtu;        /* MTU for this path */
+	u_int32_t       rmx_hopcount;   /* max hops expected */
+	int32_t         rmx_expire;     /* lifetime for route, e.g. redirect */
+	u_int32_t       rmx_recvpipe;   /* inbound delay-bandwidth product */
+	u_int32_t       rmx_sendpipe;   /* outbound delay-bandwidth product */
+	u_int32_t       rmx_ssthresh;   /* outbound gateway buffer limit */
+	u_int32_t       rmx_rtt;        /* estimated round trip time */
+	u_int32_t       rmx_rttvar;     /* estimated rtt variance */
+	u_int32_t       rmx_pksent;     /* packets sent using this route */
+	u_int32_t       rmx_state;      /* route state */
+	u_int32_t       rmx_filler[3];  /* will be used for T/TCP later */
+};
+
+/*
+ * rmx_rtt and rmx_rttvar are stored as microseconds;
+ */
+#define RTM_RTTUNIT     1000000 /* units for rtt, rttvar, as units per sec */
+
+
+
+#define RTF_UP          0x1             /* route usable */
+#define RTF_GATEWAY     0x2             /* destination is a gateway */
+#define RTF_HOST        0x4             /* host entry (net otherwise) */
+#define RTF_REJECT      0x8             /* host or net unreachable */
+#define RTF_DYNAMIC     0x10            /* created dynamically (by redirect) */
+#define RTF_MODIFIED    0x20            /* modified dynamically (by redirect) */
+#define RTF_DONE        0x40            /* message confirmed */
+#define RTF_DELCLONE    0x80            /* delete cloned route */
+#define RTF_CLONING     0x100           /* generate new routes on use */
+#define RTF_XRESOLVE    0x200           /* external daemon resolves name */
+#define RTF_LLINFO      0x400           /* DEPRECATED - exists ONLY for backward
+	                                 *  compatibility */
+#define RTF_LLDATA      0x400           /* used by apps to add/del L2 entries */
+#define RTF_STATIC      0x800           /* manually added */
+#define RTF_BLACKHOLE   0x1000          /* just discard pkts (during updates) */
+#define RTF_NOIFREF     0x2000          /* not eligible for RTF_IFREF */
+#define RTF_PROTO2      0x4000          /* protocol specific routing flag */
+#define RTF_PROTO1      0x8000          /* protocol specific routing flag */
+
+#define RTF_PRCLONING   0x10000         /* protocol requires cloning */
+#define RTF_WASCLONED   0x20000         /* route generated through cloning */
+#define RTF_PROTO3      0x40000         /* protocol specific routing flag */
+                                        /* 0x80000 unused */
+#define RTF_PINNED      0x100000        /* future use */
+#define RTF_LOCAL       0x200000        /* route represents a local address */
+#define RTF_BROADCAST   0x400000        /* route represents a bcast address */
+#define RTF_MULTICAST   0x800000        /* route represents a mcast address */
+#define RTF_IFSCOPE     0x1000000       /* has valid interface scope */
+#define RTF_CONDEMNED   0x2000000       /* defunct; no longer modifiable */
+#define RTF_IFREF       0x4000000       /* route holds a ref to interface */
+#define RTF_PROXY       0x8000000       /* proxying, no interface scope */
+#define RTF_ROUTER      0x10000000      /* host is a router */
+#define RTF_DEAD        0x20000000      /* Route entry is being freed */
+                                        /* 0x40000000 and up unassigned */
+
+#define RTPRF_OURS      RTF_PROTO3      /* set on routes we manage */
+#define RTF_BITS \
+	"\020\1UP\2GATEWAY\3HOST\4REJECT\5DYNAMIC\6MODIFIED\7DONE" \
+	"\10DELCLONE\11CLONING\12XRESOLVE\13LLINFO\14STATIC\15BLACKHOLE" \
+	"\16NOIFREF\17PROTO2\20PROTO1\21PRCLONING\22WASCLONED\23PROTO3" \
+	"\25PINNED\26LOCAL\27BROADCAST\30MULTICAST\31IFSCOPE\32CONDEMNED" \
+	"\33IFREF\34PROXY\35ROUTER"
+
+#define IS_DIRECT_HOSTROUTE(rt) \
+	(((rt)->rt_flags & (RTF_HOST | RTF_GATEWAY)) == RTF_HOST)
+/*
+ * Routing statistics.
+ */
+struct  rtstat {
+	short   rts_badredirect;        /* bogus redirect calls */
+	short   rts_dynamic;            /* routes created by redirects */
+	short   rts_newgateway;         /* routes modified by redirects */
+	short   rts_unreach;            /* lookups which failed */
+	short   rts_wildcard;           /* lookups satisfied by a wildcard */
+	short   rts_badrtgwroute;       /* route to gateway is not direct */
+};
+
+/*
+ * Structures for routing messages.
+ */
+struct rt_msghdr {
+	u_short rtm_msglen;     /* to skip over non-understood messages */
+	u_char  rtm_version;    /* future binary compatibility */
+	u_char  rtm_type;       /* message type */
+	u_short rtm_index;      /* index for associated ifp */
+	int     rtm_flags;      /* flags, incl. kern & message, e.g. DONE */
+	int     rtm_addrs;      /* bitmask identifying sockaddrs in msg */
+	pid_t   rtm_pid;        /* identify sender */
+	int     rtm_seq;        /* for sender to identify action */
+	int     rtm_errno;      /* why failed */
+	int     rtm_use;        /* from rtentry */
+	u_int32_t rtm_inits;    /* which metrics we are initializing */
+	struct rt_metrics rtm_rmx; /* metrics themselves */
+};
+
+struct rt_msghdr2 {
+	u_short rtm_msglen;     /* to skip over non-understood messages */
+	u_char  rtm_version;    /* future binary compatibility */
+	u_char  rtm_type;       /* message type */
+	u_short rtm_index;      /* index for associated ifp */
+	int     rtm_flags;      /* flags, incl. kern & message, e.g. DONE */
+	int     rtm_addrs;      /* bitmask identifying sockaddrs in msg */
+	int32_t rtm_refcnt;     /* reference count */
+	int     rtm_parentflags; /* flags of the parent route */
+	int     rtm_reserved;   /* reserved field set to 0 */
+	int     rtm_use;        /* from rtentry */
+	u_int32_t rtm_inits;    /* which metrics we are initializing */
+	struct rt_metrics rtm_rmx; /* metrics themselves */
+};
+
+
+#define RTM_VERSION     5       /* Up the ante and ignore older versions */
+
+/*
+ * Message types.
+ */
+#define RTM_ADD         0x1     /* Add Route */
+#define RTM_DELETE      0x2     /* Delete Route */
+#define RTM_CHANGE      0x3     /* Change Metrics or flags */
+#define RTM_GET         0x4     /* Report Metrics */
+#define RTM_LOSING      0x5     /* RTM_LOSING is no longer generated by xnu
+	                         *  and is deprecated */
+#define RTM_REDIRECT    0x6     /* Told to use different route */
+#define RTM_MISS        0x7     /* Lookup failed on this address */
+#define RTM_LOCK        0x8     /* fix specified metrics */
+#define RTM_OLDADD      0x9     /* caused by SIOCADDRT */
+#define RTM_OLDDEL      0xa     /* caused by SIOCDELRT */
+#define RTM_RESOLVE     0xb     /* req to resolve dst to LL addr */
+#define RTM_NEWADDR     0xc     /* address being added to iface */
+#define RTM_DELADDR     0xd     /* address being removed from iface */
+#define RTM_IFINFO      0xe     /* iface going up/down etc. */
+#define RTM_NEWMADDR    0xf     /* mcast group membership being added to if */
+#define RTM_DELMADDR    0x10    /* mcast group membership being deleted */
+#define RTM_IFINFO2     0x12    /* */
+#define RTM_NEWMADDR2   0x13    /* */
+#define RTM_GET2        0x14    /* */
+
+/*
+ * Bitmask values for rtm_inits and rmx_locks.
+ */
+#define RTV_MTU         0x1     /* init or lock _mtu */
+#define RTV_HOPCOUNT    0x2     /* init or lock _hopcount */
+#define RTV_EXPIRE      0x4     /* init or lock _expire */
+#define RTV_RPIPE       0x8     /* init or lock _recvpipe */
+#define RTV_SPIPE       0x10    /* init or lock _sendpipe */
+#define RTV_SSTHRESH    0x20    /* init or lock _ssthresh */
+#define RTV_RTT         0x40    /* init or lock _rtt */
+#define RTV_RTTVAR      0x80    /* init or lock _rttvar */
+
+/*
+ * Bitmask values for rtm_addrs.
+ */
+#define RTA_DST         0x1     /* destination sockaddr present */
+#define RTA_GATEWAY     0x2     /* gateway sockaddr present */
+#define RTA_NETMASK     0x4     /* netmask sockaddr present */
+#define RTA_GENMASK     0x8     /* cloning mask sockaddr present */
+#define RTA_IFP         0x10    /* interface name sockaddr present */
+#define RTA_IFA         0x20    /* interface addr sockaddr present */
+#define RTA_AUTHOR      0x40    /* sockaddr for author of redirect */
+#define RTA_BRD         0x80    /* for NEWADDR, broadcast or p-p dest addr */
+
+/*
+ * Index offsets for sockaddr array for alternate internal encoding.
+ */
+#define RTAX_DST        0       /* destination sockaddr present */
+#define RTAX_GATEWAY    1       /* gateway sockaddr present */
+#define RTAX_NETMASK    2       /* netmask sockaddr present */
+#define RTAX_GENMASK    3       /* cloning mask sockaddr present */
+#define RTAX_IFP        4       /* interface name sockaddr present */
+#define RTAX_IFA        5       /* interface addr sockaddr present */
+#define RTAX_AUTHOR     6       /* sockaddr for author of redirect */
+#define RTAX_BRD        7       /* for NEWADDR, broadcast or p-p dest addr */
+#define RTAX_MAX        8       /* size of array to allocate */
+
+struct rt_addrinfo {
+	int     rti_addrs;
+	struct  sockaddr *rti_info[RTAX_MAX];
+};
+
+
+#endif /* _NET_ROUTE_H_ */

net/netcheck/netcheck.go

@@ -8,7 +8,9 @@ package netcheck
 import (
 	"bufio"
 	"context"
+	"crypto/rand"
 	"crypto/tls"
+	"encoding/binary"
 	"errors"
 	"fmt"
 	"io"
@@ -16,11 +18,14 @@ import (
 	"log"
 	"net"
 	"net/http"
+	"os"
 	"sort"
+	"strconv"
 	"sync"
 	"time"
 
 	"github.com/tcnksm/go-httpstat"
+	"go4.org/mem"
 	"inet.af/netaddr"
 	"tailscale.com/derp/derphttp"
 	"tailscale.com/net/dnscache"
@@ -33,16 +38,66 @@ import (
 	"tailscale.com/types/opt"
 )
 
+// Debugging and experimentation tweakables.
+var (
+	debugNetcheck, _ = strconv.ParseBool(os.Getenv("TS_DEBUG_NETCHECK"))
+)
+
+// The various default timeouts for things.
+const (
+	// overallProbeTimeout is the maximum amount of time netcheck will
+	// spend gathering a single report.
+	overallProbeTimeout = 5 * time.Second
+	// stunTimeout is the maximum amount of time netcheck will spend
+	// probing with STUN packets without getting a reply before
+	// switching to HTTP probing, on the assumption that outbound UDP
+	// is blocked.
+	stunProbeTimeout = 3 * time.Second
+	// hairpinCheckTimeout is the amount of time we wait for a
+	// hairpinned packet to come back.
+	hairpinCheckTimeout = 100 * time.Millisecond
+	// defaultActiveRetransmitTime is the retransmit interval we use
+	// for STUN probes when we're in steady state (not in start-up),
+	// but don't have previous latency information for a DERP
+	// node. This is a somewhat conservative guess because if we have
+	// no data, likely the DERP node is very far away and we have no
+	// data because we timed out the last time we probed it.
+	defaultActiveRetransmitTime = 200 * time.Millisecond
+	// defaultInitialRetransmitTime is the retransmit interval used
+	// when netcheck first runs. We have no past context to work with,
+	// and we want answers relatively quickly, so it's biased slightly
+	// more aggressive than defaultActiveRetransmitTime. A few extra
+	// packets at startup is fine.
+	defaultInitialRetransmitTime = 100 * time.Millisecond
+	// portMapServiceProbeTimeout is the time we wait for port mapping
+	// services (UPnP, NAT-PMP, PCP) to respond before we give up and
+	// decide that they're not there. Since these services are on the
+	// same LAN as this machine and a single L3 hop away, we don't
+	// give them much time to respond.
+	portMapServiceProbeTimeout = 100 * time.Millisecond
+)
+
 type Report struct {
-	UDP                   bool                  // UDP works
-	IPv6                  bool                  // IPv6 works
-	IPv4                  bool                  // IPv4 works
-	MappingVariesByDestIP opt.Bool              // for IPv4
-	HairPinning           opt.Bool              // for IPv4
-	PreferredDERP         int                   // or 0 for unknown
-	RegionLatency         map[int]time.Duration // keyed by DERP Region ID
-	RegionV4Latency       map[int]time.Duration // keyed by DERP Region ID
-	RegionV6Latency       map[int]time.Duration // keyed by DERP Region ID
+	UDP                   bool     // UDP works
+	IPv6                  bool     // IPv6 works
+	IPv4                  bool     // IPv4 works
+	MappingVariesByDestIP opt.Bool // for IPv4
+	HairPinning           opt.Bool // for IPv4
+
+	// UPnP is whether UPnP appears present on the LAN.
+	// Empty means not checked.
+	UPnP opt.Bool
+	// PMP is whether NAT-PMP appears present on the LAN.
+	// Empty means not checked.
+	PMP opt.Bool
+	// PCP is whether PCP appears present on the LAN.
+	// Empty means not checked.
+	PCP opt.Bool
+
+	PreferredDERP   int                   // or 0 for unknown
+	RegionLatency   map[int]time.Duration // keyed by DERP Region ID
+	RegionV4Latency map[int]time.Duration // keyed by DERP Region ID
+	RegionV6Latency map[int]time.Duration // keyed by DERP Region ID
 
 	GlobalV4 string // ip:port of global IPv4
 	GlobalV6 string // [ip]:port of global IPv6
@@ -50,6 +105,11 @@ type Report struct {
 	// TODO: update Clone when adding new fields
 }
 
+// AnyPortMappingChecked reports whether any of UPnP, PMP, or PCP are non-empty.
+func (r *Report) AnyPortMappingChecked() bool {
+	return r.UPnP != "" || r.PMP != "" || r.PCP != ""
+}
+
 func (r *Report) Clone() *Report {
 	if r == nil {
 		return nil
@@ -120,7 +180,7 @@ func (c *Client) logf(format string, a ...interface{}) {
 }
 
 func (c *Client) vlogf(format string, a ...interface{}) {
-	if c.Verbose {
+	if c.Verbose || debugNetcheck {
 		c.logf(format, a...)
 	}
 }
@@ -151,6 +211,8 @@ func (c *Client) MakeNextReportFull() {
 }
 
 func (c *Client) ReceiveSTUNPacket(pkt []byte, src netaddr.IPPort) {
+	c.vlogf("received STUN packet from %s", src)
+
 	c.mu.Lock()
 	if c.handleHairSTUNLocked(pkt, src) {
 		c.mu.Unlock()
@@ -311,7 +373,7 @@ func makeProbePlan(dm *tailcfg.DERPMap, ifState *interfaces.State, last *Report)
 			n := reg.Nodes[try%len(reg.Nodes)]
 			prevLatency := last.RegionLatency[reg.RegionID] * 120 / 100
 			if prevLatency == 0 {
-				prevLatency = 200 * time.Millisecond
+				prevLatency = defaultActiveRetransmitTime
 			}
 			delay := time.Duration(try) * prevLatency
 			if do4 {
@@ -334,16 +396,12 @@ func makeProbePlan(dm *tailcfg.DERPMap, ifState *interfaces.State, last *Report)
 func makeProbePlanInitial(dm *tailcfg.DERPMap, ifState *interfaces.State) (plan probePlan) {
 	plan = make(probePlan)
 
-	// initialSTUNTimeout is only 100ms because some extra retransmits
-	// when starting up is tolerable.
-	const initialSTUNTimeout = 100 * time.Millisecond
-
 	for _, reg := range dm.Regions {
 		var p4 []probe
 		var p6 []probe
 		for try := 0; try < 3; try++ {
 			n := reg.Nodes[try%len(reg.Nodes)]
-			delay := time.Duration(try) * initialSTUNTimeout
+			delay := time.Duration(try) * defaultInitialRetransmitTime
 			if ifState.HaveV4 && nodeMight4(n) {
 				p4 = append(p4, probe{delay: delay, node: n.Name, proto: probeIPv4})
 			}
@@ -434,6 +492,7 @@ type reportState struct {
 	pc4Hair     net.PacketConn
 	incremental bool // doing a lite, follow-up netcheck
 	stopProbeCh chan struct{}
+	waitPortMap sync.WaitGroup
 
 	mu            sync.Mutex
 	sentHairCheck bool
@@ -498,7 +557,7 @@ func (rs *reportState) startHairCheckLocked(dst netaddr.IPPort) {
 	ua := dst.UDPAddr()
 	rs.pc4Hair.WriteTo(stun.Request(rs.hairTX), ua)
 	rs.c.vlogf("sent haircheck to %v", ua)
-	time.AfterFunc(500*time.Millisecond, func() { close(rs.hairTimeout) })
+	time.AfterFunc(hairpinCheckTimeout, func() { close(rs.hairTimeout) })
 }
 
 func (rs *reportState) waitHairCheck(ctx context.Context) {
@@ -519,6 +578,7 @@ func (rs *reportState) waitHairCheck(ctx context.Context) {
 	case <-rs.gotHairSTUN:
 		ret.HairPinning.Set(true)
 	case <-rs.hairTimeout:
+		rs.c.vlogf("hairCheck timeout")
 		ret.HairPinning.Set(false)
 	default:
 		select {
@@ -599,6 +659,102 @@ func (rs *reportState) stopProbes() {
 	}
 }
 
+func (rs *reportState) setOptBool(b *opt.Bool, v bool) {
+	rs.mu.Lock()
+	defer rs.mu.Unlock()
+	b.Set(v)
+}
+
+func (rs *reportState) probePortMapServices() {
+	defer rs.waitPortMap.Done()
+	gw, myIP, ok := interfaces.LikelyHomeRouterIP()
+	if !ok {
+		return
+	}
+
+	rs.setOptBool(&rs.report.UPnP, false)
+	rs.setOptBool(&rs.report.PMP, false)
+	rs.setOptBool(&rs.report.PCP, false)
+
+	port1900 := netaddr.IPPort{IP: gw, Port: 1900}.UDPAddr()
+	port5351 := netaddr.IPPort{IP: gw, Port: 5351}.UDPAddr()
+
+	rs.c.logf("probePortMapServices: me %v -> gw %v", myIP, gw)
+
+	// Create a UDP4 socket used just for querying for UPnP, NAT-PMP, and PCP.
+	uc, err := netns.Listener().ListenPacket(context.Background(), "udp4", ":0")
+	if err != nil {
+		rs.c.logf("probePortMapServices: %v", err)
+		return
+	}
+	defer uc.Close()
+	tempPort := uc.LocalAddr().(*net.UDPAddr).Port
+	uc.SetReadDeadline(time.Now().Add(portMapServiceProbeTimeout))
+
+	// Send request packets for all three protocols.
+	uc.WriteTo(uPnPPacket, port1900)
+	uc.WriteTo(pmpPacket, port5351)
+	uc.WriteTo(pcpPacket(myIP, tempPort, false), port5351)
+
+	res := make([]byte, 1500)
+	for {
+		n, addr, err := uc.ReadFrom(res)
+		if err != nil {
+			return
+		}
+		switch addr.(*net.UDPAddr).Port {
+		case 1900:
+			if mem.Contains(mem.B(res[:n]), mem.S(":InternetGatewayDevice:")) {
+				rs.setOptBool(&rs.report.UPnP, true)
+			}
+		case 5351:
+			if n == 12 && res[0] == 0x00 { // right length and version 0
+				rs.setOptBool(&rs.report.PMP, true)
+			}
+			if n == 60 && res[0] == 0x02 { // right length and version 2
+				rs.setOptBool(&rs.report.PCP, true)
+
+				// And now delete the mapping.
+				// (PCP is the only protocol of the three that requires
+				// we cause a side effect to detect whether it's present,
+				// so we need to redo that side effect now.)
+				uc.WriteTo(pcpPacket(myIP, tempPort, true), port5351)
+			}
+		}
+	}
+}
+
+var pmpPacket = []byte{0, 0} // version 0, opcode 0 = "Public address request"
+
+var uPnPPacket = []byte("M-SEARCH * HTTP/1.1\r\n" +
+	"HOST: 239.255.255.250:1900\r\n" +
+	"ST: ssdp:all\r\n" +
+	"MAN: \"ssdp:discover\"\r\n" +
+	"MX: 2\r\n\r\n")
+
+var v4unspec, _ = netaddr.ParseIP("0.0.0.0")
+
+func pcpPacket(myIP netaddr.IP, mapToLocalPort int, delete bool) []byte {
+	const udpProtoNumber = 17
+	lifetimeSeconds := uint32(1)
+	if delete {
+		lifetimeSeconds = 0
+	}
+	const opMap = 1
+	pkt := make([]byte, (32+32+128)/8+(96+8+24+16+16+128)/8)
+	pkt[0] = 2 // version
+	pkt[1] = opMap
+	binary.BigEndian.PutUint32(pkt[4:8], lifetimeSeconds)
+	myIP16 := myIP.As16()
+	copy(pkt[8:], myIP16[:])
+	rand.Read(pkt[24 : 24+12])
+	pkt[36] = udpProtoNumber
+	binary.BigEndian.PutUint16(pkt[40:], uint16(mapToLocalPort))
+	v4unspec16 := v4unspec.As16()
+	copy(pkt[40:], v4unspec16[:])
+	return pkt
+}
+
 func newReport() *Report {
 	return &Report{
 		RegionLatency:   make(map[int]time.Duration),
@@ -611,15 +767,10 @@ func newReport() *Report {
 //
 // It may not be called concurrently with itself.
 func (c *Client) GetReport(ctx context.Context, dm *tailcfg.DERPMap) (*Report, error) {
-	// Wait for STUN for 3 seconds, but then give HTTP probing
-	// another 2 seconds if all UDP failed.
-	const overallTimeout = 5 * time.Second
-	const stunTimeout = 3 * time.Second
-
 	// Mask user context with ours that we guarantee to cancel so
 	// we can depend on it being closed in goroutines later.
 	// (User ctx might be context.Background, etc)
-	ctx, cancel := context.WithTimeout(ctx, overallTimeout)
+	ctx, cancel := context.WithTimeout(ctx, overallProbeTimeout)
 	defer cancel()
 
 	if dm == nil {
@@ -671,6 +822,22 @@ func (c *Client) GetReport(ctx context.Context, dm *tailcfg.DERPMap) (*Report, e
 	}
 	defer rs.pc4Hair.Close()
 
+	rs.waitPortMap.Add(1)
+	go rs.probePortMapServices()
+
+	// At least the Apple Airport Extreme doesn't allow hairpin
+	// sends from a private socket until it's seen traffic from
+	// that src IP:port to something else out on the internet.
+	//
+	// See https://github.com/tailscale/tailscale/issues/188#issuecomment-600728643
+	//
+	// And it seems that even sending to a likely-filtered RFC 5737
+	// documentation-only IPv4 range is enough to set up the mapping.
+	// So do that for now. In the future we might want to classify networks
+	// that do and don't require this separately. But for now help it.
+	const documentationIP = "203.0.113.1"
+	rs.pc4Hair.WriteTo([]byte("tailscale netcheck; see https://github.com/tailscale/tailscale/issues/188"), &net.UDPAddr{IP: net.ParseIP(documentationIP), Port: 12345})
+
 	if f := c.GetSTUNConn4; f != nil {
 		rs.pc4 = f()
 	} else {
@@ -712,7 +879,7 @@ func (c *Client) GetReport(ctx context.Context, dm *tailcfg.DERPMap) (*Report, e
 		}(probeSet)
 	}
 
-	stunTimer := time.NewTimer(stunTimeout)
+	stunTimer := time.NewTimer(stunProbeTimeout)
 	defer stunTimer.Stop()
 
 	select {
@@ -725,6 +892,9 @@ func (c *Client) GetReport(ctx context.Context, dm *tailcfg.DERPMap) (*Report, e
 	}
 
 	rs.waitHairCheck(ctx)
+	c.vlogf("hairCheck done")
+	rs.waitPortMap.Wait()
+	c.vlogf("portMap done")
 	rs.stopTimers()
 
 	// Try HTTPS latency check if all STUN probes failed due to UDP presumably being blocked.
@@ -779,7 +949,7 @@ func (c *Client) GetReport(ctx context.Context, dm *tailcfg.DERPMap) (*Report, e
 
 func (c *Client) measureHTTPSLatency(ctx context.Context, reg *tailcfg.DERPRegion) (time.Duration, netaddr.IP, error) {
 	var result httpstat.Result
-	ctx, cancel := context.WithTimeout(httpstat.WithHTTPStat(ctx, &result), 5*time.Second)
+	ctx, cancel := context.WithTimeout(httpstat.WithHTTPStat(ctx, &result), overallProbeTimeout)
 	defer cancel()
 
 	var ip netaddr.IP
@@ -848,6 +1018,11 @@ func (c *Client) logConciseReport(r *Report, dm *tailcfg.DERPMap) {
 		fmt.Fprintf(w, " v6=%v", r.IPv6)
 		fmt.Fprintf(w, " mapvarydest=%v", r.MappingVariesByDestIP)
 		fmt.Fprintf(w, " hair=%v", r.HairPinning)
+		if r.AnyPortMappingChecked() {
+			fmt.Fprintf(w, " portmap=%v%v%v", conciseOptBool(r.UPnP, "U"), conciseOptBool(r.PMP, "M"), conciseOptBool(r.PCP, "C"))
+		} else {
+			fmt.Fprintf(w, " portmap=?")
+		}
 		if r.GlobalV4 != "" {
 			fmt.Fprintf(w, " v4a=%v", r.GlobalV4)
 		}
@@ -1008,6 +1183,20 @@ func (c *Client) nodeAddr(ctx context.Context, n *tailcfg.DERPNode, proto probeP
 	if port < 0 || port > 1<<16-1 {
 		return nil
 	}
+	if n.STUNTestIP != "" {
+		ip, err := netaddr.ParseIP(n.STUNTestIP)
+		if err != nil {
+			return nil
+		}
+		if proto == probeIPv4 && ip.Is6() {
+			return nil
+		}
+		if proto == probeIPv6 && ip.Is4() {
+			return nil
+		}
+		return netaddr.IPPort{IP: ip, Port: uint16(port)}.UDPAddr()
+	}
+
 	switch proto {
 	case probeIPv4:
 		if n.IPv4 != "" {
@@ -1015,7 +1204,7 @@ func (c *Client) nodeAddr(ctx context.Context, n *tailcfg.DERPNode, proto probeP
 			if !ip.Is4() {
 				return nil
 			}
-			return netaddr.IPPort{ip, uint16(port)}.UDPAddr()
+			return netaddr.IPPort{IP: ip, Port: uint16(port)}.UDPAddr()
 		}
 	case probeIPv6:
 		if n.IPv6 != "" {
@@ -1023,7 +1212,7 @@ func (c *Client) nodeAddr(ctx context.Context, n *tailcfg.DERPNode, proto probeP
 			if !ip.Is6() {
 				return nil
 			}
-			return netaddr.IPPort{ip, uint16(port)}.UDPAddr()
+			return netaddr.IPPort{IP: ip, Port: uint16(port)}.UDPAddr()
 		}
 	default:
 		return nil
@@ -1056,3 +1245,17 @@ func maxDurationValue(m map[int]time.Duration) (max time.Duration) {
 	}
 	return max
 }
+
+func conciseOptBool(b opt.Bool, trueVal string) string {
+	if b == "" {
+		return "_"
+	}
+	v, ok := b.Get()
+	if !ok {
+		return "x"
+	}
+	if v {
+		return trueVal
+	}
+	return ""
+}

net/netcheck/netcheck_test.go

@@ -100,6 +100,9 @@ func TestWorksWhenUDPBlocked(t *testing.T) {
 		t.Fatal(err)
 	}
 	want := newReport()
+	r.UPnP = ""
+	r.PMP = ""
+	r.PCP = ""
 
 	if !reflect.DeepEqual(r, want) {
 		t.Errorf("mismatch\n got: %+v\nwant: %+v\n", r, want)
@@ -463,7 +466,7 @@ func TestLogConciseReport(t *testing.T) {
 		{
 			name: "no_udp",
 			r:    &Report{},
-			want: "udp=false v4=false v6=false mapvarydest= hair= derp=0",
+			want: "udp=false v4=false v6=false mapvarydest= hair= portmap=? derp=0",
 		},
 		{
 			name: "ipv4_one_region",
@@ -478,7 +481,7 @@ func TestLogConciseReport(t *testing.T) {
 					1: 10 * ms,
 				},
 			},
-			want: "udp=true v6=false mapvarydest= hair= derp=1 derpdist=1v4:10ms",
+			want: "udp=true v6=false mapvarydest= hair= portmap=? derp=1 derpdist=1v4:10ms",
 		},
 		{
 			name: "ipv4_all_region",
@@ -497,7 +500,7 @@ func TestLogConciseReport(t *testing.T) {
 					3: 30 * ms,
 				},
 			},
-			want: "udp=true v6=false mapvarydest= hair= derp=1 derpdist=1v4:10ms,2v4:20ms,3v4:30ms",
+			want: "udp=true v6=false mapvarydest= hair= portmap=? derp=1 derpdist=1v4:10ms,2v4:20ms,3v4:30ms",
 		},
 		{
 			name: "ipboth_all_region",
@@ -522,7 +525,27 @@ func TestLogConciseReport(t *testing.T) {
 					3: 30 * ms,
 				},
 			},
-			want: "udp=true v6=true mapvarydest= hair= derp=1 derpdist=1v4:10ms,1v6:10ms,2v4:20ms,2v6:20ms,3v4:30ms,3v6:30ms",
+			want: "udp=true v6=true mapvarydest= hair= portmap=? derp=1 derpdist=1v4:10ms,1v6:10ms,2v4:20ms,2v6:20ms,3v4:30ms,3v6:30ms",
+		},
+		{
+			name: "portmap_all",
+			r: &Report{
+				UDP:  true,
+				UPnP: "true",
+				PMP:  "true",
+				PCP:  "true",
+			},
+			want: "udp=true v4=false v6=false mapvarydest= hair= portmap=UMC derp=0",
+		},
+		{
+			name: "portmap_some",
+			r: &Report{
+				UDP:  true,
+				UPnP: "true",
+				PMP:  "false",
+				PCP:  "true",
+			},
+			want: "udp=true v4=false v6=false mapvarydest= hair= portmap=UC derp=0",
 		},
 	}
 	for _, tt := range tests {

net/netns/netns_linux.go

@@ -5,19 +5,15 @@
 package netns
 
 import (
-	"bufio"
-	"bytes"
-	"errors"
 	"flag"
 	"fmt"
-	"io"
 	"os"
 	"os/exec"
-	"strings"
 	"sync"
 	"syscall"
 
 	"golang.org/x/sys/unix"
+	"tailscale.com/net/interfaces"
 )
 
 // tailscaleBypassMark is the mark indicating that packets originating
@@ -26,7 +22,7 @@ import (
 //
 // Keep this in sync with tailscaleBypassMark in
 // wgengine/router/router_linux.go.
-const tailscaleBypassMark = 0x20000
+const tailscaleBypassMark = 0x80000
 
 // ipRuleOnce is the sync.Once & cached value for ipRuleAvailable.
 var ipRuleOnce struct {
@@ -43,47 +39,6 @@ func ipRuleAvailable() bool {
 	return ipRuleOnce.v
 }
 
-var zeroRouteBytes = []byte("00000000")
-
-// defaultRouteInterface returns the name of the network interface that owns
-// the default route, not including any tailscale interfaces. We only use
-// this in SO_BINDTODEVICE mode.
-func defaultRouteInterface() (string, error) {
-	f, err := os.Open("/proc/net/route")
-	if err != nil {
-		return "", err
-	}
-	defer f.Close()
-	br := bufio.NewReaderSize(f, 128)
-	for {
-		line, err := br.ReadSlice('\n')
-		if err == io.EOF {
-			break
-		}
-		if err != nil {
-			return "", err
-		}
-		if !bytes.Contains(line, zeroRouteBytes) {
-			continue
-		}
-		fields := strings.Fields(string(line))
-		ifc := fields[0]
-		ip := fields[1]
-		netmask := fields[7]
-
-		if strings.HasPrefix(ifc, "tailscale") ||
-			strings.HasPrefix(ifc, "wg") {
-			continue
-		}
-		if ip == "00000000" && netmask == "00000000" {
-			// default route
-			return ifc, nil // interface name
-		}
-	}
-
-	return "", errors.New("no default routes found")
-}
-
 // ignoreErrors returns true if we should ignore setsocketopt errors in
 // this instance.
 func ignoreErrors() bool {
@@ -133,7 +88,7 @@ func setBypassMark(fd uintptr) error {
 }
 
 func bindToDevice(fd uintptr) error {
-	ifc, err := defaultRouteInterface()
+	ifc, err := interfaces.DefaultRouteInterface()
 	if err != nil {
 		// Make sure we bind to *some* interface,
 		// or we could get a routing loop.

net/netns/netns_linux_test.go

@@ -49,12 +49,3 @@ func TestBypassMarkInSync(t *testing.T) {
 	}
 	t.Errorf("tailscaleBypassMark not found in router_linux.go")
 }
-
-func BenchmarkDefaultRouteInterface(b *testing.B) {
-	b.ReportAllocs()
-	for i := 0; i < b.N; i++ {
-		if _, err := defaultRouteInterface(); err != nil {
-			b.Fatal(err)
-		}
-	}
-}

net/stun/stuntest/stuntest.go

@@ -6,6 +6,7 @@
 package stuntest
 
 import (
+	"context"
 	"fmt"
 	"net"
 	"strconv"
@@ -16,6 +17,7 @@ import (
 	"inet.af/netaddr"
 	"tailscale.com/net/stun"
 	"tailscale.com/tailcfg"
+	"tailscale.com/types/nettype"
 )
 
 type stunStats struct {
@@ -25,18 +27,22 @@ type stunStats struct {
 }
 
 func Serve(t *testing.T) (addr *net.UDPAddr, cleanupFn func()) {
+	return ServeWithPacketListener(t, nettype.Std{})
+}
+
+func ServeWithPacketListener(t *testing.T, ln nettype.PacketListener) (addr *net.UDPAddr, cleanupFn func()) {
 	t.Helper()
 
 	// TODO(crawshaw): use stats to test re-STUN logic
 	var stats stunStats
 
-	pc, err := net.ListenPacket("udp4", ":0")
+	pc, err := ln.ListenPacket(context.Background(), "udp4", ":0")
 	if err != nil {
 		t.Fatalf("failed to open STUN listener: %v", err)
 	}
-	addr = &net.UDPAddr{
-		IP:   net.ParseIP("127.0.0.1"),
-		Port: pc.LocalAddr().(*net.UDPAddr).Port,
+	addr = pc.LocalAddr().(*net.UDPAddr)
+	if len(addr.IP) == 0 || addr.IP.IsUnspecified() {
+		addr.IP = net.ParseIP("127.0.0.1")
 	}
 	doneCh := make(chan struct{})
 	go runSTUN(t, pc, &stats, doneCh)

net/tsaddr/tsaddr.go

@@ -0,0 +1,74 @@
+// Copyright (c) 2020 Tailscale Inc & AUTHORS All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+// Package tsaddr handles Tailscale-specific IPs and ranges.
+package tsaddr
+
+import (
+	"sync"
+
+	"inet.af/netaddr"
+)
+
+// ChromeOSVMRange returns the subset of the CGNAT IPv4 range used by
+// ChromeOS to interconnect the host OS to containers and VMs. We
+// avoid allocating Tailscale IPs from it, to avoid conflicts.
+func ChromeOSVMRange() netaddr.IPPrefix {
+	chromeOSRange.Do(func() { mustPrefix(&chromeOSRange.v, "100.115.92.0/23") })
+	return chromeOSRange.v
+}
+
+var chromeOSRange oncePrefix
+
+// CGNATRange returns the Carrier Grade NAT address range that
+// is the superset range that Tailscale assigns out of.
+// See https://tailscale.com/kb/1015/100.x-addresses.
+// Note that Tailscale does not assign out of the ChromeOSVMRange.
+func CGNATRange() netaddr.IPPrefix {
+	cgnatRange.Do(func() { mustPrefix(&cgnatRange.v, "100.64.0.0/10") })
+	return cgnatRange.v
+}
+
+var cgnatRange oncePrefix
+
+// TailscaleServiceIP returns the listen address of services
+// provided by Tailscale itself such as the Magic DNS proxy.
+func TailscaleServiceIP() netaddr.IP {
+	serviceIP.Do(func() { mustIP(&serviceIP.v, "100.100.100.100") })
+	return serviceIP.v
+}
+
+var serviceIP onceIP
+
+// IsTailscaleIP reports whether ip is an IP address in a range that
+// Tailscale assigns from.
+func IsTailscaleIP(ip netaddr.IP) bool {
+	return CGNATRange().Contains(ip) && !ChromeOSVMRange().Contains(ip)
+}
+
+func mustPrefix(v *netaddr.IPPrefix, prefix string) {
+	var err error
+	*v, err = netaddr.ParseIPPrefix(prefix)
+	if err != nil {
+		panic(err)
+	}
+}
+
+type oncePrefix struct {
+	sync.Once
+	v netaddr.IPPrefix
+}
+
+func mustIP(v *netaddr.IP, ip string) {
+	var err error
+	*v, err = netaddr.ParseIP(ip)
+	if err != nil {
+		panic(err)
+	}
+}
+
+type onceIP struct {
+	sync.Once
+	v netaddr.IP
+}

net/tsaddr/tsaddr_test.go

@@ -0,0 +1,19 @@
+// Copyright (c) 2020 Tailscale Inc & AUTHORS All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package tsaddr
+
+import "testing"
+
+func TestChromeOSVMRange(t *testing.T) {
+	if got, want := ChromeOSVMRange().String(), "100.115.92.0/23"; got != want {
+		t.Errorf("got %q; want %q", got, want)
+	}
+}
+
+func TestCGNATRange(t *testing.T) {
+	if got, want := CGNATRange().String(), "100.64.0.0/10"; got != want {
+		t.Errorf("got %q; want %q", got, want)
+	}
+}

paths/paths.go

@@ -11,9 +11,15 @@ import (
 	"runtime"
 )
 
-// LegacyConfigPath is the path used by the pre-tailscaled "relaynode"
-// daemon's config file.
-const LegacyConfigPath = "/var/lib/tailscale/relay.conf"
+// LegacyConfigPath returns the path used by the pre-tailscaled
+// "relaynode" daemon's config file. It returns the empty string for
+// platforms where relaynode never ran.
+func LegacyConfigPath() string {
+	if runtime.GOOS == "windows" {
+		return ""
+	}
+	return "/var/lib/tailscale/relay.conf"
+}
 
 // DefaultTailscaledSocket returns the path to the tailscaled Unix socket
 // or the empty string if there's no reasonable default.

portlist/portlist_linux.go

@@ -10,12 +10,15 @@ import (
 	"io"
 	"io/ioutil"
 	"os"
+	"runtime"
 	"sort"
 	"strconv"
 	"strings"
+	"syscall"
 	"time"
 
 	"golang.org/x/sys/unix"
+	"tailscale.com/syncs"
 )
 
 // Reading the sockfiles on Linux is very fast, so we can do it often.
@@ -26,13 +29,30 @@ const pollInterval = 1 * time.Second
 var sockfiles = []string{"/proc/net/tcp", "/proc/net/udp"}
 var protos = []string{"tcp", "udp"}
 
+var sawProcNetPermissionErr syncs.AtomicBool
+
 func listPorts() (List, error) {
+	if sawProcNetPermissionErr.Get() {
+		return nil, nil
+	}
 	l := []Port{}
 
 	for pi, fname := range sockfiles {
 		proto := protos[pi]
 
+		// Android 10+ doesn't allow access to this anymore.
+		// https://developer.android.com/about/versions/10/privacy/changes#proc-net-filesystem
+		// Ignore it rather than have the system log about our violation.
+		if runtime.GOOS == "android" && syscall.Access(fname, unix.R_OK) != nil {
+			sawProcNetPermissionErr.Set(true)
+			return nil, nil
+		}
+
 		f, err := os.Open(fname)
+		if os.IsPermission(err) {
+			sawProcNetPermissionErr.Set(true)
+			return nil, nil
+		}
 		if err != nil {
 			return nil, fmt.Errorf("%s: %s", fname, err)
 		}
@@ -96,7 +116,18 @@ func addProcesses(pl []Port) ([]Port, error) {
 	}
 
 	err := foreachPID(func(pid string) error {
-		fdDir, err := os.Open(fmt.Sprintf("/proc/%s/fd", pid))
+		fdPath := fmt.Sprintf("/proc/%s/fd", pid)
+
+		// Android logs a bunch of audit violations in logcat
+		// if we try to open things we don't have access
+		// to. So on Android only, ask if we have permission
+		// rather than just trying it to determine whether we
+		// have permission.
+		if runtime.GOOS == "android" && syscall.Access(fdPath, unix.R_OK) != nil {
+			return nil
+		}
+
+		fdDir, err := os.Open(fdPath)
 		if err != nil {
 			// Can't open fd list for this pid. Maybe
 			// don't have access. Ignore it.

safesocket/unixsocket.go

@@ -11,6 +11,7 @@ import (
 	"bytes"
 	"fmt"
 	"io"
+	"io/ioutil"
 	"log"
 	"net"
 	"os"
@@ -77,6 +78,16 @@ func listen(path string, port uint16) (ln net.Listener, _ uint16, err error) {
 //   * it also picks a random hex string that acts as an auth token
 //   * it then creates a file named "sameuserproof-$PORT-$TOKEN" and leaves
 //     that file descriptor open forever.
+//
+// Then, we do different things depending on whether the user is
+// running cmd/tailscale that they built themselves (running as
+// themselves, outside the App Sandbox), or whether the user is
+// running the CLI via the GUI binary
+// (e.g. /Applications/Tailscale.app/Contents/MacOS/Tailscale <args>),
+// in which case we're running within the App Sandbox.
+//
+// If we're outside the App Sandbox:
+//
 //   * then we come along here, running as the same UID, but outside
 //     of the sandbox, and look for it. We can run lsof on our own processes,
 //     but other users on the system can't.
@@ -86,7 +97,38 @@ func listen(path string, port uint16) (ln net.Listener, _ uint16, err error) {
 //   * server verifies $TOKEN, sends "#IPN\n" if okay.
 //   * server is now protocol switched
 //   * we return the net.Conn and the caller speaks the normal protocol
+//
+// If we're inside the App Sandbox, then TS_MACOS_CLI_SHARED_DIR has
+// been set to our shared directory. We now have to find the most
+// recent "sameuserproof" file (there should only be 1, but previous
+// versions of the macOS app didn't clean them up).
 func connectMacOSAppSandbox() (net.Conn, error) {
+	// Are we running the Tailscale.app GUI binary as a CLI, running within the App Sandbox?
+	if d := os.Getenv("TS_MACOS_CLI_SHARED_DIR"); d != "" {
+		fis, err := ioutil.ReadDir(d)
+		if err != nil {
+			return nil, fmt.Errorf("reading TS_MACOS_CLI_SHARED_DIR: %w", err)
+		}
+		var best os.FileInfo
+		for _, fi := range fis {
+			if !strings.HasPrefix(fi.Name(), "sameuserproof-") || strings.Count(fi.Name(), "-") != 2 {
+				continue
+			}
+			if best == nil || fi.ModTime().After(best.ModTime()) {
+				best = fi
+			}
+		}
+		if best == nil {
+			return nil, fmt.Errorf("no sameuserproof token found in TS_MACOS_CLI_SHARED_DIR %q", d)
+		}
+		f := strings.SplitN(best.Name(), "-", 3)
+		portStr, token := f[1], f[2]
+		return connectMacTCP(portStr, token)
+	}
+
+	// Otherwise, assume we're running the cmd/tailscale binary from outside the
+	// App Sandbox.
+
 	out, err := exec.Command("lsof",
 		"-n",                             // numeric sockets; don't do DNS lookups, etc
 		"-a",                             // logical AND remaining options
@@ -110,22 +152,26 @@ func connectMacOSAppSandbox() (net.Conn, error) {
 			continue
 		}
 		portStr, token := f[0], f[1]
-		c, err := net.Dial("tcp", "localhost:"+portStr)
-		if err != nil {
-			return nil, fmt.Errorf("error dialing IPNExtension: %w", err)
-		}
-		if _, err := io.WriteString(c, token+"\n"); err != nil {
-			return nil, fmt.Errorf("error writing auth token: %w", err)
-		}
-		buf := make([]byte, 5)
-		const authOK = "#IPN\n"
-		if _, err := io.ReadFull(c, buf); err != nil {
-			return nil, fmt.Errorf("error reading from IPNExtension post-auth: %w", err)
-		}
-		if string(buf) != authOK {
-			return nil, fmt.Errorf("invalid response reading from IPNExtension post-auth")
-		}
-		return c, nil
+		return connectMacTCP(portStr, token)
 	}
 	return nil, fmt.Errorf("failed to find Tailscale's IPNExtension process")
 }
+
+func connectMacTCP(portStr, token string) (net.Conn, error) {
+	c, err := net.Dial("tcp", "localhost:"+portStr)
+	if err != nil {
+		return nil, fmt.Errorf("error dialing IPNExtension: %w", err)
+	}
+	if _, err := io.WriteString(c, token+"\n"); err != nil {
+		return nil, fmt.Errorf("error writing auth token: %w", err)
+	}
+	buf := make([]byte, 5)
+	const authOK = "#IPN\n"
+	if _, err := io.ReadFull(c, buf); err != nil {
+		return nil, fmt.Errorf("error reading from IPNExtension post-auth: %w", err)
+	}
+	if string(buf) != authOK {
+		return nil, fmt.Errorf("invalid response reading from IPNExtension post-auth")
+	}
+	return c, nil
+}

tailcfg/derpmap.go

@@ -117,4 +117,8 @@ type DERPNode struct {
 	// of using the default port of 443. If non-zero, TLS
 	// verification is skipped.
 	DERPTestPort int `json:",omitempty"`
+
+	// STUNTestIP is used in tests to override the STUN server's IP.
+	// If empty, it's assumed to be the same as the DERP server.
+	STUNTestIP string `json:",omitempty"`
 }

tailcfg/tailcfg.go

@@ -4,6 +4,8 @@
 
 package tailcfg
 
+//go:generate go run tailscale.com/cmd/cloner -type=User,Node,Hostinfo,NetInfo -output=tailcfg_clone.go
+
 import (
 	"bytes"
 	"errors"
@@ -15,6 +17,7 @@ import (
 	"github.com/tailscale/wireguard-go/wgcfg"
 	"go4.org/mem"
 	"golang.org/x/oauth2"
+	"inet.af/netaddr"
 	"tailscale.com/types/key"
 	"tailscale.com/types/opt"
 	"tailscale.com/types/structs"
@@ -93,18 +96,6 @@ type User struct {
 	// Note: be sure to update Clone when adding new fields
 }
 
-// Clone returns a copy of u that aliases no memory with the original.
-func (u *User) Clone() *User {
-	if u == nil {
-		return nil
-	}
-	u2 := new(User)
-	*u2 = *u
-	u2.Logins = append([]LoginID(nil), u.Logins...)
-	u2.Roles = append([]RoleID(nil), u.Roles...)
-	return u2
-}
-
 type Login struct {
 	_             structs.Incomparable
 	ID            LoginID
@@ -150,23 +141,6 @@ type Node struct {
 	//       require changes to Node.Clone.
 }
 
-// Clone makes a deep copy of Node.
-// The result aliases no memory with the original.
-func (n *Node) Clone() (res *Node) {
-	res = new(Node)
-	*res = *n
-
-	res.Addresses = append([]wgcfg.CIDR{}, res.Addresses...)
-	res.AllowedIPs = append([]wgcfg.CIDR{}, res.AllowedIPs...)
-	res.Endpoints = append([]string{}, res.Endpoints...)
-	if res.LastSeen != nil {
-		lastSeen := *res.LastSeen
-		res.LastSeen = &lastSeen
-	}
-	res.Hostinfo = *res.Hostinfo.Clone()
-	return res
-}
-
 type MachineStatus int
 
 const (
@@ -284,7 +258,10 @@ type Hostinfo struct {
 	FrontendLogID string       // logtail ID of frontend instance
 	BackendLogID  string       // logtail ID of backend instance
 	OS            string       // operating system the client runs on (a version.OS value)
+	OSVersion     string       // operating system version, with optional distro prefix ("Debian 10.4", "Windows 10 Pro 10.0.19041")
+	DeviceModel   string       // mobile phone model ("Pixel 3a", "iPhone 11 Pro")
 	Hostname      string       // name of the host the client runs on
+	GoArch        string       // the host's GOARCH value (of the running binary)
 	RoutableIPs   []wgcfg.CIDR `json:",omitempty"` // set of IP ranges this client can route
 	RequestTags   []string     `json:",omitempty"` // set of ACL tags this node wants to claim
 	Services      []Service    `json:",omitempty"` // services advertised by this machine
@@ -310,6 +287,18 @@ type NetInfo struct {
 	// WorkingUDP is whether UDP works.
 	WorkingUDP opt.Bool
 
+	// UPnP is whether UPnP appears present on the LAN.
+	// Empty means not checked.
+	UPnP opt.Bool
+
+	// PMP is whether NAT-PMP appears present on the LAN.
+	// Empty means not checked.
+	PMP opt.Bool
+
+	// PCP is whether PCP appears present on the LAN.
+	// Empty means not checked.
+	PCP opt.Bool
+
 	// PreferredDERP is this node's preferred DERP server
 	// for incoming traffic. The node might be be temporarily
 	// connected to multiple DERP servers (to send to other nodes)
@@ -338,9 +327,32 @@ func (ni *NetInfo) String() string {
 	if ni == nil {
 		return "NetInfo(nil)"
 	}
-	return fmt.Sprintf("NetInfo{varies=%v hairpin=%v ipv6=%v udp=%v derp=#%v link=%q}",
+	return fmt.Sprintf("NetInfo{varies=%v hairpin=%v ipv6=%v udp=%v derp=#%v portmap=%v link=%q}",
 		ni.MappingVariesByDestIP, ni.HairPinning, ni.WorkingIPv6,
-		ni.WorkingUDP, ni.PreferredDERP, ni.LinkType)
+		ni.WorkingUDP, ni.PreferredDERP,
+		ni.portMapSummary(),
+		ni.LinkType)
+}
+
+func (ni *NetInfo) portMapSummary() string {
+	if ni.UPnP == "" && ni.PMP == "" && ni.PCP == "" {
+		return "?"
+	}
+	return conciseOptBool(ni.UPnP, "U") + conciseOptBool(ni.PMP, "M") + conciseOptBool(ni.PCP, "C")
+}
+
+func conciseOptBool(b opt.Bool, trueVal string) string {
+	if b == "" {
+		return "_"
+	}
+	v, ok := b.Get()
+	if !ok {
+		return "x"
+	}
+	if v {
+		return trueVal
+	}
+	return ""
 }
 
 // BasicallyEqual reports whether ni and ni2 are basically equal, ignoring
@@ -356,39 +368,21 @@ func (ni *NetInfo) BasicallyEqual(ni2 *NetInfo) bool {
 		ni.HairPinning == ni2.HairPinning &&
 		ni.WorkingIPv6 == ni2.WorkingIPv6 &&
 		ni.WorkingUDP == ni2.WorkingUDP &&
+		ni.UPnP == ni2.UPnP &&
+		ni.PMP == ni2.PMP &&
+		ni.PCP == ni2.PCP &&
 		ni.PreferredDERP == ni2.PreferredDERP &&
 		ni.LinkType == ni2.LinkType
 }
 
-func (ni *NetInfo) Clone() (res *NetInfo) {
-	if ni == nil {
-		return nil
-	}
-	res = new(NetInfo)
-	*res = *ni
-	if ni.DERPLatency != nil {
-		res.DERPLatency = map[string]float64{}
-		for k, v := range ni.DERPLatency {
-			res.DERPLatency[k] = v
-		}
-	}
-	return res
-}
-
-// Clone makes a deep copy of Hostinfo.
-// The result aliases no memory with the original.
-func (h *Hostinfo) Clone() (res *Hostinfo) {
-	res = new(Hostinfo)
-	*res = *h
-
-	res.RoutableIPs = append([]wgcfg.CIDR{}, h.RoutableIPs...)
-	res.Services = append([]Service{}, h.Services...)
-	res.NetInfo = h.NetInfo.Clone()
-	return res
-}
-
 // Equal reports whether h and h2 are equal.
 func (h *Hostinfo) Equal(h2 *Hostinfo) bool {
+	if h == nil && h2 == nil {
+		return true
+	}
+	if (h == nil) != (h2 == nil) {
+		return false
+	}
 	return reflect.DeepEqual(h, h2)
 }
 
@@ -415,6 +409,8 @@ type RegisterRequest struct {
 
 // Clone makes a deep copy of RegisterRequest.
 // The result aliases no memory with the original.
+//
+// TODO: extend cmd/cloner to generate this method.
 func (req *RegisterRequest) Clone() *RegisterRequest {
 	res := new(RegisterRequest)
 	*res = *req
@@ -447,11 +443,12 @@ type RegisterResponse struct {
 type MapRequest struct {
 	Version     int    // current version is 4
 	Compress    string // "zstd" or "" (no compression)
-	KeepAlive   bool   // server sends keep-alives
+	KeepAlive   bool   // whether server should send keep-alives back to us
 	NodeKey     NodeKey
 	DiscoKey    DiscoKey
 	Endpoints   []string // caller's endpoints (IPv4 or IPv6)
 	IncludeIPv6 bool     // include IPv6 endpoints in returned Node Endpoints
+	DeltaPeers  bool     // whether the 2nd+ network map in response should be deltas, using PeersChanged, PeersRemoved
 	Stream      bool     // if true, multiple MapResponse objects are returned
 	Hostinfo    *Hostinfo
 
@@ -497,15 +494,45 @@ var FilterAllowAll = []FilterRule{
 	},
 }
 
+// DNSConfig is the DNS configuration.
+type DNSConfig struct {
+	Nameservers []netaddr.IP `json:",omitempty"`
+	Domains     []string     `json:",omitempty"`
+	PerDomain   bool
+	Proxied     bool
+}
+
 type MapResponse struct {
-	KeepAlive bool // if set, all other fields are ignored
+	KeepAlive bool `json:",omitempty"` // if set, all other fields are ignored
 
 	// Networking
-	Node        *Node
-	Peers       []*Node
-	DNS         []wgcfg.IP
-	SearchPaths []string
-	DERPMap     *DERPMap
+	Node    *Node
+	DERPMap *DERPMap `json:",omitempty"` // if non-empty, a change in the DERP map.
+
+	// Peers, if non-empty, is the complete list of peers.
+	// It will be set in the first MapResponse for a long-polled request/response.
+	// Subsequent responses will be delta-encoded if DeltaPeers was set in the request.
+	// If Peers is non-empty, PeersChanged and PeersRemoved should
+	// be ignored (and should be empty).
+	// Peers is always returned sorted by Node.ID.
+	Peers []*Node `json:",omitempty"`
+	// PeersChanged are the Nodes (identified by their ID) that
+	// have changed or been added since the past update on the
+	// HTTP response. It's only set if MapRequest.DeltaPeers was true.
+	// PeersChanged is always returned sorted by Node.ID.
+	PeersChanged []*Node `json:",omitempty"`
+	// PeersRemoved are the NodeIDs that are no longer in the peer list.
+	PeersRemoved []NodeID `json:",omitempty"`
+
+	// DNS is the same as DNSConfig.Nameservers.
+	//
+	// TODO(dmytro): should be sent in DNSConfig.Nameservers once clients have updated.
+	DNS []wgcfg.IP `json:",omitempty"`
+	// SearchPaths are the same as DNSConfig.Domains.
+	//
+	// TODO(dmytro): should be sent in DNSConfig.Domains once clients have updated.
+	SearchPaths []string  `json:",omitempty"`
+	DNSConfig   DNSConfig `json:",omitempty"`
 
 	// ACLs
 	Domain       string
@@ -598,11 +625,39 @@ func (n *Node) Equal(n2 *Node) bool {
 		n.KeyExpiry.Equal(n2.KeyExpiry) &&
 		n.Machine == n2.Machine &&
 		n.DiscoKey == n2.DiscoKey &&
-		reflect.DeepEqual(n.Addresses, n2.Addresses) &&
-		reflect.DeepEqual(n.AllowedIPs, n2.AllowedIPs) &&
-		reflect.DeepEqual(n.Endpoints, n2.Endpoints) &&
-		reflect.DeepEqual(n.Hostinfo, n2.Hostinfo) &&
+		eqCIDRs(n.Addresses, n2.Addresses) &&
+		eqCIDRs(n.AllowedIPs, n2.AllowedIPs) &&
+		eqStrings(n.Endpoints, n2.Endpoints) &&
+		n.Hostinfo.Equal(&n2.Hostinfo) &&
 		n.Created.Equal(n2.Created) &&
-		reflect.DeepEqual(n.LastSeen, n2.LastSeen) &&
+		eqTimePtr(n.LastSeen, n2.LastSeen) &&
 		n.MachineAuthorized == n2.MachineAuthorized
 }
+
+func eqStrings(a, b []string) bool {
+	if len(a) != len(b) || ((a == nil) != (b == nil)) {
+		return false
+	}
+	for i, v := range a {
+		if v != b[i] {
+			return false
+		}
+	}
+	return true
+}
+
+func eqCIDRs(a, b []wgcfg.CIDR) bool {
+	if len(a) != len(b) || ((a == nil) != (b == nil)) {
+		return false
+	}
+	for i, v := range a {
+		if v != b[i] {
+			return false
+		}
+	}
+	return true
+}
+
+func eqTimePtr(a, b *time.Time) bool {
+	return ((a == nil) == (b == nil)) && (a == nil || a.Equal(*b))
+}

tailcfg/tailcfg_clone.go

@@ -0,0 +1,75 @@
+// Copyright (c) 2020 Tailscale Inc & AUTHORS All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+// Code generated by tailscale.com/cmd/cloner -type User,Node,Hostinfo,NetInfo; DO NOT EDIT.
+
+package tailcfg
+
+import (
+	"time"
+)
+
+// Clone makes a deep copy of User.
+// The result aliases no memory with the original.
+func (src *User) Clone() *User {
+	if src == nil {
+		return nil
+	}
+	dst := new(User)
+	*dst = *src
+	dst.Logins = append(src.Logins[:0:0], src.Logins...)
+	dst.Roles = append(src.Roles[:0:0], src.Roles...)
+	return dst
+}
+
+// Clone makes a deep copy of Node.
+// The result aliases no memory with the original.
+func (src *Node) Clone() *Node {
+	if src == nil {
+		return nil
+	}
+	dst := new(Node)
+	*dst = *src
+	dst.Addresses = append(src.Addresses[:0:0], src.Addresses...)
+	dst.AllowedIPs = append(src.AllowedIPs[:0:0], src.AllowedIPs...)
+	dst.Endpoints = append(src.Endpoints[:0:0], src.Endpoints...)
+	dst.Hostinfo = *src.Hostinfo.Clone()
+	if dst.LastSeen != nil {
+		dst.LastSeen = new(time.Time)
+		*dst.LastSeen = *src.LastSeen
+	}
+	return dst
+}
+
+// Clone makes a deep copy of Hostinfo.
+// The result aliases no memory with the original.
+func (src *Hostinfo) Clone() *Hostinfo {
+	if src == nil {
+		return nil
+	}
+	dst := new(Hostinfo)
+	*dst = *src
+	dst.RoutableIPs = append(src.RoutableIPs[:0:0], src.RoutableIPs...)
+	dst.RequestTags = append(src.RequestTags[:0:0], src.RequestTags...)
+	dst.Services = append(src.Services[:0:0], src.Services...)
+	dst.NetInfo = src.NetInfo.Clone()
+	return dst
+}
+
+// Clone makes a deep copy of NetInfo.
+// The result aliases no memory with the original.
+func (src *NetInfo) Clone() *NetInfo {
+	if src == nil {
+		return nil
+	}
+	dst := new(NetInfo)
+	*dst = *src
+	if dst.DERPLatency != nil {
+		dst.DERPLatency = map[string]float64{}
+		for k, v := range src.DERPLatency {
+			dst.DERPLatency[k] = v
+		}
+	}
+	return dst
+}

tailcfg/tailcfg_test.go

@@ -23,7 +23,8 @@ func fieldsOf(t reflect.Type) (fields []string) {
 
 func TestHostinfoEqual(t *testing.T) {
 	hiHandles := []string{
-		"IPNVersion", "FrontendLogID", "BackendLogID", "OS", "Hostname", "RoutableIPs", "RequestTags", "Services",
+		"IPNVersion", "FrontendLogID", "BackendLogID", "OS", "OSVersion",
+		"DeviceModel", "Hostname", "GoArch", "RoutableIPs", "RequestTags", "Services",
 		"NetInfo",
 	}
 	if have := fieldsOf(reflect.TypeOf(Hostinfo{})); !reflect.DeepEqual(have, hiHandles) {
@@ -329,6 +330,9 @@ func TestNetInfoFields(t *testing.T) {
 		"HairPinning",
 		"WorkingIPv6",
 		"WorkingUDP",
+		"UPnP",
+		"PMP",
+		"PCP",
 		"PreferredDERP",
 		"LinkType",
 		"DERPLatency",
@@ -386,3 +390,43 @@ func testKey(t *testing.T, prefix string, in keyIn, out encoding.TextUnmarshaler
 		t.Errorf("mismatch after unmarshal")
 	}
 }
+
+func TestCloneUser(t *testing.T) {
+	tests := []struct {
+		name string
+		u    *User
+	}{
+		{"nil_logins", &User{}},
+		{"zero_logins", &User{Logins: make([]LoginID, 0)}},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			u2 := tt.u.Clone()
+			if !reflect.DeepEqual(tt.u, u2) {
+				t.Errorf("not equal")
+			}
+		})
+	}
+}
+
+func TestCloneNode(t *testing.T) {
+	tests := []struct {
+		name string
+		v    *Node
+	}{
+		{"nil_fields", &Node{}},
+		{"zero_fields", &Node{
+			Addresses:  make([]wgcfg.CIDR, 0),
+			AllowedIPs: make([]wgcfg.CIDR, 0),
+			Endpoints:  make([]string, 0),
+		}},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			v2 := tt.v.Clone()
+			if !reflect.DeepEqual(tt.v, v2) {
+				t.Errorf("not equal")
+			}
+		})
+	}
+}

tempfork/pprof/README.md

@@ -0,0 +1,4 @@
+This is a fork of net/http/pprof that doesn't use init side effects
+and doesn't use html/template (which ends up calling
+reflect.Value.MethodByName, which disables some linker deadcode
+optimizations).

tempfork/pprof/pprof.go

@@ -0,0 +1,301 @@
+// Copyright 2010 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+// Package pprof serves via its HTTP server runtime profiling data
+// in the format expected by the pprof visualization tool.
+//
+// See Go's net/http/pprof for docs.
+//
+// This is a fork of net/http/pprof that doesn't use init side effects
+// and doesn't use html/template (which ends up calling
+// reflect.Value.MethodByName, which disables some linker deadcode
+// optimizations).
+package pprof
+
+import (
+	"bufio"
+	"bytes"
+	"fmt"
+	"html"
+	"io"
+	"net/http"
+	"os"
+	"runtime"
+	"runtime/pprof"
+	"runtime/trace"
+	"sort"
+	"strconv"
+	"strings"
+	"time"
+)
+
+func AddHandlers(mux *http.ServeMux) {
+	mux.HandleFunc("/debug/pprof/", Index)
+	mux.HandleFunc("/debug/pprof/cmdline", Cmdline)
+	mux.HandleFunc("/debug/pprof/profile", Profile)
+	mux.HandleFunc("/debug/pprof/symbol", Symbol)
+	mux.HandleFunc("/debug/pprof/trace", Trace)
+}
+
+// Cmdline responds with the running program's
+// command line, with arguments separated by NUL bytes.
+// The package initialization registers it as /debug/pprof/cmdline.
+func Cmdline(w http.ResponseWriter, r *http.Request) {
+	w.Header().Set("X-Content-Type-Options", "nosniff")
+	w.Header().Set("Content-Type", "text/plain; charset=utf-8")
+	fmt.Fprintf(w, strings.Join(os.Args, "\x00"))
+}
+
+func sleep(w http.ResponseWriter, d time.Duration) {
+	var clientGone <-chan bool
+	if cn, ok := w.(http.CloseNotifier); ok {
+		clientGone = cn.CloseNotify()
+	}
+	select {
+	case <-time.After(d):
+	case <-clientGone:
+	}
+}
+
+func durationExceedsWriteTimeout(r *http.Request, seconds float64) bool {
+	srv, ok := r.Context().Value(http.ServerContextKey).(*http.Server)
+	return ok && srv.WriteTimeout != 0 && seconds >= srv.WriteTimeout.Seconds()
+}
+
+func serveError(w http.ResponseWriter, status int, txt string) {
+	w.Header().Set("Content-Type", "text/plain; charset=utf-8")
+	w.Header().Set("X-Go-Pprof", "1")
+	w.Header().Del("Content-Disposition")
+	w.WriteHeader(status)
+	fmt.Fprintln(w, txt)
+}
+
+// Profile responds with the pprof-formatted cpu profile.
+// Profiling lasts for duration specified in seconds GET parameter, or for 30 seconds if not specified.
+// The package initialization registers it as /debug/pprof/profile.
+func Profile(w http.ResponseWriter, r *http.Request) {
+	w.Header().Set("X-Content-Type-Options", "nosniff")
+	sec, err := strconv.ParseInt(r.FormValue("seconds"), 10, 64)
+	if sec <= 0 || err != nil {
+		sec = 30
+	}
+
+	if durationExceedsWriteTimeout(r, float64(sec)) {
+		serveError(w, http.StatusBadRequest, "profile duration exceeds server's WriteTimeout")
+		return
+	}
+
+	// Set Content Type assuming StartCPUProfile will work,
+	// because if it does it starts writing.
+	w.Header().Set("Content-Type", "application/octet-stream")
+	w.Header().Set("Content-Disposition", `attachment; filename="profile"`)
+	if err := pprof.StartCPUProfile(w); err != nil {
+		// StartCPUProfile failed, so no writes yet.
+		serveError(w, http.StatusInternalServerError,
+			fmt.Sprintf("Could not enable CPU profiling: %s", err))
+		return
+	}
+	sleep(w, time.Duration(sec)*time.Second)
+	pprof.StopCPUProfile()
+}
+
+// Trace responds with the execution trace in binary form.
+// Tracing lasts for duration specified in seconds GET parameter, or for 1 second if not specified.
+// The package initialization registers it as /debug/pprof/trace.
+func Trace(w http.ResponseWriter, r *http.Request) {
+	w.Header().Set("X-Content-Type-Options", "nosniff")
+	sec, err := strconv.ParseFloat(r.FormValue("seconds"), 64)
+	if sec <= 0 || err != nil {
+		sec = 1
+	}
+
+	if durationExceedsWriteTimeout(r, sec) {
+		serveError(w, http.StatusBadRequest, "profile duration exceeds server's WriteTimeout")
+		return
+	}
+
+	// Set Content Type assuming trace.Start will work,
+	// because if it does it starts writing.
+	w.Header().Set("Content-Type", "application/octet-stream")
+	w.Header().Set("Content-Disposition", `attachment; filename="trace"`)
+	if err := trace.Start(w); err != nil {
+		// trace.Start failed, so no writes yet.
+		serveError(w, http.StatusInternalServerError,
+			fmt.Sprintf("Could not enable tracing: %s", err))
+		return
+	}
+	sleep(w, time.Duration(sec*float64(time.Second)))
+	trace.Stop()
+}
+
+// Symbol looks up the program counters listed in the request,
+// responding with a table mapping program counters to function names.
+// The package initialization registers it as /debug/pprof/symbol.
+func Symbol(w http.ResponseWriter, r *http.Request) {
+	w.Header().Set("X-Content-Type-Options", "nosniff")
+	w.Header().Set("Content-Type", "text/plain; charset=utf-8")
+
+	// We have to read the whole POST body before
+	// writing any output. Buffer the output here.
+	var buf bytes.Buffer
+
+	// We don't know how many symbols we have, but we
+	// do have symbol information. Pprof only cares whether
+	// this number is 0 (no symbols available) or > 0.
+	fmt.Fprintf(&buf, "num_symbols: 1\n")
+
+	var b *bufio.Reader
+	if r.Method == "POST" {
+		b = bufio.NewReader(r.Body)
+	} else {
+		b = bufio.NewReader(strings.NewReader(r.URL.RawQuery))
+	}
+
+	for {
+		word, err := b.ReadSlice('+')
+		if err == nil {
+			word = word[0 : len(word)-1] // trim +
+		}
+		pc, _ := strconv.ParseUint(string(word), 0, 64)
+		if pc != 0 {
+			f := runtime.FuncForPC(uintptr(pc))
+			if f != nil {
+				fmt.Fprintf(&buf, "%#x %s\n", pc, f.Name())
+			}
+		}
+
+		// Wait until here to check for err; the last
+		// symbol will have an err because it doesn't end in +.
+		if err != nil {
+			if err != io.EOF {
+				fmt.Fprintf(&buf, "reading request: %v\n", err)
+			}
+			break
+		}
+	}
+
+	w.Write(buf.Bytes())
+}
+
+// Handler returns an HTTP handler that serves the named profile.
+func Handler(name string) http.Handler {
+	return handler(name)
+}
+
+type handler string
+
+func (name handler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
+	w.Header().Set("X-Content-Type-Options", "nosniff")
+	p := pprof.Lookup(string(name))
+	if p == nil {
+		serveError(w, http.StatusNotFound, "Unknown profile")
+		return
+	}
+	gc, _ := strconv.Atoi(r.FormValue("gc"))
+	if name == "heap" && gc > 0 {
+		runtime.GC()
+	}
+	debug, _ := strconv.Atoi(r.FormValue("debug"))
+	if debug != 0 {
+		w.Header().Set("Content-Type", "text/plain; charset=utf-8")
+	} else {
+		w.Header().Set("Content-Type", "application/octet-stream")
+		w.Header().Set("Content-Disposition", fmt.Sprintf(`attachment; filename="%s"`, name))
+	}
+	p.WriteTo(w, debug)
+}
+
+var profileDescriptions = map[string]string{
+	"allocs":       "A sampling of all past memory allocations",
+	"block":        "Stack traces that led to blocking on synchronization primitives",
+	"cmdline":      "The command line invocation of the current program",
+	"goroutine":    "Stack traces of all current goroutines",
+	"heap":         "A sampling of memory allocations of live objects. You can specify the gc GET parameter to run GC before taking the heap sample.",
+	"mutex":        "Stack traces of holders of contended mutexes",
+	"profile":      "CPU profile. You can specify the duration in the seconds GET parameter. After you get the profile file, use the go tool pprof command to investigate the profile.",
+	"threadcreate": "Stack traces that led to the creation of new OS threads",
+	"trace":        "A trace of execution of the current program. You can specify the duration in the seconds GET parameter. After you get the trace file, use the go tool trace command to investigate the trace.",
+}
+
+// Index responds with the pprof-formatted profile named by the request.
+// For example, "/debug/pprof/heap" serves the "heap" profile.
+// Index responds to a request for "/debug/pprof/" with an HTML page
+// listing the available profiles.
+func Index(w http.ResponseWriter, r *http.Request) {
+	if strings.HasPrefix(r.URL.Path, "/debug/pprof/") {
+		name := strings.TrimPrefix(r.URL.Path, "/debug/pprof/")
+		if name != "" {
+			handler(name).ServeHTTP(w, r)
+			return
+		}
+	}
+
+	type profile struct {
+		Name  string
+		Href  string
+		Desc  string
+		Count int
+	}
+	var profiles []profile
+	for _, p := range pprof.Profiles() {
+		profiles = append(profiles, profile{
+			Name:  p.Name(),
+			Href:  p.Name() + "?debug=1",
+			Desc:  profileDescriptions[p.Name()],
+			Count: p.Count(),
+		})
+	}
+
+	// Adding other profiles exposed from within this package
+	for _, p := range []string{"cmdline", "profile", "trace"} {
+		profiles = append(profiles, profile{
+			Name: p,
+			Href: p,
+			Desc: profileDescriptions[p],
+		})
+	}
+
+	sort.Slice(profiles, func(i, j int) bool {
+		return profiles[i].Name < profiles[j].Name
+	})
+
+	io.WriteString(w, `<html>
+<head>
+<title>/debug/pprof/</title>
+<style>
+.profile-name{
+	display:inline-block;
+	width:6rem;
+}
+</style>
+</head>
+<body>
+/debug/pprof/<br>
+<br>
+Types of profiles available:
+<table>
+<thead><td>Count</td><td>Profile</td></thead>
+`)
+	for _, p := range profiles {
+		fmt.Fprintf(w, "<tr><td>%d</td><td><a href=%v>%v</td></tr>\n",
+			p.Count, html.EscapeString(p.Href), html.EscapeString(p.Name))
+	}
+	io.WriteString(w, `</table>
+<a href="goroutine?debug=2">full goroutine stack dump</a>
+<br/>
+<p>
+Profile Descriptions:
+<ul>
+`)
+	for _, p := range profiles {
+		fmt.Fprintf(w, "<li><div class=profile-name>%s:</div> %s</li>\n",
+			html.EscapeString(p.Name), html.EscapeString(p.Desc))
+	}
+	io.WriteString(w, `
+</ul>
+</p>
+</body>
+</html>
+`)
+}

tempfork/pprof/pprof_test.go

@@ -0,0 +1,81 @@
+// Copyright 2018 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package pprof
+
+import (
+	"bytes"
+	"io/ioutil"
+	"net/http"
+	"net/http/httptest"
+	"runtime/pprof"
+	"testing"
+)
+
+// TestDescriptions checks that the profile names under runtime/pprof package
+// have a key in the description map.
+func TestDescriptions(t *testing.T) {
+	for _, p := range pprof.Profiles() {
+		_, ok := profileDescriptions[p.Name()]
+		if ok != true {
+			t.Errorf("%s does not exist in profileDescriptions map\n", p.Name())
+		}
+	}
+}
+
+func TestHandlers(t *testing.T) {
+	testCases := []struct {
+		path               string
+		handler            http.HandlerFunc
+		statusCode         int
+		contentType        string
+		contentDisposition string
+		resp               []byte
+	}{
+		{"/debug/pprof/<script>scripty<script>", Index, http.StatusNotFound, "text/plain; charset=utf-8", "", []byte("Unknown profile\n")},
+		{"/debug/pprof/heap", Index, http.StatusOK, "application/octet-stream", `attachment; filename="heap"`, nil},
+		{"/debug/pprof/heap?debug=1", Index, http.StatusOK, "text/plain; charset=utf-8", "", nil},
+		{"/debug/pprof/cmdline", Cmdline, http.StatusOK, "text/plain; charset=utf-8", "", nil},
+		{"/debug/pprof/profile?seconds=1", Profile, http.StatusOK, "application/octet-stream", `attachment; filename="profile"`, nil},
+		{"/debug/pprof/symbol", Symbol, http.StatusOK, "text/plain; charset=utf-8", "", nil},
+		{"/debug/pprof/trace", Trace, http.StatusOK, "application/octet-stream", `attachment; filename="trace"`, nil},
+	}
+	for _, tc := range testCases {
+		t.Run(tc.path, func(t *testing.T) {
+			req := httptest.NewRequest("GET", "http://example.com"+tc.path, nil)
+			w := httptest.NewRecorder()
+			tc.handler(w, req)
+
+			resp := w.Result()
+			if got, want := resp.StatusCode, tc.statusCode; got != want {
+				t.Errorf("status code: got %d; want %d", got, want)
+			}
+
+			body, err := ioutil.ReadAll(resp.Body)
+			if err != nil {
+				t.Errorf("when reading response body, expected non-nil err; got %v", err)
+			}
+			if got, want := resp.Header.Get("X-Content-Type-Options"), "nosniff"; got != want {
+				t.Errorf("X-Content-Type-Options: got %q; want %q", got, want)
+			}
+			if got, want := resp.Header.Get("Content-Type"), tc.contentType; got != want {
+				t.Errorf("Content-Type: got %q; want %q", got, want)
+			}
+			if got, want := resp.Header.Get("Content-Disposition"), tc.contentDisposition; got != want {
+				t.Errorf("Content-Disposition: got %q; want %q", got, want)
+			}
+
+			if resp.StatusCode == http.StatusOK {
+				return
+			}
+			if got, want := resp.Header.Get("X-Go-Pprof"), "1"; got != want {
+				t.Errorf("X-Go-Pprof: got %q; want %q", got, want)
+			}
+			if !bytes.Equal(body, tc.resp) {
+				t.Errorf("response: got %q; want %q", body, tc.resp)
+			}
+		})
+	}
+
+}

tstest/clock.go

@@ -30,16 +30,27 @@ type Clock struct {
 func (c *Clock) Now() time.Time {
 	c.Lock()
 	defer c.Unlock()
+	c.initLocked()
+	step := c.Step
+	ret := c.Present
+	c.Present = c.Present.Add(step)
+	return ret
+}
+
+func (c *Clock) Advance(d time.Duration) {
+	c.Lock()
+	defer c.Unlock()
+	c.initLocked()
+	c.Present = c.Present.Add(d)
+}
+
+func (c *Clock) initLocked() {
 	if c.Start.IsZero() {
 		c.Start = time.Now()
 	}
 	if c.Present.Before(c.Start) {
 		c.Present = c.Start
 	}
-	step := c.Step
-	ret := c.Present
-	c.Present = c.Present.Add(step)
-	return ret
 }
 
 // Reset rewinds the virtual clock to its start time.

tstest/natlab/firewall.go

@@ -0,0 +1,157 @@
+// Copyright (c) 2020 Tailscale Inc & AUTHORS All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package natlab
+
+import (
+	"fmt"
+	"sync"
+	"time"
+
+	"inet.af/netaddr"
+)
+
+// FirewallType is the type of filtering a stateful firewall
+// does. Values express different modes defined by RFC 4787.
+type FirewallType int
+
+const (
+	// AddressAndPortDependentFirewall specifies a destination
+	// address-and-port dependent firewall. Outbound traffic to an
+	// ip:port authorizes traffic from that ip:port exactly, and
+	// nothing else.
+	AddressAndPortDependentFirewall FirewallType = iota
+	// AddressDependentFirewall specifies a destination address
+	// dependent firewall. Once outbound traffic has been seen to an
+	// IP address, that IP address can talk back from any port.
+	AddressDependentFirewall
+	// EndpointIndependentFirewall specifies a destination endpoint
+	// independent firewall. Once outbound traffic has been seen from
+	// a source, anyone can talk back to that source.
+	EndpointIndependentFirewall
+)
+
+// fwKey is the lookup key for a firewall session. While it contains a
+// 4-tuple ({src,dst} {ip,port}), some FirewallTypes will zero out
+// some fields, so in practice the key is either a 2-tuple (src only),
+// 3-tuple (src ip+port and dst ip) or 4-tuple (src+dst ip+port).
+type fwKey struct {
+	src netaddr.IPPort
+	dst netaddr.IPPort
+}
+
+// key returns an fwKey for the given src and dst, trimmed according
+// to the FirewallType. fwKeys are always constructed from the
+// "outbound" point of view (i.e. src is the "trusted" side of the
+// world), it's the caller's responsibility to swap src and dst in the
+// call to key when processing packets inbound from the "untrusted"
+// world.
+func (s FirewallType) key(src, dst netaddr.IPPort) fwKey {
+	k := fwKey{src: src}
+	switch s {
+	case EndpointIndependentFirewall:
+	case AddressDependentFirewall:
+		k.dst.IP = dst.IP
+	case AddressAndPortDependentFirewall:
+		k.dst = dst
+	default:
+		panic(fmt.Sprintf("unknown firewall selectivity %v", s))
+	}
+	return k
+}
+
+// DefaultSessionTimeout is the default timeout for a firewall
+// session.
+const DefaultSessionTimeout = 30 * time.Second
+
+// Firewall is a simple stateful firewall that allows all outbound
+// traffic and filters inbound traffic based on recently seen outbound
+// traffic. Its HandlePacket method should be attached to a Machine to
+// give it a stateful firewall.
+type Firewall struct {
+	// SessionTimeout is the lifetime of idle sessions in the firewall
+	// state. Packets transiting from the TrustedInterface reset the
+	// session lifetime to SessionTimeout. If zero,
+	// DefaultSessionTimeout is used.
+	SessionTimeout time.Duration
+	// Type specifies how precisely return traffic must match
+	// previously seen outbound traffic to be allowed. Defaults to
+	// AddressAndPortDependentFirewall.
+	Type FirewallType
+	// TrustedInterface is an optional interface that is considered
+	// trusted in addition to PacketConns local to the Machine. All
+	// other interfaces can only respond to traffic from
+	// TrustedInterface or the local host.
+	TrustedInterface *Interface
+	// TimeNow is a function returning the current time. If nil,
+	// time.Now is used.
+	TimeNow func() time.Time
+
+	// TODO: refresh directionality: outbound-only, both
+
+	mu   sync.Mutex
+	seen map[fwKey]time.Time // session -> deadline
+}
+
+func (f *Firewall) timeNow() time.Time {
+	if f.TimeNow != nil {
+		return f.TimeNow()
+	}
+	return time.Now()
+}
+
+func (f *Firewall) init() {
+	if f.seen == nil {
+		f.seen = map[fwKey]time.Time{}
+	}
+}
+
+func (f *Firewall) HandleOut(p *Packet, oif *Interface) *Packet {
+	f.mu.Lock()
+	defer f.mu.Unlock()
+	f.init()
+
+	k := f.Type.key(p.Src, p.Dst)
+	f.seen[k] = f.timeNow().Add(f.sessionTimeoutLocked())
+	p.Trace("firewall out ok")
+	return p
+}
+
+func (f *Firewall) HandleIn(p *Packet, iif *Interface) *Packet {
+	f.mu.Lock()
+	defer f.mu.Unlock()
+	f.init()
+
+	// reverse src and dst because the session table is from the POV
+	// of outbound packets.
+	k := f.Type.key(p.Dst, p.Src)
+	now := f.timeNow()
+	if now.After(f.seen[k]) {
+		p.Trace("firewall drop")
+		return nil
+	}
+	p.Trace("firewall in ok")
+	return p
+}
+
+func (f *Firewall) HandleForward(p *Packet, iif *Interface, oif *Interface) *Packet {
+	if iif == f.TrustedInterface {
+		// Treat just like a locally originated packet
+		return f.HandleOut(p, oif)
+	}
+	if oif != f.TrustedInterface {
+		// Not a possible return packet from our trusted interface, drop.
+		p.Trace("firewall drop, unexpected oif")
+		return nil
+	}
+	// Otherwise, a session must exist, same as HandleIn.
+	return f.HandleIn(p, iif)
+}
+
+func (f *Firewall) sessionTimeoutLocked() time.Duration {
+	if f.SessionTimeout == 0 {
+		return DefaultSessionTimeout
+	}
+	return f.SessionTimeout
+}

tstest/natlab/nat.go

@@ -0,0 +1,257 @@
+// Copyright (c) 2020 Tailscale Inc & AUTHORS All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package natlab
+
+import (
+	"context"
+	"fmt"
+	"net"
+	"sync"
+	"time"
+
+	"inet.af/netaddr"
+)
+
+// mapping is the state of an allocated NAT session.
+type mapping struct {
+	lanSrc   netaddr.IPPort
+	lanDst   netaddr.IPPort
+	wanSrc   netaddr.IPPort
+	deadline time.Time
+
+	// pc is a PacketConn that reserves an outbound port on the NAT's
+	// WAN interface. We do this because ListenPacket already has
+	// random port selection logic built in. Additionally this means
+	// that concurrent use of ListenPacket for connections originating
+	// from the NAT box won't conflict with NAT mappings, since both
+	// use PacketConn to reserve ports on the machine.
+	pc net.PacketConn
+}
+
+// NATType is the mapping behavior of a NAT device. Values express
+// different modes defined by RFC 4787.
+type NATType int
+
+const (
+	// EndpointIndependentNAT specifies a destination endpoint
+	// independent NAT. All traffic from a source ip:port gets mapped
+	// to a single WAN ip:port.
+	EndpointIndependentNAT NATType = iota
+	// AddressDependentNAT specifies a destination address dependent
+	// NAT. Every distinct destination IP gets its own WAN ip:port
+	// allocation.
+	AddressDependentNAT
+	// AddressAndPortDependentNAT specifies a destination
+	// address-and-port dependent NAT. Every distinct destination
+	// ip:port gets its own WAN ip:port allocation.
+	AddressAndPortDependentNAT
+)
+
+// natKey is the lookup key for a NAT session. While it contains a
+// 4-tuple ({src,dst} {ip,port}), some NATTypes will zero out some
+// fields, so in practice the key is either a 2-tuple (src only),
+// 3-tuple (src ip+port and dst ip) or 4-tuple (src+dst ip+port).
+type natKey struct {
+	src, dst netaddr.IPPort
+}
+
+func (t NATType) key(src, dst netaddr.IPPort) natKey {
+	k := natKey{src: src}
+	switch t {
+	case EndpointIndependentNAT:
+	case AddressDependentNAT:
+		k.dst.IP = dst.IP
+	case AddressAndPortDependentNAT:
+		k.dst = dst
+	default:
+		panic(fmt.Sprintf("unknown NAT type %v", t))
+	}
+	return k
+}
+
+// DefaultMappingTimeout is the default timeout for a NAT mapping.
+const DefaultMappingTimeout = 30 * time.Second
+
+// SNAT44 implements an IPv4-to-IPv4 source NAT (SNAT) translator, with
+// optional builtin firewall.
+type SNAT44 struct {
+	// Machine is the machine to which this NAT is attached. Altered
+	// packets are injected back into this Machine for processing.
+	Machine *Machine
+	// ExternalInterface is the "WAN" interface of Machine. Packets
+	// from other sources get NATed onto this interface.
+	ExternalInterface *Interface
+	// Type specifies the mapping allocation behavior for this NAT.
+	Type NATType
+	// MappingTimeout is the lifetime of individual NAT sessions. Once
+	// a session expires, the mapped port effectively "closes" to new
+	// traffic. If MappingTimeout is 0, DefaultMappingTimeout is used.
+	MappingTimeout time.Duration
+	// Firewall is an optional packet handler that will be invoked as
+	// a firewall during NAT translation. The firewall always sees
+	// packets in their "LAN form", i.e. before translation in the
+	// outbound direction and after translation in the inbound
+	// direction.
+	Firewall PacketHandler
+	// TimeNow is a function that returns the current time. If
+	// nil, time.Now is used.
+	TimeNow func() time.Time
+
+	mu    sync.Mutex
+	byLAN map[natKey]*mapping         // lookup by outbound packet tuple
+	byWAN map[netaddr.IPPort]*mapping // lookup by wan ip:port only
+}
+
+func (n *SNAT44) timeNow() time.Time {
+	if n.TimeNow != nil {
+		return n.TimeNow()
+	}
+	return time.Now()
+}
+
+func (n *SNAT44) mappingTimeout() time.Duration {
+	if n.MappingTimeout == 0 {
+		return DefaultMappingTimeout
+	}
+	return n.MappingTimeout
+}
+
+func (n *SNAT44) initLocked() {
+	if n.byLAN == nil {
+		n.byLAN = map[natKey]*mapping{}
+		n.byWAN = map[netaddr.IPPort]*mapping{}
+	}
+	if n.ExternalInterface.Machine() != n.Machine {
+		panic(fmt.Sprintf("NAT given interface %s that is not part of given machine %s", n.ExternalInterface, n.Machine.Name))
+	}
+}
+
+func (n *SNAT44) HandleOut(p *Packet, oif *Interface) *Packet {
+	// NATs don't affect locally originated packets.
+	if n.Firewall != nil {
+		return n.Firewall.HandleOut(p, oif)
+	}
+	return p
+}
+
+func (n *SNAT44) HandleIn(p *Packet, iif *Interface) *Packet {
+	if iif != n.ExternalInterface {
+		// NAT can't apply, defer to firewall.
+		if n.Firewall != nil {
+			return n.Firewall.HandleIn(p, iif)
+		}
+		return p
+	}
+
+	n.mu.Lock()
+	defer n.mu.Unlock()
+	n.initLocked()
+
+	now := n.timeNow()
+	mapping := n.byWAN[p.Dst]
+	if mapping == nil || now.After(mapping.deadline) {
+		// NAT didn't hit, defer to firewall or allow in for local
+		// socket handling.
+		if n.Firewall != nil {
+			return n.Firewall.HandleIn(p, iif)
+		}
+		return p
+	}
+
+	p.Dst = mapping.lanSrc
+	p.Trace("dnat to %v", p.Dst)
+	// Don't process firewall here. We mutated the packet such that
+	// it's no longer destined locally, so we'll get reinvoked as
+	// HandleForward and need to process the altered packet there.
+	return p
+}
+
+func (n *SNAT44) HandleForward(p *Packet, iif, oif *Interface) *Packet {
+	switch {
+	case oif == n.ExternalInterface:
+		if p.Src.IP == oif.V4() {
+			// Packet already NATed and is just retraversing Forward,
+			// don't touch it again.
+			return p
+		}
+
+		if n.Firewall != nil {
+			p2 := n.Firewall.HandleForward(p, iif, oif)
+			if p2 == nil {
+				// firewall dropped, done
+				return nil
+			}
+			if !p.Equivalent(p2) {
+				// firewall mutated packet? Weird, but okay.
+				return p2
+			}
+		}
+
+		n.mu.Lock()
+		defer n.mu.Unlock()
+		n.initLocked()
+
+		k := n.Type.key(p.Src, p.Dst)
+		now := n.timeNow()
+		m := n.byLAN[k]
+		if m == nil || now.After(m.deadline) {
+			pc, wanAddr := n.allocateMappedPort()
+			m = &mapping{
+				lanSrc: p.Src,
+				lanDst: p.Dst,
+				wanSrc: wanAddr,
+				pc:     pc,
+			}
+			n.byLAN[k] = m
+			n.byWAN[wanAddr] = m
+		}
+		m.deadline = now.Add(n.mappingTimeout())
+		p.Src = m.wanSrc
+		p.Trace("snat from %v", p.Src)
+		return p
+	case iif == n.ExternalInterface:
+		// Packet was already un-NAT-ed, we just need to either
+		// firewall it or let it through.
+		if n.Firewall != nil {
+			return n.Firewall.HandleForward(p, iif, oif)
+		}
+		return p
+	default:
+		// No NAT applies, invoke firewall or drop.
+		if n.Firewall != nil {
+			return n.Firewall.HandleForward(p, iif, oif)
+		}
+		return nil
+	}
+}
+
+func (n *SNAT44) allocateMappedPort() (net.PacketConn, netaddr.IPPort) {
+	// Clean up old entries before trying to allocate, to free up any
+	// expired ports.
+	n.gc()
+
+	ip := n.ExternalInterface.V4()
+	pc, err := n.Machine.ListenPacket(context.Background(), "udp", net.JoinHostPort(ip.String(), "0"))
+	if err != nil {
+		panic(fmt.Sprintf("ran out of NAT ports: %v", err))
+	}
+	addr := netaddr.IPPort{
+		IP:   ip,
+		Port: uint16(pc.LocalAddr().(*net.UDPAddr).Port),
+	}
+	return pc, addr
+}
+
+func (n *SNAT44) gc() {
+	now := n.timeNow()
+	for _, m := range n.byLAN {
+		if !now.After(m.deadline) {
+			continue
+		}
+		m.pc.Close()
+		delete(n.byLAN, n.Type.key(m.lanSrc, m.lanDst))
+		delete(n.byWAN, m.wanSrc)
+	}
+}

tstest/natlab/natlab.go

@@ -12,10 +12,13 @@
 package natlab
 
 import (
+	"bytes"
 	"context"
 	"crypto/sha256"
 	"encoding/base64"
+	"errors"
 	"fmt"
+	"math/rand"
 	"net"
 	"os"
 	"sort"
@@ -26,23 +29,57 @@ import (
 	"inet.af/netaddr"
 )
 
-var traceOn = os.Getenv("NATLAB_TRACE")
+var traceOn, _ = strconv.ParseBool(os.Getenv("NATLAB_TRACE"))
 
-func trace(p []byte, msg string, args ...interface{}) {
-	if traceOn == "" {
-		return
-	}
-	id := packetShort(p)
-	as := []interface{}{id}
-	as = append(as, args...)
-	fmt.Fprintf(os.Stderr, "[%s] "+msg+"\n", as...)
+// Packet represents a UDP packet flowing through the virtual network.
+type Packet struct {
+	Src, Dst netaddr.IPPort
+	Payload  []byte
+
+	// Prefix set by various internal methods of natlab, to locate
+	// where in the network a trace occured.
+	locator string
 }
 
-// packetShort returns a short identifier for a packet payload,
-// suitable for pritning trace information.
-func packetShort(p []byte) string {
-	s := sha256.Sum256(p)
-	return base64.RawStdEncoding.EncodeToString(s[:])[:4]
+// Equivalent returns true if Src, Dst and Payload are the same in p
+// and p2.
+func (p *Packet) Equivalent(p2 *Packet) bool {
+	return p.Src == p2.Src && p.Dst == p2.Dst && bytes.Equal(p.Payload, p2.Payload)
+}
+
+// Clone returns a copy of p that shares nothing with p.
+func (p *Packet) Clone() *Packet {
+	return &Packet{
+		Src:     p.Src,
+		Dst:     p.Dst,
+		Payload: append([]byte(nil), p.Payload...),
+		locator: p.locator,
+	}
+}
+
+// short returns a short identifier for a packet payload,
+// suitable for printing trace information.
+func (p *Packet) short() string {
+	s := sha256.Sum256(p.Payload)
+	payload := base64.RawStdEncoding.EncodeToString(s[:])[:2]
+
+	s = sha256.Sum256([]byte(p.Src.String() + "_" + p.Dst.String()))
+	tuple := base64.RawStdEncoding.EncodeToString(s[:])[:2]
+
+	return fmt.Sprintf("%s/%s", payload, tuple)
+}
+
+func (p *Packet) Trace(msg string, args ...interface{}) {
+	if !traceOn {
+		return
+	}
+	allArgs := []interface{}{p.short(), p.locator, p.Src, p.Dst}
+	allArgs = append(allArgs, args...)
+	fmt.Fprintf(os.Stderr, "[%s]%s src=%s dst=%s "+msg+"\n", allArgs...)
+}
+
+func (p *Packet) setLocator(msg string, args ...interface{}) {
+	p.locator = fmt.Sprintf(" "+msg, args...)
 }
 
 func mustPrefix(s string) netaddr.IPPrefix {
@@ -56,9 +93,10 @@ func mustPrefix(s string) netaddr.IPPrefix {
 // NewInternet returns a network that simulates the internet.
 func NewInternet() *Network {
 	return &Network{
-		Name:    "internet",
-		Prefix4: mustPrefix("203.0.113.0/24"), // documentation netblock that looks Internet-y
-		Prefix6: mustPrefix("fc00:52::/64"),
+		Name: "internet",
+		// easily recognizable internett-y addresses
+		Prefix4: mustPrefix("1.0.0.0/24"),
+		Prefix6: mustPrefix("1111::/64"),
 	}
 }
 
@@ -68,29 +106,32 @@ type Network struct {
 	Prefix6 netaddr.IPPrefix
 
 	mu        sync.Mutex
-	machine   map[netaddr.IP]*Machine
-	defaultGW *Machine // optional
+	machine   map[netaddr.IP]*Interface
+	defaultGW *Interface // optional
 	lastV4    netaddr.IP
 	lastV6    netaddr.IP
 }
 
-func (n *Network) SetDefaultGateway(gw *Machine) {
+func (n *Network) SetDefaultGateway(gwIf *Interface) {
 	n.mu.Lock()
 	defer n.mu.Unlock()
-	n.defaultGW = gw
+	if gwIf.net != n {
+		panic(fmt.Sprintf("can't set if=%s as net=%s's default gw, if not connected to net", gwIf.name, gwIf.net.Name))
+	}
+	n.defaultGW = gwIf
 }
 
-func (n *Network) addMachineLocked(ip netaddr.IP, m *Machine) {
-	if m == nil {
+func (n *Network) addMachineLocked(ip netaddr.IP, iface *Interface) {
+	if iface == nil {
 		return // for tests
 	}
 	if n.machine == nil {
-		n.machine = map[netaddr.IP]*Machine{}
+		n.machine = map[netaddr.IP]*Interface{}
 	}
-	n.machine[ip] = m
+	n.machine[ip] = iface
 }
 
-func (n *Network) allocIPv4(m *Machine) netaddr.IP {
+func (n *Network) allocIPv4(iface *Interface) netaddr.IP {
 	n.mu.Lock()
 	defer n.mu.Unlock()
 	if n.Prefix4.IsZero() {
@@ -105,11 +146,11 @@ func (n *Network) allocIPv4(m *Machine) netaddr.IP {
 	if !n.Prefix4.Contains(n.lastV4) {
 		panic("pool exhausted")
 	}
-	n.addMachineLocked(n.lastV4, m)
+	n.addMachineLocked(n.lastV4, iface)
 	return n.lastV4
 }
 
-func (n *Network) allocIPv6(m *Machine) netaddr.IP {
+func (n *Network) allocIPv6(iface *Interface) netaddr.IP {
 	n.mu.Lock()
 	defer n.mu.Unlock()
 	if n.Prefix6.IsZero() {
@@ -124,7 +165,7 @@ func (n *Network) allocIPv6(m *Machine) netaddr.IP {
 	if !n.Prefix6.Contains(n.lastV6) {
 		panic("pool exhausted")
 	}
-	n.addMachineLocked(n.lastV6, m)
+	n.addMachineLocked(n.lastV6, iface)
 	return n.lastV6
 }
 
@@ -137,30 +178,51 @@ func addOne(a *[16]byte, index int) {
 	}
 }
 
-func (n *Network) write(p []byte, dst, src netaddr.IPPort) (num int, err error) {
+func (n *Network) write(p *Packet) (num int, err error) {
+	p.setLocator("net=%s", n.Name)
+
 	n.mu.Lock()
 	defer n.mu.Unlock()
-	m, ok := n.machine[dst.IP]
+	iface, ok := n.machine[p.Dst.IP]
 	if !ok {
-		if n.defaultGW == nil {
-			trace(p, "net=%s dropped, no route to %v", n.Name, dst.IP)
-			return len(p), nil
+		// If the destination is within the network's authoritative
+		// range, no route to host.
+		if p.Dst.IP.Is4() && n.Prefix4.Contains(p.Dst.IP) {
+			p.Trace("no route to %v", p.Dst.IP)
+			return len(p.Payload), nil
 		}
-		m = n.defaultGW
+		if p.Dst.IP.Is6() && n.Prefix6.Contains(p.Dst.IP) {
+			p.Trace("no route to %v", p.Dst.IP)
+			return len(p.Payload), nil
+		}
+
+		if n.defaultGW == nil {
+			p.Trace("no route to %v", p.Dst.IP)
+			return len(p.Payload), nil
+		}
+		iface = n.defaultGW
 	}
 
 	// Pretend it went across the network. Make a copy so nobody
 	// can later mess with caller's memory.
-	trace(p, "net=%s src=%v dst=%v -> mach=%s", n.Name, src, dst, m.Name)
-	pcopy := append([]byte(nil), p...)
-	go m.deliverIncomingPacket(pcopy, dst, src)
-	return len(p), nil
+	p.Trace("-> mach=%s if=%s", iface.machine.Name, iface.name)
+	go iface.machine.deliverIncomingPacket(p, iface)
+	return len(p.Payload), nil
 }
 
 type Interface struct {
-	net  *Network
-	name string       // optional
-	ips  []netaddr.IP // static; not mutated once created
+	machine *Machine
+	net     *Network
+	name    string       // optional
+	ips     []netaddr.IP // static; not mutated once created
+}
+
+func (f *Interface) Machine() *Machine {
+	return f.machine
+}
+
+func (f *Interface) Network() *Network {
+	return f.net
 }
 
 // V4 returns the machine's first IPv4 address, or the zero value if none.
@@ -223,8 +285,41 @@ func (v PacketVerdict) String() string {
 	}
 }
 
-// A PacketHandler is a function that can process packets.
-type PacketHandler func(p []byte, dst, src netaddr.IPPort) PacketVerdict
+// A PacketHandler can look at packets arriving at, departing, and
+// transiting a Machine, and filter or mutate them.
+//
+// Each method is invoked with a Packet that natlab would like to keep
+// processing. Handlers can return that same Packet to allow
+// processing to continue; nil to drop the Packet; or a different
+// Packet that should be processed instead of the original.
+//
+// Packets passed to handlers share no state with anything else, and
+// are therefore safe to mutate. It's safe to return the original
+// packet mutated in-place, or a brand new packet initialized from
+// scratch.
+//
+// Packets mutated by a PacketHandler are processed anew by the
+// associated Machine, as if the packet had always been the mutated
+// one. For example, if HandleForward is invoked with a Packet, and
+// the handler changes the destination IP address to one of the
+// Machine's own IPs, the Machine restarts delivery, but this time
+// going to a local PacketConn (which in turn will invoke HandleIn,
+// since the packet is now destined for local delivery).
+type PacketHandler interface {
+	// HandleIn processes a packet arriving on iif, whose destination
+	// is an IP address owned by the attached Machine. If p is
+	// returned unmodified, the Machine will go on to deliver the
+	// Packet to the appropriate listening PacketConn, if one exists.
+	HandleIn(p *Packet, iif *Interface) *Packet
+	// HandleOut processes a packet about to depart on oif from a
+	// local PacketConn. If p is returned unmodified, the Machine will
+	// transmit the Packet on oif.
+	HandleOut(p *Packet, oif *Interface) *Packet
+	// HandleForward is called when the Machine wants to forward a
+	// packet from iif to oif. If p is returned unmodified, the
+	// Machine will transmit the packet on oif.
+	HandleForward(p *Packet, iif, oif *Interface) *Packet
+}
 
 // A Machine is a representation of an operating system's network
 // stack. It has a network routing table and can have multiple
@@ -235,13 +330,14 @@ type Machine struct {
 	// not be globally unique.
 	Name string
 
-	// HandlePacket, if not nil, is a function that gets invoked for
-	// every packet this Machine receives. Returns a verdict for how
-	// the packet should continue to be handled (or not).
+	// PacketHandler, if not nil, is a PacketHandler implementation
+	// that inspects all packets arriving, departing, or transiting
+	// the Machine. See the definition of the PacketHandler interface
+	// for semantics.
 	//
-	// This can be used to implement things like stateful firewalls
-	// and NAT boxes.
-	HandlePacket PacketHandler
+	// If PacketHandler is nil, the machine allows all inbound
+	// traffic, all outbound traffic, and drops forwarded packets.
+	PacketHandler PacketHandler
 
 	mu         sync.Mutex
 	interfaces []*Interface
@@ -251,22 +347,42 @@ type Machine struct {
 	conns6 map[netaddr.IPPort]*conn // conns that want IPv6 packets
 }
 
-// Inject transmits p from src to dst, without the need for a local socket.
-// It's useful for implementing e.g. NAT boxes that need to mangle IPs.
-func (m *Machine) Inject(p []byte, dst, src netaddr.IPPort) error {
-	trace(p, "mach=%s src=%s dst=%s packet injected", m.Name, src, dst)
-	_, err := m.writePacket(p, dst, src)
-	return err
+func (m *Machine) isLocalIP(ip netaddr.IP) bool {
+	m.mu.Lock()
+	defer m.mu.Unlock()
+	for _, intf := range m.interfaces {
+		for _, iip := range intf.ips {
+			if ip == iip {
+				return true
+			}
+		}
+	}
+	return false
 }
 
-func (m *Machine) deliverIncomingPacket(p []byte, dst, src netaddr.IPPort) {
+func (m *Machine) deliverIncomingPacket(p *Packet, iface *Interface) {
+	p.setLocator("mach=%s if=%s", m.Name, iface.name)
+
+	if m.isLocalIP(p.Dst.IP) {
+		m.deliverLocalPacket(p, iface)
+	} else {
+		m.forwardPacket(p, iface)
+	}
+}
+
+func (m *Machine) deliverLocalPacket(p *Packet, iface *Interface) {
 	// TODO: can't hold lock while handling packet. This is safe as
 	// long as you set HandlePacket before traffic starts flowing.
-	if m.HandlePacket != nil {
-		verdict := m.HandlePacket(p, dst, src)
-		trace(p, "mach=%s src=%v dst=%v packethandler verdict=%s", m.Name, src, dst, verdict)
-		if verdict == Drop {
-			// Custom packet handler ate the packet, we're done.
+	if m.PacketHandler != nil {
+		p2 := m.PacketHandler.HandleIn(p.Clone(), iface)
+		if p2 == nil {
+			// Packet dropped, nothing left to do.
+			return
+		}
+		if !p.Equivalent(p2) {
+			// Restart delivery, this packet might be a forward packet
+			// now.
+			m.deliverIncomingPacket(p2, iface)
 			return
 		}
 	}
@@ -275,13 +391,13 @@ func (m *Machine) deliverIncomingPacket(p []byte, dst, src netaddr.IPPort) {
 	defer m.mu.Unlock()
 
 	conns := m.conns4
-	if dst.IP.Is6() {
+	if p.Dst.IP.Is6() {
 		conns = m.conns6
 	}
 	possibleDsts := []netaddr.IPPort{
-		dst,
-		netaddr.IPPort{IP: v6unspec, Port: dst.Port},
-		netaddr.IPPort{IP: v4unspec, Port: dst.Port},
+		p.Dst,
+		netaddr.IPPort{IP: v6unspec, Port: p.Dst.Port},
+		netaddr.IPPort{IP: v4unspec, Port: p.Dst.Port},
 	}
 	for _, dest := range possibleDsts {
 		c, ok := conns[dest]
@@ -289,15 +405,44 @@ func (m *Machine) deliverIncomingPacket(p []byte, dst, src netaddr.IPPort) {
 			continue
 		}
 		select {
-		case c.in <- incomingPacket{src: src, p: p}:
-			trace(p, "mach=%s src=%v dst=%v queued to conn", m.Name, src, dst)
+		case c.in <- p:
+			p.Trace("queued to conn")
 		default:
-			trace(p, "mach=%s src=%v dst=%v dropped, queue overflow", m.Name, src, dst)
+			p.Trace("dropped, queue overflow")
 			// Queue overflow. Just drop it.
 		}
 		return
 	}
-	trace(p, "mach=%s src=%v dst=%v dropped, no listening conn", m.Name, src, dst)
+	p.Trace("dropped, no listening conn")
+}
+
+func (m *Machine) forwardPacket(p *Packet, iif *Interface) {
+	oif, err := m.interfaceForIP(p.Dst.IP)
+	if err != nil {
+		p.Trace("%v", err)
+		return
+	}
+
+	if m.PacketHandler == nil {
+		// Forwarding not allowed by default
+		p.Trace("drop, forwarding not allowed")
+		return
+	}
+	p2 := m.PacketHandler.HandleForward(p.Clone(), iif, oif)
+	if p2 == nil {
+		p.Trace("drop")
+		// Packet dropped, done.
+		return
+	}
+	if !p.Equivalent(p2) {
+		// Packet changed, restart delivery.
+		p2.Trace("PacketHandler mutated packet")
+		m.deliverIncomingPacket(p2, iif)
+		return
+	}
+
+	p.Trace("-> net=%s oif=%s", oif.net.Name, oif)
+	oif.net.write(p)
 }
 
 func unspecOf(ip netaddr.IP) netaddr.IP {
@@ -316,13 +461,14 @@ func unspecOf(ip netaddr.IP) netaddr.IP {
 // default route.
 func (m *Machine) Attach(interfaceName string, n *Network) *Interface {
 	f := &Interface{
-		net:  n,
-		name: interfaceName,
+		machine: m,
+		net:     n,
+		name:    interfaceName,
 	}
-	if ip := n.allocIPv4(m); !ip.IsZero() {
+	if ip := n.allocIPv4(f); !ip.IsZero() {
 		f.ips = append(f.ips, ip)
 	}
-	if ip := n.allocIPv6(m); !ip.IsZero() {
+	if ip := n.allocIPv6(f); !ip.IsZero() {
 		f.ips = append(f.ips, ip)
 	}
 
@@ -366,38 +512,56 @@ var (
 	v6unspec = netaddr.IPv6Unspecified()
 )
 
-func (m *Machine) writePacket(p []byte, dst, src netaddr.IPPort) (n int, err error) {
-	iface, err := m.interfaceForIP(dst.IP)
+func (m *Machine) writePacket(p *Packet) (n int, err error) {
+	p.setLocator("mach=%s", m.Name)
+
+	iface, err := m.interfaceForIP(p.Dst.IP)
 	if err != nil {
-		trace(p, "%v", err)
+		p.Trace("%v", err)
 		return 0, err
 	}
-	origSrcIP := src.IP
+	origSrcIP := p.Src.IP
 	switch {
-	case src.IP == v4unspec:
-		src.IP = iface.V4()
-	case src.IP == v6unspec:
+	case p.Src.IP == v4unspec:
+		p.Trace("assigning srcIP=%s", iface.V4())
+		p.Src.IP = iface.V4()
+	case p.Src.IP == v6unspec:
 		// v6unspec in Go means "any src, but match address families"
-		if dst.IP.Is6() {
-			src.IP = iface.V6()
-		} else if dst.IP.Is4() {
-			src.IP = iface.V4()
+		if p.Dst.IP.Is6() {
+			p.Trace("assigning srcIP=%s", iface.V6())
+			p.Src.IP = iface.V6()
+		} else if p.Dst.IP.Is4() {
+			p.Trace("assigning srcIP=%s", iface.V4())
+			p.Src.IP = iface.V4()
 		}
 	default:
-		if !iface.Contains(src.IP) {
-			err := fmt.Errorf("can't send to %v with src %v on interface %v", dst.IP, src.IP, iface)
-			trace(p, "%v", err)
+		if !iface.Contains(p.Src.IP) {
+			err := fmt.Errorf("can't send to %v with src %v on interface %v", p.Dst.IP, p.Src.IP, iface)
+			p.Trace("%v", err)
 			return 0, err
 		}
 	}
-	if src.IP.IsZero() {
+	if p.Src.IP.IsZero() {
 		err := fmt.Errorf("no matching address for address family for %v", origSrcIP)
-		trace(p, "%v", err)
+		p.Trace("%v", err)
 		return 0, err
 	}
 
-	trace(p, "mach=%s src=%s dst=%s -> net=%s", m.Name, src, dst, iface.net.Name)
-	return iface.net.write(p, dst, src)
+	if m.PacketHandler != nil {
+		p2 := m.PacketHandler.HandleOut(p.Clone(), iface)
+		if p2 == nil {
+			// Packet dropped, done.
+			return len(p.Payload), nil
+		}
+		if !p.Equivalent(p2) {
+			// Restart transmission, src may have changed weirdly
+			m.writePacket(p2)
+			return
+		}
+	}
+
+	p.Trace("-> net=%s if=%s", iface.net.Name, iface)
+	return iface.net.write(p)
 }
 
 func (m *Machine) interfaceForIP(ip netaddr.IP) (*Interface, error) {
@@ -424,6 +588,32 @@ func (m *Machine) hasv6() bool {
 	return false
 }
 
+func (m *Machine) pickEphemPort() (port uint16, err error) {
+	m.mu.Lock()
+	defer m.mu.Unlock()
+	for tries := 0; tries < 500; tries++ {
+		port := uint16(rand.Intn(32<<10) + 32<<10)
+		if !m.portInUseLocked(port) {
+			return port, nil
+		}
+	}
+	return 0, errors.New("failed to find an ephemeral port")
+}
+
+func (m *Machine) portInUseLocked(port uint16) bool {
+	for ipp := range m.conns4 {
+		if ipp.Port == port {
+			return true
+		}
+	}
+	for ipp := range m.conns6 {
+		if ipp.Port == port {
+			return true
+		}
+	}
+	return false
+}
+
 func (m *Machine) registerConn4(c *conn) error {
 	m.mu.Lock()
 	defer m.mu.Unlock()
@@ -467,7 +657,7 @@ func registerConn(conns *map[netaddr.IPPort]*conn, c *conn) error {
 
 func (m *Machine) AddNetwork(n *Network) {}
 
-func (m *Machine) ListenPacket(network, address string) (net.PacketConn, error) {
+func (m *Machine) ListenPacket(ctx context.Context, network, address string) (net.PacketConn, error) {
 	// if udp4, udp6, etc... look at address IP vs unspec
 	var (
 		fam uint8
@@ -496,18 +686,34 @@ func (m *Machine) ListenPacket(network, address string) (net.PacketConn, error)
 		if err != nil {
 			return nil, err
 		}
+		if fam == 0 && (ip != v4unspec && ip != v6unspec) {
+			// We got an explicit IP address, need to switch the
+			// family to the right one.
+			if ip.Is4() {
+				fam = 4
+			} else {
+				fam = 6
+			}
+		}
 	}
-	port, err := strconv.ParseUint(portStr, 10, 16)
+	porti, err := strconv.ParseUint(portStr, 10, 16)
 	if err != nil {
 		return nil, err
 	}
-	ipp := netaddr.IPPort{IP: ip, Port: uint16(port)}
+	port := uint16(porti)
+	if port == 0 {
+		port, err = m.pickEphemPort()
+		if err != nil {
+			return nil, nil
+		}
+	}
+	ipp := netaddr.IPPort{IP: ip, Port: port}
 
 	c := &conn{
 		m:   m,
 		fam: fam,
 		ipp: ipp,
-		in:  make(chan incomingPacket, 100), // arbitrary
+		in:  make(chan *Packet, 100), // arbitrary
 	}
 	switch c.fam {
 	case 0:
@@ -540,23 +746,24 @@ type conn struct {
 	closed       bool
 	readDeadline time.Time
 	activeReads  map[*activeRead]bool
-	in           chan incomingPacket
-}
-
-type incomingPacket struct {
-	p   []byte
-	src netaddr.IPPort
+	in           chan *Packet
 }
 
 type activeRead struct {
 	cancel context.CancelFunc
 }
 
-// readDeadlineExceeded reports whether the read deadline is set and has already passed.
-func (c *conn) readDeadlineExceeded() bool {
+// canRead reports whether we can do a read.
+func (c *conn) canRead() error {
 	c.mu.Lock()
 	defer c.mu.Unlock()
-	return !c.readDeadline.IsZero() && c.readDeadline.Before(time.Now())
+	if c.closed {
+		return errors.New("closed network connection") // sadface: magic string used by other; don't change
+	}
+	if !c.readDeadline.IsZero() && c.readDeadline.Before(time.Now()) {
+		return errors.New("read deadline exceeded")
+	}
+	return nil
 }
 
 func (c *conn) registerActiveRead(ar *activeRead, active bool) {
@@ -609,8 +816,8 @@ func (c *conn) ReadFrom(p []byte) (n int, addr net.Addr, err error) {
 
 	ar := &activeRead{cancel: cancel}
 
-	if c.readDeadlineExceeded() {
-		return 0, nil, context.DeadlineExceeded
+	if err := c.canRead(); err != nil {
+		return 0, nil, err
 	}
 
 	c.registerActiveRead(ar, true)
@@ -618,9 +825,9 @@ func (c *conn) ReadFrom(p []byte) (n int, addr net.Addr, err error) {
 
 	select {
 	case pkt := <-c.in:
-		n = copy(p, pkt.p)
-		trace(pkt.p, "mach=%s src=%s PacketConn.ReadFrom", c.m.Name, pkt.src)
-		return n, pkt.src.UDPAddr(), nil
+		n = copy(p, pkt.Payload)
+		pkt.Trace("PacketConn.ReadFrom")
+		return n, pkt.Src.UDPAddr(), nil
 	case <-ctx.Done():
 		return 0, nil, context.DeadlineExceeded
 	}
@@ -631,7 +838,14 @@ func (c *conn) WriteTo(p []byte, addr net.Addr) (n int, err error) {
 	if err != nil {
 		return 0, fmt.Errorf("bogus addr %T %q", addr, addr.String())
 	}
-	return c.m.writePacket(p, ipp, c.ipp)
+	pkt := &Packet{
+		Src:     c.ipp,
+		Dst:     ipp,
+		Payload: append([]byte(nil), p...),
+	}
+	pkt.setLocator("mach=%s", c.m.Name)
+	pkt.Trace("PacketConn.WriteTo")
+	return c.m.writePacket(pkt)
 }
 
 func (c *conn) SetDeadline(t time.Time) error {

tstest/natlab/natlab_test.go

@@ -5,17 +5,20 @@
 package natlab
 
 import (
+	"context"
 	"fmt"
 	"testing"
+	"time"
 
 	"inet.af/netaddr"
+	"tailscale.com/tstest"
 )
 
 func TestAllocIPs(t *testing.T) {
 	n := NewInternet()
 	saw := map[netaddr.IP]bool{}
 	for i := 0; i < 255; i++ {
-		for _, f := range []func(*Machine) netaddr.IP{n.allocIPv4, n.allocIPv6} {
+		for _, f := range []func(*Interface) netaddr.IP{n.allocIPv4, n.allocIPv6} {
 			ip := f(nil)
 			if saw[ip] {
 				t.Fatalf("got duplicate %v", ip)
@@ -49,11 +52,12 @@ func TestSendPacket(t *testing.T) {
 	fooAddr := netaddr.IPPort{IP: ifFoo.V4(), Port: 123}
 	barAddr := netaddr.IPPort{IP: ifBar.V4(), Port: 456}
 
-	fooPC, err := foo.ListenPacket("udp4", fooAddr.String())
+	ctx := context.Background()
+	fooPC, err := foo.ListenPacket(ctx, "udp4", fooAddr.String())
 	if err != nil {
 		t.Fatal(err)
 	}
-	barPC, err := bar.ListenPacket("udp4", barAddr.String())
+	barPC, err := bar.ListenPacket(ctx, "udp4", barAddr.String())
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -93,15 +97,16 @@ func TestMultiNetwork(t *testing.T) {
 	ifNATLAN := nat.Attach("ethlan", lan)
 	ifServer := server.Attach("eth0", internet)
 
-	clientPC, err := client.ListenPacket("udp", ":123")
+	ctx := context.Background()
+	clientPC, err := client.ListenPacket(ctx, "udp", ":123")
 	if err != nil {
 		t.Fatal(err)
 	}
-	natPC, err := nat.ListenPacket("udp", ":456")
+	natPC, err := nat.ListenPacket(ctx, "udp", ":456")
 	if err != nil {
 		t.Fatal(err)
 	}
-	serverPC, err := server.ListenPacket("udp", ":789")
+	serverPC, err := server.ListenPacket(ctx, "udp", ":789")
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -143,6 +148,38 @@ func TestMultiNetwork(t *testing.T) {
 	}
 }
 
+type trivialNAT struct {
+	clientIP     netaddr.IP
+	lanIf, wanIf *Interface
+}
+
+func (n *trivialNAT) HandleIn(p *Packet, iface *Interface) *Packet {
+	if iface == n.wanIf && p.Dst.IP == n.wanIf.V4() {
+		p.Dst.IP = n.clientIP
+	}
+	return p
+}
+
+func (n trivialNAT) HandleOut(p *Packet, iface *Interface) *Packet {
+	return p
+}
+
+func (n *trivialNAT) HandleForward(p *Packet, iif, oif *Interface) *Packet {
+	// Outbound from LAN -> apply NAT, continue
+	if iif == n.lanIf && oif == n.wanIf {
+		if p.Src.IP == n.clientIP {
+			p.Src.IP = n.wanIf.V4()
+		}
+		return p
+	}
+	// Return traffic to LAN, allow if right dst.
+	if iif == n.wanIf && oif == n.lanIf && p.Dst.IP == n.clientIP {
+		return p
+	}
+	// Else drop.
+	return nil
+}
+
 func TestPacketHandler(t *testing.T) {
 	lan := &Network{
 		Name:    "lan",
@@ -153,42 +190,27 @@ func TestPacketHandler(t *testing.T) {
 
 	client := &Machine{Name: "client"}
 	nat := &Machine{Name: "nat"}
-	lan.SetDefaultGateway(nat)
 	server := &Machine{Name: "server"}
 
 	ifClient := client.Attach("eth0", lan)
 	ifNATWAN := nat.Attach("wan", internet)
-	_ = nat.Attach("lan", lan)
+	ifNATLAN := nat.Attach("lan", lan)
 	ifServer := server.Attach("server", internet)
 
-	// This HandlePacket implements a basic (some might say "broken")
-	// 1:1 NAT, where client's IP gets replaced with the NAT's WAN IP,
-	// and vice versa.
-	//
-	// This NAT is not suitable for actual use, since it doesn't do
-	// port remappings or any other things that NATs usually to. But
-	// it works as a demonstrator for a single client behind the NAT,
-	// where the NAT box itself doesn't also make PacketConns.
-	nat.HandlePacket = func(p []byte, dst, src netaddr.IPPort) PacketVerdict {
-		switch {
-		case dst.IP.Is6():
-			return Continue // no NAT for ipv6
-		case src.IP == ifClient.V4():
-			nat.Inject(p, dst, netaddr.IPPort{IP: ifNATWAN.V4(), Port: src.Port})
-			return Drop
-		case dst.IP == ifNATWAN.V4():
-			nat.Inject(p, netaddr.IPPort{IP: ifClient.V4(), Port: dst.Port}, src)
-			return Drop
-		default:
-			return Continue
-		}
+	lan.SetDefaultGateway(ifNATLAN)
+
+	nat.PacketHandler = &trivialNAT{
+		clientIP: ifClient.V4(),
+		lanIf:    ifNATLAN,
+		wanIf:    ifNATWAN,
 	}
 
-	clientPC, err := client.ListenPacket("udp4", ":123")
+	ctx := context.Background()
+	clientPC, err := client.ListenPacket(ctx, "udp4", ":123")
 	if err != nil {
 		t.Fatal(err)
 	}
-	serverPC, err := server.ListenPacket("udp4", ":456")
+	serverPC, err := server.ListenPacket(ctx, "udp4", ":456")
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -212,5 +234,276 @@ func TestPacketHandler(t *testing.T) {
 	if addr.String() != mappedAddr.String() {
 		t.Errorf("addr = %q; want %q", addr, mappedAddr)
 	}
-
+}
+
+func TestFirewall(t *testing.T) {
+	wan := NewInternet()
+	lan := &Network{
+		Name:    "lan",
+		Prefix4: mustPrefix("10.0.0.0/8"),
+	}
+	m := &Machine{Name: "test"}
+	trust := m.Attach("trust", lan)
+	untrust := m.Attach("untrust", wan)
+
+	client := ipp("192.168.0.2:1234")
+	serverA := ipp("2.2.2.2:5678")
+	serverB1 := ipp("7.7.7.7:9012")
+	serverB2 := ipp("7.7.7.7:3456")
+
+	t.Run("ip_port_dependent", func(t *testing.T) {
+		f := &Firewall{
+			TrustedInterface: trust,
+			SessionTimeout:   30 * time.Second,
+			Type:             AddressAndPortDependentFirewall,
+		}
+		testFirewall(t, f, []fwTest{
+			// client -> A authorizes A -> client
+			{trust, untrust, client, serverA, true},
+			{untrust, trust, serverA, client, true},
+			{untrust, trust, serverA, client, true},
+
+			// B1 -> client fails until client -> B1
+			{untrust, trust, serverB1, client, false},
+			{trust, untrust, client, serverB1, true},
+			{untrust, trust, serverB1, client, true},
+
+			// B2 -> client still fails
+			{untrust, trust, serverB2, client, false},
+		})
+	})
+	t.Run("ip_dependent", func(t *testing.T) {
+		f := &Firewall{
+			TrustedInterface: trust,
+			SessionTimeout:   30 * time.Second,
+			Type:             AddressDependentFirewall,
+		}
+		testFirewall(t, f, []fwTest{
+			// client -> A authorizes A -> client
+			{trust, untrust, client, serverA, true},
+			{untrust, trust, serverA, client, true},
+			{untrust, trust, serverA, client, true},
+
+			// B1 -> client fails until client -> B1
+			{untrust, trust, serverB1, client, false},
+			{trust, untrust, client, serverB1, true},
+			{untrust, trust, serverB1, client, true},
+
+			// B2 -> client also works now
+			{untrust, trust, serverB2, client, true},
+		})
+	})
+	t.Run("endpoint_independent", func(t *testing.T) {
+		f := &Firewall{
+			TrustedInterface: trust,
+			SessionTimeout:   30 * time.Second,
+			Type:             EndpointIndependentFirewall,
+		}
+		testFirewall(t, f, []fwTest{
+			// client -> A authorizes A -> client
+			{trust, untrust, client, serverA, true},
+			{untrust, trust, serverA, client, true},
+			{untrust, trust, serverA, client, true},
+
+			// B1 -> client also works
+			{untrust, trust, serverB1, client, true},
+
+			// B2 -> client also works
+			{untrust, trust, serverB2, client, true},
+		})
+	})
+}
+
+type fwTest struct {
+	iif, oif *Interface
+	src, dst netaddr.IPPort
+	ok       bool
+}
+
+func testFirewall(t *testing.T, f *Firewall, tests []fwTest) {
+	t.Helper()
+	clock := &tstest.Clock{}
+	f.TimeNow = clock.Now
+	for _, test := range tests {
+		clock.Advance(time.Second)
+		p := &Packet{
+			Src:     test.src,
+			Dst:     test.dst,
+			Payload: []byte{},
+		}
+		got := f.HandleForward(p, test.iif, test.oif)
+		gotOK := got != nil
+		if gotOK != test.ok {
+			t.Errorf("iif=%s oif=%s src=%s dst=%s got ok=%v, want ok=%v", test.iif, test.oif, test.src, test.dst, gotOK, test.ok)
+		}
+	}
+}
+
+func ipp(str string) netaddr.IPPort {
+	ipp, err := netaddr.ParseIPPort(str)
+	if err != nil {
+		panic(err)
+	}
+	return ipp
+}
+
+func TestNAT(t *testing.T) {
+	internet := NewInternet()
+	lan := &Network{
+		Name:    "LAN",
+		Prefix4: mustPrefix("192.168.0.0/24"),
+	}
+	m := &Machine{Name: "NAT"}
+	wanIf := m.Attach("wan", internet)
+	lanIf := m.Attach("lan", lan)
+
+	t.Run("endpoint_independent_mapping", func(t *testing.T) {
+		n := &SNAT44{
+			Machine:           m,
+			ExternalInterface: wanIf,
+			Type:              EndpointIndependentNAT,
+			Firewall: &Firewall{
+				TrustedInterface: lanIf,
+			},
+		}
+		testNAT(t, n, lanIf, wanIf, []natTest{
+			{
+				src:            ipp("192.168.0.20:1234"),
+				dst:            ipp("2.2.2.2:5678"),
+				wantNewMapping: true,
+			},
+			{
+				src:            ipp("192.168.0.20:1234"),
+				dst:            ipp("7.7.7.7:9012"),
+				wantNewMapping: false,
+			},
+			{
+				src:            ipp("192.168.0.20:2345"),
+				dst:            ipp("7.7.7.7:9012"),
+				wantNewMapping: true,
+			},
+		})
+	})
+
+	t.Run("address_dependent_mapping", func(t *testing.T) {
+		n := &SNAT44{
+			Machine:           m,
+			ExternalInterface: wanIf,
+			Type:              AddressDependentNAT,
+			Firewall: &Firewall{
+				TrustedInterface: lanIf,
+			},
+		}
+		testNAT(t, n, lanIf, wanIf, []natTest{
+			{
+				src:            ipp("192.168.0.20:1234"),
+				dst:            ipp("2.2.2.2:5678"),
+				wantNewMapping: true,
+			},
+			{
+				src:            ipp("192.168.0.20:1234"),
+				dst:            ipp("2.2.2.2:9012"),
+				wantNewMapping: false,
+			},
+			{
+				src:            ipp("192.168.0.20:1234"),
+				dst:            ipp("7.7.7.7:9012"),
+				wantNewMapping: true,
+			},
+			{
+				src:            ipp("192.168.0.20:1234"),
+				dst:            ipp("7.7.7.7:1234"),
+				wantNewMapping: false,
+			},
+		})
+	})
+
+	t.Run("address_and_port_dependent_mapping", func(t *testing.T) {
+		n := &SNAT44{
+			Machine:           m,
+			ExternalInterface: wanIf,
+			Type:              AddressAndPortDependentNAT,
+			Firewall: &Firewall{
+				TrustedInterface: lanIf,
+			},
+		}
+		testNAT(t, n, lanIf, wanIf, []natTest{
+			{
+				src:            ipp("192.168.0.20:1234"),
+				dst:            ipp("2.2.2.2:5678"),
+				wantNewMapping: true,
+			},
+			{
+				src:            ipp("192.168.0.20:1234"),
+				dst:            ipp("2.2.2.2:9012"),
+				wantNewMapping: true,
+			},
+			{
+				src:            ipp("192.168.0.20:1234"),
+				dst:            ipp("7.7.7.7:9012"),
+				wantNewMapping: true,
+			},
+			{
+				src:            ipp("192.168.0.20:1234"),
+				dst:            ipp("7.7.7.7:1234"),
+				wantNewMapping: true,
+			},
+		})
+	})
+}
+
+type natTest struct {
+	src, dst       netaddr.IPPort
+	wantNewMapping bool
+}
+
+func testNAT(t *testing.T, n *SNAT44, lanIf, wanIf *Interface, tests []natTest) {
+	clock := &tstest.Clock{}
+	n.TimeNow = clock.Now
+
+	mappings := map[netaddr.IPPort]bool{}
+	for _, test := range tests {
+		clock.Advance(time.Second)
+		p := &Packet{
+			Src:     test.src,
+			Dst:     test.dst,
+			Payload: []byte("foo"),
+		}
+		gotPacket := n.HandleForward(p.Clone(), lanIf, wanIf)
+		if gotPacket == nil {
+			t.Errorf("n.HandleForward(%v) dropped packet", p)
+			continue
+		}
+
+		if gotPacket.Dst != p.Dst {
+			t.Errorf("n.HandleForward(%v) mutated dest ip:port, got %v", p, gotPacket.Dst)
+		}
+		gotNewMapping := !mappings[gotPacket.Src]
+		if gotNewMapping != test.wantNewMapping {
+			t.Errorf("n.HandleForward(%v) mapping was new=%v, want %v", p, gotNewMapping, test.wantNewMapping)
+		}
+		mappings[gotPacket.Src] = true
+
+		// Check that the return path works and translates back
+		// correctly.
+		clock.Advance(time.Second)
+		p2 := &Packet{
+			Src:     test.dst,
+			Dst:     gotPacket.Src,
+			Payload: []byte("bar"),
+		}
+		gotPacket2 := n.HandleIn(p2.Clone(), wanIf)
+
+		if gotPacket2 == nil {
+			t.Errorf("return packet was dropped")
+			continue
+		}
+
+		if gotPacket2.Src != test.dst {
+			t.Errorf("return packet has src=%v, want %v", gotPacket2.Src, test.dst)
+		}
+		if gotPacket2.Dst != test.src {
+			t.Errorf("return packet has dst=%v, want %v", gotPacket2.Dst, test.src)
+		}
+	}
 }

tsweb/jsonhandler.go

@@ -32,8 +32,8 @@ func responseError(e string) *response {
 
 func writeResponse(w http.ResponseWriter, s int, resp *response) {
 	b, _ := json.Marshal(resp)
-	w.WriteHeader(s)
 	w.Header().Set("Content-Type", "application/json")
+	w.WriteHeader(s)
 	w.Write(b)
 }
 

tsweb/jsonhandler_test.go

@@ -44,6 +44,10 @@ func TestNewJSONHandler(t *testing.T) {
 			t.Fatalf("wrong status: %s %s", d.Status, status)
 		}
 
+		if w.Header().Get("Content-Type") != "application/json" {
+			t.Fatalf("wrong content type: %s", w.Header().Get("Content-Type"))
+		}
+
 		return d
 	}
 

types/logger/logger_test.go

@@ -60,7 +60,7 @@ func TestRateLimiter(t *testing.T) {
 		lg("templated format string no. %d", i)
 		if i == 4 {
 			lg("Make sure this string makes it through the rest (that are blocked) %d", i)
-			prefixed = WithPrefix(lg, string('0'+i))
+			prefixed = WithPrefix(lg, string(rune('0'+i)))
 			prefixed(" shouldn't get filtered.")
 		}
 	}

types/nettype/nettype.go

@@ -0,0 +1,25 @@
+// Copyright (c) 2020 Tailscale Inc & AUTHORS All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+// Package nettype defines an interface that doesn't exist in the Go net package.
+package nettype
+
+import (
+	"context"
+	"net"
+)
+
+// PacketListener defines the ListenPacket method as implemented
+// by net.ListenConfig, net.ListenPacket, and tstest/natlab.
+type PacketListener interface {
+	ListenPacket(ctx context.Context, network, address string) (net.PacketConn, error)
+}
+
+// Std implements PacketListener using the Go net package's ListenPacket func.
+type Std struct{}
+
+func (Std) ListenPacket(ctx context.Context, network, address string) (net.PacketConn, error) {
+	var conf net.ListenConfig
+	return conf.ListenPacket(ctx, network, address)
+}

util/lineread/lineread.go

@@ -0,0 +1,33 @@
+// Copyright (c) 2020 Tailscale Inc & AUTHORS All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+// Package lineread reads lines from files. It's not fancy, but it got repetitive.
+package lineread
+
+import (
+	"bufio"
+	"io"
+	"os"
+)
+
+// File opens name and calls fn for each line. It returns an error if the Open failed
+// or once fn returns an error.
+func File(name string, fn func(line []byte) error) error {
+	f, err := os.Open(name)
+	if err != nil {
+		return err
+	}
+	defer f.Close()
+	return Reader(f, fn)
+}
+
+func Reader(r io.Reader, fn func(line []byte) error) error {
+	bs := bufio.NewScanner(r)
+	for bs.Scan() {
+		if err := fn(bs.Bytes()); err != nil {
+			return err
+		}
+	}
+	return bs.Err()
+}

version/cmdname.go

@@ -9,6 +9,7 @@ package version
 import (
 	"os"
 	"path"
+	"path/filepath"
 	"strings"
 
 	"rsc.io/goversion/version"
@@ -22,22 +23,27 @@ func CmdName() string {
 	if err != nil {
 		return "cmd"
 	}
+
+	// fallbackName, the lowercase basename of the executable, is what we return if
+	// we can't find the Go module metadata embedded in the file.
+	fallbackName := filepath.Base(strings.TrimSuffix(strings.ToLower(e), ".exe"))
+
 	var ret string
 	v, err := version.ReadExe(e)
 	if err != nil {
-		ret = strings.TrimSuffix(strings.ToLower(e), ".exe")
-	} else {
-		// v is like:
-		// "path\ttailscale.com/cmd/tailscale\nmod\ttailscale.com\t(devel)\t\ndep\tgithub.com/apenwarr/fixconsole\tv0.0.0-20191012055117-5a9f6489cc29\th1:muXWUcay7DDy1/hEQWrYlBy+g0EuwT70sBHg65SeUc4=\ndep\tgithub....
-		for _, line := range strings.Split(v.ModuleInfo, "\n") {
-			if strings.HasPrefix(line, "path\t") {
-				ret = path.Base(strings.TrimPrefix(line, "path\t"))
-				break
-			}
+		return fallbackName
+	}
+	// v is like:
+	// "path\ttailscale.com/cmd/tailscale\nmod\ttailscale.com\t(devel)\t\ndep\tgithub.com/apenwarr/fixconsole\tv0.0.0-20191012055117-5a9f6489cc29\th1:muXWUcay7DDy1/hEQWrYlBy+g0EuwT70sBHg65SeUc4=\ndep\tgithub....
+	for _, line := range strings.Split(v.ModuleInfo, "\n") {
+		if strings.HasPrefix(line, "path\t") {
+			goPkg := strings.TrimPrefix(line, "path\t") // like "tailscale.com/cmd/tailscale"
+			ret = path.Base(goPkg)                      // goPkg is always forward slashes; use path, not filepath
+			break
 		}
 	}
 	if ret == "" {
-		return "cmd"
+		return fallbackName
 	}
 	return ret
 }

version/describe.txt.do

@@ -1,4 +1,4 @@
-describe=$(cd ../.. && git describe --long)
+describe=$(cd ../.. && git describe --long --abbrev=9)
 echo "$describe" >$3
 redo-always
 redo-stamp <$3

version/mkversion.sh

@@ -5,89 +5,107 @@ set -eu
 mode=$1
 describe=$2
 
-long() {
-    ver="${describe#v}"
-    stem="${ver%%-*}"
-    case "$stem" in
-        *.*.*)
-            # Full SemVer, nothing to do.
-            semver="${stem}"
+# Git describe output overall looks like
+# MAJOR.MINOR.PATCH-NUMCOMMITS-GITHASH. Depending on the tag being
+# described and the state of the repo, ver can be missing PATCH,
+# NUMCOMMITS, or both.
+#
+# Valid values look like: 1.2.3-1234-abcdef, 0.98-1234-abcdef,
+# 1.0.0-abcdef, 0.99-abcdef.
+ver="${describe#v}"
+stem="${ver%%-*}" # Just the semver-ish bit e.g. 1.2.3, 0.98
+suffix="${ver#$stem}" # The rest e.g. -23-abcdef, -abcdef
+
+# Normalize the stem into a full major.minor.patch semver. We might
+# not use all those pieces depending on what kind of version we're
+# making, but it's good to have them all on hand.
+case "$stem" in
+    *.*.*)
+        # Full SemVer, nothing to do
+        stem="$stem"
         ;;
-        *.*)
-            # Old style major.minor, add a .0
-            semver="${stem}.0"
-            ;;
-        *)
-            echo "Unparseable version $stem" >&2
-            exit 1
-            ;;
-    esac
-    suffix="${ver#$stem}"
-    case "$suffix" in
-        -*-*)
-            # Has a change count in addition to the commit hash.
+    *.*)
+        # Old style major.minor, add a .0
+        stem="${stem}.0"
+        ;;
+    *)
+        echo "Unparseable version $stem" >&2
+        exit 1
+        ;;
+esac
+major=$(echo "$stem" | cut -f1 -d.)
+minor=$(echo "$stem" | cut -f2 -d.)
+patch=$(echo "$stem" | cut -f3 -d.)
+
+# Extract the change count and git ID from the suffix.
+case "$suffix" in
+    -*-*)
+        # Has both a change count and a commit hash.
+        changecount=$(echo "$suffix" | cut -f2 -d-)
+        githash=$(echo "$suffix" | cut -f3 -d-)
+        ;;
+    -*)
+        # Git hash only, change count is zero.
+        changecount="0"
+        githash=$(echo "$suffix" | cut -f2 -d-)
+        ;;
+    *)
+        echo "Unparseable version suffix $suffix" >&2
+        exit 1
+        ;;
+esac
+
+# Validate that the version data makes sense. Rules:
+#  - Odd number minors are unstable. Patch must be 0, and gets
+#    replaced by changecount.
+#  - Even number minors are stable. Changecount must be 0, and
+#    gets removed.
+#
+# After this section, we only use major/minor/patch, which have been
+# tweaked as needed.
+if expr "$minor" : "[0-9]*[13579]$" >/dev/null; then
+    # Unstable
+    if [ "$patch" != "0" ]; then
+        # This is a fatal error, because a non-zero patch number
+        # indicates that we created an unstable git tag in violation
+        # of our versioning policy, and we want to blow up loudly to
+        # get that fixed.
+        echo "Unstable release $describe has a non-zero patch number, which is not allowed" >&2
+        exit 1
+    fi
+    patch="$changecount"
+else
+    # Stable
+    if [ "$changecount" != "0" ]; then
+        # This is a commit that's sitting between two stable
+        # releases. We never want to release such a commit to the
+        # pbulic, but it's useful to be able to build it for
+        # debugging. Just force the version to 0.0.0, so that we're
+        # forced to rely on the git commit hash.
+        major=0
+        minor=0
+        patch=0
+    fi
+fi
+
+case "$1" in
+    long)
+        echo "${major}.${minor}.${patch}-${githash}"
+        ;;
+    short)
+        echo "${major}.${minor}.${patch}"
+        ;;
+    xcode)
+        # CFBundleShortVersionString: the "short name" used in the App
+        # Store.  eg. 0.92.98
+        echo "VERSION_NAME = ${major}.${minor}.${patch}"
+        # CFBundleVersion: the build number. Needs to be 3 numeric
+        # sections that increment for each release according to SemVer
+        # rules.
+        #
+        # We start counting at 100 because we submitted using raw
+        # build numbers before, and Apple doesn't let you start over.
+        # e.g. 0.98.3 -> 100.98.3
+        echo "VERSION_ID = $((major + 100)).${minor}.${patch}"        
         ;;
-        -*)
-            # Missing change count, add one.
-            suffix="-0${suffix}"
-            ;;
-        *)
-            echo "Unexpected version suffix" >&2
-            exit 1
-    esac
-    echo "${semver}${suffix}"
-}
-
-short() {
-    ver="$(long)"
-    case "$ver" in
-        *-*-*)
-            echo "${ver%-*}"
-            ;;
-        *-*)
-            echo "$ver"
-            ;;
-        *)
-            echo "Long version in invalid format" >&2
-            exit 1
-            ;;
-    esac
-}
-
-xcode() {
-    ver=$(short | sed -e 's/-/./')
-    major=$(echo "$ver" | cut -f1 -d.)
-    minor=$(echo "$ver" | cut -f2 -d.)
-    patch=$(echo "$ver" | cut -f3 -d.)
-    changecount=$(echo "$ver" | cut -f4 -d.)
-
-    # Apple version numbers must be major.minor.patch. We have 4 fields
-    # because we need major.minor.patch for go module compatibility, and
-    # changecount for automatic version numbering of unstable builds. To
-    # resolve this, for Apple builds, we combine changecount into patch:
-    patch=$((patch*10000 + changecount))
-
-    # CFBundleShortVersionString: the "short name" used in the App Store.
-    # eg. 0.92.98
-    echo "VERSION_NAME = $major.$minor.$patch"
-    # CFBundleVersion: the build number. Needs to be 3 numeric sections
-    # that increment for each release according to SemVer rules.
-    #
-    # We start counting at 100 because we submitted using raw build
-    # numbers before, and Apple doesn't let you start over.
-    # e.g. 0.98.3-123 -> 100.98.3123
-    major=$((major + 100))
-    echo "VERSION_ID = $major.$minor.$patch"
-}
-
-case "$mode" in
-    long)
-        long
-    ;;
-    short)
-        short
-    ;;
-    xcode)
-        xcode
-    ;;
 esac

version/mkversion_test.go

@@ -15,42 +15,60 @@ func xcode(short, long string) string {
 	return fmt.Sprintf("VERSION_NAME = %s\nVERSION_ID = %s", short, long)
 }
 
-func mkversion(t *testing.T, mode, in string) string {
+func mkversion(t *testing.T, mode, in string) (string, bool) {
 	t.Helper()
 	bs, err := exec.Command("./mkversion.sh", mode, in).CombinedOutput()
 	if err != nil {
 		t.Logf("mkversion.sh output: %s", string(bs))
-		t.Fatalf("mkversion.sh %s %s: %v", mode, in, err)
+		return "", false
 	}
-	return strings.TrimSpace(string(bs))
+	return strings.TrimSpace(string(bs)), true
 }
 
 func TestMkversion(t *testing.T) {
 	tests := []struct {
 		in    string
+		ok    bool
 		long  string
 		short string
 		xcode string
 	}{
-		{"v0.98-abcdef", "0.98.0-0-abcdef", "0.98.0-0", xcode("0.98.0", "100.98.0")},
-		{"v0.98-123-abcdef", "0.98.0-123-abcdef", "0.98.0-123", xcode("0.98.123", "100.98.123")},
-		{"v0.99.5-123-abcdef", "0.99.5-123-abcdef", "0.99.5-123", xcode("0.99.50123", "100.99.50123")},
-		{"v0.99.5-123-abcdef", "0.99.5-123-abcdef", "0.99.5-123", xcode("0.99.50123", "100.99.50123")},
-		{"v2.3-0-abcdef", "2.3.0-0-abcdef", "2.3.0-0", xcode("2.3.0", "102.3.0")},
-		{"1.2.3-4-abcdef", "1.2.3-4-abcdef", "1.2.3-4", xcode("1.2.30004", "101.2.30004")},
+		{"v0.98-abcdef", true, "0.98.0-abcdef", "0.98.0", xcode("0.98.0", "100.98.0")},
+		{"v0.98.1-abcdef", true, "0.98.1-abcdef", "0.98.1", xcode("0.98.1", "100.98.1")},
+		{"v1.1.0-37-abcdef", true, "1.1.37-abcdef", "1.1.37", xcode("1.1.37", "101.1.37")},
+		{"v1.2.9-abcdef", true, "1.2.9-abcdef", "1.2.9", xcode("1.2.9", "101.2.9")},
+		{"v1.2.9-0-abcdef", true, "1.2.9-abcdef", "1.2.9", xcode("1.2.9", "101.2.9")},
+		{"v1.15.0-129-abcdef", true, "1.15.129-abcdef", "1.15.129", xcode("1.15.129", "101.15.129")},
+
+		{"v0.98-123-abcdef", true, "0.0.0-abcdef", "0.0.0", xcode("0.0.0", "100.0.0")},
+		{"v1.0.0-37-abcdef", true, "0.0.0-abcdef", "0.0.0", xcode("0.0.0", "100.0.0")},
+
+		{"v0.99.5-0-abcdef", false, "", "", ""},   // unstable, patch not allowed
+		{"v0.99.5-123-abcdef", false, "", "", ""}, // unstable, patch not allowed
+		{"v1-abcdef", false, "", "", ""},          // bad semver
+		{"v1.0", false, "", "", ""},               // missing suffix
 	}
 
 	for _, test := range tests {
-		gotlong := mkversion(t, "long", test.in)
-		gotshort := mkversion(t, "short", test.in)
-		gotxcode := mkversion(t, "xcode", test.in)
-		if gotlong != test.long {
+		gotlong, longOK := mkversion(t, "long", test.in)
+		if longOK != test.ok {
+			t.Errorf("mkversion.sh long %q ok=%v, want %v", test.in, longOK, test.ok)
+		}
+		gotshort, shortOK := mkversion(t, "short", test.in)
+		if shortOK != test.ok {
+			t.Errorf("mkversion.sh short %q ok=%v, want %v", test.in, shortOK, test.ok)
+		}
+		gotxcode, xcodeOK := mkversion(t, "xcode", test.in)
+		if xcodeOK != test.ok {
+			t.Errorf("mkversion.sh xcode %q ok=%v, want %v", test.in, xcodeOK, test.ok)
+		}
+		if longOK && gotlong != test.long {
 			t.Errorf("mkversion.sh long %q: got %q, want %q", test.in, gotlong, test.long)
 		}
-		if gotshort != test.short {
+		if shortOK && gotshort != test.short {
 			t.Errorf("mkversion.sh short %q: got %q, want %q", test.in, gotshort, test.short)
 		}
-		if gotxcode != test.xcode {
+		if xcodeOK && gotxcode != test.xcode {
 			t.Errorf("mkversion.sh xcode %q: got %q, want %q", test.in, gotxcode, test.xcode)
 		}
 	}

version/version.go

@@ -7,5 +7,5 @@
 // Package version provides the version that the binary was built at.
 package version
 
-const LONG = "date.20200703"
+const LONG = "date.20200806"
 const SHORT = LONG

wgengine/filter/filter.go

@@ -6,6 +6,7 @@
 package filter
 
 import (
+	"fmt"
 	"sync"
 	"time"
 
@@ -136,9 +137,13 @@ func maybeHexdump(flag RunFlags, b []byte) string {
 var acceptBucket = rate.NewLimiter(rate.Every(10*time.Second), 3)
 var dropBucket = rate.NewLimiter(rate.Every(5*time.Second), 10)
 
-func (f *Filter) logRateLimit(runflags RunFlags, q *packet.ParsedPacket, r Response, why string) {
+func (f *Filter) logRateLimit(runflags RunFlags, q *packet.ParsedPacket, dir direction, r Response, why string) {
 	var verdict string
 
+	if r == Drop && omitDropLogging(q, dir) {
+		return
+	}
+
 	if r == Drop && (runflags&LogDrops) != 0 && dropBucket.Allow() {
 		verdict = "Drop"
 		runflags &= HexdumpDrops
@@ -157,26 +162,28 @@ func (f *Filter) logRateLimit(runflags RunFlags, q *packet.ParsedPacket, r Respo
 
 // RunIn determines whether this node is allowed to receive q from a Tailscale peer.
 func (f *Filter) RunIn(q *packet.ParsedPacket, rf RunFlags) Response {
-	r := f.pre(q, rf)
+	dir := in
+	r := f.pre(q, rf, dir)
 	if r == Accept || r == Drop {
 		// already logged
 		return r
 	}
 
 	r, why := f.runIn(q)
-	f.logRateLimit(rf, q, r, why)
+	f.logRateLimit(rf, q, dir, r, why)
 	return r
 }
 
 // RunOut determines whether this node is allowed to send q to a Tailscale peer.
 func (f *Filter) RunOut(q *packet.ParsedPacket, rf RunFlags) Response {
-	r := f.pre(q, rf)
+	dir := out
+	r := f.pre(q, rf, dir)
 	if r == Drop || r == Accept {
 		// already logged
 		return r
 	}
 	r, why := f.runOut(q)
-	f.logRateLimit(rf, q, r, why)
+	f.logRateLimit(rf, q, dir, r, why)
 	return r
 }
 
@@ -188,6 +195,11 @@ func (f *Filter) runIn(q *packet.ParsedPacket) (r Response, why string) {
 		return Drop, "destination not allowed"
 	}
 
+	if q.IPVersion == 6 {
+		// TODO: support IPv6.
+		return Drop, "no rules matched"
+	}
+
 	switch q.IPProto {
 	case packet.ICMP:
 		if q.IsEchoResponse() || q.IsError() {
@@ -247,30 +259,106 @@ func (f *Filter) runOut(q *packet.ParsedPacket) (r Response, why string) {
 	return Accept, "ok out"
 }
 
-func (f *Filter) pre(q *packet.ParsedPacket, rf RunFlags) Response {
+// direction is whether a packet was flowing in to this machine, or flowing out.
+type direction int
+
+const (
+	in direction = iota
+	out
+)
+
+func (d direction) String() string {
+	switch d {
+	case in:
+		return "in"
+	case out:
+		return "out"
+	default:
+		return fmt.Sprintf("[??dir=%d]", int(d))
+	}
+}
+
+func (f *Filter) pre(q *packet.ParsedPacket, rf RunFlags, dir direction) Response {
 	if len(q.Buffer()) == 0 {
 		// wireguard keepalive packet, always permit.
 		return Accept
 	}
 	if len(q.Buffer()) < 20 {
-		f.logRateLimit(rf, q, Drop, "too short")
+		f.logRateLimit(rf, q, dir, Drop, "too short")
 		return Drop
 	}
 
+	if q.IPVersion == 6 {
+		f.logRateLimit(rf, q, dir, Drop, "ipv6")
+		return Drop
+	}
 	switch q.IPProto {
 	case packet.Unknown:
 		// Unknown packets are dangerous; always drop them.
-		f.logRateLimit(rf, q, Drop, "unknown")
-		return Drop
-	case packet.IPv6:
-		f.logRateLimit(rf, q, Drop, "ipv6")
+		f.logRateLimit(rf, q, dir, Drop, "unknown")
 		return Drop
 	case packet.Fragment:
 		// Fragments after the first always need to be passed through.
 		// Very small fragments are considered Junk by ParsedPacket.
-		f.logRateLimit(rf, q, Accept, "fragment")
+		f.logRateLimit(rf, q, dir, Accept, "fragment")
 		return Accept
 	}
 
 	return noVerdict
 }
+
+const (
+	// ipv6AllRoutersLinkLocal is ff02::2 (All link-local routers)
+	ipv6AllRoutersLinkLocal = "\xff\x02\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x02"
+	// ipv6AllMLDv2CapableRouters is ff02::16 (All MLDv2-capable routers)
+	ipv6AllMLDv2CapableRouters = "\xff\x02\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x16"
+)
+
+// omitDropLogging reports whether packet p, which has already been
+// deemded a packet to Drop, should bypass the [rate-limited] logging.
+// We don't want to log scary & spammy reject warnings for packets that
+// are totally normal, like IPv6 route announcements.
+func omitDropLogging(p *packet.ParsedPacket, dir direction) bool {
+	b := p.Buffer()
+	switch dir {
+	case out:
+		switch p.IPVersion {
+		case 4:
+			// ParsedPacket.Decode zeros out ParsedPacket.IPProtocol for protocols
+			// it doesn't know about, so parse it out ourselves if needed.
+			ipProto := p.IPProto
+			if ipProto == 0 && len(b) > 8 {
+				ipProto = packet.IPProto(b[9])
+			}
+			// Omit logging about outgoing IGMP.
+			if ipProto == packet.IGMP {
+				return true
+			}
+		case 6:
+			if len(b) < 40 {
+				return false
+			}
+			src, dst := b[8:8+16], b[24:24+16]
+			// Omit logging for outgoing IPv6 ICMP-v6 queries to ff02::2,
+			// as sent by the OS, looking for routers.
+			if p.IPProto == packet.ICMPv6 {
+				if isLinkLocalV6(src) && string(dst) == ipv6AllRoutersLinkLocal {
+					return true
+				}
+			}
+			if string(dst) == ipv6AllMLDv2CapableRouters {
+				return true
+			}
+			// Actually, just catch all multicast.
+			if dst[0] == 0xff {
+				return true
+			}
+		}
+	}
+	return false
+}
+
+// isLinkLocalV6 reports whether src is in fe80::/10.
+func isLinkLocalV6(src []byte) bool {
+	return len(src) == 16 && src[0] == 0xfe && src[1]>>6 == 0x80>>6
+}

wgengine/filter/filter_test.go

@@ -6,7 +6,9 @@ package filter
 
 import (
 	"encoding/binary"
+	"encoding/hex"
 	"encoding/json"
+	"strings"
 	"testing"
 
 	"tailscale.com/types/logger"
@@ -219,7 +221,7 @@ func TestPreFilter(t *testing.T) {
 	for _, testPacket := range packets {
 		p := &ParsedPacket{}
 		p.Decode(testPacket.b)
-		got := f.pre(p, LogDrops|LogAccepts)
+		got := f.pre(p, LogDrops|LogAccepts, in)
 		if got != testPacket.want {
 			t.Errorf("%q got=%v want=%v packet:\n%s", testPacket.desc, got, testPacket.want, packet.Hexdump(testPacket.b))
 		}
@@ -298,3 +300,63 @@ func rawdefault(proto packet.IPProto, trimLength int) []byte {
 	port := uint16(53)
 	return rawpacket(proto, ip, ip, port, port, trimLength)
 }
+
+func parseHexPkt(t *testing.T, h string) *packet.ParsedPacket {
+	t.Helper()
+	b, err := hex.DecodeString(strings.ReplaceAll(h, " ", ""))
+	if err != nil {
+		t.Fatalf("failed to read hex %q: %v", h, err)
+	}
+	p := new(packet.ParsedPacket)
+	p.Decode(b)
+	return p
+}
+
+func TestOmitDropLogging(t *testing.T) {
+	tests := []struct {
+		name string
+		pkt  *packet.ParsedPacket
+		dir  direction
+		want bool
+	}{
+		{
+			name: "v4_tcp_out",
+			pkt:  &packet.ParsedPacket{IPVersion: 4, IPProto: packet.TCP},
+			dir:  out,
+			want: false,
+		},
+		{
+			name: "v6_icmp_out", // as seen on Linux
+			pkt:  parseHexPkt(t, "60 00 00 00 00 00 3a 00   fe800000000000000000000000000000 ff020000000000000000000000000002"),
+			dir:  out,
+			want: true,
+		},
+		{
+			name: "v6_to_MLDv2_capable_routers", // as seen on Windows
+			pkt:  parseHexPkt(t, "60 00 00 00 00 24 00 01 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 ff 02 00 00 00 00 00 00 00 00 00 00 00 00 00 16 3a 00 05 02 00 00 01 00 8f 00 6e 80 00 00 00 01 04 00 00 00 ff 02 00 00 00 00 00 00 00 00 00 00 00 00 00 0c"),
+			dir:  out,
+			want: true,
+		},
+		{
+			name: "v4_igmp_out", // on Windows, from https://github.com/tailscale/tailscale/issues/618
+			pkt:  parseHexPkt(t, "46 00 00 30 37 3a 00 00 01 02 10 0e a9 fe 53 6b e0 00 00 16 94 04 00 00 22 00 14 05 00 00 00 02 04 00 00 00 e0 00 00 fb 04 00 00 00 e0 00 00 fc"),
+			dir:  out,
+			want: true,
+		},
+		{
+			name: "v6_udp_multicast",
+			pkt:  parseHexPkt(t, "60 00 00 00 00 00 11 00  fe800000000000007dc6bc04499262a3 ff120000000000000000000000008384"),
+			dir:  out,
+			want: true,
+		},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			got := omitDropLogging(tt.pkt, tt.dir)
+			if got != tt.want {
+				t.Errorf("got %v; want %v\npacket: %#v\n%s", got, tt.want, tt.pkt, packet.Hexdump(tt.pkt.Buffer()))
+			}
+		})
+	}
+}

wgengine/monitor/monitor.go

@@ -3,7 +3,8 @@
 // license that can be found in the LICENSE file.
 
 // Package monitor provides facilities for monitoring network
-// interface changes.
+// interface and route changes. It primarily exists to know when
+// portable devices move between different networks.
 package monitor
 
 import (
@@ -14,10 +15,10 @@ import (
 )
 
 // message represents a message returned from an osMon.
-//
-// TODO: currently messages are being discarded, so the properties of
-// the message haven't been defined.
-type message interface{}
+type message interface {
+	// Ignore is whether we should ignore this message.
+	ignore() bool
+}
 
 // osMon is the interface that each operating system-specific
 // implementation of the link monitor must implement.
@@ -52,7 +53,8 @@ type Mon struct {
 // are propagated to the callback function.
 // The returned monitor is inactive until it's started by the Start method.
 func New(logf logger.Logf, callback ChangeFunc) (*Mon, error) {
-	om, err := newOSMon()
+	logf = logger.WithPrefix(logf, "monitor: ")
+	om, err := newOSMon(logf)
 	if err != nil {
 		return nil, err
 	}
@@ -100,7 +102,7 @@ func (m *Mon) Close() error {
 func (m *Mon) pump() {
 	defer m.goroutines.Done()
 	for {
-		_, err := m.om.Receive()
+		msg, err := m.om.Receive()
 		if err != nil {
 			select {
 			case <-m.stop:
@@ -108,10 +110,13 @@ func (m *Mon) pump() {
 			default:
 			}
 			// Keep retrying while we're not closed.
-			m.logf("Error receiving from connection: %v", err)
+			m.logf("error receiving from connection: %v", err)
 			time.Sleep(time.Second)
 			continue
 		}
+		if msg.ignore() {
+			continue
+		}
 		select {
 		case m.change <- struct{}{}:
 		case <-m.stop:

wgengine/monitor/monitor_freebsd.go

@@ -9,14 +9,23 @@ import (
 	"fmt"
 	"net"
 	"strings"
+
+	"tailscale.com/types/logger"
 )
 
+// unspecifiedMessage is a minimal message implementation that should not
+// be ignored. In general, OS-specific implementations should use better
+// types and avoid this if they can.
+type unspecifiedMessage struct{}
+
+func (unspecifiedMessage) ignore() bool { return false }
+
 // devdConn implements osMon using devd(8).
 type devdConn struct {
 	conn net.Conn
 }
 
-func newOSMon() (osMon, error) {
+func newOSMon(logf logger.Logf) (osMon, error) {
 	conn, err := net.Dial("unixpacket", "/var/run/devd.seqpacket.pipe")
 	if err != nil {
 		return nil, fmt.Errorf("devd dial error: %v", err)
@@ -41,8 +50,8 @@ func (c *devdConn) Receive() (message, error) {
 		if !strings.Contains(msg, "system=IFNET") {
 			continue
 		}
-		// TODO(]|[): this is where the devd-specific message would
+		// TODO: this is where the devd-specific message would
 		// get converted into a "standard" event message and returned.
-		return nil, nil
+		return unspecifiedMessage{}, nil
 	}
 }

wgengine/monitor/monitor_linux.go

@@ -8,22 +8,36 @@ package monitor
 
 import (
 	"fmt"
+	"net"
 	"time"
 
+	"github.com/jsimonetti/rtnetlink"
 	"github.com/mdlayher/netlink"
 	"golang.org/x/sys/unix"
+	"inet.af/netaddr"
+	"tailscale.com/net/tsaddr"
+	"tailscale.com/types/logger"
 )
 
+// unspecifiedMessage is a minimal message implementation that should not
+// be ignored. In general, OS-specific implementations should use better
+// types and avoid this if they can.
+type unspecifiedMessage struct{}
+
+func (unspecifiedMessage) ignore() bool { return false }
+
 // nlConn wraps a *netlink.Conn and returns a monitor.Message
 // instead of a netlink.Message. Currently, messages are discarded,
 // but down the line, when messages trigger different logic depending
 // on the type of event, this provides the capability of handling
 // each architecture-specific message in a generic fashion.
 type nlConn struct {
-	conn *netlink.Conn
+	logf     logger.Logf
+	conn     *netlink.Conn
+	buffered []netlink.Message
 }
 
-func newOSMon() (osMon, error) {
+func newOSMon(logf logger.Logf) (osMon, error) {
 	conn, err := netlink.Dial(unix.NETLINK_ROUTE, &netlink.Config{
 		// IPv4 address and route changes. Routes get us most of the
 		// events of interest, but we need address as well to cover
@@ -35,7 +49,7 @@ func newOSMon() (osMon, error) {
 	if err != nil {
 		return nil, fmt.Errorf("dialing netlink socket: %v", err)
 	}
-	return &nlConn{conn}, nil
+	return &nlConn{logf: logf, conn: conn}, nil
 }
 
 func (c *nlConn) Close() error {
@@ -44,12 +58,89 @@ func (c *nlConn) Close() error {
 }
 
 func (c *nlConn) Receive() (message, error) {
-	// currently ignoring the message
-	_, err := c.conn.Receive()
-	if err != nil {
-		return nil, err
+	if len(c.buffered) == 0 {
+		var err error
+		c.buffered, err = c.conn.Receive()
+		if err != nil {
+			return nil, err
+		}
+		if len(c.buffered) == 0 {
+			// Unexpected. Not seen in wild, but sleep defensively.
+			time.Sleep(time.Second)
+			return ignoreMessage{}, nil
+		}
+	}
+	msg := c.buffered[0]
+	c.buffered = c.buffered[1:]
+
+	// See https://github.com/torvalds/linux/blob/master/include/uapi/linux/rtnetlink.h
+	// And https://man7.org/linux/man-pages/man7/rtnetlink.7.html
+	switch msg.Header.Type {
+	case unix.RTM_NEWADDR, unix.RTM_DELADDR:
+		var rmsg rtnetlink.AddressMessage
+		if err := rmsg.UnmarshalBinary(msg.Data); err != nil {
+			c.logf("failed to parse type %v: %v", msg.Header.Type, err)
+			return unspecifiedMessage{}, nil
+		}
+		return &newAddrMessage{
+			Label:  rmsg.Attributes.Label,
+			Addr:   netaddrIP(rmsg.Attributes.Local),
+			Delete: msg.Header.Type == unix.RTM_DELADDR,
+		}, nil
+	case unix.RTM_NEWROUTE:
+		var rmsg rtnetlink.RouteMessage
+		if err := rmsg.UnmarshalBinary(msg.Data); err != nil {
+			c.logf("RTM_NEWROUTE: failed to parse: %v", err)
+			return unspecifiedMessage{}, nil
+		}
+		return &newRouteMessage{
+			Table:   rmsg.Table,
+			Src:     netaddrIP(rmsg.Attributes.Src),
+			Dst:     netaddrIP(rmsg.Attributes.Dst),
+			Gateway: netaddrIP(rmsg.Attributes.Gateway),
+		}, nil
+	case unix.RTM_DELROUTE:
+		var rmsg rtnetlink.RouteMessage
+		if err := rmsg.UnmarshalBinary(msg.Data); err != nil {
+			c.logf("RTM_DELROUTE: failed to parse: %v", err)
+			return unspecifiedMessage{}, nil
+		}
+		// Just log it for now, but don't bubble it up.
+		// (Debugging https://github.com/tailscale/tailscale/issues/643)
+		c.logf("RTM_DELROUTE: %+v", rmsg)
+		return unspecifiedMessage{}, nil
+	default:
+		c.logf("unhandled netlink msg type %+v, %q", msg.Header, msg.Data)
+		return unspecifiedMessage{}, nil
 	}
-	// TODO(]|[): this is where the NetLink-specific message would
-	// get converted into a "standard" event message and returned.
-	return nil, nil
 }
+
+func netaddrIP(std net.IP) netaddr.IP {
+	ip, _ := netaddr.FromStdIP(std)
+	return ip
+}
+
+// newRouteMessage is a message for a new route being added.
+type newRouteMessage struct {
+	Src, Dst, Gateway netaddr.IP
+	Table             uint8
+}
+
+func (m *newRouteMessage) ignore() bool {
+	return m.Table == 88 || tsaddr.IsTailscaleIP(m.Dst)
+}
+
+// newAddrMessage is a message for a new address being added.
+type newAddrMessage struct {
+	Delete bool
+	Addr   netaddr.IP
+	Label  string // netlink Label attribute (e.g. "tailscale0")
+}
+
+func (m *newAddrMessage) ignore() bool {
+	return tsaddr.IsTailscaleIP(m.Addr)
+}
+
+type ignoreMessage struct{}
+
+func (ignoreMessage) ignore() bool { return true }

wgengine/monitor/monitor_unsupported.go

@@ -6,4 +6,6 @@
 
 package monitor
 
-func newOSMon() (osMon, error) { return nil, nil }
+import "tailscale.com/types/logger"
+
+func newOSMon(logger.Logf) (osMon, error) { return nil, nil }

wgengine/packet/ip.go

@@ -7,6 +7,8 @@ package packet
 import (
 	"fmt"
 	"net"
+
+	"inet.af/netaddr"
 )
 
 // IP is an IPv4 address.
@@ -22,6 +24,17 @@ func NewIP(b net.IP) IP {
 	return IP(get32(b4))
 }
 
+// IPFromNetaddr converts a netaddr.IP to an IP.
+func IPFromNetaddr(ip netaddr.IP) IP {
+	ipbytes := ip.As4()
+	return IP(get32(ipbytes[:]))
+}
+
+// Netaddr converts an IP to a netaddr.IP.
+func (ip IP) Netaddr() netaddr.IP {
+	return netaddr.IPv4(byte(ip>>24), byte(ip>>16), byte(ip>>8), byte(ip))
+}
+
 func (ip IP) String() string {
 	return fmt.Sprintf("%d.%d.%d.%d", byte(ip>>24), byte(ip>>16), byte(ip>>8), byte(ip))
 }
@@ -34,12 +47,13 @@ const (
 	// Unknown represents an unknown or unsupported protocol; it's deliberately the zero value.
 	Unknown IPProto = 0x00
 	ICMP    IPProto = 0x01
+	IGMP    IPProto = 0x02
+	ICMPv6  IPProto = 0x3a
 	TCP     IPProto = 0x06
 	UDP     IPProto = 0x11
-	// IPv6 and Fragment are special values. They're not really IPProto values
-	// so we're using the unassigned 0xFE and 0xFF values for them.
+	// Fragment is a special value. It's not really an IPProto value
+	// so we're using the unassigned 0xFF value.
 	// TODO(dmytro): special values should be taken out of here.
-	IPv6     IPProto = 0xFE
 	Fragment IPProto = 0xFF
 )
 
@@ -53,8 +67,6 @@ func (p IPProto) String() string {
 		return "UDP"
 	case TCP:
 		return "TCP"
-	case IPv6:
-		return "IPv6"
 	default:
 		return "Unknown"
 	}

wgengine/packet/packet.go

@@ -30,6 +30,8 @@ var (
 )
 
 // ParsedPacket is a minimal decoding of a packet suitable for use in filters.
+//
+// In general, it only supports IPv4. The IPv6 parsing is very minimal.
 type ParsedPacket struct {
 	// b is the byte buffer that this decodes.
 	b []byte
@@ -41,27 +43,32 @@ type ParsedPacket struct {
 	// This is not the same as len(b) because b can have trailing zeros.
 	length int
 
-	IPProto  IPProto // IP subprotocol (UDP, TCP, etc)
-	SrcIP    IP      // IP source address
-	DstIP    IP      // IP destination address
-	SrcPort  uint16  // TCP/UDP source port
-	DstPort  uint16  // TCP/UDP destination port
-	TCPFlags uint8   // TCP flags (SYN, ACK, etc)
+	IPVersion uint8   // 4, 6, or 0
+	IPProto   IPProto // IP subprotocol (UDP, TCP, etc); the NextHeader field for IPv6
+	SrcIP     IP      // IP source address (not used for IPv6)
+	DstIP     IP      // IP destination address (not used for IPv6)
+	SrcPort   uint16  // TCP/UDP source port
+	DstPort   uint16  // TCP/UDP destination port
+	TCPFlags  uint8   // TCP flags (SYN, ACK, etc)
 }
 
-func (q *ParsedPacket) String() string {
-	switch q.IPProto {
-	case IPv6:
-		return "IPv6{???}"
+// NextHeader
+type NextHeader uint8
+
+func (p *ParsedPacket) String() string {
+	if p.IPVersion == 6 {
+		return fmt.Sprintf("IPv6{Proto=%d}", p.IPProto)
+	}
+	switch p.IPProto {
 	case Unknown:
 		return "Unknown{???}"
 	}
 	sb := strbuilder.Get()
-	sb.WriteString(q.IPProto.String())
+	sb.WriteString(p.IPProto.String())
 	sb.WriteByte('{')
-	writeIPPort(sb, q.SrcIP, q.SrcPort)
+	writeIPPort(sb, p.SrcIP, p.SrcPort)
 	sb.WriteString(" > ")
-	writeIPPort(sb, q.DstIP, q.DstPort)
+	writeIPPort(sb, p.DstIP, p.DstPort)
 	sb.WriteByte('}')
 	return sb.String()
 }
@@ -105,20 +112,22 @@ func (q *ParsedPacket) Decode(b []byte) {
 	q.b = b
 
 	if len(b) < ipHeaderLength {
+		q.IPVersion = 0
 		q.IPProto = Unknown
 		return
 	}
 
 	// Check that it's IPv4.
 	// TODO(apenwarr): consider IPv6 support
-	switch (b[0] & 0xF0) >> 4 {
+	q.IPVersion = (b[0] & 0xF0) >> 4
+	switch q.IPVersion {
 	case 4:
 		q.IPProto = IPProto(b[9])
-		// continue
 	case 6:
-		q.IPProto = IPv6
+		q.IPProto = IPProto(b[6]) // "Next Header" field
 		return
 	default:
+		q.IPVersion = 0
 		q.IPProto = Unknown
 		return
 	}