net/portmapper: fix UPnP probing, work against all ports

Prior to Tailscale 1.12 it detected UPnP on any port.
Starting with Tailscale 1.11.x, it stopped detecting UPnP on all ports.

Then start plumbing its discovered Location header port number to the
code that was assuming port 5000.

Fixes #2109

Signed-off-by: Brad Fitzpatrick <bradfitz@tailscale.com>

.github/workflows/go_generate.yml

@@ -0,0 +1,34 @@
+name: go generate
+
+on:
+  push:
+    branches:
+      - main
+      - "release-branch/*"
+  pull_request:
+    branches:
+      - "*"
+
+jobs:
+  check:
+    runs-on: ubuntu-latest
+
+    steps:
+      - name: Set up Go
+        uses: actions/setup-go@v1
+        with:
+          go-version: 1.16
+
+      - name: Check out code
+        uses: actions/checkout@v2
+        with:
+          fetch-depth: 0
+
+      - name: check 'go generate' is clean
+        run: |
+          mkdir gentools
+          go build -o gentools/stringer golang.org/x/tools/cmd/stringer
+          PATH="$PATH:$(pwd)/gentools" go generate ./...
+          echo
+          echo
+          git diff --name-only --exit-code || (echo "The files above need updating. Please run 'go generate'."; exit 1)

.github/workflows/xe-experimental-vm-test.yml

@@ -1,16 +1,11 @@
 name: "integration-vms"
 
 on:
-  # # NOTE(Xe): uncomment this region when testing the test
-  # pull_request:
-  #   branches:
-  #     - 'main'
+  pull_request:
+    paths:
+      - "tstest/integration/vms/**"
   release:
     types: [ created ]
-  schedule:
-    # At minute 0 past hour 6 and 18
-    # https://crontab.guru/#00_6,18_*_*_*
-    - cron: '00 6,18 * * *'
 
 jobs:
   experimental-linux-vm-test:
@@ -24,7 +19,7 @@ jobs:
         uses: actions/checkout@v1
 
       - name: Download VM Images
-        run: go test ./tstest/integration/vms -run-vm-tests -run=Download -timeout=60m
+        run: go test ./tstest/integration/vms -run-vm-tests -run=Download -timeout=60m -no-s3
         env:
           XDG_CACHE_HOME: "/var/lib/ghrunner/cache"
 
@@ -34,3 +29,17 @@ jobs:
           TMPDIR: "/tmp"
           XDG_CACHE_HOME: "/var/lib/ghrunner/cache"
 
+      - uses: k0kubun/action-slack@v2.0.0
+        with:
+          payload: |
+            {
+              "attachments": [{
+                "text": "${{ job.status }}: ${{ github.workflow }} <https://github.com/${{ github.repository }}/commit/${{ github.sha }}/checks|${{ env.COMMIT_DATE }} #${{ env.COMMIT_NUMBER_OF_DAY }}> " +
+                        "(<https://github.com/${{ github.repository }}/commit/${{ github.sha }}|" + "${{ github.sha }}".substring(0, 10) + ">) " +
+                        "of ${{ github.repository }}@" + "${{ github.ref }}".split('/').reverse()[0] + " by ${{ github.event.head_commit.committer.name }}",
+                "color": "danger"
+              }]
+            }
+        env:
+          SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK_URL }}
+        if: failure() && github.event_name == 'push'

Dockerfile

@@ -42,8 +42,7 @@ FROM golang:1.16-alpine AS build-env
 
 WORKDIR /go/src/tailscale
 
-COPY go.mod .
-COPY go.sum .
+COPY go.mod go.sum ./
 RUN go mod download
 
 COPY . .

VERSION.txt

@@ -1 +1 @@
-1.11.0
+1.13.0

api.md

@@ -471,15 +471,16 @@ Determines what rules match for a user on an ACL without saving the ACL to the s
 ##### Parameters
 
 ###### Query Parameters
-`user` - A user's email. The provided ACL is queried with this user to determine which rules match.
+`type` - can be 'user' or 'ipport'
+`previewFor` - if type=user, a user's email. If type=ipport, a IP address + port like "10.0.0.1:80".
+The provided ACL is queried with this paramater to determine which rules match.
 
 ###### POST Body
 ACL JSON or HuJSON (see https://tailscale.com/kb/1018/acls)
 
 ##### Example
 ```
-POST /api/v2/tailnet/example.com/acl/preiew
-curl 'https://api.tailscale.com/api/v2/tailnet/example.com/acl?user=user1@example.com' \
+curl 'https://api.tailscale.com/api/v2/tailnet/example.com/acl/preview?previewFor=user1@example.com&type=user' \
   -u "tskey-yourapikey123:" \
   --data-binary '// Example/default ACLs for unrestricted connections.
 {

build_docker.sh

@@ -25,7 +25,7 @@
 
 set -eu
 
-eval $(./version/version.sh)
+eval $(./build_dist.sh shellvars)
 
 docker build \
   --build-arg VERSION_LONG=$VERSION_LONG \

client/tailscale/tailscale.go

@@ -24,6 +24,7 @@ import (
 	"tailscale.com/ipn/ipnstate"
 	"tailscale.com/paths"
 	"tailscale.com/safesocket"
+	"tailscale.com/tailcfg"
 )
 
 // TailscaledSocket is the tailscaled Unix socket.
@@ -278,3 +279,17 @@ func SetDNS(ctx context.Context, name, value string) error {
 	_, err := send(ctx, "POST", "/localapi/v0/set-dns?"+v.Encode(), 200, nil)
 	return err
 }
+
+// CurrentDERPMap returns the current DERPMap that is being used by the local tailscaled.
+// It is intended to be used with netcheck to see availability of DERPs.
+func CurrentDERPMap(ctx context.Context) (*tailcfg.DERPMap, error) {
+	var derpMap tailcfg.DERPMap
+	res, err := send(ctx, "GET", "/localapi/v0/derpmap", 200, nil)
+	if err != nil {
+		return nil, err
+	}
+	if err = json.Unmarshal(res, &derpMap); err != nil {
+		return nil, fmt.Errorf("invalid derp map json: %w", err)
+	}
+	return &derpMap, nil
+}

cmd/addlicense/main.go

@@ -0,0 +1,77 @@
+// Copyright (c) 2021 Tailscale Inc & AUTHORS All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+// Program addlicense adds a license header to a file.
+// It is intended for use with 'go generate',
+// so it has a slightly weird usage.
+package main
+
+import (
+	"flag"
+	"fmt"
+	"os"
+	"os/exec"
+)
+
+var (
+	year = flag.Int("year", 0, "copyright year")
+	file = flag.String("file", "", "file to modify")
+)
+
+func usage() {
+	fmt.Fprintf(os.Stderr, `
+usage: addlicense -year YEAR -file FILE <subcommand args...>
+`[1:])
+
+	flag.PrintDefaults()
+	fmt.Fprintf(os.Stderr, `
+addlicense adds a Tailscale license to the beginning of file,
+using year as the copyright year.
+
+It is intended for use with 'go generate', so it also runs a subcommand,
+which presumably creates the file.
+
+Sample usage:
+
+addlicense -year 2021 -file pull_strings.go stringer -type=pull
+`[1:])
+	os.Exit(2)
+}
+
+func main() {
+	flag.Usage = usage
+	flag.Parse()
+	if len(flag.Args()) == 0 {
+		flag.Usage()
+	}
+	cmd := exec.Command(flag.Arg(0), flag.Args()[1:]...)
+	cmd.Stdout = os.Stdout
+	cmd.Stderr = os.Stderr
+	err := cmd.Run()
+	check(err)
+	b, err := os.ReadFile(*file)
+	check(err)
+	f, err := os.OpenFile(*file, os.O_TRUNC|os.O_WRONLY, 0644)
+	check(err)
+	_, err = fmt.Fprintf(f, license, *year)
+	check(err)
+	_, err = f.Write(b)
+	check(err)
+	err = f.Close()
+	check(err)
+}
+
+func check(err error) {
+	if err != nil {
+		fmt.Fprintln(os.Stderr, err)
+		os.Exit(1)
+	}
+}
+
+var license = `
+// Copyright (c) %d Tailscale Inc & AUTHORS All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+`[1:]

cmd/derper/derper.go

@@ -58,7 +58,12 @@ func loadConfig() config {
 		return config{PrivateKey: mustNewKey()}
 	}
 	if *configPath == "" {
-		log.Fatalf("derper: -c <config path> not specified")
+		if os.Getuid() == 0 {
+			*configPath = "/var/lib/derper/derper.key"
+		} else {
+			log.Fatalf("derper: -c <config path> not specified")
+		}
+		log.Printf("no config path specified; using %s", *configPath)
 	}
 	b, err := ioutil.ReadFile(*configPath)
 	switch {

cmd/derper/mesh.go

@@ -9,7 +9,9 @@ import (
 	"errors"
 	"fmt"
 	"log"
+	"net"
 	"strings"
+	"time"
 
 	"tailscale.com/derp"
 	"tailscale.com/derp/derphttp"
@@ -39,6 +41,34 @@ func startMeshWithHost(s *derp.Server, host string) error {
 		return err
 	}
 	c.MeshKey = s.MeshKey()
+
+	// For meshed peers within a region, connect via VPC addresses.
+	c.SetURLDialer(func(ctx context.Context, network, addr string) (net.Conn, error) {
+		host, port, err := net.SplitHostPort(addr)
+		if err != nil {
+			return nil, err
+		}
+		var d net.Dialer
+		var r net.Resolver
+		if port == "443" && strings.HasSuffix(host, ".tailscale.com") {
+			base := strings.TrimSuffix(host, ".tailscale.com")
+			subCtx, cancel := context.WithTimeout(ctx, 2*time.Second)
+			defer cancel()
+			vpcHost := base + "-vpc.tailscale.com"
+			ips, _ := r.LookupIP(subCtx, "ip", vpcHost)
+			if len(ips) > 0 {
+				vpcAddr := net.JoinHostPort(ips[0].String(), port)
+				c, err := d.DialContext(subCtx, network, vpcAddr)
+				if err == nil {
+					log.Printf("connected to %v (%v) instead of %v", vpcHost, ips[0], base)
+					return c, nil
+				}
+				log.Printf("failed to connect to %v (%v): %v; trying non-VPC route", vpcHost, ips[0], err)
+			}
+		}
+		return d.DialContext(ctx, network, addr)
+	})
+
 	add := func(k key.Public) { s.AddPacketForwarder(k, c) }
 	remove := func(k key.Public) { s.RemovePacketForwarder(k, c) }
 	go c.RunWatchConnectionLoop(context.Background(), s.PublicKey(), logf, add, remove)

cmd/derpprobe/derpprobe.go

@@ -0,0 +1,354 @@
+// Copyright (c) 2021 Tailscale Inc & AUTHORS All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+// The derpprobe binary probes derpers.
+package main // import "tailscale.com/cmd/derper/derpprobe"
+
+import (
+	"bytes"
+	"context"
+	crand "crypto/rand"
+	"encoding/json"
+	"flag"
+	"fmt"
+	"html"
+	"io"
+	"log"
+	"net/http"
+	"sort"
+	"sync"
+	"time"
+
+	"tailscale.com/derp"
+	"tailscale.com/derp/derphttp"
+	"tailscale.com/tailcfg"
+	"tailscale.com/types/key"
+)
+
+var (
+	derpMapURL = flag.String("derp-map", "https://login.tailscale.com/derpmap/default", "URL to DERP map (https:// or file://)")
+	listen     = flag.String("listen", ":8030", "HTTP listen address")
+)
+
+var (
+	mu            sync.Mutex
+	state         = map[nodePair]pairStatus{}
+	lastDERPMap   *tailcfg.DERPMap
+	lastDERPMapAt time.Time
+)
+
+func main() {
+	flag.Parse()
+	go probeLoop()
+	log.Fatal(http.ListenAndServe(*listen, http.HandlerFunc(serve)))
+}
+
+type overallStatus struct {
+	good, bad []string
+}
+
+func (st *overallStatus) addBadf(format string, a ...interface{}) {
+	st.bad = append(st.bad, fmt.Sprintf(format, a...))
+}
+
+func (st *overallStatus) addGoodf(format string, a ...interface{}) {
+	st.good = append(st.good, fmt.Sprintf(format, a...))
+}
+
+func getOverallStatus() (o overallStatus) {
+	mu.Lock()
+	defer mu.Unlock()
+	if lastDERPMap == nil {
+		o.addBadf("no DERP map")
+		return
+	}
+	now := time.Now()
+	if age := now.Sub(lastDERPMapAt); age > time.Minute {
+		o.addBadf("DERPMap hasn't been successfully refreshed in %v", age.Round(time.Second))
+	}
+	for _, reg := range sortedRegions(lastDERPMap) {
+		for _, from := range reg.Nodes {
+			for _, to := range reg.Nodes {
+				pair := nodePair{from.Name, to.Name}
+				st, ok := state[pair]
+				age := now.Sub(st.at).Round(time.Second)
+				switch {
+				case !ok:
+					o.addBadf("no state for %v", pair)
+				case st.err != nil:
+					o.addBadf("%v: %v", pair, st.err)
+				case age > 90*time.Second:
+					o.addBadf("%v: update is %v old", pair, age)
+				default:
+					o.addGoodf("%v: %v, %v ago", pair, st.latency.Round(time.Millisecond), age)
+				}
+			}
+		}
+	}
+	return
+}
+
+func serve(w http.ResponseWriter, r *http.Request) {
+	st := getOverallStatus()
+	summary := "All good"
+	if len(st.bad) > 0 {
+		w.WriteHeader(500)
+		summary = fmt.Sprintf("%d problems", len(st.bad))
+	}
+	io.WriteString(w, "<html><head><style>.bad { font-weight: bold; color: #700; }</style></head>\n")
+	fmt.Fprintf(w, "<body><h1>derp probe</h1>\n%s:<ul>", summary)
+	for _, s := range st.bad {
+		fmt.Fprintf(w, "<li class=bad>%s</li>\n", html.EscapeString(s))
+	}
+	for _, s := range st.good {
+		fmt.Fprintf(w, "<li>%s</li>\n", html.EscapeString(s))
+	}
+	io.WriteString(w, "</ul></body></html>\n")
+}
+
+func sortedRegions(dm *tailcfg.DERPMap) []*tailcfg.DERPRegion {
+	ret := make([]*tailcfg.DERPRegion, 0, len(dm.Regions))
+	for _, r := range dm.Regions {
+		ret = append(ret, r)
+	}
+	sort.Slice(ret, func(i, j int) bool { return ret[i].RegionID < ret[j].RegionID })
+	return ret
+}
+
+type nodePair struct {
+	from, to string // DERPNode.Name
+}
+
+func (p nodePair) String() string { return fmt.Sprintf("(%s→%s)", p.from, p.to) }
+
+type pairStatus struct {
+	err     error
+	latency time.Duration
+	at      time.Time
+}
+
+func setDERPMap(dm *tailcfg.DERPMap) {
+	mu.Lock()
+	defer mu.Unlock()
+	lastDERPMap = dm
+	lastDERPMapAt = time.Now()
+}
+
+func setState(p nodePair, latency time.Duration, err error) {
+	mu.Lock()
+	defer mu.Unlock()
+	st := pairStatus{
+		err:     err,
+		latency: latency,
+		at:      time.Now(),
+	}
+	state[p] = st
+	if err != nil {
+		log.Printf("%+v error: %v", p, err)
+	} else {
+		log.Printf("%+v: %v", p, latency.Round(time.Millisecond))
+	}
+}
+
+func probeLoop() {
+	ticker := time.NewTicker(15 * time.Second)
+	for {
+		err := probe()
+		if err != nil {
+			log.Printf("probe: %v", err)
+		}
+		<-ticker.C
+	}
+}
+
+func probe() error {
+	ctx, cancel := context.WithTimeout(context.Background(), 60*time.Second)
+	defer cancel()
+	dm, err := getDERPMap(ctx)
+	if err != nil {
+		return err
+	}
+
+	var wg sync.WaitGroup
+	wg.Add(len(dm.Regions))
+	for _, reg := range dm.Regions {
+		reg := reg
+		go func() {
+			defer wg.Done()
+			for _, from := range reg.Nodes {
+				for _, to := range reg.Nodes {
+					latency, err := probeNodePair(ctx, dm, from, to)
+					setState(nodePair{from.Name, to.Name}, latency, err)
+				}
+			}
+		}()
+	}
+
+	wg.Wait()
+	return ctx.Err()
+}
+
+func probeNodePair(ctx context.Context, dm *tailcfg.DERPMap, from, to *tailcfg.DERPNode) (latency time.Duration, err error) {
+	// The passed in context is a minute for the whole region. The
+	// idea is that each node pair in the region will be done
+	// serially and regularly in the future, reusing connections
+	// (at least in the happy path). For now they don't reuse
+	// connections and probe at most once every 15 seconds. We
+	// bound the duration of a single node pair within a region
+	// so one bad one can't starve others.
+	ctx, cancel := context.WithTimeout(ctx, 10*time.Second)
+	defer cancel()
+
+	fromc, err := newConn(ctx, dm, from)
+	if err != nil {
+		return 0, err
+	}
+	defer fromc.Close()
+	toc, err := newConn(ctx, dm, to)
+	if err != nil {
+		return 0, err
+	}
+	defer toc.Close()
+
+	// Wait a bit for from's node to hear about to existing on the
+	// other node in the region, in the case where the two nodes
+	// are different.
+	if from.Name != to.Name {
+		time.Sleep(100 * time.Millisecond) // pretty arbitrary
+	}
+
+	// Make a random packet
+	pkt := make([]byte, 8)
+	crand.Read(pkt)
+
+	t0 := time.Now()
+
+	// Send the random packet.
+	sendc := make(chan error, 1)
+	go func() {
+		sendc <- fromc.Send(toc.SelfPublicKey(), pkt)
+	}()
+	select {
+	case <-ctx.Done():
+		return 0, fmt.Errorf("timeout sending via %q: %w", from.Name, ctx.Err())
+	case err := <-sendc:
+		if err != nil {
+			return 0, fmt.Errorf("error sending via %q: %w", from.Name, err)
+		}
+	}
+
+	// Receive the random packet.
+	recvc := make(chan interface{}, 1) // either derp.ReceivedPacket or error
+	go func() {
+		for {
+			m, err := toc.Recv()
+			if err != nil {
+				recvc <- err
+				return
+			}
+			switch v := m.(type) {
+			case derp.ReceivedPacket:
+				recvc <- v
+			default:
+				log.Printf("%v: ignoring Recv frame type %T", to.Name, v)
+				// Loop.
+			}
+		}
+	}()
+	select {
+	case <-ctx.Done():
+		return 0, fmt.Errorf("timeout receiving from %q: %w", to.Name, ctx.Err())
+	case v := <-recvc:
+		if err, ok := v.(error); ok {
+			return 0, fmt.Errorf("error receiving from %q: %w", to.Name, err)
+		}
+		p := v.(derp.ReceivedPacket)
+		if p.Source != fromc.SelfPublicKey() {
+			return 0, fmt.Errorf("got data packet from unexpected source, %v", p.Source)
+		}
+		if !bytes.Equal(p.Data, pkt) {
+			return 0, fmt.Errorf("unexpected data packet %q", p.Data)
+		}
+	}
+	return time.Since(t0), nil
+}
+
+func newConn(ctx context.Context, dm *tailcfg.DERPMap, n *tailcfg.DERPNode) (*derphttp.Client, error) {
+	priv := key.NewPrivate()
+	dc := derphttp.NewRegionClient(priv, log.Printf, func() *tailcfg.DERPRegion {
+		rid := n.RegionID
+		return &tailcfg.DERPRegion{
+			RegionID:   rid,
+			RegionCode: fmt.Sprintf("%s-%s", dm.Regions[rid].RegionCode, n.Name),
+			RegionName: dm.Regions[rid].RegionName,
+			Nodes:      []*tailcfg.DERPNode{n},
+		}
+	})
+	dc.IsProber = true
+	err := dc.Connect(ctx)
+	if err != nil {
+		return nil, err
+	}
+	errc := make(chan error, 1)
+	go func() {
+		m, err := dc.Recv()
+		if err != nil {
+			errc <- err
+			return
+		}
+		switch m.(type) {
+		case derp.ServerInfoMessage:
+			errc <- nil
+		default:
+			errc <- fmt.Errorf("unexpected first message type %T", errc)
+		}
+	}()
+	select {
+	case err := <-errc:
+		if err != nil {
+			go dc.Close()
+			return nil, err
+		}
+	case <-ctx.Done():
+		go dc.Close()
+		return nil, fmt.Errorf("timeout waiting for ServerInfoMessage: %w", ctx.Err())
+	}
+	return dc, nil
+}
+
+var httpOrFileClient = &http.Client{Transport: httpOrFileTransport()}
+
+func httpOrFileTransport() http.RoundTripper {
+	tr := http.DefaultTransport.(*http.Transport).Clone()
+	tr.RegisterProtocol("file", http.NewFileTransport(http.Dir("/")))
+	return tr
+}
+
+func getDERPMap(ctx context.Context) (*tailcfg.DERPMap, error) {
+	req, err := http.NewRequestWithContext(ctx, "GET", *derpMapURL, nil)
+	if err != nil {
+		return nil, err
+	}
+	res, err := httpOrFileClient.Do(req)
+	if err != nil {
+		mu.Lock()
+		defer mu.Unlock()
+		if lastDERPMap != nil && time.Since(lastDERPMapAt) < 10*time.Minute {
+			// Assume that control is restarting and use
+			// the same one for a bit.
+			return lastDERPMap, nil
+		}
+		return nil, err
+	}
+	defer res.Body.Close()
+	if res.StatusCode != 200 {
+		return nil, fmt.Errorf("fetching %s: %s", *derpMapURL, res.Status)
+	}
+	dm := new(tailcfg.DERPMap)
+	if err := json.NewDecoder(res.Body).Decode(dm); err != nil {
+		return nil, fmt.Errorf("decoding %s JSON: %v", *derpMapURL, err)
+	}
+	setDERPMap(dm)
+	return dm, nil
+}

cmd/speedtest/speedtest.go

@@ -0,0 +1,121 @@
+// Copyright (c) 2021 Tailscale Inc & AUTHORS All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+// Program speedtest provides the speedtest command. The reason to keep it separate from
+// the normal tailscale cli is because it is not yet ready to go in the tailscale binary.
+// It will be included in the tailscale cli after it has been added to tailscaled.
+
+// Example usage for client command: go run cmd/speedtest -host 127.0.0.1:20333 -t 5s
+// This will connect to the server on 127.0.0.1:20333 and start a 5 second download speedtest.
+// Example usage for server command: go run cmd/speedtest -s -host :20333
+// This will start a speedtest server on port 20333.
+package main
+
+import (
+	"context"
+	"errors"
+	"flag"
+	"fmt"
+	"net"
+	"os"
+	"strconv"
+	"text/tabwriter"
+	"time"
+
+	"github.com/peterbourgon/ff/v2/ffcli"
+	"tailscale.com/net/speedtest"
+)
+
+// Runs the speedtest command as a commandline program
+func main() {
+	args := os.Args[1:]
+	if err := speedtestCmd.Parse(args); err != nil {
+		fmt.Fprintln(os.Stderr, err.Error())
+		os.Exit(1)
+	}
+
+	err := speedtestCmd.Run(context.Background())
+	if errors.Is(err, flag.ErrHelp) {
+		fmt.Fprintln(os.Stderr, speedtestCmd.ShortUsage)
+		os.Exit(2)
+	}
+	if err != nil {
+		fmt.Fprintln(os.Stderr, err.Error())
+		os.Exit(1)
+	}
+}
+
+// speedtestCmd is the root command. It runs either the server or client depending on the
+// flags passed to it.
+var speedtestCmd = &ffcli.Command{
+	Name:       "speedtest",
+	ShortUsage: "speedtest [-host <host:port>] [-s] [-r] [-t <test duration>]",
+	ShortHelp:  "Run a speed test",
+	FlagSet: (func() *flag.FlagSet {
+		fs := flag.NewFlagSet("speedtest", flag.ExitOnError)
+		fs.StringVar(&speedtestArgs.host, "host", ":20333", "host:port pair to connect to or listen on")
+		fs.DurationVar(&speedtestArgs.testDuration, "t", speedtest.DefaultDuration, "duration of the speed test")
+		fs.BoolVar(&speedtestArgs.runServer, "s", false, "run a speedtest server")
+		fs.BoolVar(&speedtestArgs.reverse, "r", false, "run in reverse mode (server sends, client receives)")
+		return fs
+	})(),
+	Exec: runSpeedtest,
+}
+
+var speedtestArgs struct {
+	host         string
+	testDuration time.Duration
+	runServer    bool
+	reverse      bool
+}
+
+func runSpeedtest(ctx context.Context, args []string) error {
+
+	if _, _, err := net.SplitHostPort(speedtestArgs.host); err != nil {
+		var addrErr *net.AddrError
+		if errors.As(err, &addrErr) && addrErr.Err == "missing port in address" {
+			// if no port is provided, append the default port
+			speedtestArgs.host = net.JoinHostPort(speedtestArgs.host, strconv.Itoa(speedtest.DefaultPort))
+		}
+	}
+
+	if speedtestArgs.runServer {
+		listener, err := net.Listen("tcp", speedtestArgs.host)
+		if err != nil {
+			return err
+		}
+
+		fmt.Printf("listening on %v\n", listener.Addr())
+
+		return speedtest.Serve(listener)
+	}
+
+	// Ensure the duration is within the allowed range
+	if speedtestArgs.testDuration < speedtest.MinDuration || speedtestArgs.testDuration > speedtest.MaxDuration {
+		return fmt.Errorf("test duration must be within %v and %v", speedtest.MinDuration, speedtest.MaxDuration)
+	}
+
+	dir := speedtest.Download
+	if speedtestArgs.reverse {
+		dir = speedtest.Upload
+	}
+
+	fmt.Printf("Starting a %s test with %s\n", dir, speedtestArgs.host)
+	results, err := speedtest.RunClient(dir, speedtestArgs.testDuration, speedtestArgs.host)
+	if err != nil {
+		return err
+	}
+
+	w := tabwriter.NewWriter(os.Stdout, 12, 0, 0, ' ', tabwriter.TabIndent)
+	fmt.Println("Results:")
+	fmt.Fprintln(w, "Interval\t\tTransfer\t\tBandwidth\t\t")
+	for _, r := range results {
+		if r.Total {
+			fmt.Fprintln(w, "-------------------------------------------------------------------------")
+		}
+		fmt.Fprintf(w, "%.2f-%.2f\tsec\t%.4f\tMBits\t%.4f\tMbits/sec\t\n", r.IntervalStart.Seconds(), r.IntervalEnd.Seconds(), r.MegaBits(), r.MBitsPerSecond())
+	}
+	w.Flush()
+	return nil
+}

cmd/tailscale/cli/cli_test.go

@@ -13,9 +13,11 @@ import (
 	"strings"
 	"testing"
 
+	"github.com/google/go-cmp/cmp"
 	"inet.af/netaddr"
 	"tailscale.com/ipn"
 	"tailscale.com/ipn/ipnstate"
+	"tailscale.com/types/persist"
 	"tailscale.com/types/preftype"
 )
 
@@ -402,6 +404,28 @@ func TestCheckForAccidentalSettingReverts(t *testing.T) {
 			},
 			want: accidentalUpPrefix + " --hostname=foo --exit-node=100.64.5.7",
 		},
+		{
+			name:  "ignore_login_server_synonym",
+			flags: []string{"--login-server=https://controlplane.tailscale.com"},
+			curPrefs: &ipn.Prefs{
+				ControlURL:       "https://login.tailscale.com",
+				AllowSingleHosts: true,
+				CorpDNS:          true,
+				NetfilterMode:    preftype.NetfilterOn,
+			},
+			want: "", // not an error
+		},
+		{
+			name:  "ignore_login_server_synonym_on_other_change",
+			flags: []string{"--netfilter-mode=off"},
+			curPrefs: &ipn.Prefs{
+				ControlURL:       "https://login.tailscale.com",
+				AllowSingleHosts: true,
+				CorpDNS:          false,
+				NetfilterMode:    preftype.NetfilterOn,
+			},
+			want: accidentalUpPrefix + " --netfilter-mode=off --accept-dns=false",
+		},
 	}
 	for _, tt := range tests {
 		t.Run(tt.name, func(t *testing.T) {
@@ -418,8 +442,9 @@ func TestCheckForAccidentalSettingReverts(t *testing.T) {
 			}
 			applyImplicitPrefs(newPrefs, tt.curPrefs, tt.curUser)
 			var got string
-			if err := checkForAccidentalSettingReverts(flagSet, tt.curPrefs, newPrefs, upCheckEnv{
+			if err := checkForAccidentalSettingReverts(newPrefs, tt.curPrefs, upCheckEnv{
 				goos:          goos,
+				flagSet:       flagSet,
 				curExitNodeIP: tt.curExitNodeIP,
 			}); err != nil {
 				got = err.Error()
@@ -666,3 +691,108 @@ func TestFlagAppliesToOS(t *testing.T) {
 		})
 	}
 }
+
+func TestUpdatePrefs(t *testing.T) {
+	tests := []struct {
+		name     string
+		flags    []string // argv to be parsed into env.flagSet and env.upArgs
+		curPrefs *ipn.Prefs
+		env      upCheckEnv // empty goos means "linux"
+
+		wantSimpleUp   bool
+		wantJustEditMP *ipn.MaskedPrefs
+		wantErrSubtr   string
+	}{
+		{
+			name:  "bare_up_means_up",
+			flags: []string{},
+			curPrefs: &ipn.Prefs{
+				ControlURL:  ipn.DefaultControlURL,
+				WantRunning: false,
+				Hostname:    "foo",
+			},
+		},
+		{
+			name:  "just_up",
+			flags: []string{},
+			curPrefs: &ipn.Prefs{
+				ControlURL: ipn.DefaultControlURL,
+				Persist:    &persist.Persist{LoginName: "crawshaw.github"},
+			},
+			env: upCheckEnv{
+				backendState: "Stopped",
+			},
+			wantSimpleUp: true,
+		},
+		{
+			name:  "just_edit",
+			flags: []string{},
+			curPrefs: &ipn.Prefs{
+				ControlURL: ipn.DefaultControlURL,
+				Persist:    &persist.Persist{LoginName: "crawshaw.github"},
+			},
+			env:            upCheckEnv{backendState: "Running"},
+			wantSimpleUp:   true,
+			wantJustEditMP: &ipn.MaskedPrefs{WantRunningSet: true},
+		},
+		{
+			name:  "control_synonym",
+			flags: []string{},
+			curPrefs: &ipn.Prefs{
+				ControlURL: "https://login.tailscale.com",
+				Persist:    &persist.Persist{LoginName: "crawshaw.github"},
+			},
+			env:            upCheckEnv{backendState: "Running"},
+			wantSimpleUp:   true,
+			wantJustEditMP: &ipn.MaskedPrefs{WantRunningSet: true},
+		},
+		{
+			name:  "change_login_server",
+			flags: []string{"--login-server=https://localhost:1000"},
+			curPrefs: &ipn.Prefs{
+				ControlURL:       "https://login.tailscale.com",
+				Persist:          &persist.Persist{LoginName: "crawshaw.github"},
+				AllowSingleHosts: true,
+				CorpDNS:          true,
+				NetfilterMode:    preftype.NetfilterOn,
+			},
+			env:            upCheckEnv{backendState: "Running"},
+			wantSimpleUp:   true,
+			wantJustEditMP: &ipn.MaskedPrefs{WantRunningSet: true},
+			wantErrSubtr:   "can't change --login-server without --force-reauth",
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			if tt.env.goos == "" {
+				tt.env.goos = "linux"
+			}
+			tt.env.flagSet = newUpFlagSet(tt.env.goos, &tt.env.upArgs)
+			tt.env.flagSet.Parse(tt.flags)
+
+			newPrefs, err := prefsFromUpArgs(tt.env.upArgs, t.Logf, new(ipnstate.Status), tt.env.goos)
+			if err != nil {
+				t.Fatal(err)
+			}
+			simpleUp, justEditMP, err := updatePrefs(newPrefs, tt.curPrefs, tt.env)
+			if err != nil {
+				if tt.wantErrSubtr != "" {
+					if !strings.Contains(err.Error(), tt.wantErrSubtr) {
+						t.Fatalf("want error %q, got: %v", tt.wantErrSubtr, err)
+					}
+					return
+				}
+				t.Fatal(err)
+			}
+			if simpleUp != tt.wantSimpleUp {
+				t.Fatalf("simpleUp=%v, want %v", simpleUp, tt.wantSimpleUp)
+			}
+			if justEditMP != nil {
+				justEditMP.Prefs = ipn.Prefs{} // uninteresting
+			}
+			if !reflect.DeepEqual(justEditMP, tt.wantJustEditMP) {
+				t.Fatalf("justEditMP: %v", cmp.Diff(justEditMP, tt.wantJustEditMP))
+			}
+		})
+	}
+}

cmd/tailscale/cli/debug.go

@@ -31,6 +31,7 @@ var debugCmd = &ffcli.Command{
 		fs.BoolVar(&debugArgs.goroutines, "daemon-goroutines", false, "If true, dump the tailscaled daemon's goroutines")
 		fs.BoolVar(&debugArgs.ipn, "ipn", false, "If true, subscribe to IPN notifications")
 		fs.BoolVar(&debugArgs.prefs, "prefs", false, "If true, dump active prefs")
+		fs.BoolVar(&debugArgs.derpMap, "derp", false, "If true, dump DERP map")
 		fs.BoolVar(&debugArgs.pretty, "pretty", false, "If true, pretty-print output (for --prefs)")
 		fs.BoolVar(&debugArgs.netMap, "netmap", true, "whether to include netmap in --ipn mode")
 		fs.BoolVar(&debugArgs.localCreds, "local-creds", false, "print how to connect to local tailscaled")
@@ -44,6 +45,7 @@ var debugArgs struct {
 	goroutines bool
 	ipn        bool
 	netMap     bool
+	derpMap    bool
 	file       string
 	prefs      bool
 	pretty     bool
@@ -87,6 +89,18 @@ func runDebug(ctx context.Context, args []string) error {
 		os.Stdout.Write(goroutines)
 		return nil
 	}
+	if debugArgs.derpMap {
+		dm, err := tailscale.CurrentDERPMap(ctx)
+		if err != nil {
+			return fmt.Errorf(
+				"failed to get local derp map, instead `curl %s/derpmap/default`: %w", ipn.DefaultControlURL, err,
+			)
+		}
+		enc := json.NewEncoder(os.Stdout)
+		enc.SetIndent("", "\t")
+		enc.Encode(dm)
+		return nil
+	}
 	if debugArgs.ipn {
 		c, bc, ctx, cancel := connect(ctx)
 		defer cancel()

cmd/tailscale/cli/diag.go

@@ -0,0 +1,55 @@
+// Copyright (c) 2021 Tailscale Inc & AUTHORS All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+// +build linux windows darwin
+
+package cli
+
+import (
+	"fmt"
+	"path/filepath"
+	"runtime"
+	"strings"
+
+	ps "github.com/mitchellh/go-ps"
+)
+
+// fixTailscaledConnectError is called when the local tailscaled has
+// been determined unreachable due to the provided origErr value. It
+// returns either the same error or a better one to help the user
+// understand why tailscaled isn't running for their platform.
+func fixTailscaledConnectError(origErr error) error {
+	procs, err := ps.Processes()
+	if err != nil {
+		return fmt.Errorf("failed to connect to local Tailscaled process and failed to enumerate processes while looking for it")
+	}
+	found := false
+	for _, proc := range procs {
+		base := filepath.Base(proc.Executable())
+		if base == "tailscaled" {
+			found = true
+			break
+		}
+		if runtime.GOOS == "darwin" && base == "IPNExtension" {
+			found = true
+			break
+		}
+		if runtime.GOOS == "windows" && strings.EqualFold(base, "tailscaled.exe") {
+			found = true
+			break
+		}
+	}
+	if !found {
+		switch runtime.GOOS {
+		case "windows":
+			return fmt.Errorf("failed to connect to local tailscaled process; is the Tailscale service running?")
+		case "darwin":
+			return fmt.Errorf("failed to connect to local Tailscale service; is Tailscale running?")
+		case "linux":
+			return fmt.Errorf("failed to connect to local tailscaled; it doesn't appear to be running (sudo systemctl start tailscaled ?)")
+		}
+		return fmt.Errorf("failed to connect to local tailscaled process; it doesn't appear to be running")
+	}
+	return fmt.Errorf("failed to connect to local tailscaled (which appears to be running). Got error: %w", origErr)
+}

cmd/tailscale/cli/diag_other.go

@@ -0,0 +1,16 @@
+// Copyright (c) 2021 Tailscale Inc & AUTHORS All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+// +build !linux,!windows,!darwin
+
+package cli
+
+import "fmt"
+
+// The github.com/mitchellh/go-ps package doesn't work on all platforms,
+// so just don't diagnose connect failures.
+
+func fixTailscaledConnectError(origErr error) error {
+	return fmt.Errorf("failed to connect to local tailscaled process (is it running?); got: %w", origErr)
+}

cmd/tailscale/cli/netcheck.go

@@ -9,14 +9,18 @@ import (
 	"encoding/json"
 	"flag"
 	"fmt"
+	"io"
+	"io/ioutil"
 	"log"
+	"net/http"
 	"os"
 	"sort"
 	"strings"
 	"time"
 
 	"github.com/peterbourgon/ff/v2/ffcli"
-	"tailscale.com/derp/derpmap"
+	"tailscale.com/client/tailscale"
+	"tailscale.com/ipn"
 	"tailscale.com/net/netcheck"
 	"tailscale.com/net/portmapper"
 	"tailscale.com/tailcfg"
@@ -46,7 +50,7 @@ var netcheckArgs struct {
 func runNetcheck(ctx context.Context, args []string) error {
 	c := &netcheck.Client{
 		UDPBindAddr: os.Getenv("TS_DEBUG_NETCHECK_UDP_BIND"),
-		PortMapper:  portmapper.NewClient(logger.WithPrefix(log.Printf, "portmap: ")),
+		PortMapper:  portmapper.NewClient(logger.WithPrefix(log.Printf, "portmap: "), nil),
 	}
 	if netcheckArgs.verbose {
 		c.Logf = logger.WithPrefix(log.Printf, "netcheck: ")
@@ -59,7 +63,13 @@ func runNetcheck(ctx context.Context, args []string) error {
 		fmt.Fprintln(os.Stderr, "# Warning: this JSON format is not yet considered a stable interface")
 	}
 
-	dm := derpmap.Prod()
+	dm, err := tailscale.CurrentDERPMap(ctx)
+	if err != nil {
+		dm, err = prodDERPMap(ctx, http.DefaultClient)
+		if err != nil {
+			return err
+		}
+	}
 	for {
 		t0 := time.Now()
 		report, err := c.GetReport(ctx, dm)
@@ -176,3 +186,27 @@ func portMapping(r *netcheck.Report) string {
 	}
 	return strings.Join(got, ", ")
 }
+
+func prodDERPMap(ctx context.Context, httpc *http.Client) (*tailcfg.DERPMap, error) {
+	req, err := http.NewRequestWithContext(ctx, "GET", ipn.DefaultControlURL+"/derpmap/default", nil)
+	if err != nil {
+		return nil, fmt.Errorf("create prodDERPMap request: %w", err)
+	}
+	res, err := httpc.Do(req)
+	if err != nil {
+		return nil, fmt.Errorf("fetch prodDERPMap failed: %w", err)
+	}
+	defer res.Body.Close()
+	b, err := ioutil.ReadAll(io.LimitReader(res.Body, 1<<20))
+	if err != nil {
+		return nil, fmt.Errorf("fetch prodDERPMap failed: %w", err)
+	}
+	if res.StatusCode != 200 {
+		return nil, fmt.Errorf("fetch prodDERPMap: %v: %s", res.Status, b)
+	}
+	var derpMap tailcfg.DERPMap
+	if err = json.Unmarshal(b, &derpMap); err != nil {
+		return nil, fmt.Errorf("fetch prodDERPMap: %w", err)
+	}
+	return &derpMap, nil
+}

cmd/tailscale/cli/status.go

@@ -23,6 +23,7 @@ import (
 	"tailscale.com/ipn"
 	"tailscale.com/ipn/ipnstate"
 	"tailscale.com/net/interfaces"
+	"tailscale.com/tstime/mono"
 	"tailscale.com/util/dnsname"
 )
 
@@ -57,7 +58,7 @@ var statusArgs struct {
 func runStatus(ctx context.Context, args []string) error {
 	st, err := tailscale.Status(ctx)
 	if err != nil {
-		return err
+		return fixTailscaledConnectError(err)
 	}
 	if statusArgs.json {
 		if statusArgs.active {
@@ -193,7 +194,7 @@ func runStatus(ctx context.Context, args []string) error {
 //
 // TODO: have the server report this bool instead.
 func peerActive(ps *ipnstate.PeerStatus) bool {
-	return !ps.LastWrite.IsZero() && time.Since(ps.LastWrite) < 2*time.Minute
+	return !ps.LastWrite.IsZero() && mono.Since(ps.LastWrite) < 2*time.Minute
 }
 
 func dnsOrQuoteHostname(st *ipnstate.Status, ps *ipnstate.PeerStatus) string {

cmd/tailscale/cli/up.go

@@ -51,7 +51,14 @@ flag is also used.
 	Exec:    runUp,
 }
 
-var upFlagSet = newUpFlagSet(runtime.GOOS, &upArgs)
+func effectiveGOOS() string {
+	if v := os.Getenv("TS_DEBUG_UP_FLAG_GOOS"); v != "" {
+		return v
+	}
+	return runtime.GOOS
+}
+
+var upFlagSet = newUpFlagSet(effectiveGOOS(), &upArgs)
 
 func newUpFlagSet(goos string, upArgs *upArgsT) *flag.FlagSet {
 	upf := flag.NewFlagSet("up", flag.ExitOnError)
@@ -63,13 +70,13 @@ func newUpFlagSet(goos string, upArgs *upArgsT) *flag.FlagSet {
 	upf.BoolVar(&upArgs.acceptRoutes, "accept-routes", false, "accept routes advertised by other Tailscale nodes")
 	upf.BoolVar(&upArgs.acceptDNS, "accept-dns", true, "accept DNS configuration from the admin panel")
 	upf.BoolVar(&upArgs.singleRoutes, "host-routes", true, "install host routes to other Tailscale nodes")
-	upf.StringVar(&upArgs.exitNodeIP, "exit-node", "", "Tailscale IP of the exit node for internet traffic")
+	upf.StringVar(&upArgs.exitNodeIP, "exit-node", "", "Tailscale IP of the exit node for internet traffic, or empty string to not use an exit node")
 	upf.BoolVar(&upArgs.exitNodeAllowLANAccess, "exit-node-allow-lan-access", false, "Allow direct access to the local network when routing traffic via an exit node")
 	upf.BoolVar(&upArgs.shieldsUp, "shields-up", false, "don't allow incoming connections")
 	upf.StringVar(&upArgs.advertiseTags, "advertise-tags", "", "comma-separated ACL tags to request; each must start with \"tag:\" (e.g. \"tag:eng,tag:montreal,tag:ssh\")")
 	upf.StringVar(&upArgs.authKey, "authkey", "", "node authorization key")
 	upf.StringVar(&upArgs.hostname, "hostname", "", "hostname to use instead of the one provided by the OS")
-	upf.StringVar(&upArgs.advertiseRoutes, "advertise-routes", "", "routes to advertise to other nodes (comma-separated, e.g. \"10.0.0.0/8,192.168.0.0/24\")")
+	upf.StringVar(&upArgs.advertiseRoutes, "advertise-routes", "", "routes to advertise to other nodes (comma-separated, e.g. \"10.0.0.0/8,192.168.0.0/24\") or empty string to not advertise routes")
 	upf.BoolVar(&upArgs.advertiseDefaultRoute, "advertise-exit-node", false, "offer to be an exit node for internet traffic for the tailnet")
 	if safesocket.GOOSUsesPeerCreds(goos) {
 		upf.StringVar(&upArgs.opUser, "operator", "", "Unix username to allow to operate on tailscaled without sudo")
@@ -240,6 +247,53 @@ func prefsFromUpArgs(upArgs upArgsT, warnf logger.Logf, st *ipnstate.Status, goo
 	return prefs, nil
 }
 
+// updatePrefs updates prefs based on curPrefs
+//
+// It returns a non-nil justEditMP if we're already running and none of
+// the flags require a restart, so we can just do an EditPrefs call and
+// change the prefs at runtime (e.g. changing hostname, changing
+// advertised tags, routes, etc).
+//
+// It returns simpleUp if we're running a simple "tailscale up" to
+// transition to running from a previously-logged-in but down state,
+// without changing any settings.
+func updatePrefs(prefs, curPrefs *ipn.Prefs, env upCheckEnv) (simpleUp bool, justEditMP *ipn.MaskedPrefs, err error) {
+	if !env.upArgs.reset {
+		applyImplicitPrefs(prefs, curPrefs, env.user)
+
+		if err := checkForAccidentalSettingReverts(prefs, curPrefs, env); err != nil {
+			return false, nil, err
+		}
+	}
+
+	controlURLChanged := curPrefs.ControlURL != prefs.ControlURL &&
+		!(ipn.IsLoginServerSynonym(curPrefs.ControlURL) && ipn.IsLoginServerSynonym(prefs.ControlURL))
+	if controlURLChanged && env.backendState == ipn.Running.String() && !env.upArgs.forceReauth {
+		return false, nil, fmt.Errorf("can't change --login-server without --force-reauth")
+	}
+
+	simpleUp = env.flagSet.NFlag() == 0 &&
+		curPrefs.Persist != nil &&
+		curPrefs.Persist.LoginName != "" &&
+		env.backendState != ipn.NeedsLogin.String()
+
+	justEdit := env.backendState == ipn.Running.String() &&
+		!env.upArgs.forceReauth &&
+		!env.upArgs.reset &&
+		env.upArgs.authKey == "" &&
+		!controlURLChanged
+	if justEdit {
+		justEditMP = new(ipn.MaskedPrefs)
+		justEditMP.WantRunningSet = true
+		justEditMP.Prefs = *prefs
+		env.flagSet.Visit(func(f *flag.Flag) {
+			updateMaskedPrefsFromUpFlag(justEditMP, f.Name)
+		})
+	}
+
+	return simpleUp, justEditMP, nil
+}
+
 func runUp(ctx context.Context, args []string) error {
 	if len(args) > 0 {
 		fatalf("too many non-flag arguments: %q", args)
@@ -247,7 +301,7 @@ func runUp(ctx context.Context, args []string) error {
 
 	st, err := tailscale.Status(ctx)
 	if err != nil {
-		fatalf("can't fetch status from tailscaled: %v", err)
+		return fixTailscaledConnectError(err)
 	}
 	origAuthURL := st.AuthURL
 
@@ -280,7 +334,7 @@ func runUp(ctx context.Context, args []string) error {
 		}
 	}
 
-	prefs, err := prefsFromUpArgs(upArgs, warnf, st, runtime.GOOS)
+	prefs, err := prefsFromUpArgs(upArgs, warnf, st, effectiveGOOS())
 	if err != nil {
 		fatalf("%s", err)
 	}
@@ -296,51 +350,23 @@ func runUp(ctx context.Context, args []string) error {
 		return err
 	}
 
-	if !upArgs.reset {
-		applyImplicitPrefs(prefs, curPrefs, os.Getenv("USER"))
-
-		if err := checkForAccidentalSettingReverts(upFlagSet, curPrefs, prefs, upCheckEnv{
-			goos:          runtime.GOOS,
-			curExitNodeIP: exitNodeIP(prefs, st),
-		}); err != nil {
-			fatalf("%s", err)
-		}
+	env := upCheckEnv{
+		goos:          effectiveGOOS(),
+		user:          os.Getenv("USER"),
+		flagSet:       upFlagSet,
+		upArgs:        upArgs,
+		backendState:  st.BackendState,
+		curExitNodeIP: exitNodeIP(prefs, st),
 	}
-
-	controlURLChanged := curPrefs.ControlURL != prefs.ControlURL
-	if controlURLChanged && st.BackendState == ipn.Running.String() && !upArgs.forceReauth {
-		fatalf("can't change --login-server without --force-reauth")
+	simpleUp, justEditMP, err := updatePrefs(prefs, curPrefs, env)
+	if err != nil {
+		fatalf("%s", err)
 	}
-
-	// If we're already running and none of the flags require a
-	// restart, we can just do an EditPrefs call and change the
-	// prefs at runtime (e.g. changing hostname, changing
-	// advertised tags, routes, etc)
-	justEdit := st.BackendState == ipn.Running.String() &&
-		!upArgs.forceReauth &&
-		!upArgs.reset &&
-		upArgs.authKey == "" &&
-		!controlURLChanged
-	if justEdit {
-		mp := new(ipn.MaskedPrefs)
-		mp.WantRunningSet = true
-		mp.Prefs = *prefs
-		upFlagSet.Visit(func(f *flag.Flag) {
-			updateMaskedPrefsFromUpFlag(mp, f.Name)
-		})
-
-		_, err := tailscale.EditPrefs(ctx, mp)
+	if justEditMP != nil {
+		_, err := tailscale.EditPrefs(ctx, justEditMP)
 		return err
 	}
 
-	// simpleUp is whether we're running a simple "tailscale up"
-	// to transition to running from a previously-logged-in but
-	// down state, without changing any settings.
-	simpleUp := upFlagSet.NFlag() == 0 &&
-		curPrefs.Persist != nil &&
-		curPrefs.Persist.LoginName != "" &&
-		st.BackendState != ipn.NeedsLogin.String()
-
 	// At this point we need to subscribe to the IPN bus to watch
 	// for state transitions and possible need to authenticate.
 	c, bc, pumpCtx, cancel := connect(ctx)
@@ -365,7 +391,7 @@ func runUp(ctx context.Context, args []string) error {
 		if n.ErrMessage != nil {
 			msg := *n.ErrMessage
 			if msg == ipn.ErrMsgPermissionDenied {
-				switch runtime.GOOS {
+				switch effectiveGOOS() {
 				case "windows":
 					msg += " (Tailscale service in use by other user?)"
 				default:
@@ -381,7 +407,7 @@ func runUp(ctx context.Context, args []string) error {
 				startLoginInteractive()
 			case ipn.NeedsMachineAuth:
 				printed = true
-				fmt.Fprintf(os.Stderr, "\nTo authorize your machine, visit (as admin):\n\n\t%s/admin/machines\n\n", upArgs.server)
+				fmt.Fprintf(os.Stderr, "\nTo authorize your machine, visit (as admin):\n\n\t%s\n\n", prefs.AdminPageURL())
 			case ipn.Starting, ipn.Running:
 				// Done full authentication process
 				if printed {
@@ -439,7 +465,7 @@ func runUp(ctx context.Context, args []string) error {
 		// Windows service (~tailscaled) is the one that computes the
 		// StateKey based on the connection identity. So for now, just
 		// do as the Windows GUI's always done:
-		if runtime.GOOS == "windows" {
+		if effectiveGOOS() == "windows" {
 			// The Windows service will set this as needed based
 			// on our connection's identity.
 			opts.StateKey = ""
@@ -538,6 +564,10 @@ const accidentalUpPrefix = "Error: changing settings via 'tailscale up' requires
 // needed by checkForAccidentalSettingReverts and friends.
 type upCheckEnv struct {
 	goos          string
+	user          string
+	flagSet       *flag.FlagSet
+	upArgs        upArgsT
+	backendState  string
 	curExitNodeIP netaddr.IP
 }
 
@@ -555,14 +585,14 @@ type upCheckEnv struct {
 //
 // mp is the mask of settings actually set, where mp.Prefs is the new
 // preferences to set, including any values set from implicit flags.
-func checkForAccidentalSettingReverts(flagSet *flag.FlagSet, curPrefs, newPrefs *ipn.Prefs, env upCheckEnv) error {
+func checkForAccidentalSettingReverts(newPrefs, curPrefs *ipn.Prefs, env upCheckEnv) error {
 	if curPrefs.ControlURL == "" {
 		// Don't validate things on initial "up" before a control URL has been set.
 		return nil
 	}
 
 	flagIsSet := map[string]bool{}
-	flagSet.Visit(func(f *flag.Flag) {
+	env.flagSet.Visit(func(f *flag.Flag) {
 		flagIsSet[f.Name] = true
 	})
 
@@ -586,6 +616,9 @@ func checkForAccidentalSettingReverts(flagSet *flag.FlagSet, curPrefs, newPrefs
 		if reflect.DeepEqual(valCur, valNew) {
 			continue
 		}
+		if flagName == "login-server" && ipn.IsLoginServerSynonym(valCur) && ipn.IsLoginServerSynonym(valNew) {
+			continue
+		}
 		missing = append(missing, fmtFlagValueArg(flagName, valCur))
 	}
 	if len(missing) == 0 {
@@ -596,7 +629,7 @@ func checkForAccidentalSettingReverts(flagSet *flag.FlagSet, curPrefs, newPrefs
 	// Compute the stringification of the explicitly provided args in flagSet
 	// to prepend to the command to run.
 	var explicit []string
-	flagSet.Visit(func(f *flag.Flag) {
+	env.flagSet.Visit(func(f *flag.Flag) {
 		type isBool interface {
 			IsBoolFlag() bool
 		}

cmd/tailscale/cli/web.go

@@ -15,6 +15,7 @@ import (
 	"html/template"
 	"io/ioutil"
 	"log"
+	"net"
 	"net/http"
 	"net/http/cgi"
 	"net/url"
@@ -60,6 +61,14 @@ var webCmd = &ffcli.Command{
 	ShortUsage: "web [flags]",
 	ShortHelp:  "Run a web server for controlling Tailscale",
 
+	LongHelp: strings.TrimSpace(`
+"tailscale web" runs a webserver for controlling the Tailscale daemon.
+
+It's primarily intended for use on Synology, QNAP, and other
+NAS devices where a web interface is the natural place to control
+Tailscale, as opposed to a CLI or a native app.
+`),
+
 	FlagSet: (func() *flag.FlagSet {
 		webf := flag.NewFlagSet("web", flag.ExitOnError)
 		webf.StringVar(&webArgs.listen, "listen", "localhost:8088", "listen address; use port 0 for automatic")
@@ -86,9 +95,20 @@ func runWeb(ctx context.Context, args []string) error {
 		}
 		return nil
 	}
+
+	log.Printf("web server running on: %s", urlOfListenAddr(webArgs.listen))
 	return http.ListenAndServe(webArgs.listen, http.HandlerFunc(webHandler))
 }
 
+// urlOfListenAddr parses a given listen address into a formatted URL
+func urlOfListenAddr(addr string) string {
+	host, port, _ := net.SplitHostPort(addr)
+	if host == "" {
+		host = "127.0.0.1"
+	}
+	return fmt.Sprintf("http://%s", net.JoinHostPort(host, port))
+}
+
 // authorize returns the name of the user accessing the web UI after verifying
 // whether the user has access to the web UI. The function will write the
 // error to the provided http.ResponseWriter.

cmd/tailscale/cli/web_test.go

@@ -0,0 +1,44 @@
+// Copyright (c) 2021 Tailscale Inc & AUTHORS All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package cli
+
+import "testing"
+
+func TestUrlOfListenAddr(t *testing.T) {
+	tests := []struct {
+		name     string
+		in, want string
+	}{
+		{
+			name: "TestLocalhost",
+			in:   "localhost:8088",
+			want: "http://localhost:8088",
+		},
+		{
+			name: "TestNoHost",
+			in:   ":8088",
+			want: "http://127.0.0.1:8088",
+		},
+		{
+			name: "TestExplicitHost",
+			in:   "127.0.0.2:8088",
+			want: "http://127.0.0.2:8088",
+		},
+		{
+			name: "TestIPv6",
+			in:   "[::1]:8088",
+			want: "http://[::1]:8088",
+		},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			url := urlOfListenAddr(tt.in)
+			if url != tt.want {
+				t.Errorf("expected url: %q, got: %q", tt.want, url)
+			}
+		})
+	}
+}

cmd/tailscale/depaware.txt

@@ -4,8 +4,15 @@ tailscale.com/cmd/tailscale dependencies: (generated by github.com/tailscale/dep
    W    github.com/alexbrainman/sspi/internal/common                 from github.com/alexbrainman/sspi/negotiate
    W 💣 github.com/alexbrainman/sspi/negotiate                       from tailscale.com/net/tshttpproxy
         github.com/kballard/go-shellquote                            from tailscale.com/cmd/tailscale/cli
+     💣 github.com/mitchellh/go-ps                                   from tailscale.com/cmd/tailscale/cli
         github.com/peterbourgon/ff/v2                                from github.com/peterbourgon/ff/v2/ffcli
         github.com/peterbourgon/ff/v2/ffcli                          from tailscale.com/cmd/tailscale/cli
+        github.com/tailscale/goupnp                                  from github.com/tailscale/goupnp/dcps/internetgateway2
+        github.com/tailscale/goupnp/dcps/internetgateway2            from tailscale.com/net/portmapper
+        github.com/tailscale/goupnp/httpu                            from github.com/tailscale/goupnp+
+        github.com/tailscale/goupnp/scpd                             from github.com/tailscale/goupnp
+        github.com/tailscale/goupnp/soap                             from github.com/tailscale/goupnp+
+        github.com/tailscale/goupnp/ssdp                             from github.com/tailscale/goupnp
         github.com/tcnksm/go-httpstat                                from tailscale.com/net/netcheck
         github.com/toqueteos/webbrowser                              from tailscale.com/cmd/tailscale/cli
      💣 go4.org/intern                                               from inet.af/netaddr
@@ -18,9 +25,9 @@ tailscale.com/cmd/tailscale dependencies: (generated by github.com/tailscale/dep
         tailscale.com/client/tailscale                               from tailscale.com/cmd/tailscale/cli+
         tailscale.com/client/tailscale/apitype                       from tailscale.com/client/tailscale+
         tailscale.com/cmd/tailscale/cli                              from tailscale.com/cmd/tailscale
+        tailscale.com/control/controlknobs                           from tailscale.com/net/portmapper
         tailscale.com/derp                                           from tailscale.com/derp/derphttp
         tailscale.com/derp/derphttp                                  from tailscale.com/net/netcheck
-        tailscale.com/derp/derpmap                                   from tailscale.com/cmd/tailscale/cli
         tailscale.com/disco                                          from tailscale.com/derp
         tailscale.com/hostinfo                                       from tailscale.com/net/interfaces
         tailscale.com/ipn                                            from tailscale.com/cmd/tailscale/cli+
@@ -42,6 +49,8 @@ tailscale.com/cmd/tailscale dependencies: (generated by github.com/tailscale/dep
         tailscale.com/syncs                                          from tailscale.com/net/interfaces+
         tailscale.com/tailcfg                                        from tailscale.com/cmd/tailscale/cli+
    W    tailscale.com/tsconst                                        from tailscale.com/net/interfaces
+     💣 tailscale.com/tstime/mono                                    from tailscale.com/cmd/tailscale/cli+
+        tailscale.com/tstime/rate                                    from tailscale.com/wgengine/filter
         tailscale.com/types/empty                                    from tailscale.com/ipn
         tailscale.com/types/ipproto                                  from tailscale.com/net/flowtrack+
         tailscale.com/types/key                                      from tailscale.com/derp+
@@ -77,7 +86,7 @@ tailscale.com/cmd/tailscale dependencies: (generated by github.com/tailscale/dep
         golang.org/x/net/idna                                        from golang.org/x/net/http/httpguts+
         golang.org/x/net/proxy                                       from tailscale.com/net/netns
    D    golang.org/x/net/route                                       from net+
-        golang.org/x/sync/errgroup                                   from tailscale.com/derp
+        golang.org/x/sync/errgroup                                   from tailscale.com/derp+
         golang.org/x/sync/singleflight                               from tailscale.com/net/dnscache
         golang.org/x/sys/cpu                                         from golang.org/x/crypto/blake2b+
   LD    golang.org/x/sys/unix                                        from tailscale.com/net/netns+
@@ -127,7 +136,7 @@ tailscale.com/cmd/tailscale dependencies: (generated by github.com/tailscale/dep
         encoding/hex                                                 from crypto/x509+
         encoding/json                                                from expvar+
         encoding/pem                                                 from crypto/tls+
-        encoding/xml                                                 from tailscale.com/cmd/tailscale/cli
+        encoding/xml                                                 from tailscale.com/cmd/tailscale/cli+
         errors                                                       from bufio+
         expvar                                                       from tailscale.com/derp+
         flag                                                         from github.com/peterbourgon/ff/v2+

cmd/tailscaled/debug.go

@@ -11,19 +11,26 @@ import (
 	"errors"
 	"flag"
 	"fmt"
+	"io"
+	"io/ioutil"
 	"log"
+	"net"
 	"net/http"
 	"net/http/httptrace"
 	"net/url"
 	"os"
+	"strings"
 	"time"
 
+	"inet.af/netaddr"
 	"tailscale.com/derp/derphttp"
-	"tailscale.com/derp/derpmap"
+	"tailscale.com/ipn"
 	"tailscale.com/net/interfaces"
+	"tailscale.com/net/portmapper"
 	"tailscale.com/net/tshttpproxy"
 	"tailscale.com/tailcfg"
 	"tailscale.com/types/key"
+	"tailscale.com/types/logger"
 	"tailscale.com/wgengine/monitor"
 )
 
@@ -31,6 +38,7 @@ var debugArgs struct {
 	monitor   bool
 	getURL    string
 	derpCheck string
+	portmap   bool
 }
 
 var debugModeFunc = debugMode // so it can be addressable
@@ -38,6 +46,7 @@ var debugModeFunc = debugMode // so it can be addressable
 func debugMode(args []string) error {
 	fs := flag.NewFlagSet("debug", flag.ExitOnError)
 	fs.BoolVar(&debugArgs.monitor, "monitor", false, "If true, run link monitor forever. Precludes all other options.")
+	fs.BoolVar(&debugArgs.portmap, "portmap", false, "If true, run portmap debugging. Precludes all other options.")
 	fs.StringVar(&debugArgs.getURL, "get-url", "", "If non-empty, fetch provided URL.")
 	fs.StringVar(&debugArgs.derpCheck, "derp", "", "if non-empty, test a DERP ping via named region code")
 	if err := fs.Parse(args); err != nil {
@@ -53,6 +62,9 @@ func debugMode(args []string) error {
 	if debugArgs.monitor {
 		return runMonitor(ctx)
 	}
+	if debugArgs.portmap {
+		return debugPortmap(ctx)
+	}
 	if debugArgs.getURL != "" {
 		return getURL(ctx, debugArgs.getURL)
 	}
@@ -131,7 +143,26 @@ func getURL(ctx context.Context, urlStr string) error {
 }
 
 func checkDerp(ctx context.Context, derpRegion string) error {
-	dmap := derpmap.Prod()
+	req, err := http.NewRequestWithContext(ctx, "GET", ipn.DefaultControlURL+"/derpmap/default", nil)
+	if err != nil {
+		return fmt.Errorf("create derp map request: %w", err)
+	}
+	res, err := http.DefaultClient.Do(req)
+	if err != nil {
+		return fmt.Errorf("fetch derp map failed: %w", err)
+	}
+	defer res.Body.Close()
+	b, err := ioutil.ReadAll(io.LimitReader(res.Body, 1<<20))
+	if err != nil {
+		return fmt.Errorf("fetch derp map failed: %w", err)
+	}
+	if res.StatusCode != 200 {
+		return fmt.Errorf("fetch derp map: %v: %s", res.Status, b)
+	}
+	var dmap tailcfg.DERPMap
+	if err = json.Unmarshal(b, &dmap); err != nil {
+		return fmt.Errorf("fetch DERP map: %w", err)
+	}
 	getRegion := func() *tailcfg.DERPRegion {
 		for _, r := range dmap.Regions {
 			if r.RegionCode == derpRegion {
@@ -170,3 +201,83 @@ func checkDerp(ctx context.Context, derpRegion string) error {
 	log.Printf("ok")
 	return err
 }
+
+func debugPortmap(ctx context.Context) error {
+	ctx, cancel := context.WithTimeout(ctx, 3*time.Second)
+	defer cancel()
+
+	portmapper.VerboseLogs = true
+
+	done := make(chan bool, 1)
+
+	var c *portmapper.Client
+	logf := log.Printf
+	c = portmapper.NewClient(logger.WithPrefix(logf, "portmapper: "), func() {
+		logf("portmapping changed.")
+		logf("have mapping: %v", c.HaveMapping())
+
+		if ext, ok := c.GetCachedMappingOrStartCreatingOne(); ok {
+			logf("cb: mapping: %v", ext)
+			select {
+			case done <- true:
+			default:
+			}
+			return
+		}
+		logf("cb: no mapping")
+	})
+	linkMon, err := monitor.New(logger.WithPrefix(logf, "monitor: "))
+	if err != nil {
+		return err
+	}
+
+	gatewayAndSelfIP := func() (gw, self netaddr.IP, ok bool) {
+		if v := os.Getenv("TS_DEBUG_GW_SELF"); strings.Contains(v, "/") {
+			i := strings.Index(v, "/")
+			gw = netaddr.MustParseIP(v[:i])
+			self = netaddr.MustParseIP(v[i+1:])
+			return gw, self, true
+		}
+		return linkMon.GatewayAndSelfIP()
+	}
+
+	c.SetGatewayLookupFunc(gatewayAndSelfIP)
+
+	gw, selfIP, ok := gatewayAndSelfIP()
+	if !ok {
+		logf("no gateway or self IP; %v", linkMon.InterfaceState())
+		return nil
+	}
+	logf("gw=%v; self=%v", gw, selfIP)
+
+	res, err := c.Probe(ctx)
+	if err != nil {
+		return fmt.Errorf("Probe: %v", err)
+	}
+	logf("Probe: %+v", res)
+
+	if !res.PCP && !res.PMP && !res.UPnP {
+		logf("no portmapping services available")
+		return nil
+	}
+
+	uc, err := net.ListenPacket("udp", "0.0.0.0:0")
+	if err != nil {
+		return err
+	}
+	defer uc.Close()
+	c.SetLocalPort(uint16(uc.LocalAddr().(*net.UDPAddr).Port))
+
+	if ext, ok := c.GetCachedMappingOrStartCreatingOne(); ok {
+		logf("mapping: %v", ext)
+	} else {
+		logf("no mapping")
+	}
+
+	select {
+	case <-done:
+		return nil
+	case <-ctx.Done():
+		return ctx.Err()
+	}
+}

cmd/tailscaled/depaware.txt

@@ -23,6 +23,12 @@ tailscale.com/cmd/tailscaled dependencies: (generated by github.com/tailscale/de
    L 💣 github.com/mdlayher/socket                                   from github.com/mdlayher/netlink
    W    github.com/pkg/errors                                        from github.com/tailscale/certstore
    W 💣 github.com/tailscale/certstore                               from tailscale.com/control/controlclient
+        github.com/tailscale/goupnp                                  from github.com/tailscale/goupnp/dcps/internetgateway2
+        github.com/tailscale/goupnp/dcps/internetgateway2            from tailscale.com/net/portmapper
+        github.com/tailscale/goupnp/httpu                            from github.com/tailscale/goupnp+
+        github.com/tailscale/goupnp/scpd                             from github.com/tailscale/goupnp
+        github.com/tailscale/goupnp/soap                             from github.com/tailscale/goupnp+
+        github.com/tailscale/goupnp/ssdp                             from github.com/tailscale/goupnp
         github.com/tcnksm/go-httpstat                                from tailscale.com/net/netcheck
      💣 go4.org/intern                                               from inet.af/netaddr
      💣 go4.org/mem                                                  from tailscale.com/derp+
@@ -79,13 +85,12 @@ tailscale.com/cmd/tailscaled dependencies: (generated by github.com/tailscale/de
         tailscale.com/client/tailscale                               from tailscale.com/derp
         tailscale.com/client/tailscale/apitype                       from tailscale.com/ipn/ipnlocal+
         tailscale.com/control/controlclient                          from tailscale.com/ipn/ipnlocal+
+        tailscale.com/control/controlknobs                           from tailscale.com/control/controlclient+
         tailscale.com/derp                                           from tailscale.com/derp/derphttp+
         tailscale.com/derp/derphttp                                  from tailscale.com/net/netcheck+
-        tailscale.com/derp/derpmap                                   from tailscale.com/cmd/tailscaled
         tailscale.com/disco                                          from tailscale.com/derp+
         tailscale.com/health                                         from tailscale.com/control/controlclient+
         tailscale.com/hostinfo                                       from tailscale.com/control/controlclient+
-        tailscale.com/internal/deephash                              from tailscale.com/ipn/ipnlocal+
         tailscale.com/ipn                                            from tailscale.com/ipn/ipnserver+
         tailscale.com/ipn/ipnlocal                                   from tailscale.com/ipn/ipnserver+
         tailscale.com/ipn/ipnserver                                  from tailscale.com/cmd/tailscaled
@@ -110,7 +115,8 @@ tailscale.com/cmd/tailscaled dependencies: (generated by github.com/tailscale/de
      💣 tailscale.com/net/netstat                                    from tailscale.com/ipn/ipnserver
         tailscale.com/net/packet                                     from tailscale.com/wgengine+
         tailscale.com/net/portmapper                                 from tailscale.com/net/netcheck+
-        tailscale.com/net/socks5                                     from tailscale.com/cmd/tailscaled
+        tailscale.com/net/socks5                                     from tailscale.com/net/socks5/tssocks
+        tailscale.com/net/socks5/tssocks                             from tailscale.com/cmd/tailscaled
         tailscale.com/net/stun                                       from tailscale.com/net/netcheck+
         tailscale.com/net/tlsdial                                    from tailscale.com/control/controlclient+
         tailscale.com/net/tsaddr                                     from tailscale.com/ipn/ipnlocal+
@@ -124,6 +130,8 @@ tailscale.com/cmd/tailscaled dependencies: (generated by github.com/tailscale/de
         tailscale.com/tailcfg                                        from tailscale.com/control/controlclient+
    W    tailscale.com/tsconst                                        from tailscale.com/net/interfaces
         tailscale.com/tstime                                         from tailscale.com/wgengine/magicsock
+     💣 tailscale.com/tstime/mono                                    from tailscale.com/net/tstun+
+        tailscale.com/tstime/rate                                    from tailscale.com/wgengine/filter
         tailscale.com/types/empty                                    from tailscale.com/control/controlclient+
         tailscale.com/types/flagtype                                 from tailscale.com/cmd/tailscaled
         tailscale.com/types/ipproto                                  from tailscale.com/net/flowtrack+
@@ -137,6 +145,7 @@ tailscale.com/cmd/tailscaled dependencies: (generated by github.com/tailscale/de
         tailscale.com/types/structs                                  from tailscale.com/control/controlclient+
         tailscale.com/types/wgkey                                    from tailscale.com/control/controlclient+
    L    tailscale.com/util/cmpver                                    from tailscale.com/net/dns
+     💣 tailscale.com/util/deephash                                  from tailscale.com/ipn/ipnlocal+
         tailscale.com/util/dnsname                                   from tailscale.com/ipn/ipnstate+
   LW    tailscale.com/util/endian                                    from tailscale.com/net/netns+
         tailscale.com/util/groupmember                               from tailscale.com/ipn/ipnserver
@@ -154,7 +163,7 @@ tailscale.com/cmd/tailscaled dependencies: (generated by github.com/tailscale/de
         tailscale.com/wgengine/filter                                from tailscale.com/control/controlclient+
         tailscale.com/wgengine/magicsock                             from tailscale.com/wgengine+
         tailscale.com/wgengine/monitor                               from tailscale.com/wgengine+
-        tailscale.com/wgengine/netstack                              from tailscale.com/cmd/tailscaled
+        tailscale.com/wgengine/netstack                              from tailscale.com/cmd/tailscaled+
         tailscale.com/wgengine/router                                from tailscale.com/cmd/tailscaled+
         tailscale.com/wgengine/wgcfg                                 from tailscale.com/ipn/ipnlocal+
         tailscale.com/wgengine/wgcfg/nmcfg                           from tailscale.com/ipn/ipnlocal
@@ -182,7 +191,7 @@ tailscale.com/cmd/tailscaled dependencies: (generated by github.com/tailscale/de
         golang.org/x/net/ipv6                                        from golang.zx2c4.com/wireguard/device+
         golang.org/x/net/proxy                                       from tailscale.com/net/netns
    D    golang.org/x/net/route                                       from net+
-        golang.org/x/sync/errgroup                                   from tailscale.com/derp
+        golang.org/x/sync/errgroup                                   from tailscale.com/derp+
         golang.org/x/sync/singleflight                               from tailscale.com/net/dnscache
         golang.org/x/sys/cpu                                         from golang.org/x/crypto/blake2b+
   LD    golang.org/x/sys/unix                                        from github.com/mdlayher/netlink+
@@ -236,6 +245,7 @@ tailscale.com/cmd/tailscaled dependencies: (generated by github.com/tailscale/de
         encoding/hex                                                 from crypto/x509+
         encoding/json                                                from expvar+
         encoding/pem                                                 from crypto/tls+
+        encoding/xml                                                 from github.com/tailscale/goupnp+
         errors                                                       from bufio+
         expvar                                                       from tailscale.com/derp+
         flag                                                         from tailscale.com/cmd/tailscaled+

cmd/tailscaled/tailscaled.go

@@ -24,22 +24,19 @@ import (
 	"runtime/debug"
 	"strconv"
 	"strings"
-	"sync"
 	"syscall"
 	"time"
 
 	"github.com/go-multierror/multierror"
-	"inet.af/netaddr"
+	"tailscale.com/ipn"
 	"tailscale.com/ipn/ipnserver"
 	"tailscale.com/logpolicy"
 	"tailscale.com/net/dns"
-	"tailscale.com/net/socks5"
-	"tailscale.com/net/tsaddr"
+	"tailscale.com/net/socks5/tssocks"
 	"tailscale.com/net/tstun"
 	"tailscale.com/paths"
 	"tailscale.com/types/flagtype"
 	"tailscale.com/types/logger"
-	"tailscale.com/types/netmap"
 	"tailscale.com/util/osshare"
 	"tailscale.com/version"
 	"tailscale.com/version/distro"
@@ -49,15 +46,6 @@ import (
 	"tailscale.com/wgengine/router"
 )
 
-// globalStateKey is the ipn.StateKey that tailscaled loads on
-// startup.
-//
-// We have to support multiple state keys for other OSes (Windows in
-// particular), but right now Unix daemons run with a single
-// node-global state. To keep open the option of having per-user state
-// later, the global state key doesn't look like a username.
-const globalStateKey = "_daemon"
-
 // defaultTunName returns the default tun device name for the platform.
 func defaultTunName() string {
 	switch runtime.GOOS {
@@ -171,6 +159,28 @@ func main() {
 	}
 }
 
+func ipnServerOpts() (o ipnserver.Options) {
+	// Allow changing the OS-specific IPN behavior for tests
+	// so we can e.g. test Windows-specific behaviors on Linux.
+	goos := os.Getenv("TS_DEBUG_TAILSCALED_IPN_GOOS")
+	if goos == "" {
+		goos = runtime.GOOS
+	}
+
+	o.Port = 41112
+	o.StatePath = args.statepath
+	o.SocketPath = args.socketpath // even for goos=="windows", for tests
+
+	switch goos {
+	default:
+		o.SurviveDisconnects = true
+		o.AutostartStateKey = ipn.GlobalDaemonStateKey
+	case "windows":
+		// Not those.
+	}
+	return o
+}
+
 func run() error {
 	var err error
 
@@ -200,6 +210,9 @@ func run() error {
 	logf = logger.RateLimitedFn(logf, 5*time.Second, 5, 100)
 
 	if args.cleanup {
+		if os.Getenv("TS_PLEASE_PANIC") != "" {
+			panic("TS_PLEASE_PANIC asked us to panic")
+		}
 		dns.Cleanup(logf, args.tunname)
 		router.Cleanup(logf, args.tunname)
 		return nil
@@ -228,6 +241,11 @@ func run() error {
 		if err != nil {
 			log.Fatalf("SOCKS5 listener: %v", err)
 		}
+		if strings.HasSuffix(args.socksAddr, ":0") {
+			// Log kernel-selected port number so integration tests
+			// can find it portably.
+			log.Printf("SOCKS5 listening on %v", socksListener.Addr())
+		}
 	}
 
 	e, useNetstack, err := createEngine(logf, linkMon)
@@ -243,35 +261,7 @@ func run() error {
 	}
 
 	if socksListener != nil {
-		srv := &socks5.Server{
-			Logf: logger.WithPrefix(logf, "socks5: "),
-		}
-		var (
-			mu  sync.Mutex // guards the following field
-			dns netstack.DNSMap
-		)
-		e.AddNetworkMapCallback(func(nm *netmap.NetworkMap) {
-			mu.Lock()
-			defer mu.Unlock()
-			dns = netstack.DNSMapFromNetworkMap(nm)
-		})
-		useNetstackForIP := func(ip netaddr.IP) bool {
-			// TODO(bradfitz): this isn't exactly right.
-			// We should also support subnets when the
-			// prefs are configured as such.
-			return tsaddr.IsTailscaleIP(ip)
-		}
-		srv.Dialer = func(ctx context.Context, network, addr string) (net.Conn, error) {
-			ipp, err := dns.Resolve(ctx, addr)
-			if err != nil {
-				return nil, err
-			}
-			if ns != nil && useNetstackForIP(ipp.IP()) {
-				return ns.DialContextTCP(ctx, addr)
-			}
-			var d net.Dialer
-			return d.DialContext(ctx, network, ipp.String())
-		}
+		srv := tssocks.NewServer(logger.WithPrefix(logf, "socks5: "), e, ns)
 		go func() {
 			log.Fatalf("SOCKS5 server exited: %v", srv.Serve(socksListener))
 		}()
@@ -298,14 +288,8 @@ func run() error {
 		}
 	}()
 
-	opts := ipnserver.Options{
-		SocketPath:         args.socketpath,
-		Port:               41112,
-		StatePath:          args.statepath,
-		AutostartStateKey:  globalStateKey,
-		SurviveDisconnects: runtime.GOOS != "windows",
-		DebugMux:           debugMux,
-	}
+	opts := ipnServerOpts()
+	opts.DebugMux = debugMux
 	err = ipnserver.Run(ctx, logf, pol.PublicID.String(), ipnserver.FixedEngine(e), opts)
 	// Cancelation is not an error: it is the only way to stop ipnserver.
 	if err != nil && err != context.Canceled {
@@ -368,7 +352,7 @@ func tryEngine(logf logger.Logf, linkMon *monitor.Mon, name string) (e wgengine.
 			return nil, false, err
 		}
 		conf.Tun = dev
-		r, err := router.New(logf, dev)
+		r, err := router.New(logf, dev, linkMon)
 		if err != nil {
 			dev.Close()
 			return nil, false, err

cmd/tailscaled/tailscaled.openrc

@@ -1,6 +1,8 @@
 #!/sbin/openrc-run
 
+set -a
 source /etc/default/tailscaled
+set +a
 
 command="/usr/sbin/tailscaled"
 command_args="--state=/var/lib/tailscale/tailscaled.state --port=$PORT --socket=/var/run/tailscale/tailscaled.sock $FLAGS"

cmd/tailscaled/tailscaled_windows.go

@@ -19,6 +19,7 @@ package main // import "tailscale.com/cmd/tailscaled"
 
 import (
 	"context"
+	"encoding/json"
 	"fmt"
 	"log"
 	"os"
@@ -27,6 +28,7 @@ import (
 	"golang.org/x/sys/windows"
 	"golang.org/x/sys/windows/svc"
 	"golang.zx2c4.com/wireguard/windows/tunnel/winipcfg"
+	"inet.af/netaddr"
 	"tailscale.com/ipn/ipnserver"
 	"tailscale.com/logpolicy"
 	"tailscale.com/net/dns"
@@ -126,16 +128,6 @@ func beFirewallKillswitch() bool {
 	log.SetFlags(0)
 	log.Printf("killswitch subprocess starting, tailscale GUID is %s", os.Args[2])
 
-	go func() {
-		b := make([]byte, 16)
-		for {
-			_, err := os.Stdin.Read(b)
-			if err != nil {
-				log.Fatalf("parent process died or requested exit, exiting (%v)", err)
-			}
-		}
-	}()
-
 	guid, err := windows.GUIDFromString(os.Args[2])
 	if err != nil {
 		log.Fatalf("invalid GUID %q: %v", os.Args[2], err)
@@ -147,13 +139,25 @@ func beFirewallKillswitch() bool {
 	}
 
 	start := time.Now()
-	if _, err := wf.New(uint64(luid)); err != nil {
-		log.Fatalf("filewall creation failed: %v", err)
+	fw, err := wf.New(uint64(luid))
+	if err != nil {
+		log.Fatalf("failed to enable firewall: %v", err)
 	}
 	log.Printf("killswitch enabled, took %s", time.Since(start))
 
-	// Block until the monitor goroutine shuts us down.
-	select {}
+	// Note(maisem): when local lan access toggled, tailscaled needs to
+	// inform the firewall to let local routes through. The set of routes
+	// is passed in via stdin encoded in json.
+	dcd := json.NewDecoder(os.Stdin)
+	for {
+		var routes []netaddr.IPPrefix
+		if err := dcd.Decode(&routes); err != nil {
+			log.Fatalf("parent process died or requested exit, exiting (%v)", err)
+		}
+		if err := fw.UpdatePermittedRoutes(routes); err != nil {
+			log.Fatalf("failed to update routes (%v)", err)
+		}
+	}
 }
 
 func startIPNServer(ctx context.Context, logid string) error {
@@ -164,7 +168,7 @@ func startIPNServer(ctx context.Context, logid string) error {
 		if err != nil {
 			return nil, fmt.Errorf("TUN: %w", err)
 		}
-		r, err := router.New(logf, dev)
+		r, err := router.New(logf, dev, nil)
 		if err != nil {
 			dev.Close()
 			return nil, fmt.Errorf("router: %w", err)
@@ -229,12 +233,6 @@ func startIPNServer(ctx context.Context, logid string) error {
 		}
 	}()
 
-	opts := ipnserver.Options{
-		Port:               41112,
-		SurviveDisconnects: false,
-		StatePath:          args.statepath,
-	}
-
 	// getEngine is called by ipnserver to get the engine. It's
 	// not called concurrently and is not called again once it
 	// successfully returns an engine.
@@ -259,7 +257,7 @@ func startIPNServer(ctx context.Context, logid string) error {
 			return nil, fmt.Errorf("%w\n\nlogid: %v", res.Err, logid)
 		}
 	}
-	err := ipnserver.Run(ctx, logf, logid, getEngine, opts)
+	err := ipnserver.Run(ctx, logf, logid, getEngine, ipnServerOpts())
 	if err != nil {
 		logf("ipnserver.Run: %v", err)
 	}

cmd/tsshd/tsshd.go

@@ -29,8 +29,8 @@ import (
 	"time"
 	"unsafe"
 
+	"github.com/creack/pty"
 	"github.com/gliderlabs/ssh"
-	"github.com/kr/pty"
 	gossh "golang.org/x/crypto/ssh"
 	"inet.af/netaddr"
 	"tailscale.com/net/interfaces"

control/controlclient/debug.go

@@ -11,7 +11,6 @@ import (
 	"fmt"
 	"log"
 	"net/http"
-	"regexp"
 	"runtime"
 	"strconv"
 	"time"
@@ -42,28 +41,82 @@ func dumpGoroutinesToURL(c *http.Client, targetURL string) {
 	}
 }
 
-var reHexArgs = regexp.MustCompile(`\b0x[0-9a-f]+\b`)
-
 // scrubbedGoroutineDump returns the list of all current goroutines, but with the actual
 // values of arguments scrubbed out, lest it contain some private key material.
 func scrubbedGoroutineDump() []byte {
-	buf := make([]byte, 1<<20)
-	buf = buf[:runtime.Stack(buf, true)]
+	var buf []byte
+	// Grab stacks multiple times into increasingly larger buffer sizes
+	// to minimize the risk that we blow past our iOS memory limit.
+	for size := 1 << 10; size <= 1<<20; size += 1 << 10 {
+		buf = make([]byte, size)
+		buf = buf[:runtime.Stack(buf, true)]
+		if len(buf) < size {
+			// It fit.
+			break
+		}
+	}
+	return scrubHex(buf)
+}
 
+func scrubHex(buf []byte) []byte {
 	saw := map[string][]byte{} // "0x123" => "v1%3" (unique value 1 and its value mod 8)
-	return reHexArgs.ReplaceAllFunc(buf, func(in []byte) []byte {
+
+	foreachHexAddress(buf, func(in []byte) {
 		if string(in) == "0x0" {
-			return in
+			return
 		}
 		if v, ok := saw[string(in)]; ok {
-			return v
+			for i := range in {
+				in[i] = '_'
+			}
+			copy(in, v)
+			return
 		}
+		inStr := string(in)
 		u64, err := strconv.ParseUint(string(in[2:]), 16, 64)
+		for i := range in {
+			in[i] = '_'
+		}
 		if err != nil {
-			return []byte("??")
+			in[0] = '?'
+			return
 		}
 		v := []byte(fmt.Sprintf("v%d%%%d", len(saw)+1, u64%8))
-		saw[string(in)] = v
-		return v
+		saw[inStr] = v
+		copy(in, v)
 	})
+	return buf
+}
+
+var ohx = []byte("0x")
+
+// foreachHexAddress calls f with each subslice of b that matches
+// regexp `0x[0-9a-f]*`.
+func foreachHexAddress(b []byte, f func([]byte)) {
+	for len(b) > 0 {
+		i := bytes.Index(b, ohx)
+		if i == -1 {
+			return
+		}
+		b = b[i:]
+		hx := hexPrefix(b)
+		f(hx)
+		b = b[len(hx):]
+	}
+}
+
+func hexPrefix(b []byte) []byte {
+	for i, c := range b {
+		if i < 2 {
+			continue
+		}
+		if !isHexByte(c) {
+			return b[:i]
+		}
+	}
+	return b
+}
+
+func isHexByte(b byte) bool {
+	return '0' <= b && b <= '9' || 'a' <= b && b <= 'f' || 'A' <= b && b <= 'F'
 }

control/controlclient/debug_test.go

@@ -9,3 +9,22 @@ import "testing"
 func TestScrubbedGoroutineDump(t *testing.T) {
 	t.Logf("Got:\n%s\n", scrubbedGoroutineDump())
 }
+
+func TestScrubHex(t *testing.T) {
+	tests := []struct {
+		in, want string
+	}{
+		{"foo", "foo"},
+		{"", ""},
+		{"0x", "?_"},
+		{"0x001 and same 0x001", "v1%1_ and same v1%1_"},
+		{"0x008 and same 0x008", "v1%0_ and same v1%0_"},
+		{"0x001 and diff 0x002", "v1%1_ and diff v2%2_"},
+	}
+	for _, tt := range tests {
+		got := scrubHex([]byte(tt.in))
+		if string(got) != tt.want {
+			t.Errorf("for input:\n%s\n\ngot:\n%s\n\nwant:\n%s\n", tt.in, got, tt.want)
+		}
+	}
+}

control/controlclient/direct.go

@@ -31,6 +31,7 @@ import (
 
 	"golang.org/x/crypto/nacl/box"
 	"inet.af/netaddr"
+	"tailscale.com/control/controlknobs"
 	"tailscale.com/health"
 	"tailscale.com/ipn/ipnstate"
 	"tailscale.com/log/logheap"
@@ -219,6 +220,13 @@ func packageType() string {
 		// Using tailscaled or IPNExtension?
 		exe, _ := os.Executable()
 		return filepath.Base(exe)
+	case "linux":
+		// Report whether this is in a snap.
+		// See https://snapcraft.io/docs/environment-variables
+		// We just look at two somewhat arbitrarily.
+		if os.Getenv("SNAP_NAME") != "" && os.Getenv("SNAP") != "" {
+			return "snap"
+		}
 	}
 	return ""
 }
@@ -354,6 +362,7 @@ func (c *Direct) doLogin(ctx context.Context, opt loginOpt) (mustRegen bool, new
 		if err != nil {
 			return regen, opt.URL, err
 		}
+		c.logf("control server key %s from %s", serverKey.HexString(), c.serverURL)
 
 		c.mu.Lock()
 		c.serverKey = serverKey
@@ -796,7 +805,10 @@ func (c *Direct) sendMapRequest(ctx context.Context, maxPolls int, cb func(*netm
 			continue
 		}
 
-		if resp.Debug != nil {
+		hasDebug := resp.Debug != nil
+		// being conservative here, if Debug not present set to False
+		controlknobs.SetDisableUPnP(hasDebug && resp.Debug.DisableUPnP.EqualBool(true))
+		if hasDebug {
 			if resp.Debug.LogHeapPprof {
 				go logheap.LogHeap(resp.Debug.LogHeapURL)
 			}

control/controlclient/map.go

@@ -6,8 +6,11 @@ package controlclient
 
 import (
 	"log"
+	"os"
 	"sort"
+	"strconv"
 
+	"inet.af/netaddr"
 	"tailscale.com/tailcfg"
 	"tailscale.com/types/logger"
 	"tailscale.com/types/netmap"
@@ -124,7 +127,7 @@ func (ms *mapSession) netmapForResponse(resp *tailcfg.MapResponse) *netmap.Netwo
 		nm.SelfNode = node
 		nm.Expiry = node.KeyExpiry
 		nm.Name = node.Name
-		nm.Addresses = node.Addresses
+		nm.Addresses = filterSelfAddresses(node.Addresses)
 		nm.User = node.User
 		nm.Hostinfo = node.Hostinfo
 		if node.MachineAuthorized {
@@ -280,3 +283,19 @@ func cloneNodes(v1 []*tailcfg.Node) []*tailcfg.Node {
 	}
 	return v2
 }
+
+var debugSelfIPv6Only, _ = strconv.ParseBool(os.Getenv("TS_DEBUG_SELF_V6_ONLY"))
+
+func filterSelfAddresses(in []netaddr.IPPrefix) (ret []netaddr.IPPrefix) {
+	switch {
+	default:
+		return in
+	case debugSelfIPv6Only:
+		for _, a := range in {
+			if a.IP().Is6() {
+				ret = append(ret, a)
+			}
+		}
+		return ret
+	}
+}

control/controlknobs/controlknobs.go

@@ -0,0 +1,34 @@
+// Copyright (c) 2021 Tailscale Inc & AUTHORS All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+// Package controlknobs contains client options configurable from control which can be turned on
+// or off. The ability to turn options on and off is for incrementally adding features in.
+package controlknobs
+
+import (
+	"os"
+	"strconv"
+
+	"tailscale.com/syncs"
+)
+
+// disableUPnP indicates whether to attempt UPnP mapping.
+var disableUPnP syncs.AtomicBool
+
+func init() {
+	v, _ := strconv.ParseBool(os.Getenv("TS_DISABLE_UPNP"))
+	SetDisableUPnP(v)
+}
+
+// DisableUPnP reports the last reported value from control
+// whether UPnP portmapping should be disabled.
+func DisableUPnP() bool {
+	return disableUPnP.Get()
+}
+
+// SetDisableUPnP sets whether control says that UPnP should be
+// disabled.
+func SetDisableUPnP(v bool) {
+	disableUPnP.Set(v)
+}

derp/derp_client.go

@@ -29,6 +29,7 @@ type Client struct {
 	br          *bufio.Reader
 	meshKey     string
 	canAckPings bool
+	isProber    bool
 
 	wmu sync.Mutex // hold while writing to bw
 	bw  *bufio.Writer
@@ -52,6 +53,7 @@ type clientOpt struct {
 	MeshKey     string
 	ServerPub   key.Public
 	CanAckPings bool
+	IsProber    bool
 }
 
 // MeshKey returns a ClientOpt to pass to the DERP server during connect to get
@@ -60,6 +62,10 @@ type clientOpt struct {
 // An empty key means to not use a mesh key.
 func MeshKey(key string) ClientOpt { return clientOptFunc(func(o *clientOpt) { o.MeshKey = key }) }
 
+// IsProber returns a ClientOpt to pass to the DERP server during connect to
+// declare that this client is a a prober.
+func IsProber(v bool) ClientOpt { return clientOptFunc(func(o *clientOpt) { o.IsProber = v }) }
+
 // ServerPublicKey returns a ClientOpt to declare that the server's DERP public key is known.
 // If key is the zero value, the returned ClientOpt is a no-op.
 func ServerPublicKey(key key.Public) ClientOpt {
@@ -93,6 +99,7 @@ func newClient(privateKey key.Private, nc Conn, brw *bufio.ReadWriter, logf logg
 		bw:          brw.Writer,
 		meshKey:     opt.MeshKey,
 		canAckPings: opt.CanAckPings,
+		isProber:    opt.IsProber,
 	}
 	if opt.ServerPub.IsZero() {
 		if err := c.recvServerKey(); err != nil {
@@ -160,6 +167,9 @@ type clientInfo struct {
 	// CanAckPings is whether the client declares it's able to ack
 	// pings.
 	CanAckPings bool
+
+	// IsProber is whether this client is a prober.
+	IsProber bool `json:",omitempty"`
 }
 
 func (c *Client) sendClientKey() error {
@@ -171,6 +181,7 @@ func (c *Client) sendClientKey() error {
 		Version:     ProtocolVersion,
 		MeshKey:     c.meshKey,
 		CanAckPings: c.canAckPings,
+		IsProber:    c.isProber,
 	})
 	if err != nil {
 		return err

derp/derp_server.go

@@ -36,6 +36,7 @@ import (
 	"go4.org/mem"
 	"golang.org/x/crypto/nacl/box"
 	"golang.org/x/sync/errgroup"
+	"golang.org/x/time/rate"
 	"inet.af/netaddr"
 	"tailscale.com/client/tailscale"
 	"tailscale.com/disco"
@@ -97,47 +98,46 @@ type Server struct {
 	metaCert    []byte // the encoded x509 cert to send after LetsEncrypt cert+intermediate
 
 	// Counters:
-	_                        [pad32bit]byte
-	packetsSent, bytesSent   expvar.Int
-	packetsRecv, bytesRecv   expvar.Int
-	packetsRecvByKind        metrics.LabelMap
-	packetsRecvDisco         *expvar.Int
-	packetsRecvOther         *expvar.Int
-	_                        [pad32bit]byte
-	packetsDropped           expvar.Int
-	packetsDroppedReason     metrics.LabelMap
-	packetsDroppedUnknown    *expvar.Int // unknown dst pubkey
-	packetsDroppedFwdUnknown *expvar.Int // unknown dst pubkey on forward
-	packetsDroppedGone       *expvar.Int // dst conn shutting down
-	packetsDroppedQueueHead  *expvar.Int // queue full, drop head packet
-	packetsDroppedQueueTail  *expvar.Int // queue full, drop tail packet
-	packetsDroppedWrite      *expvar.Int // error writing to dst conn
-	_                        [pad32bit]byte
-	packetsForwardedOut      expvar.Int
-	packetsForwardedIn       expvar.Int
-	peerGoneFrames           expvar.Int // number of peer gone frames sent
-	accepts                  expvar.Int
-	curClients               expvar.Int
-	curHomeClients           expvar.Int // ones with preferred
-	clientsReplaced          expvar.Int
-	unknownFrames            expvar.Int
-	homeMovesIn              expvar.Int // established clients announce home server moves in
-	homeMovesOut             expvar.Int // established clients announce home server moves out
-	multiForwarderCreated    expvar.Int
-	multiForwarderDeleted    expvar.Int
-	removePktForwardOther    expvar.Int
-	avgQueueDuration         *uint64 // In milliseconds; accessed atomically
+	_                            [pad32bit]byte
+	packetsSent, bytesSent       expvar.Int
+	packetsRecv, bytesRecv       expvar.Int
+	packetsRecvByKind            metrics.LabelMap
+	packetsRecvDisco             *expvar.Int
+	packetsRecvOther             *expvar.Int
+	_                            [pad32bit]byte
+	packetsDropped               expvar.Int
+	packetsDroppedReason         metrics.LabelMap
+	packetsDroppedReasonCounters []*expvar.Int // indexed by dropReason
+	packetsDroppedType           metrics.LabelMap
+	packetsDroppedTypeDisco      *expvar.Int
+	packetsDroppedTypeOther      *expvar.Int
+	_                            [pad32bit]byte
+	packetsForwardedOut          expvar.Int
+	packetsForwardedIn           expvar.Int
+	peerGoneFrames               expvar.Int // number of peer gone frames sent
+	accepts                      expvar.Int
+	curClients                   expvar.Int
+	curHomeClients               expvar.Int // ones with preferred
+	clientsReplaced              expvar.Int
+	clientsReplaceLimited        expvar.Int
+	clientsReplaceSleeping       expvar.Int
+	unknownFrames                expvar.Int
+	homeMovesIn                  expvar.Int // established clients announce home server moves in
+	homeMovesOut                 expvar.Int // established clients announce home server moves out
+	multiForwarderCreated        expvar.Int
+	multiForwarderDeleted        expvar.Int
+	removePktForwardOther        expvar.Int
+	avgQueueDuration             *uint64 // In milliseconds; accessed atomically
 
 	// verifyClients only accepts client connections to the DERP server if the clientKey is a
 	// known peer in the network, as specified by a running tailscaled's client's local api.
 	verifyClients bool
 
-	mu          sync.Mutex
-	closed      bool
-	netConns    map[Conn]chan struct{} // chan is closed when conn closes
-	clients     map[key.Public]*sclient
-	clientsEver map[key.Public]bool // never deleted from, for stats; fine for now
-	watchers    map[*sclient]bool   // mesh peer -> true
+	mu       sync.Mutex
+	closed   bool
+	netConns map[Conn]chan struct{} // chan is closed when conn closes
+	clients  map[key.Public]*sclient
+	watchers map[*sclient]bool // mesh peer -> true
 	// clientsMesh tracks all clients in the cluster, both locally
 	// and to mesh peers.  If the value is nil, that means the
 	// peer is only local (and thus in the clients Map, but not
@@ -167,7 +167,7 @@ type PacketForwarder interface {
 // Conn is the subset of the underlying net.Conn the DERP Server needs.
 // It is a defined type so that non-net connections can be used.
 type Conn interface {
-	io.Closer
+	io.WriteCloser
 
 	// The *Deadline methods follow the semantics of net.Conn.
 
@@ -189,8 +189,8 @@ func NewServer(privateKey key.Private, logf logger.Logf) *Server {
 		limitedLogf:          logger.RateLimitedFn(logf, 30*time.Second, 5, 100),
 		packetsRecvByKind:    metrics.LabelMap{Label: "kind"},
 		packetsDroppedReason: metrics.LabelMap{Label: "reason"},
+		packetsDroppedType:   metrics.LabelMap{Label: "type"},
 		clients:              map[key.Public]*sclient{},
-		clientsEver:          map[key.Public]bool{},
 		clientsMesh:          map[key.Public]PacketForwarder{},
 		netConns:             map[Conn]chan struct{}{},
 		memSys0:              ms.Sys,
@@ -202,12 +202,16 @@ func NewServer(privateKey key.Private, logf logger.Logf) *Server {
 	s.initMetacert()
 	s.packetsRecvDisco = s.packetsRecvByKind.Get("disco")
 	s.packetsRecvOther = s.packetsRecvByKind.Get("other")
-	s.packetsDroppedUnknown = s.packetsDroppedReason.Get("unknown_dest")
-	s.packetsDroppedFwdUnknown = s.packetsDroppedReason.Get("unknown_dest_on_fwd")
-	s.packetsDroppedGone = s.packetsDroppedReason.Get("gone")
-	s.packetsDroppedQueueHead = s.packetsDroppedReason.Get("queue_head")
-	s.packetsDroppedQueueTail = s.packetsDroppedReason.Get("queue_tail")
-	s.packetsDroppedWrite = s.packetsDroppedReason.Get("write_error")
+	s.packetsDroppedReasonCounters = []*expvar.Int{
+		s.packetsDroppedReason.Get("unknown_dest"),
+		s.packetsDroppedReason.Get("unknown_dest_on_fwd"),
+		s.packetsDroppedReason.Get("gone"),
+		s.packetsDroppedReason.Get("queue_head"),
+		s.packetsDroppedReason.Get("queue_tail"),
+		s.packetsDroppedReason.Get("write_error"),
+	}
+	s.packetsDroppedTypeDisco = s.packetsDroppedType.Get("disco")
+	s.packetsDroppedTypeOther = s.packetsDroppedType.Get("other")
 	return s
 }
 
@@ -345,26 +349,40 @@ func (s *Server) initMetacert() {
 func (s *Server) MetaCert() []byte { return s.metaCert }
 
 // registerClient notes that client c is now authenticated and ready for packets.
-// If c's public key was already connected with a different connection, the prior one is closed.
-func (s *Server) registerClient(c *sclient) {
+//
+// If c's public key was already connected with a different
+// connection, the prior one is closed, unless it's fighting rapidly
+// with another client with the same key, in which case the returned
+// ok is false, and the caller should wait the provided duration
+// before trying again.
+func (s *Server) registerClient(c *sclient) (ok bool, d time.Duration) {
 	s.mu.Lock()
 	defer s.mu.Unlock()
 	old := s.clients[c.key]
 	if old == nil {
 		c.logf("adding connection")
 	} else {
+		// Take over the old rate limiter, discarding the one
+		// our caller just made.
+		c.replaceLimiter = old.replaceLimiter
+		if rr := c.replaceLimiter.ReserveN(timeNow(), 1); rr.OK() {
+			if d := rr.DelayFrom(timeNow()); d > 0 {
+				s.clientsReplaceLimited.Add(1)
+				return false, d
+			}
+		}
 		s.clientsReplaced.Add(1)
 		c.logf("adding connection, replacing %s", old.remoteAddr)
 		go old.nc.Close()
 	}
 	s.clients[c.key] = c
-	s.clientsEver[c.key] = true
 	if _, ok := s.clientsMesh[c.key]; !ok {
 		s.clientsMesh[c.key] = nil // just for varz of total users in cluster
 	}
 	s.keyOfAddr[c.remoteIPPort] = c.key
 	s.curClients.Add(1)
 	s.broadcastPeerStateChangeLocked(c.key, true)
+	return true, 0
 }
 
 // broadcastPeerStateChangeLocked enqueues a message to all watchers
@@ -452,8 +470,9 @@ func (s *Server) addWatcher(c *sclient) {
 }
 
 func (s *Server) accept(nc Conn, brw *bufio.ReadWriter, remoteAddr string, connNum int64) error {
-	br, bw := brw.Reader, brw.Writer
+	br := brw.Reader
 	nc.SetDeadline(time.Now().Add(10 * time.Second))
+	bw := &lazyBufioWriter{w: nc, lbw: brw.Writer}
 	if err := s.sendServerKey(bw); err != nil {
 		return fmt.Errorf("send server key: %v", err)
 	}
@@ -475,21 +494,29 @@ func (s *Server) accept(nc Conn, brw *bufio.ReadWriter, remoteAddr string, connN
 	remoteIPPort, _ := netaddr.ParseIPPort(remoteAddr)
 
 	c := &sclient{
-		connNum:      connNum,
-		s:            s,
-		key:          clientKey,
-		nc:           nc,
-		br:           br,
-		bw:           bw,
-		logf:         logger.WithPrefix(s.logf, fmt.Sprintf("derp client %v/%x: ", remoteAddr, clientKey)),
-		done:         ctx.Done(),
-		remoteAddr:   remoteAddr,
-		remoteIPPort: remoteIPPort,
-		connectedAt:  time.Now(),
-		sendQueue:    make(chan pkt, perClientSendQueueDepth),
-		peerGone:     make(chan key.Public),
-		canMesh:      clientInfo.MeshKey != "" && clientInfo.MeshKey == s.meshKey,
+		connNum:        connNum,
+		s:              s,
+		key:            clientKey,
+		nc:             nc,
+		br:             br,
+		bw:             bw,
+		logf:           logger.WithPrefix(s.logf, fmt.Sprintf("derp client %v/%x: ", remoteAddr, clientKey)),
+		done:           ctx.Done(),
+		remoteAddr:     remoteAddr,
+		remoteIPPort:   remoteIPPort,
+		connectedAt:    time.Now(),
+		sendQueue:      make(chan pkt, perClientSendQueueDepth),
+		discoSendQueue: make(chan pkt, perClientSendQueueDepth),
+		peerGone:       make(chan key.Public),
+		canMesh:        clientInfo.MeshKey != "" && clientInfo.MeshKey == s.meshKey,
+
+		// Allow kicking out previous connections once a
+		// minute, with a very high burst of 100. Once a
+		// minute is less than the client's 2 minute
+		// inactivity timeout.
+		replaceLimiter: rate.NewLimiter(rate.Every(time.Minute), 100),
 	}
+
 	if c.canMesh {
 		c.meshUpdate = make(chan struct{})
 	}
@@ -497,10 +524,18 @@ func (s *Server) accept(nc Conn, brw *bufio.ReadWriter, remoteAddr string, connN
 		c.info = *clientInfo
 	}
 
-	s.registerClient(c)
+	for {
+		ok, d := s.registerClient(c)
+		if ok {
+			break
+		}
+		s.clientsReplaceSleeping.Add(1)
+		timeSleep(d)
+		s.clientsReplaceSleeping.Add(-1)
+	}
 	defer s.unregisterClient(c)
 
-	err = s.sendServerInfo(bw, clientKey)
+	err = s.sendServerInfo(c.bw, clientKey)
 	if err != nil {
 		return fmt.Errorf("send server info: %v", err)
 	}
@@ -508,6 +543,12 @@ func (s *Server) accept(nc Conn, brw *bufio.ReadWriter, remoteAddr string, connN
 	return c.run(ctx)
 }
 
+// for testing
+var (
+	timeSleep = time.Sleep
+	timeNow   = time.Now
+)
+
 // run serves the client until there's an error.
 // If the client hangs up or the server is closed, run returns nil, otherwise run returns an error.
 func (c *sclient) run(ctx context.Context) error {
@@ -631,11 +672,7 @@ func (c *sclient) handleFrameForwardPacket(ft frameType, fl uint32) error {
 	s.mu.Unlock()
 
 	if dst == nil {
-		s.packetsDropped.Add(1)
-		s.packetsDroppedFwdUnknown.Add(1)
-		if debug {
-			c.logf("dropping forwarded packet for unknown %x", dstKey)
-		}
+		s.recordDrop(contents, srcKey, dstKey, dropReasonUnknownDestOnFwd)
 		return nil
 	}
 
@@ -686,11 +723,7 @@ func (c *sclient) handleFrameSendPacket(ft frameType, fl uint32) error {
 			}
 			return nil
 		}
-		s.packetsDropped.Add(1)
-		s.packetsDroppedUnknown.Add(1)
-		if debug {
-			c.logf("dropping packet for unknown %x", dstKey)
-		}
+		s.recordDrop(contents, c.key, dstKey, dropReasonUnknownDest)
 		return nil
 	}
 
@@ -702,6 +735,41 @@ func (c *sclient) handleFrameSendPacket(ft frameType, fl uint32) error {
 	return c.sendPkt(dst, p)
 }
 
+// dropReason is why we dropped a DERP frame.
+type dropReason int
+
+//go:generate go run tailscale.com/cmd/addlicense -year 2021 -file dropreason_string.go go run golang.org/x/tools/cmd/stringer -type=dropReason -trimprefix=dropReason
+
+const (
+	dropReasonUnknownDest      dropReason = iota // unknown destination pubkey
+	dropReasonUnknownDestOnFwd                   // unknown destination pubkey on a derp-forwarded packet
+	dropReasonGone                               // destination tailscaled disconnected before we could send
+	dropReasonQueueHead                          // destination queue is full, dropped packet at queue head
+	dropReasonQueueTail                          // destination queue is full, dropped packet at queue tail
+	dropReasonWriteError                         // OS write() failed
+)
+
+func (s *Server) recordDrop(packetBytes []byte, srcKey, dstKey key.Public, reason dropReason) {
+	s.packetsDropped.Add(1)
+	s.packetsDroppedReasonCounters[reason].Add(1)
+	if disco.LooksLikeDiscoWrapper(packetBytes) {
+		s.packetsDroppedTypeDisco.Add(1)
+	} else {
+		s.packetsDroppedTypeOther.Add(1)
+	}
+	if verboseDropKeys[dstKey] {
+		// Preformat the log string prior to calling limitedLogf. The
+		// limiter acts based on the format string, and we want to
+		// rate-limit per src/dst keys, not on the generic "dropped
+		// stuff" message.
+		msg := fmt.Sprintf("drop (%s) %s -> %s", srcKey.ShortString(), reason, dstKey.ShortString())
+		s.limitedLogf(msg)
+	}
+	if debug {
+		s.logf("dropping packet reason=%s dst=%s disco=%v", reason, dstKey, disco.LooksLikeDiscoWrapper(packetBytes))
+	}
+}
+
 func (c *sclient) sendPkt(dst *sclient, p pkt) error {
 	s := c.s
 	dstKey := dst.key
@@ -709,54 +777,34 @@ func (c *sclient) sendPkt(dst *sclient, p pkt) error {
 	// Attempt to queue for sending up to 3 times. On each attempt, if
 	// the queue is full, try to drop from queue head to prioritize
 	// fresher packets.
+	sendQueue := dst.sendQueue
+	if disco.LooksLikeDiscoWrapper(p.bs) {
+		sendQueue = dst.discoSendQueue
+	}
 	for attempt := 0; attempt < 3; attempt++ {
 		select {
 		case <-dst.done:
-			s.packetsDropped.Add(1)
-			s.packetsDroppedGone.Add(1)
-			if debug {
-				c.logf("dropping packet for shutdown client %x", dstKey)
-			}
+			s.recordDrop(p.bs, c.key, dstKey, dropReasonGone)
 			return nil
 		default:
 		}
 		select {
-		case dst.sendQueue <- p:
+		case sendQueue <- p:
 			return nil
 		default:
 		}
 
 		select {
-		case pkt := <-dst.sendQueue:
-			s.packetsDropped.Add(1)
-			s.packetsDroppedQueueHead.Add(1)
-			if verboseDropKeys[dstKey] {
-				// Generate a full string including src and dst, so
-				// the limiter kicks in once per src.
-				msg := fmt.Sprintf("tail drop %s -> %s", p.src.ShortString(), dstKey.ShortString())
-				c.s.limitedLogf(msg)
-			}
+		case pkt := <-sendQueue:
+			s.recordDrop(pkt.bs, c.key, dstKey, dropReasonQueueHead)
 			c.recordQueueTime(pkt.enqueuedAt)
-			if debug {
-				c.logf("dropping packet from client %x queue head", dstKey)
-			}
 		default:
 		}
 	}
 	// Failed to make room for packet. This can happen in a heavily
 	// contended queue with racing writers. Give up and tail-drop in
 	// this case to keep reader unblocked.
-	s.packetsDropped.Add(1)
-	s.packetsDroppedQueueTail.Add(1)
-	if verboseDropKeys[dstKey] {
-		// Generate a full string including src and dst, so
-		// the limiter kicks in once per src.
-		msg := fmt.Sprintf("head drop %s -> %s", p.src.ShortString(), dstKey.ShortString())
-		c.s.limitedLogf(msg)
-	}
-	if debug {
-		c.logf("dropping packet from client %x queue tail", dstKey)
-	}
+	s.recordDrop(p.bs, c.key, dstKey, dropReasonQueueTail)
 
 	return nil
 }
@@ -789,6 +837,9 @@ func (s *Server) verifyClient(clientKey key.Public, info *clientInfo) error {
 	if err != nil {
 		return fmt.Errorf("failed to query local tailscaled status: %w", err)
 	}
+	if clientKey == status.Self.PublicKey {
+		return nil
+	}
 	if _, exists := status.Peer[clientKey]; !exists {
 		return fmt.Errorf("client %v not in set of peers", clientKey)
 	}
@@ -796,18 +847,20 @@ func (s *Server) verifyClient(clientKey key.Public, info *clientInfo) error {
 	return nil
 }
 
-func (s *Server) sendServerKey(bw *bufio.Writer) error {
+func (s *Server) sendServerKey(lw *lazyBufioWriter) error {
 	buf := make([]byte, 0, len(magic)+len(s.publicKey))
 	buf = append(buf, magic...)
 	buf = append(buf, s.publicKey[:]...)
-	return writeFrame(bw, frameServerKey, buf)
+	err := writeFrame(lw.bw(), frameServerKey, buf)
+	lw.Flush() // redundant (no-op) flush to release bufio.Writer
+	return err
 }
 
 type serverInfo struct {
 	Version int `json:"version,omitempty"`
 }
 
-func (s *Server) sendServerInfo(bw *bufio.Writer, clientKey key.Public) error {
+func (s *Server) sendServerInfo(bw *lazyBufioWriter, clientKey key.Public) error {
 	var nonce [24]byte
 	if _, err := crand.Read(nonce[:]); err != nil {
 		return err
@@ -818,7 +871,7 @@ func (s *Server) sendServerInfo(bw *bufio.Writer, clientKey key.Public) error {
 	}
 
 	msgbox := box.Seal(nil, msg, &nonce, clientKey.B32(), s.privateKey.B32())
-	if err := writeFrameHeader(bw, frameServerInfo, nonceLen+uint32(len(msgbox))); err != nil {
+	if err := writeFrameHeader(bw.bw(), frameServerInfo, nonceLen+uint32(len(msgbox))); err != nil {
 		return err
 	}
 	if _, err := bw.Write(nonce[:]); err != nil {
@@ -926,19 +979,25 @@ func (s *Server) recvForwardPacket(br *bufio.Reader, frameLen uint32) (srcKey, d
 // (The "s" prefix is to more explicitly distinguish it from Client in derp_client.go)
 type sclient struct {
 	// Static after construction.
-	connNum      int64 // process-wide unique counter, incremented each Accept
-	s            *Server
-	nc           Conn
-	key          key.Public
-	info         clientInfo
-	logf         logger.Logf
-	done         <-chan struct{} // closed when connection closes
-	remoteAddr   string          // usually ip:port from net.Conn.RemoteAddr().String()
-	remoteIPPort netaddr.IPPort  // zero if remoteAddr is not ip:port.
-	sendQueue    chan pkt        // packets queued to this client; never closed
-	peerGone     chan key.Public // write request that a previous sender has disconnected (not used by mesh peers)
-	meshUpdate   chan struct{}   // write request to write peerStateChange
-	canMesh      bool            // clientInfo had correct mesh token for inter-region routing
+	connNum        int64 // process-wide unique counter, incremented each Accept
+	s              *Server
+	nc             Conn
+	key            key.Public
+	info           clientInfo
+	logf           logger.Logf
+	done           <-chan struct{} // closed when connection closes
+	remoteAddr     string          // usually ip:port from net.Conn.RemoteAddr().String()
+	remoteIPPort   netaddr.IPPort  // zero if remoteAddr is not ip:port.
+	sendQueue      chan pkt        // packets queued to this client; never closed
+	discoSendQueue chan pkt        // important packets queued to this client; never closed
+	peerGone       chan key.Public // write request that a previous sender has disconnected (not used by mesh peers)
+	meshUpdate     chan struct{}   // write request to write peerStateChange
+	canMesh        bool            // clientInfo had correct mesh token for inter-region routing
+
+	// replaceLimiter controls how quickly two connections with
+	// the same client key can kick each other off the server by
+	// taking over ownership of a key.
+	replaceLimiter *rate.Limiter
 
 	// Owned by run, not thread-safe.
 	br          *bufio.Reader
@@ -946,7 +1005,7 @@ type sclient struct {
 	preferred   bool
 
 	// Owned by sender, not thread-safe.
-	bw *bufio.Writer
+	bw *lazyBufioWriter
 
 	// Guarded by s.mu
 	//
@@ -1031,12 +1090,10 @@ func (c *sclient) sendLoop(ctx context.Context) error {
 		// Drain the send queue to count dropped packets
 		for {
 			select {
-			case <-c.sendQueue:
-				c.s.packetsDropped.Add(1)
-				c.s.packetsDroppedGone.Add(1)
-				if debug {
-					c.logf("dropping packet for shutdown %x", c.key)
-				}
+			case pkt := <-c.sendQueue:
+				c.s.recordDrop(pkt.bs, pkt.src, c.key, dropReasonGone)
+			case pkt := <-c.discoSendQueue:
+				c.s.recordDrop(pkt.bs, pkt.src, c.key, dropReasonGone)
 			default:
 				return
 			}
@@ -1067,6 +1124,10 @@ func (c *sclient) sendLoop(ctx context.Context) error {
 			werr = c.sendPacket(msg.src, msg.bs)
 			c.recordQueueTime(msg.enqueuedAt)
 			continue
+		case msg := <-c.discoSendQueue:
+			werr = c.sendPacket(msg.src, msg.bs)
+			c.recordQueueTime(msg.enqueuedAt)
+			continue
 		case <-keepAliveTick.C:
 			werr = c.sendKeepAlive()
 			continue
@@ -1090,6 +1151,9 @@ func (c *sclient) sendLoop(ctx context.Context) error {
 		case msg := <-c.sendQueue:
 			werr = c.sendPacket(msg.src, msg.bs)
 			c.recordQueueTime(msg.enqueuedAt)
+		case msg := <-c.discoSendQueue:
+			werr = c.sendPacket(msg.src, msg.bs)
+			c.recordQueueTime(msg.enqueuedAt)
 		case <-keepAliveTick.C:
 			werr = c.sendKeepAlive()
 		}
@@ -1103,14 +1167,14 @@ func (c *sclient) setWriteDeadline() {
 // sendKeepAlive sends a keep-alive frame, without flushing.
 func (c *sclient) sendKeepAlive() error {
 	c.setWriteDeadline()
-	return writeFrameHeader(c.bw, frameKeepAlive, 0)
+	return writeFrameHeader(c.bw.bw(), frameKeepAlive, 0)
 }
 
 // sendPeerGone sends a peerGone frame, without flushing.
 func (c *sclient) sendPeerGone(peer key.Public) error {
 	c.s.peerGoneFrames.Add(1)
 	c.setWriteDeadline()
-	if err := writeFrameHeader(c.bw, framePeerGone, keyLen); err != nil {
+	if err := writeFrameHeader(c.bw.bw(), framePeerGone, keyLen); err != nil {
 		return err
 	}
 	_, err := c.bw.Write(peer[:])
@@ -1120,7 +1184,7 @@ func (c *sclient) sendPeerGone(peer key.Public) error {
 // sendPeerPresent sends a peerPresent frame, without flushing.
 func (c *sclient) sendPeerPresent(peer key.Public) error {
 	c.setWriteDeadline()
-	if err := writeFrameHeader(c.bw, framePeerPresent, keyLen); err != nil {
+	if err := writeFrameHeader(c.bw.bw(), framePeerPresent, keyLen); err != nil {
 		return err
 	}
 	_, err := c.bw.Write(peer[:])
@@ -1179,11 +1243,7 @@ func (c *sclient) sendPacket(srcKey key.Public, contents []byte) (err error) {
 	defer func() {
 		// Stats update.
 		if err != nil {
-			c.s.packetsDropped.Add(1)
-			c.s.packetsDroppedWrite.Add(1)
-			if debug {
-				c.logf("dropping packet to %x: %v", c.key, err)
-			}
+			c.s.recordDrop(contents, srcKey, c.key, dropReasonWriteError)
 		} else {
 			c.s.packetsSent.Add(1)
 			c.s.bytesSent.Add(int64(len(contents)))
@@ -1197,11 +1257,11 @@ func (c *sclient) sendPacket(srcKey key.Public, contents []byte) (err error) {
 	if withKey {
 		pktLen += len(srcKey)
 	}
-	if err = writeFrameHeader(c.bw, frameRecvPacket, uint32(pktLen)); err != nil {
+	if err = writeFrameHeader(c.bw.bw(), frameRecvPacket, uint32(pktLen)); err != nil {
 		return err
 	}
 	if withKey {
-		err := writePublicKey(c.bw, &srcKey)
+		err := writePublicKey(c.bw.bw(), &srcKey)
 		if err != nil {
 			return err
 		}
@@ -1329,7 +1389,6 @@ func (s *Server) expVarFunc(f func() interface{}) expvar.Func {
 // ExpVar returns an expvar variable suitable for registering with expvar.Publish.
 func (s *Server) ExpVar() expvar.Var {
 	m := new(metrics.Set)
-	m.Set("counter_unique_clients_ever", s.expVarFunc(func() interface{} { return len(s.clientsEver) }))
 	m.Set("gauge_memstats_sys0", expvar.Func(func() interface{} { return int64(s.memSys0) }))
 	m.Set("gauge_watchers", s.expVarFunc(func() interface{} { return len(s.watchers) }))
 	m.Set("gauge_current_connections", &s.curClients)
@@ -1339,10 +1398,13 @@ func (s *Server) ExpVar() expvar.Var {
 	m.Set("gauge_clients_remote", expvar.Func(func() interface{} { return len(s.clientsMesh) - len(s.clients) }))
 	m.Set("accepts", &s.accepts)
 	m.Set("clients_replaced", &s.clientsReplaced)
+	m.Set("clients_replace_limited", &s.clientsReplaceLimited)
+	m.Set("gauge_clients_replace_sleeping", &s.clientsReplaceSleeping)
 	m.Set("bytes_received", &s.bytesRecv)
 	m.Set("bytes_sent", &s.bytesSent)
 	m.Set("packets_dropped", &s.packetsDropped)
 	m.Set("counter_packets_dropped_reason", &s.packetsDroppedReason)
+	m.Set("counter_packets_dropped_type", &s.packetsDroppedType)
 	m.Set("counter_packets_received_kind", &s.packetsRecvByKind)
 	m.Set("packets_sent", &s.packetsSent)
 	m.Set("packets_received", &s.packetsRecv)
@@ -1514,3 +1576,45 @@ func (s *Server) ServeDebugTraffic(w http.ResponseWriter, r *http.Request) {
 		time.Sleep(minTimeBetweenLogs)
 	}
 }
+
+var bufioWriterPool = &sync.Pool{
+	New: func() interface{} {
+		return bufio.NewWriterSize(ioutil.Discard, 2<<10)
+	},
+}
+
+// lazyBufioWriter is a bufio.Writer-like wrapping writer that lazily
+// allocates its actual bufio.Writer from a sync.Pool, releasing it to
+// the pool upon flush.
+//
+// We do this to reduce memory overhead; most DERP connections are
+// idle and the idle bufio.Writers were 30% of overall memory usage.
+type lazyBufioWriter struct {
+	w   io.Writer     // underlying
+	lbw *bufio.Writer // lazy; nil means it needs an associated buffer
+}
+
+func (w *lazyBufioWriter) bw() *bufio.Writer {
+	if w.lbw == nil {
+		w.lbw = bufioWriterPool.Get().(*bufio.Writer)
+		w.lbw.Reset(w.w)
+	}
+	return w.lbw
+}
+
+func (w *lazyBufioWriter) Available() int { return w.bw().Available() }
+
+func (w *lazyBufioWriter) Write(p []byte) (int, error) { return w.bw().Write(p) }
+
+func (w *lazyBufioWriter) Flush() error {
+	if w.lbw == nil {
+		return nil
+	}
+	err := w.lbw.Flush()
+
+	w.lbw.Reset(ioutil.Discard)
+	bufioWriterPool.Put(w.lbw)
+	w.lbw = nil
+
+	return err
+}

derp/derp_test.go

@@ -23,6 +23,7 @@ import (
 	"testing"
 	"time"
 
+	"golang.org/x/time/rate"
 	"tailscale.com/net/nettest"
 	"tailscale.com/types/key"
 	"tailscale.com/types/logger"
@@ -849,6 +850,136 @@ func TestClientSendPong(t *testing.T) {
 
 }
 
+func TestServerReplaceClients(t *testing.T) {
+	defer func() {
+		timeSleep = time.Sleep
+		timeNow = time.Now
+	}()
+
+	var (
+		mu     sync.Mutex
+		now    = time.Unix(123, 0)
+		sleeps int
+		slept  time.Duration
+	)
+	timeSleep = func(d time.Duration) {
+		mu.Lock()
+		defer mu.Unlock()
+		sleeps++
+		slept += d
+		now = now.Add(d)
+	}
+	timeNow = func() time.Time {
+		mu.Lock()
+		defer mu.Unlock()
+		return now
+	}
+
+	serverPrivateKey := newPrivateKey(t)
+	var logger logger.Logf = logger.Discard
+	const debug = false
+	if debug {
+		logger = t.Logf
+	}
+
+	s := NewServer(serverPrivateKey, logger)
+	defer s.Close()
+
+	priv := newPrivateKey(t)
+
+	ln, err := net.Listen("tcp", "127.0.0.1:0")
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer ln.Close()
+
+	connNum := 0
+	connect := func() *Client {
+		connNum++
+		cout, err := net.Dial("tcp", ln.Addr().String())
+		if err != nil {
+			t.Fatal(err)
+		}
+
+		cin, err := ln.Accept()
+		if err != nil {
+			t.Fatal(err)
+		}
+
+		brwServer := bufio.NewReadWriter(bufio.NewReader(cin), bufio.NewWriter(cin))
+		go s.Accept(cin, brwServer, fmt.Sprintf("test-client-%d", connNum))
+
+		brw := bufio.NewReadWriter(bufio.NewReader(cout), bufio.NewWriter(cout))
+		c, err := NewClient(priv, cout, brw, logger)
+		if err != nil {
+			t.Fatalf("client %d: %v", connNum, err)
+		}
+		return c
+	}
+
+	wantVar := func(v *expvar.Int, want int64) {
+		t.Helper()
+		if got := v.Value(); got != want {
+			t.Errorf("got %d; want %d", got, want)
+		}
+	}
+
+	wantClosed := func(c *Client) {
+		t.Helper()
+		for {
+			m, err := c.Recv()
+			if err != nil {
+				t.Logf("got expected error: %v", err)
+				return
+			}
+			switch m.(type) {
+			case ServerInfoMessage:
+				continue
+			default:
+				t.Fatalf("client got %T; wanted an error", m)
+			}
+		}
+	}
+
+	c1 := connect()
+	waitConnect(t, c1)
+	c2 := connect()
+	waitConnect(t, c2)
+	wantVar(&s.clientsReplaced, 1)
+	wantClosed(c1)
+
+	for i := 0; i < 100+5; i++ {
+		c := connect()
+		defer c.nc.Close()
+		if s.clientsReplaceLimited.Value() == 0 && i < 90 {
+			continue
+		}
+		t.Logf("for %d: replaced=%d, limited=%d, sleeping=%d", i,
+			s.clientsReplaced.Value(),
+			s.clientsReplaceLimited.Value(),
+			s.clientsReplaceSleeping.Value(),
+		)
+	}
+
+	mu.Lock()
+	defer mu.Unlock()
+	if sleeps == 0 {
+		t.Errorf("no sleeps")
+	}
+	if slept == 0 {
+		t.Errorf("total sleep duration was 0")
+	}
+}
+
+func TestLimiter(t *testing.T) {
+	rl := rate.NewLimiter(rate.Every(time.Minute), 100)
+	for i := 0; i < 200; i++ {
+		r := rl.Reserve()
+		d := r.Delay()
+		t.Logf("i=%d, allow=%v, d=%v", i, r.OK(), d)
+	}
+}
+
 func BenchmarkSendRecv(b *testing.B) {
 	for _, size := range []int{10, 100, 1000, 10000} {
 		b.Run(fmt.Sprintf("msgsize=%d", size), func(b *testing.B) { benchmarkSendRecvSize(b, size) })

derp/derphttp/derphttp_client.go

@@ -50,9 +50,11 @@ type Client struct {
 	TLSConfig *tls.Config        // optional; nil means default
 	DNSCache  *dnscache.Resolver // optional; nil means no caching
 	MeshKey   string             // optional; for trusted clients
+	IsProber  bool               // optional; for probers to optional declare themselves as such
 
 	privateKey key.Private
 	logf       logger.Logf
+	dialer     func(ctx context.Context, network, addr string) (net.Conn, error)
 
 	// Either url or getRegion is non-nil:
 	url       *url.URL
@@ -130,6 +132,11 @@ func (c *Client) ServerPublicKey() key.Public {
 	return c.serverPubKey
 }
 
+// SelfPublicKey returns our own public key.
+func (c *Client) SelfPublicKey() key.Public {
+	return c.privateKey.Public()
+}
+
 func urlPort(u *url.URL) string {
 	if p := u.Port(); p != "" {
 		return p
@@ -338,6 +345,7 @@ func (c *Client) connect(ctx context.Context, caller string) (client *derp.Clien
 		derp.MeshKey(c.MeshKey),
 		derp.ServerPublicKey(serverPub),
 		derp.CanAckPings(c.canAckPings),
+		derp.IsProber(c.IsProber),
 	)
 	if err != nil {
 		return nil, 0, err
@@ -356,14 +364,26 @@ func (c *Client) connect(ctx context.Context, caller string) (client *derp.Clien
 	return c.client, c.connGen, nil
 }
 
+// SetURLDialer sets the dialer to use for dialing URLs.
+// This dialer is only use for clients created with NewClient, not NewRegionClient.
+// If unset or nil, the default dialer is used.
+//
+// The primary use for this is the derper mesh mode to connect to each
+// other over a VPC network.
+func (c *Client) SetURLDialer(dialer func(ctx context.Context, network, addr string) (net.Conn, error)) {
+	c.dialer = dialer
+}
+
 func (c *Client) dialURL(ctx context.Context) (net.Conn, error) {
 	host := c.url.Hostname()
+	if c.dialer != nil {
+		return c.dialer(ctx, "tcp", net.JoinHostPort(host, urlPort(c.url)))
+	}
 	hostOrIP := host
-
 	dialer := netns.NewDialer()
 
 	if c.DNSCache != nil {
-		ip, _, err := c.DNSCache.LookupIP(ctx, host)
+		ip, _, _, err := c.DNSCache.LookupIP(ctx, host)
 		if err == nil {
 			hostOrIP = ip.String()
 		}
@@ -410,9 +430,7 @@ func (c *Client) dialRegion(ctx context.Context, reg *tailcfg.DERPRegion) (net.C
 func (c *Client) tlsClient(nc net.Conn, node *tailcfg.DERPNode) *tls.Conn {
 	tlsConf := tlsdial.Config(c.tlsServerName(node), c.TLSConfig)
 	if node != nil {
-		if node.DERPTestPort != 0 {
-			tlsConf.InsecureSkipVerify = true
-		}
+		tlsConf.InsecureSkipVerify = node.InsecureForTests
 		if node.CertName != "" {
 			tlsdial.SetConfigExpectedCert(tlsConf, node.CertName)
 		}
@@ -511,8 +529,8 @@ func (c *Client) dialNode(ctx context.Context, n *tailcfg.DERPNode) (net.Conn, e
 				dst = n.HostName
 			}
 			port := "443"
-			if n.DERPTestPort != 0 {
-				port = fmt.Sprint(n.DERPTestPort)
+			if n.DERPPort != 0 {
+				port = fmt.Sprint(n.DERPPort)
 			}
 			c, err := c.dialContext(ctx, proto, net.JoinHostPort(dst, port))
 			select {

derp/derpmap/derpmap.go

@@ -1,91 +0,0 @@
-// Copyright (c) 2020 Tailscale Inc & AUTHORS All rights reserved.
-// Use of this source code is governed by a BSD-style
-// license that can be found in the LICENSE file.
-
-// Package derpmap contains information about Tailscale.com's production DERP nodes.
-//
-// This package is only used by the "tailscale netcheck" command for debugging.
-// In normal operation the Tailscale nodes get this sent to them from the control
-// server.
-//
-// TODO: remove this package and make "tailscale netcheck" get the
-// list from the control server too.
-package derpmap
-
-import (
-	"fmt"
-	"strings"
-
-	"tailscale.com/tailcfg"
-)
-
-func derpNode(suffix, v4, v6 string) *tailcfg.DERPNode {
-	return &tailcfg.DERPNode{
-		Name:     suffix, // updated later
-		RegionID: 0,      // updated later
-		IPv4:     v4,
-		IPv6:     v6,
-	}
-}
-
-func derpRegion(id int, code, name string, nodes ...*tailcfg.DERPNode) *tailcfg.DERPRegion {
-	region := &tailcfg.DERPRegion{
-		RegionID:   id,
-		RegionName: name,
-		RegionCode: code,
-		Nodes:      nodes,
-	}
-	for _, n := range nodes {
-		n.Name = fmt.Sprintf("%d%s", id, n.Name)
-		n.RegionID = id
-		n.HostName = fmt.Sprintf("derp%s.tailscale.com", strings.TrimSuffix(n.Name, "a"))
-	}
-	return region
-}
-
-// Prod returns Tailscale's map of relay servers.
-//
-// This list is only used by cmd/tailscale's netcheck subcommand. In
-// normal operation the Tailscale nodes get this sent to them from the
-// control server.
-//
-// This list is subject to change and should not be relied on.
-func Prod() *tailcfg.DERPMap {
-	return &tailcfg.DERPMap{
-		Regions: map[int]*tailcfg.DERPRegion{
-			1: derpRegion(1, "nyc", "New York City",
-				derpNode("a", "159.89.225.99", "2604:a880:400:d1::828:b001"),
-			),
-			2: derpRegion(2, "sfo", "San Francisco",
-				derpNode("a", "167.172.206.31", "2604:a880:2:d1::c5:7001"),
-			),
-			3: derpRegion(3, "sin", "Singapore",
-				derpNode("a", "68.183.179.66", "2400:6180:0:d1::67d:8001"),
-			),
-			4: derpRegion(4, "fra", "Frankfurt",
-				derpNode("a", "167.172.182.26", "2a03:b0c0:3:e0::36e:9001"),
-			),
-			5: derpRegion(5, "syd", "Sydney",
-				derpNode("a", "103.43.75.49", "2001:19f0:5801:10b7:5400:2ff:feaa:284c"),
-			),
-			6: derpRegion(6, "blr", "Bangalore",
-				derpNode("a", "68.183.90.120", "2400:6180:100:d0::982:d001"),
-			),
-			7: derpRegion(7, "tok", "Tokyo",
-				derpNode("a", "167.179.89.145", "2401:c080:1000:467f:5400:2ff:feee:22aa"),
-			),
-			8: derpRegion(8, "lhr", "London",
-				derpNode("a", "167.71.139.179", "2a03:b0c0:1:e0::3cc:e001"),
-			),
-			9: derpRegion(9, "dfw", "Dallas",
-				derpNode("a", "207.148.3.137", "2001:19f0:6401:1d9c:5400:2ff:feef:bb82"),
-			),
-			10: derpRegion(10, "sea", "Seattle",
-				derpNode("a", "137.220.36.168", "2001:19f0:8001:2d9:5400:2ff:feef:bbb1"),
-			),
-			11: derpRegion(11, "sao", "São Paulo",
-				derpNode("a", "18.230.97.74", "2600:1f1e:ee4:5611:ec5c:1736:d43b:a454"),
-			),
-		},
-	}
-}

derp/dropreason_string.go

@@ -0,0 +1,32 @@
+// Copyright (c) 2021 Tailscale Inc & AUTHORS All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+// Code generated by "stringer -type=dropReason -trimprefix=dropReason"; DO NOT EDIT.
+
+package derp
+
+import "strconv"
+
+func _() {
+	// An "invalid array index" compiler error signifies that the constant values have changed.
+	// Re-run the stringer command to generate them again.
+	var x [1]struct{}
+	_ = x[dropReasonUnknownDest-0]
+	_ = x[dropReasonUnknownDestOnFwd-1]
+	_ = x[dropReasonGone-2]
+	_ = x[dropReasonQueueHead-3]
+	_ = x[dropReasonQueueTail-4]
+	_ = x[dropReasonWriteError-5]
+}
+
+const _dropReason_name = "UnknownDestUnknownDestOnFwdGoneQueueHeadQueueTailWriteError"
+
+var _dropReason_index = [...]uint8{0, 11, 27, 31, 40, 49, 59}
+
+func (i dropReason) String() string {
+	if i < 0 || i >= dropReason(len(_dropReason_index)-1) {
+		return "dropReason(" + strconv.FormatInt(int64(i), 10) + ")"
+	}
+	return _dropReason_name[_dropReason_index[i]:_dropReason_index[i+1]]
+}

go.mod

@@ -7,6 +7,8 @@ require (
 	github.com/anmitsu/go-shlex v0.0.0-20200514113438-38f4b401e2be // indirect
 	github.com/aws/aws-sdk-go v1.38.52
 	github.com/coreos/go-iptables v0.6.0
+	github.com/creack/pty v1.1.9
+	github.com/dave/jennifer v1.4.1
 	github.com/frankban/quicktest v1.13.0
 	github.com/gliderlabs/ssh v0.3.2
 	github.com/go-multierror/multierror v1.0.2
@@ -16,32 +18,35 @@ require (
 	github.com/google/goexpect v0.0.0-20210430020637-ab937bf7fd6f
 	github.com/google/uuid v1.1.2
 	github.com/goreleaser/nfpm v1.10.3
+	github.com/iancoleman/strcase v0.2.0
 	github.com/jsimonetti/rtnetlink v0.0.0-20210525051524-4cc836578190
 	github.com/kballard/go-shellquote v0.0.0-20180428030007-95032a82bc51
 	github.com/klauspost/compress v1.12.2
-	github.com/kr/pty v1.1.8
 	github.com/mdlayher/netlink v1.4.1
 	github.com/mdlayher/sdnotify v0.0.0-20210228150836-ea3ec207d697
 	github.com/miekg/dns v1.1.42
+	github.com/mitchellh/go-ps v1.0.0
 	github.com/pborman/getopt v1.1.0
 	github.com/peterbourgon/ff/v2 v2.0.0
 	github.com/pkg/sftp v1.13.0
 	github.com/tailscale/certstore v0.0.0-20210528134328-066c94b793d3
 	github.com/tailscale/depaware v0.0.0-20201214215404-77d1e9757027
+	github.com/tailscale/goupnp v1.0.1-0.20210804011211-c64d0f06ea05
+	github.com/tailscale/hujson v0.0.0-20200924210142-dde312d0d6a2
 	github.com/tcnksm/go-httpstat v0.2.0
 	github.com/toqueteos/webbrowser v1.2.0
 	go4.org/mem v0.0.0-20201119185036-c04c5a6ff174
-	golang.org/x/crypto v0.0.0-20210513164829-c07d793c2f9a
-	golang.org/x/net v0.0.0-20210525063256-abc453219eb5
+	golang.org/x/crypto v0.0.0-20210616213533-5ff15b29337e
+	golang.org/x/net v0.0.0-20210614182718-04defd469f4e
 	golang.org/x/sync v0.0.0-20210220032951-036812b2e83c
 	golang.org/x/sys v0.0.0-20210616094352-59db8d763f22
 	golang.org/x/term v0.0.0-20210503060354-a79de5458b56
 	golang.org/x/time v0.0.0-20210611083556-38a9dc6acbc6
 	golang.org/x/tools v0.1.2
-	golang.zx2c4.com/wireguard v0.0.0-20210525143454-64cb82f2b3f5
-	golang.zx2c4.com/wireguard/windows v0.3.15-0.20210525143335-94c0476d63e3
+	golang.zx2c4.com/wireguard v0.0.0-20210624150102-15b24b6179e0
+	golang.zx2c4.com/wireguard/windows v0.3.16
 	honnef.co/go/tools v0.1.4
-	inet.af/netaddr v0.0.0-20210602152128-50f8686885e3
+	inet.af/netaddr v0.0.0-20210721214506-ce7a8ad02cc1
 	inet.af/netstack v0.0.0-20210622165351-29b14ebc044e
 	inet.af/peercred v0.0.0-20210318190834-4259e17bb763
 	inet.af/wf v0.0.0-20210516214145-a5343001b756

go.sum

@@ -82,12 +82,13 @@ github.com/coreos/go-semver v0.3.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3Ee
 github.com/coreos/go-systemd v0.0.0-20190321100706-95778dfbb74e/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4=
 github.com/coreos/pkg v0.0.0-20180928190104-399ea9e2e55f/go.mod h1:E3G3o1h8I7cfcXa63jLwjI0eiQQMgzzUDFVpN/nH/eA=
 github.com/cpuguy83/go-md2man/v2 v2.0.0/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU=
-github.com/creack/pty v1.1.7/go.mod h1:lj5s0c3V2DBrqTV7llrYr5NG6My20zk30Fl46Y7DoTY=
 github.com/creack/pty v1.1.9 h1:uDmaGzcdjhF4i/plgjmEsriH11Y0o7RKapEf/LDaM3w=
 github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E=
 github.com/daixiang0/gci v0.2.4/go.mod h1:+AV8KmHTGxxwp/pY84TLQfFKp2vuKXXJVzF3kD/hfR4=
 github.com/daixiang0/gci v0.2.7 h1:bosLNficubzJZICsVzxuyNc6oAbdz0zcqLG2G/RxtY4=
 github.com/daixiang0/gci v0.2.7/go.mod h1:+4dZ7TISfSmqfAGv59ePaHfNzgGtIkHAhhdKggP1JAc=
+github.com/dave/jennifer v1.4.1 h1:XyqG6cn5RQsTj3qlWQTKlRGAyrTcsk1kUmWdZBzRjDw=
+github.com/dave/jennifer v1.4.1/go.mod h1:7jEdnm+qBcxl8PC0zyp7vxcpSRnzXSt9r39tpTVGlwA=
 github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
 github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
@@ -296,6 +297,8 @@ github.com/hashicorp/serf v0.8.2/go.mod h1:6hOLApaqBFA1NXqRQAsxw9QxuDEvNxSQRwA/J
 github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU=
 github.com/huandu/xstrings v1.3.2 h1:L18LIDzqlW6xN2rEkpdV8+oL/IXWJ1APd+vsdYy4Wdw=
 github.com/huandu/xstrings v1.3.2/go.mod h1:y5/lhBue+AyNmUVz9RLU9xbLR0o4KIIExikq4ovT0aE=
+github.com/iancoleman/strcase v0.2.0 h1:05I4QRnGpI0m37iZQRuskXh+w77mr6Z41lwQzuHLwW0=
+github.com/iancoleman/strcase v0.2.0/go.mod h1:iwCmte+B7n89clKwxIoIXy/HfoL7AsD47ZCWhYzw7ho=
 github.com/imdario/mergo v0.3.9/go.mod h1:2EnlNZ0deacrJVfApfmtdGgDfMuh/nq6Ok1EcJh5FfA=
 github.com/imdario/mergo v0.3.11 h1:3tnifQM4i+fbajXKBHXWEH+KvNHqojZ778UH75j3bGA=
 github.com/imdario/mergo v0.3.11/go.mod h1:jmQim1M+e3UYxmgPu/WyfjB3N3VflVyUjjjwH0dnCYA=
@@ -355,8 +358,6 @@ github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORN
 github.com/kr/pretty v0.2.1 h1:Fmg33tUaq4/8ym9TJN1x7sLJnHVwhP33CNkpYV/7rwI=
 github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI=
 github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
-github.com/kr/pty v1.1.8 h1:AkaSdXYQOWeaO3neb8EM634ahkXXe3jYbVh/F9lq+GI=
-github.com/kr/pty v1.1.8/go.mod h1:O1sed60cT9XZ5uDucP5qwvh+TE3NnUj51EiZO/lmSfw=
 github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
 github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY=
 github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE=
@@ -418,6 +419,7 @@ github.com/mitchellh/copystructure v1.0.0/go.mod h1:SNtv71yrdKgLRyLFxmLdkAbkKEFW
 github.com/mitchellh/go-homedir v1.0.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0=
 github.com/mitchellh/go-homedir v1.1.0 h1:lukF9ziXFxDFPkA1vsr5zpc1XuPDn/wFntq5mG+4E0Y=
 github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0=
+github.com/mitchellh/go-ps v1.0.0 h1:i6ampVEEF4wQFF+bkYfwYgY+F/uYJDktmvLPf7qIgjc=
 github.com/mitchellh/go-ps v1.0.0/go.mod h1:J4lOc8z8yJs6vUwklHw2XEIiT4z4C40KtWVN3nvg8Pg=
 github.com/mitchellh/go-testing-interface v1.0.0/go.mod h1:kRemZodwjscx+RGhAo8eIhFbs2+BFgRtFPeD/KE+zxI=
 github.com/mitchellh/gox v0.4.0/go.mod h1:Sd9lOJ0+aimLBi73mGofS1ycjY8lL3uZM3JPS42BGNg=
@@ -579,6 +581,10 @@ github.com/tailscale/certstore v0.0.0-20210528134328-066c94b793d3 h1:fEubocuQkrl
 github.com/tailscale/certstore v0.0.0-20210528134328-066c94b793d3/go.mod h1:2P+hpOwd53e7JMX/L4f3VXkv1G+33ES6IWZSrkIeWNs=
 github.com/tailscale/depaware v0.0.0-20201214215404-77d1e9757027 h1:lK99QQdH3yBWY6aGilF+IRlQIdmhzLrsEmF6JgN+Ryw=
 github.com/tailscale/depaware v0.0.0-20201214215404-77d1e9757027/go.mod h1:p9lPsd+cx33L3H9nNoecRRxPssFKUwwI50I3pZ0yT+8=
+github.com/tailscale/goupnp v1.0.1-0.20210804011211-c64d0f06ea05 h1:4chzWmimtJPxRs2O36yuGRW3f9SYV+bMTTvMBI0EKio=
+github.com/tailscale/goupnp v1.0.1-0.20210804011211-c64d0f06ea05/go.mod h1:PdCqy9JzfWMJf1H5UJW2ip33/d4YkoKN0r67yKH1mG8=
+github.com/tailscale/hujson v0.0.0-20200924210142-dde312d0d6a2 h1:reREUgl2FG+o7YCsrZB8XLjnuKv5hEIWtnOdAbRAXZI=
+github.com/tailscale/hujson v0.0.0-20200924210142-dde312d0d6a2/go.mod h1:STqf+YV0ADdzk4ejtXFsGqDpATP9JoL0OB+hiFQbkdE=
 github.com/tcnksm/go-httpstat v0.2.0 h1:rP7T5e5U2HfmOBmZzGgGZjBQ5/GluWUylujl0tJ04I0=
 github.com/tcnksm/go-httpstat v0.2.0/go.mod h1:s3JVJFtQxtBEBC9dwcdTTXS9xFnM3SXAZwPG41aurT8=
 github.com/tdakkota/asciicheck v0.0.0-20200416190851-d7f85be797a2/go.mod h1:yHp0ai0Z9gUljN3o0xMhYJnH/IcvkdTBOX2fmJ93JEM=
@@ -650,8 +656,8 @@ golang.org/x/crypto v0.0.0-20201016220609-9e8e0b390897/go.mod h1:LzIPMQfyMNhhGPh
 golang.org/x/crypto v0.0.0-20201208171446-5f87f3452ae9/go.mod h1:jdWPYTVW3xRLrWPugEBEK3UY2ZEsg3UU495nc5E+M+I=
 golang.org/x/crypto v0.0.0-20201221181555-eec23a3978ad/go.mod h1:jdWPYTVW3xRLrWPugEBEK3UY2ZEsg3UU495nc5E+M+I=
 golang.org/x/crypto v0.0.0-20210220033148-5ea612d1eb83/go.mod h1:jdWPYTVW3xRLrWPugEBEK3UY2ZEsg3UU495nc5E+M+I=
-golang.org/x/crypto v0.0.0-20210513164829-c07d793c2f9a h1:kr2P4QFmQr29mSLA43kwrOcgcReGTfbE9N577tCTuBc=
-golang.org/x/crypto v0.0.0-20210513164829-c07d793c2f9a/go.mod h1:P+XmwS30IXTQdn5tA2iutPOUgjI07+tq3H3K9MVA1s8=
+golang.org/x/crypto v0.0.0-20210616213533-5ff15b29337e h1:gsTQYXdTw2Gq7RBsWvlQ91b+aEQ6bXFUngBGuR8sPpI=
+golang.org/x/crypto v0.0.0-20210616213533-5ff15b29337e/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc=
 golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
 golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
 golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
@@ -716,8 +722,9 @@ golang.org/x/net v0.0.0-20201224014010-6772e930b67b/go.mod h1:m0MpNAwzfU5UDzcl9v
 golang.org/x/net v0.0.0-20210119194325-5f4716e94777/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg=
 golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg=
 golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM=
-golang.org/x/net v0.0.0-20210525063256-abc453219eb5 h1:wjuX4b5yYQnEQHzd+CBcrcC6OVR2J1CN6mUy0oSxIPo=
 golang.org/x/net v0.0.0-20210525063256-abc453219eb5/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
+golang.org/x/net v0.0.0-20210614182718-04defd469f4e h1:XpT3nA5TvE525Ne3hInMh6+GETgn27Zfm9dxsThnX2Q=
+golang.org/x/net v0.0.0-20210614182718-04defd469f4e/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
 golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
 golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
 golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
@@ -783,12 +790,12 @@ golang.org/x/sys v0.0.0-20210216163648-f7da38b97c65/go.mod h1:h1NjWce9XRLGQEsW7w
 golang.org/x/sys v0.0.0-20210301091718-77cc2087c03b/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20210303074136-134d130e1a04/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20210305230114-8fe3ee5dd75b/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20210309040221-94ec62e08169/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20210330210617-4fbd30eecc44/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20210403161142-5e06dd20ab57/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20210525143221-35b2ab0089ea/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20210616094352-59db8d763f22 h1:RqytpXGR1iVNX7psjB3ff8y7sNFinVFvkx1c8SjBkio=
 golang.org/x/sys v0.0.0-20210616094352-59db8d763f22/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw=
@@ -869,11 +876,10 @@ golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8T
 golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
 golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 h1:go1bK/D/BFZV2I8cIQd1NKEZ+0owSTG1fDTci4IqFcE=
 golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
-golang.zx2c4.com/wireguard v0.0.0-20210521230051-c27ff9b9f6f7/go.mod h1:a057zjmoc00UN7gVkaJt2sXVK523kMJcogDTEvPIasg=
-golang.zx2c4.com/wireguard v0.0.0-20210525143454-64cb82f2b3f5 h1:5D3v3AKu7ktIhDlqZhZ4+YeNKsW+dnc2+zfFAdhwa8M=
-golang.zx2c4.com/wireguard v0.0.0-20210525143454-64cb82f2b3f5/go.mod h1:laHzsbfMhGSobUmruXWAyMKKHSqvIcrqZJMyHD+/3O8=
-golang.zx2c4.com/wireguard/windows v0.3.15-0.20210525143335-94c0476d63e3 h1:Xw0ZuZcvq981iPGZoLrUXhrK2jOJAw/B6gZxc6g8FsU=
-golang.zx2c4.com/wireguard/windows v0.3.15-0.20210525143335-94c0476d63e3/go.mod h1:f/UVhQ6vXZKDodGB3Glgwu9B3djRxR14jIbcuxD8NBw=
+golang.zx2c4.com/wireguard v0.0.0-20210624150102-15b24b6179e0 h1:qINUmOnDCCF7i14oomDDkGmlda7BSDTGfge77/aqdfk=
+golang.zx2c4.com/wireguard v0.0.0-20210624150102-15b24b6179e0/go.mod h1:laHzsbfMhGSobUmruXWAyMKKHSqvIcrqZJMyHD+/3O8=
+golang.zx2c4.com/wireguard/windows v0.3.16 h1:S42i0kp3SFHZm1mMFTtiU3OnEQJ0GRVOVlMkBhSDTZI=
+golang.zx2c4.com/wireguard/windows v0.3.16/go.mod h1:f80rkFY2CKQklps1GHE15k/M4Tq78aofbr1iQM5MTVY=
 google.golang.org/api v0.4.0/go.mod h1:8k5glujaEP+g9n7WNsDg8QP6cUVNI86fCNMcbazEtwE=
 google.golang.org/api v0.7.0/go.mod h1:WtwebWUNSVBH/HAw79HIFXZNqEvBhG+Ra+ax0hx3E3M=
 google.golang.org/api v0.8.0/go.mod h1:o4eAsZoiT+ibD93RtjEohWalFOjRDx6CVaqeizhEnKg=
@@ -949,8 +955,8 @@ honnef.co/go/tools v0.0.1-2020.1.6/go.mod h1:pyyisuGw24ruLjrr1ddx39WE0y9OooInRzE
 honnef.co/go/tools v0.1.4 h1:SadWOkti5uVN1FAMgxn165+Mw00fuQKyk4Gyn/inxNQ=
 honnef.co/go/tools v0.1.4/go.mod h1:NgwopIslSNH47DimFoV78dnkksY2EFtX0ajyb3K/las=
 inet.af/netaddr v0.0.0-20210515010201-ad03edc7c841/go.mod h1:z0nx+Dh+7N7CC8V5ayHtHGpZpxLQZZxkIaaz6HN65Ls=
-inet.af/netaddr v0.0.0-20210602152128-50f8686885e3 h1:RlarOdsmOUCCvy7Xm1JchJIGuQsuKwD/Lo1bjYmfuQI=
-inet.af/netaddr v0.0.0-20210602152128-50f8686885e3/go.mod h1:z0nx+Dh+7N7CC8V5ayHtHGpZpxLQZZxkIaaz6HN65Ls=
+inet.af/netaddr v0.0.0-20210721214506-ce7a8ad02cc1 h1:mxmfTV6kjXTlFqqFETnG9FQZzNFc6AKunZVAgQ3b7WA=
+inet.af/netaddr v0.0.0-20210721214506-ce7a8ad02cc1/go.mod h1:z0nx+Dh+7N7CC8V5ayHtHGpZpxLQZZxkIaaz6HN65Ls=
 inet.af/netstack v0.0.0-20210622165351-29b14ebc044e h1:z11NK94NQcI3DA+a3pUC/2dRYTph1kPX6B0FnCaMDzk=
 inet.af/netstack v0.0.0-20210622165351-29b14ebc044e/go.mod h1:fG3G1dekmK8oDX3iVzt8c0zICLMLSN8SjdxbXVt0WjU=
 inet.af/peercred v0.0.0-20210318190834-4259e17bb763 h1:gPSJmmVzmdy4kHhlCMx912GdiUz3k/RzJGg0ADqy1dg=

hostinfo/hostinfo.go

@@ -27,6 +27,7 @@ const (
 	AWSLambda       = EnvType("lm")
 	Heroku          = EnvType("hr")
 	AzureAppService = EnvType("az")
+	AWSFargate      = EnvType("fg")
 )
 
 var envType atomic.Value // of EnvType
@@ -53,6 +54,9 @@ func getEnvType() EnvType {
 	if inAzureAppService() {
 		return AzureAppService
 	}
+	if inAWSFargate() {
+		return AWSFargate
+	}
 	return ""
 }
 
@@ -115,3 +119,10 @@ func inAzureAppService() bool {
 	}
 	return false
 }
+
+func inAWSFargate() bool {
+	if os.Getenv("AWS_EXECUTION_ENV") == "AWS_ECS_FARGATE" {
+		return true
+	}
+	return false
+}

internal/deephash/deephash.go

@@ -1,226 +0,0 @@
-// Copyright (c) 2020 Tailscale Inc & AUTHORS All rights reserved.
-// Use of this source code is governed by a BSD-style
-// license that can be found in the LICENSE file.
-
-// Package deephash hashes a Go value recursively, in a predictable
-// order, without looping.
-package deephash
-
-import (
-	"bufio"
-	"crypto/sha256"
-	"encoding/hex"
-	"fmt"
-	"hash"
-	"reflect"
-	"strconv"
-	"sync"
-)
-
-func calcHash(v interface{}) string {
-	h := sha256.New()
-	b := bufio.NewWriterSize(h, h.BlockSize())
-	scratch := make([]byte, 0, 128)
-	printTo(b, v, scratch)
-	b.Flush()
-	scratch = h.Sum(scratch[:0])
-	hex.Encode(scratch[:cap(scratch)], scratch[:sha256.Size])
-	return string(scratch[:sha256.Size*2])
-}
-
-// UpdateHash sets last to the hash of v and reports whether its value changed.
-func UpdateHash(last *string, v ...interface{}) (changed bool) {
-	sig := calcHash(v)
-	if *last != sig {
-		*last = sig
-		return true
-	}
-	return false
-}
-
-func printTo(w *bufio.Writer, v interface{}, scratch []byte) {
-	print(w, reflect.ValueOf(v), make(map[uintptr]bool), scratch)
-}
-
-var appenderToType = reflect.TypeOf((*appenderTo)(nil)).Elem()
-
-type appenderTo interface {
-	AppendTo([]byte) []byte
-}
-
-// print hashes v into w.
-// It reports whether it was able to do so without hitting a cycle.
-func print(w *bufio.Writer, v reflect.Value, visited map[uintptr]bool, scratch []byte) (acyclic bool) {
-	if !v.IsValid() {
-		return true
-	}
-
-	if v.CanInterface() {
-		// Use AppendTo methods, if available and cheap.
-		if v.CanAddr() && v.Type().Implements(appenderToType) {
-			a := v.Addr().Interface().(appenderTo)
-			scratch = a.AppendTo(scratch[:0])
-			w.Write(scratch)
-			return true
-		}
-	}
-
-	// Generic handling.
-	switch v.Kind() {
-	default:
-		panic(fmt.Sprintf("unhandled kind %v for type %v", v.Kind(), v.Type()))
-	case reflect.Ptr:
-		ptr := v.Pointer()
-		if visited[ptr] {
-			return false
-		}
-		visited[ptr] = true
-		return print(w, v.Elem(), visited, scratch)
-	case reflect.Struct:
-		acyclic = true
-		w.WriteString("struct{\n")
-		for i, n := 0, v.NumField(); i < n; i++ {
-			fmt.Fprintf(w, " [%d]: ", i)
-			if !print(w, v.Field(i), visited, scratch) {
-				acyclic = false
-			}
-			w.WriteString("\n")
-		}
-		w.WriteString("}\n")
-		return acyclic
-	case reflect.Slice, reflect.Array:
-		if v.Type().Elem().Kind() == reflect.Uint8 && v.CanInterface() {
-			fmt.Fprintf(w, "%q", v.Interface())
-			return true
-		}
-		fmt.Fprintf(w, "[%d]{\n", v.Len())
-		acyclic = true
-		for i, ln := 0, v.Len(); i < ln; i++ {
-			fmt.Fprintf(w, " [%d]: ", i)
-			if !print(w, v.Index(i), visited, scratch) {
-				acyclic = false
-			}
-			w.WriteString("\n")
-		}
-		w.WriteString("}\n")
-		return acyclic
-	case reflect.Interface:
-		return print(w, v.Elem(), visited, scratch)
-	case reflect.Map:
-		if hashMapAcyclic(w, v, visited, scratch) {
-			return true
-		}
-		return hashMapFallback(w, v, visited, scratch)
-	case reflect.String:
-		w.WriteString(v.String())
-	case reflect.Bool:
-		fmt.Fprintf(w, "%v", v.Bool())
-	case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64:
-		fmt.Fprintf(w, "%v", v.Int())
-	case reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64, reflect.Uintptr:
-		scratch = strconv.AppendUint(scratch[:0], v.Uint(), 10)
-		w.Write(scratch)
-	case reflect.Float32, reflect.Float64:
-		fmt.Fprintf(w, "%v", v.Float())
-	case reflect.Complex64, reflect.Complex128:
-		fmt.Fprintf(w, "%v", v.Complex())
-	}
-	return true
-}
-
-type mapHasher struct {
-	xbuf [sha256.Size]byte // XOR'ed accumulated buffer
-	ebuf [sha256.Size]byte // scratch buffer
-	s256 hash.Hash         // sha256 hash.Hash
-	bw   *bufio.Writer     // to hasher into ebuf
-	val  valueCache        // re-usable values for map iteration
-	iter *reflect.MapIter  // re-usable map iterator
-}
-
-func (mh *mapHasher) Reset() {
-	for i := range mh.xbuf {
-		mh.xbuf[i] = 0
-	}
-}
-
-func (mh *mapHasher) startEntry() {
-	for i := range mh.ebuf {
-		mh.ebuf[i] = 0
-	}
-	mh.bw.Flush()
-	mh.s256.Reset()
-}
-
-func (mh *mapHasher) endEntry() {
-	mh.bw.Flush()
-	for i, b := range mh.s256.Sum(mh.ebuf[:0]) {
-		mh.xbuf[i] ^= b
-	}
-}
-
-var mapHasherPool = &sync.Pool{
-	New: func() interface{} {
-		mh := new(mapHasher)
-		mh.s256 = sha256.New()
-		mh.bw = bufio.NewWriter(mh.s256)
-		mh.val = make(valueCache)
-		mh.iter = new(reflect.MapIter)
-		return mh
-	},
-}
-
-type valueCache map[reflect.Type]reflect.Value
-
-func (c valueCache) get(t reflect.Type) reflect.Value {
-	v, ok := c[t]
-	if !ok {
-		v = reflect.New(t).Elem()
-		c[t] = v
-	}
-	return v
-}
-
-// hashMapAcyclic is the faster sort-free version of map hashing. If
-// it detects a cycle it returns false and guarantees that nothing was
-// written to w.
-func hashMapAcyclic(w *bufio.Writer, v reflect.Value, visited map[uintptr]bool, scratch []byte) (acyclic bool) {
-	mh := mapHasherPool.Get().(*mapHasher)
-	defer mapHasherPool.Put(mh)
-	mh.Reset()
-	iter := mapIter(mh.iter, v)
-	defer mapIter(mh.iter, reflect.Value{}) // avoid pinning v from mh.iter when we return
-	k := mh.val.get(v.Type().Key())
-	e := mh.val.get(v.Type().Elem())
-	for iter.Next() {
-		key := iterKey(iter, k)
-		val := iterVal(iter, e)
-		mh.startEntry()
-		if !print(mh.bw, key, visited, scratch) {
-			return false
-		}
-		if !print(mh.bw, val, visited, scratch) {
-			return false
-		}
-		mh.endEntry()
-	}
-	w.Write(mh.xbuf[:])
-	return true
-}
-
-func hashMapFallback(w *bufio.Writer, v reflect.Value, visited map[uintptr]bool, scratch []byte) (acyclic bool) {
-	acyclic = true
-	sm := newSortedMap(v)
-	fmt.Fprintf(w, "map[%d]{\n", len(sm.Key))
-	for i, k := range sm.Key {
-		if !print(w, k, visited, scratch) {
-			acyclic = false
-		}
-		w.WriteString(": ")
-		if !print(w, sm.Value[i], visited, scratch) {
-			acyclic = false
-		}
-		w.WriteString("\n")
-	}
-	w.WriteString("}\n")
-	return acyclic
-}

internal/deephash/deephash_test.go

@@ -1,136 +0,0 @@
-// Copyright (c) 2020 Tailscale Inc & AUTHORS All rights reserved.
-// Use of this source code is governed by a BSD-style
-// license that can be found in the LICENSE file.
-
-package deephash
-
-import (
-	"bufio"
-	"bytes"
-	"fmt"
-	"reflect"
-	"testing"
-
-	"inet.af/netaddr"
-	"tailscale.com/tailcfg"
-	"tailscale.com/util/dnsname"
-	"tailscale.com/wgengine/router"
-	"tailscale.com/wgengine/wgcfg"
-)
-
-func TestDeepHash(t *testing.T) {
-	// v contains the types of values we care about for our current callers.
-	// Mostly we're just testing that we don't panic on handled types.
-	v := getVal()
-
-	hash1 := calcHash(v)
-	t.Logf("hash: %v", hash1)
-	for i := 0; i < 20; i++ {
-		hash2 := calcHash(getVal())
-		if hash1 != hash2 {
-			t.Error("second hash didn't match")
-		}
-	}
-}
-
-func getVal() []interface{} {
-	return []interface{}{
-		&wgcfg.Config{
-			Name:      "foo",
-			Addresses: []netaddr.IPPrefix{netaddr.IPPrefixFrom(netaddr.IPFrom16([16]byte{3: 3}), 5)},
-			Peers: []wgcfg.Peer{
-				{
-					Endpoints: wgcfg.Endpoints{
-						IPPorts: wgcfg.NewIPPortSet(netaddr.MustParseIPPort("42.42.42.42:5")),
-					},
-				},
-			},
-		},
-		&router.Config{
-			Routes: []netaddr.IPPrefix{
-				netaddr.MustParseIPPrefix("1.2.3.0/24"),
-				netaddr.MustParseIPPrefix("1234::/64"),
-			},
-		},
-		map[dnsname.FQDN][]netaddr.IP{
-			dnsname.FQDN("a."): {netaddr.MustParseIP("1.2.3.4"), netaddr.MustParseIP("4.3.2.1")},
-			dnsname.FQDN("b."): {netaddr.MustParseIP("8.8.8.8"), netaddr.MustParseIP("9.9.9.9")},
-			dnsname.FQDN("c."): {netaddr.MustParseIP("6.6.6.6"), netaddr.MustParseIP("7.7.7.7")},
-			dnsname.FQDN("d."): {netaddr.MustParseIP("6.7.6.6"), netaddr.MustParseIP("7.7.7.8")},
-			dnsname.FQDN("e."): {netaddr.MustParseIP("6.8.6.6"), netaddr.MustParseIP("7.7.7.9")},
-			dnsname.FQDN("f."): {netaddr.MustParseIP("6.9.6.6"), netaddr.MustParseIP("7.7.7.0")},
-		},
-		map[dnsname.FQDN][]netaddr.IPPort{
-			dnsname.FQDN("a."): {netaddr.MustParseIPPort("1.2.3.4:11"), netaddr.MustParseIPPort("4.3.2.1:22")},
-			dnsname.FQDN("b."): {netaddr.MustParseIPPort("8.8.8.8:11"), netaddr.MustParseIPPort("9.9.9.9:22")},
-			dnsname.FQDN("c."): {netaddr.MustParseIPPort("8.8.8.8:12"), netaddr.MustParseIPPort("9.9.9.9:23")},
-			dnsname.FQDN("d."): {netaddr.MustParseIPPort("8.8.8.8:13"), netaddr.MustParseIPPort("9.9.9.9:24")},
-			dnsname.FQDN("e."): {netaddr.MustParseIPPort("8.8.8.8:14"), netaddr.MustParseIPPort("9.9.9.9:25")},
-		},
-		map[tailcfg.DiscoKey]bool{
-			{1: 1}: true,
-			{1: 2}: false,
-			{2: 3}: true,
-			{3: 4}: false,
-		},
-	}
-}
-
-func BenchmarkHash(b *testing.B) {
-	b.ReportAllocs()
-	v := getVal()
-	for i := 0; i < b.N; i++ {
-		calcHash(v)
-	}
-}
-
-func TestHashMapAcyclic(t *testing.T) {
-	m := map[int]string{}
-	for i := 0; i < 100; i++ {
-		m[i] = fmt.Sprint(i)
-	}
-	got := map[string]bool{}
-
-	var buf bytes.Buffer
-	bw := bufio.NewWriter(&buf)
-
-	for i := 0; i < 20; i++ {
-		visited := map[uintptr]bool{}
-		scratch := make([]byte, 0, 64)
-		v := reflect.ValueOf(m)
-		buf.Reset()
-		bw.Reset(&buf)
-		if !hashMapAcyclic(bw, v, visited, scratch) {
-			t.Fatal("returned false")
-		}
-		if got[string(buf.Bytes())] {
-			continue
-		}
-		got[string(buf.Bytes())] = true
-	}
-	if len(got) != 1 {
-		t.Errorf("got %d results; want 1", len(got))
-	}
-}
-
-func BenchmarkHashMapAcyclic(b *testing.B) {
-	b.ReportAllocs()
-	m := map[int]string{}
-	for i := 0; i < 100; i++ {
-		m[i] = fmt.Sprint(i)
-	}
-
-	var buf bytes.Buffer
-	bw := bufio.NewWriter(&buf)
-	visited := map[uintptr]bool{}
-	scratch := make([]byte, 0, 64)
-	v := reflect.ValueOf(m)
-
-	for i := 0; i < b.N; i++ {
-		buf.Reset()
-		bw.Reset(&buf)
-		if !hashMapAcyclic(bw, v, visited, scratch) {
-			b.Fatal("returned false")
-		}
-	}
-}

internal/deephash/fmtsort.go

@@ -1,224 +0,0 @@
-// Copyright (c) 2020 Tailscale Inc & AUTHORS All rights reserved.
-// Use of this source code is governed by a BSD-style
-// license that can be found in the LICENSE file.
-
-// and
-
-// Copyright 2018 The Go Authors. All rights reserved.
-// Use of this source code is governed by a BSD-style
-// license that can be found in the LICENSE file.
-
-// This is a slightly modified fork of Go's src/internal/fmtsort/sort.go
-
-package deephash
-
-import (
-	"reflect"
-	"sort"
-)
-
-// Note: Throughout this package we avoid calling reflect.Value.Interface as
-// it is not always legal to do so and it's easier to avoid the issue than to face it.
-
-// sortedMap represents a map's keys and values. The keys and values are
-// aligned in index order: Value[i] is the value in the map corresponding to Key[i].
-type sortedMap struct {
-	Key   []reflect.Value
-	Value []reflect.Value
-}
-
-func (o *sortedMap) Len() int           { return len(o.Key) }
-func (o *sortedMap) Less(i, j int) bool { return compare(o.Key[i], o.Key[j]) < 0 }
-func (o *sortedMap) Swap(i, j int) {
-	o.Key[i], o.Key[j] = o.Key[j], o.Key[i]
-	o.Value[i], o.Value[j] = o.Value[j], o.Value[i]
-}
-
-// Sort accepts a map and returns a sortedMap that has the same keys and
-// values but in a stable sorted order according to the keys, modulo issues
-// raised by unorderable key values such as NaNs.
-//
-// The ordering rules are more general than with Go's < operator:
-//
-//  - when applicable, nil compares low
-//  - ints, floats, and strings order by <
-//  - NaN compares less than non-NaN floats
-//  - bool compares false before true
-//  - complex compares real, then imag
-//  - pointers compare by machine address
-//  - channel values compare by machine address
-//  - structs compare each field in turn
-//  - arrays compare each element in turn.
-//    Otherwise identical arrays compare by length.
-//  - interface values compare first by reflect.Type describing the concrete type
-//    and then by concrete value as described in the previous rules.
-//
-func newSortedMap(mapValue reflect.Value) *sortedMap {
-	if mapValue.Type().Kind() != reflect.Map {
-		return nil
-	}
-	// Note: this code is arranged to not panic even in the presence
-	// of a concurrent map update. The runtime is responsible for
-	// yelling loudly if that happens. See issue 33275.
-	n := mapValue.Len()
-	key := make([]reflect.Value, 0, n)
-	value := make([]reflect.Value, 0, n)
-	iter := mapValue.MapRange()
-	for iter.Next() {
-		key = append(key, iter.Key())
-		value = append(value, iter.Value())
-	}
-	sorted := &sortedMap{
-		Key:   key,
-		Value: value,
-	}
-	sort.Stable(sorted)
-	return sorted
-}
-
-// compare compares two values of the same type. It returns -1, 0, 1
-// according to whether a > b (1), a == b (0), or a < b (-1).
-// If the types differ, it returns -1.
-// See the comment on Sort for the comparison rules.
-func compare(aVal, bVal reflect.Value) int {
-	aType, bType := aVal.Type(), bVal.Type()
-	if aType != bType {
-		return -1 // No good answer possible, but don't return 0: they're not equal.
-	}
-	switch aVal.Kind() {
-	case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64:
-		a, b := aVal.Int(), bVal.Int()
-		switch {
-		case a < b:
-			return -1
-		case a > b:
-			return 1
-		default:
-			return 0
-		}
-	case reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64, reflect.Uintptr:
-		a, b := aVal.Uint(), bVal.Uint()
-		switch {
-		case a < b:
-			return -1
-		case a > b:
-			return 1
-		default:
-			return 0
-		}
-	case reflect.String:
-		a, b := aVal.String(), bVal.String()
-		switch {
-		case a < b:
-			return -1
-		case a > b:
-			return 1
-		default:
-			return 0
-		}
-	case reflect.Float32, reflect.Float64:
-		return floatCompare(aVal.Float(), bVal.Float())
-	case reflect.Complex64, reflect.Complex128:
-		a, b := aVal.Complex(), bVal.Complex()
-		if c := floatCompare(real(a), real(b)); c != 0 {
-			return c
-		}
-		return floatCompare(imag(a), imag(b))
-	case reflect.Bool:
-		a, b := aVal.Bool(), bVal.Bool()
-		switch {
-		case a == b:
-			return 0
-		case a:
-			return 1
-		default:
-			return -1
-		}
-	case reflect.Ptr:
-		a, b := aVal.Pointer(), bVal.Pointer()
-		switch {
-		case a < b:
-			return -1
-		case a > b:
-			return 1
-		default:
-			return 0
-		}
-	case reflect.Chan:
-		if c, ok := nilCompare(aVal, bVal); ok {
-			return c
-		}
-		ap, bp := aVal.Pointer(), bVal.Pointer()
-		switch {
-		case ap < bp:
-			return -1
-		case ap > bp:
-			return 1
-		default:
-			return 0
-		}
-	case reflect.Struct:
-		for i := 0; i < aVal.NumField(); i++ {
-			if c := compare(aVal.Field(i), bVal.Field(i)); c != 0 {
-				return c
-			}
-		}
-		return 0
-	case reflect.Array:
-		for i := 0; i < aVal.Len(); i++ {
-			if c := compare(aVal.Index(i), bVal.Index(i)); c != 0 {
-				return c
-			}
-		}
-		return 0
-	case reflect.Interface:
-		if c, ok := nilCompare(aVal, bVal); ok {
-			return c
-		}
-		c := compare(reflect.ValueOf(aVal.Elem().Type()), reflect.ValueOf(bVal.Elem().Type()))
-		if c != 0 {
-			return c
-		}
-		return compare(aVal.Elem(), bVal.Elem())
-	default:
-		// Certain types cannot appear as keys (maps, funcs, slices), but be explicit.
-		panic("bad type in compare: " + aType.String())
-	}
-}
-
-// nilCompare checks whether either value is nil. If not, the boolean is false.
-// If either value is nil, the boolean is true and the integer is the comparison
-// value. The comparison is defined to be 0 if both are nil, otherwise the one
-// nil value compares low. Both arguments must represent a chan, func,
-// interface, map, pointer, or slice.
-func nilCompare(aVal, bVal reflect.Value) (int, bool) {
-	if aVal.IsNil() {
-		if bVal.IsNil() {
-			return 0, true
-		}
-		return -1, true
-	}
-	if bVal.IsNil() {
-		return 1, true
-	}
-	return 0, false
-}
-
-// floatCompare compares two floating-point values. NaNs compare low.
-func floatCompare(a, b float64) int {
-	switch {
-	case isNaN(a):
-		return -1 // No good answer if b is a NaN so don't bother checking.
-	case isNaN(b):
-		return 1
-	case a < b:
-		return -1
-	case a > b:
-		return 1
-	}
-	return 0
-}
-
-func isNaN(a float64) bool {
-	return a != a
-}

ipn/handle.go

@@ -156,7 +156,7 @@ func (h *Handle) Expiry() time.Time {
 }
 
 func (h *Handle) AdminPageURL() string {
-	return h.prefsCache.ControlURLOrDefault() + "/admin/machines"
+	return h.prefsCache.AdminPageURL()
 }
 
 func (h *Handle) StartLoginInteractive() {

ipn/ipnlocal/local.go

@@ -29,7 +29,6 @@ import (
 	"tailscale.com/client/tailscale/apitype"
 	"tailscale.com/control/controlclient"
 	"tailscale.com/health"
-	"tailscale.com/internal/deephash"
 	"tailscale.com/ipn"
 	"tailscale.com/ipn/ipnstate"
 	"tailscale.com/ipn/policy"
@@ -46,6 +45,7 @@ import (
 	"tailscale.com/types/persist"
 	"tailscale.com/types/preftype"
 	"tailscale.com/types/wgkey"
+	"tailscale.com/util/deephash"
 	"tailscale.com/util/dnsname"
 	"tailscale.com/util/osshare"
 	"tailscale.com/util/systemd"
@@ -95,7 +95,7 @@ type LocalBackend struct {
 	serverURL             string           // tailcontrol URL
 	newDecompressor       func() (controlclient.Decompressor, error)
 
-	filterHash string
+	filterHash deephash.Sum
 
 	// The mutex protects the following elements.
 	mu             sync.Mutex
@@ -179,6 +179,7 @@ func NewLocalBackend(logf logger.Logf, logid string, store ipn.StateStore, e wge
 		gotPortPollRes: make(chan struct{}),
 	}
 	b.statusChanged = sync.NewCond(&b.statusLock)
+	b.e.SetStatusCallback(b.setWgengineStatus)
 
 	linkMon := e.GetLinkMonitor()
 	b.prevIfState = linkMon.InterfaceState()
@@ -214,6 +215,15 @@ func (b *LocalBackend) SetDirectFileRoot(dir string) {
 	b.directFileRoot = dir
 }
 
+// b.mu must be held.
+func (b *LocalBackend) maybePauseControlClientLocked() {
+	if b.cc == nil {
+		return
+	}
+	networkUp := b.prevIfState.AnyInterfaceUp()
+	b.cc.SetPaused((b.state == ipn.Stopped && b.netMap != nil) || !networkUp)
+}
+
 // linkChange is our link monitor callback, called whenever the network changes.
 // major is whether ifst is different than earlier.
 func (b *LocalBackend) linkChange(major bool, ifst *interfaces.State) {
@@ -222,11 +232,7 @@ func (b *LocalBackend) linkChange(major bool, ifst *interfaces.State) {
 
 	hadPAC := b.prevIfState.HasPAC()
 	b.prevIfState = ifst
-
-	networkUp := ifst.AnyInterfaceUp()
-	if b.cc != nil {
-		go b.cc.SetPaused((b.state == ipn.Stopped && b.netMap != nil) || !networkUp)
-	}
+	b.maybePauseControlClientLocked()
 
 	// If the PAC-ness of the network changed, reconfig wireguard+route to
 	// add/remove subnets.
@@ -246,8 +252,8 @@ func (b *LocalBackend) linkChange(major bool, ifst *interfaces.State) {
 
 	if peerAPIListenAsync && b.netMap != nil && b.state == ipn.Running {
 		want := len(b.netMap.Addresses)
-		b.logf("linkChange: peerAPIListeners too low; trying again")
 		if len(b.peerAPIListeners) < want {
+			b.logf("linkChange: peerAPIListeners too low; trying again")
 			go b.initPeerAPIListener()
 		}
 	}
@@ -605,8 +611,8 @@ func (b *LocalBackend) setWgengineStatus(s *wgengine.Status, err error) {
 
 	if cc != nil {
 		cc.UpdateEndpoints(0, s.LocalAddrs)
+		b.stateMachine()
 	}
-	b.stateMachine()
 
 	b.statusLock.Lock()
 	b.statusChanged.Broadcast()
@@ -757,6 +763,12 @@ func (b *LocalBackend) Start(opts ipn.Options) error {
 		newPrefs := opts.UpdatePrefs
 		newPrefs.Persist = b.prefs.Persist
 		b.prefs = newPrefs
+
+		if opts.StateKey != "" {
+			if err := b.store.WriteState(opts.StateKey, b.prefs.ToBytes()); err != nil {
+				b.logf("failed to save UpdatePrefs state: %v", err)
+			}
+		}
 	}
 
 	wantRunning := b.prefs.WantRunning
@@ -857,7 +869,6 @@ func (b *LocalBackend) Start(opts ipn.Options) error {
 	}
 
 	cc.SetStatusFunc(b.setClientStatus)
-	b.e.SetStatusCallback(b.setWgengineStatus)
 	b.e.SetNetInfoCallback(b.setNetInfo)
 
 	b.mu.Lock()
@@ -933,7 +944,7 @@ func (b *LocalBackend) updateFilter(netMap *netmap.NetworkMap, prefs *ipn.Prefs)
 	localNets, _ := localNetsB.IPSet()
 	logNets, _ := logNetsB.IPSet()
 
-	changed := deephash.UpdateHash(&b.filterHash, haveNetmap, addrs, packetFilter, localNets.Ranges(), logNets.Ranges(), shieldsUp)
+	changed := deephash.Update(&b.filterHash, haveNetmap, addrs, packetFilter, localNets.Ranges(), logNets.Ranges(), shieldsUp)
 	if !changed {
 		return
 	}
@@ -973,6 +984,44 @@ var removeFromDefaultRoute = []netaddr.IPPrefix{
 	tsaddr.TailscaleULARange(),
 }
 
+// internalAndExternalInterfaces splits interface routes into "internal"
+// and "external" sets. Internal routes are those of virtual ethernet
+// network interfaces used by guest VMs and containers, such as WSL and
+// Docker.
+//
+// Given that "internal" routes don't leave the device, we choose to
+// trust them more, allowing access to them when an Exit Node is enabled.
+func internalAndExternalInterfaces() (internal, external []netaddr.IPPrefix, err error) {
+	if err := interfaces.ForeachInterfaceAddress(func(iface interfaces.Interface, pfx netaddr.IPPrefix) {
+		if tsaddr.IsTailscaleIP(pfx.IP()) {
+			return
+		}
+		if pfx.IsSingleIP() {
+			return
+		}
+		if runtime.GOOS == "windows" {
+			// Windows Hyper-V prefixes all MAC addresses with 00:15:5d.
+			// https://docs.microsoft.com/en-us/troubleshoot/windows-server/virtualization/default-limit-256-dynamic-mac-addresses
+			//
+			// This includes WSL2 vEthernet.
+			// Importantly: by default WSL2 /etc/resolv.conf points to
+			// a stub resolver running on the host vEthernet IP.
+			// So enabling exit nodes with the default tailnet
+			// configuration breaks WSL2 DNS without this.
+			mac := iface.Interface.HardwareAddr
+			if len(mac) == 6 && mac[0] == 0x00 && mac[1] == 0x15 && mac[2] == 0x5d {
+				internal = append(internal, pfx)
+				return
+			}
+		}
+		external = append(external, pfx)
+	}); err != nil {
+		return nil, nil, err
+	}
+
+	return internal, external, nil
+}
+
 func interfaceRoutes() (ips *netaddr.IPSet, hostIPs []netaddr.IP, err error) {
 	var b netaddr.IPSetBuilder
 	if err := interfaces.ForeachInterfaceAddress(func(_ interfaces.Interface, pfx netaddr.IPPrefix) {
@@ -2108,18 +2157,21 @@ func (b *LocalBackend) routerConfig(cfg *wgcfg.Config, prefs *ipn.Prefs) *router
 		if !default6 {
 			rs.Routes = append(rs.Routes, ipv6Default)
 		}
-		if runtime.GOOS == "linux" || runtime.GOOS == "darwin" {
-			// Only allow local lan access on linux machines for now.
-			ips, _, err := interfaceRoutes()
-			if err != nil {
-				b.logf("failed to discover interface ips: %v", err)
-			}
+		internalIPs, externalIPs, err := internalAndExternalInterfaces()
+		if err != nil {
+			b.logf("failed to discover interface ips: %v", err)
+		}
+		if runtime.GOOS == "linux" || runtime.GOOS == "darwin" || runtime.GOOS == "windows" {
+			rs.LocalRoutes = internalIPs // unconditionally allow access to guest VM networks
 			if prefs.ExitNodeAllowLANAccess {
-				rs.LocalRoutes = ips.Prefixes()
+				rs.LocalRoutes = append(rs.LocalRoutes, externalIPs...)
+				if len(externalIPs) != 0 {
+					b.logf("allowing exit node access to internal IPs: %v", internalIPs)
+				}
 			} else {
 				// Explicitly add routes to the local network so that we do not
 				// leak any traffic.
-				rs.Routes = append(rs.Routes, ips.Prefixes()...)
+				rs.Routes = append(rs.Routes, externalIPs...)
 			}
 		}
 	}
@@ -2148,6 +2200,18 @@ func applyPrefsToHostinfo(hi *tailcfg.Hostinfo, prefs *ipn.Prefs) {
 	}
 	if v := prefs.OSVersion; v != "" {
 		hi.OSVersion = v
+
+		// The Android app annotates when Google Play Services
+		// aren't available by tacking on a string to the
+		// OSVersion. Promote that to the Hostinfo.Package
+		// field instead, rather than adding a new pref, as
+		// this applyPrefsToHostinfo mechanism is mostly
+		// abused currently. TODO(bradfitz): instead let
+		// frontends update Hostinfo, without using Prefs.
+		if runtime.GOOS == "android" && strings.HasSuffix(v, " [nogoogle]") {
+			hi.Package = "nogoogle"
+			hi.OSVersion = strings.TrimSuffix(v, " [nogoogle]")
+		}
 	}
 	if m := prefs.DeviceModel; m != "" {
 		hi.DeviceModel = m
@@ -2167,9 +2231,7 @@ func (b *LocalBackend) enterState(newState ipn.State) {
 	oldState := b.state
 	b.state = newState
 	prefs := b.prefs
-	cc := b.cc
 	netMap := b.netMap
-	networkUp := b.prevIfState.AnyInterfaceUp()
 	activeLogin := b.activeLogin
 	authURL := b.authURL
 	if newState == ipn.Running {
@@ -2179,6 +2241,7 @@ func (b *LocalBackend) enterState(newState ipn.State) {
 		// Transitioning away from running.
 		b.closePeerAPIListenersLocked()
 	}
+	b.maybePauseControlClientLocked()
 	b.mu.Unlock()
 
 	if oldState == newState {
@@ -2189,10 +2252,6 @@ func (b *LocalBackend) enterState(newState ipn.State) {
 	health.SetIPNState(newState.String(), prefs.WantRunning)
 	b.send(ipn.Notify{State: &newState})
 
-	if cc != nil {
-		cc.SetPaused((newState == ipn.Stopped && netMap != nil) || !networkUp)
-	}
-
 	switch newState {
 	case ipn.NeedsLogin:
 		systemd.Status("Needs login: %s", authURL)
@@ -2213,7 +2272,7 @@ func (b *LocalBackend) enterState(newState ipn.State) {
 		b.e.RequestStatus()
 	case ipn.Running:
 		var addrs []string
-		for _, addr := range b.netMap.Addresses {
+		for _, addr := range netMap.Addresses {
 			addrs = append(addrs, addr.IP().String())
 		}
 		systemd.Status("Connected; %s; %s", activeLogin, strings.Join(addrs, " "))
@@ -2453,6 +2512,7 @@ func (b *LocalBackend) setNetMapLocked(nm *netmap.NetworkMap) {
 		b.logf("active login: %v", login)
 		b.activeLogin = login
 	}
+	b.maybePauseControlClientLocked()
 
 	// Determine if file sharing is enabled
 	fs := hasCapability(nm, tailcfg.CapabilityFileSharing)
@@ -2724,17 +2784,18 @@ func (b *LocalBackend) CheckIPForwarding() error {
 		return nil
 	}
 
+	const suffix = "\nSubnet routes won't work without IP forwarding.\nSee https://tailscale.com/kb/1104/enable-ip-forwarding/"
 	for _, key := range keys {
 		bs, err := exec.Command("sysctl", "-n", key).Output()
 		if err != nil {
-			return fmt.Errorf("couldn't check %s (%v).\nSubnet routes won't work without IP forwarding.", key, err)
+			return fmt.Errorf("couldn't check %s (%v)%s", key, err, suffix)
 		}
 		on, err := strconv.ParseBool(string(bytes.TrimSpace(bs)))
 		if err != nil {
-			return fmt.Errorf("couldn't parse %s (%v).\nSubnet routes won't work without IP forwarding.", key, err)
+			return fmt.Errorf("couldn't parse %s (%v)%s.", key, err, suffix)
 		}
 		if !on {
-			return fmt.Errorf("%s is disabled. Subnet routes won't work.", key)
+			return fmt.Errorf("%s is disabled.%s", key, suffix)
 		}
 	}
 	return nil
@@ -2752,3 +2813,13 @@ func (b *LocalBackend) PeerDialControlFunc() func(network, address string, c sys
 	}
 	return nil
 }
+
+// DERPMap returns the current DERPMap in use, or nil if not connected.
+func (b *LocalBackend) DERPMap() *tailcfg.DERPMap {
+	b.mu.Lock()
+	defer b.mu.Unlock()
+	if b.netMap == nil {
+		return nil
+	}
+	return b.netMap.DERPMap
+}

ipn/ipnlocal/state_test.go

@@ -14,6 +14,7 @@ import (
 
 	"tailscale.com/control/controlclient"
 	"tailscale.com/ipn"
+	"tailscale.com/syncs"
 	"tailscale.com/tailcfg"
 	"tailscale.com/types/empty"
 	"tailscale.com/types/logger"
@@ -277,7 +278,7 @@ func TestStateMachine(t *testing.T) {
 	c := qt.New(t)
 
 	logf := t.Logf
-	store := new(ipn.MemoryStore)
+	store := new(testStateStorage)
 	e, err := wgengine.NewFakeUserspaceEngine(logf, 0)
 	if err != nil {
 		t.Fatalf("NewFakeUserspaceEngine: %v", err)
@@ -353,7 +354,7 @@ func TestStateMachine(t *testing.T) {
 	c.Assert(b.Start(ipn.Options{StateKey: ipn.GlobalDaemonStateKey}), qt.IsNil)
 	{
 		// BUG: strictly, it should pause, not unpause, here, since !WantRunning.
-		c.Assert([]string{"Shutdown", "New", "unpause"}, qt.DeepEquals, cc.getCalls())
+		c.Assert([]string{"Shutdown", "unpause", "New", "unpause"}, qt.DeepEquals, cc.getCalls())
 
 		nn := notifies.drain(2)
 		c.Assert(cc.getCalls(), qt.HasLen, 0)
@@ -388,7 +389,7 @@ func TestStateMachine(t *testing.T) {
 	url1 := "http://localhost:1/1"
 	cc.send(nil, url1, false, nil)
 	{
-		c.Assert(cc.getCalls(), qt.DeepEquals, []string{})
+		c.Assert(cc.getCalls(), qt.DeepEquals, []string{"unpause"})
 
 		// ...but backend eats that notification, because the user
 		// didn't explicitly request interactive login yet, and
@@ -413,7 +414,7 @@ func TestStateMachine(t *testing.T) {
 		// We're still not logged in so there's nothing we can do
 		// with it. (And empirically, it's providing an empty list
 		// of endpoints.)
-		c.Assert([]string{"UpdateEndpoints"}, qt.DeepEquals, cc.getCalls())
+		c.Assert([]string{"UpdateEndpoints", "unpause"}, qt.DeepEquals, cc.getCalls())
 		c.Assert(nn[0].BrowseToURL, qt.Not(qt.IsNil))
 		c.Assert(url1, qt.Equals, *nn[0].BrowseToURL)
 	}
@@ -439,7 +440,7 @@ func TestStateMachine(t *testing.T) {
 	cc.send(nil, url2, false, nil)
 	{
 		// BUG: UpdateEndpoints again, this is getting silly.
-		c.Assert([]string{"UpdateEndpoints"}, qt.DeepEquals, cc.getCalls())
+		c.Assert([]string{"UpdateEndpoints", "unpause", "unpause"}, qt.DeepEquals, cc.getCalls())
 
 		// This time, backend should emit it to the UI right away,
 		// because the UI is anxiously awaiting a new URL to visit.
@@ -469,7 +470,7 @@ func TestStateMachine(t *testing.T) {
 		// wait until it gets into Starting.
 		// TODO: (Currently this test doesn't detect that bug, but
 		// it's visible in the logs)
-		c.Assert([]string{"unpause", "UpdateEndpoints"}, qt.DeepEquals, cc.getCalls())
+		c.Assert([]string{"unpause", "unpause", "UpdateEndpoints", "unpause"}, qt.DeepEquals, cc.getCalls())
 		c.Assert(nn[0].LoginFinished, qt.Not(qt.IsNil))
 		c.Assert(nn[1].Prefs, qt.Not(qt.IsNil))
 		c.Assert(nn[2].State, qt.Not(qt.IsNil))
@@ -482,7 +483,7 @@ func TestStateMachine(t *testing.T) {
 	notifies.expect(1)
 	// BUG: the real controlclient sends LoginFinished with every
 	// notification while it's in StateAuthenticated, but not StateSynced.
-	// We should send it exactly once, or every time we're authenticated,
+	// It should send it exactly once, or every time we're authenticated,
 	// but the current code is brittle.
 	// (ie. I suspect it would be better to change false->true in send()
 	// below, and do the same in the real controlclient.)
@@ -491,7 +492,7 @@ func TestStateMachine(t *testing.T) {
 	})
 	{
 		nn := notifies.drain(1)
-		c.Assert([]string{"unpause", "UpdateEndpoints"}, qt.DeepEquals, cc.getCalls())
+		c.Assert([]string{"unpause", "unpause", "UpdateEndpoints", "unpause"}, qt.DeepEquals, cc.getCalls())
 		c.Assert(nn[0].State, qt.Not(qt.IsNil))
 		c.Assert(ipn.Starting, qt.Equals, *nn[0].State)
 	}
@@ -523,6 +524,7 @@ func TestStateMachine(t *testing.T) {
 
 	// The user changes their preference to WantRunning after all.
 	t.Logf("\n\nWantRunning -> true")
+	store.awaitWrite()
 	notifies.expect(2)
 	b.EditPrefs(&ipn.MaskedPrefs{
 		WantRunningSet: true,
@@ -532,15 +534,17 @@ func TestStateMachine(t *testing.T) {
 		nn := notifies.drain(2)
 		// BUG: UpdateEndpoints isn't needed here.
 		// BUG: Login isn't needed here. We never logged out.
-		c.Assert([]string{"Login", "unpause", "UpdateEndpoints"}, qt.DeepEquals, cc.getCalls())
+		c.Assert([]string{"Login", "unpause", "UpdateEndpoints", "unpause"}, qt.DeepEquals, cc.getCalls())
 		// BUG: I would expect Prefs to change first, and state after.
 		c.Assert(nn[0].State, qt.Not(qt.IsNil))
 		c.Assert(nn[1].Prefs, qt.Not(qt.IsNil))
 		c.Assert(ipn.Starting, qt.Equals, *nn[0].State)
+		c.Assert(store.sawWrite(), qt.IsTrue)
 	}
 
 	// Test the fast-path frontend reconnection.
-	// This one is very finicky, so we have to force State==Running.
+	// This one is very finicky, so we have to force State==Running
+	// or it won't use the fast path.
 	// TODO: actually get to State==Running, rather than cheating.
 	//  That'll require spinning up a fake DERP server and putting it in
 	//  the netmap.
@@ -561,18 +565,20 @@ func TestStateMachine(t *testing.T) {
 	b.state = ipn.Starting
 
 	// User wants to logout.
+	store.awaitWrite()
 	t.Logf("\n\nLogout (async)")
 	notifies.expect(2)
 	b.Logout()
 	{
 		nn := notifies.drain(2)
-		c.Assert([]string{"pause", "StartLogout"}, qt.DeepEquals, cc.getCalls())
+		c.Assert([]string{"pause", "StartLogout", "pause"}, qt.DeepEquals, cc.getCalls())
 		c.Assert(nn[0].State, qt.Not(qt.IsNil))
 		c.Assert(nn[1].Prefs, qt.Not(qt.IsNil))
 		c.Assert(ipn.Stopped, qt.Equals, *nn[0].State)
 		c.Assert(nn[1].Prefs.LoggedOut, qt.IsTrue)
 		c.Assert(nn[1].Prefs.WantRunning, qt.IsFalse)
 		c.Assert(ipn.Stopped, qt.Equals, b.State())
+		c.Assert(store.sawWrite(), qt.IsTrue)
 	}
 
 	// Let's make the logout succeed.
@@ -582,7 +588,7 @@ func TestStateMachine(t *testing.T) {
 	cc.send(nil, "", false, nil)
 	{
 		nn := notifies.drain(1)
-		c.Assert([]string{"unpause"}, qt.DeepEquals, cc.getCalls())
+		c.Assert([]string{"unpause", "unpause"}, qt.DeepEquals, cc.getCalls())
 		c.Assert(nn[0].State, qt.Not(qt.IsNil))
 		c.Assert(ipn.NeedsLogin, qt.Equals, *nn[0].State)
 		c.Assert(b.Prefs().LoggedOut, qt.IsTrue)
@@ -598,7 +604,7 @@ func TestStateMachine(t *testing.T) {
 		notifies.drain(0)
 		// BUG: the backend has already called StartLogout, and we're
 		// still logged out. So it shouldn't call it again.
-		c.Assert([]string{"StartLogout"}, qt.DeepEquals, cc.getCalls())
+		c.Assert([]string{"StartLogout", "unpause"}, qt.DeepEquals, cc.getCalls())
 		c.Assert(cc.getCalls(), qt.HasLen, 0)
 		c.Assert(b.Prefs().LoggedOut, qt.IsTrue)
 		c.Assert(b.Prefs().WantRunning, qt.IsFalse)
@@ -612,8 +618,7 @@ func TestStateMachine(t *testing.T) {
 	cc.send(nil, "", false, nil)
 	{
 		notifies.drain(0)
-		c.Assert(cc.getCalls(), qt.HasLen, 0)
-		c.Assert(cc.getCalls(), qt.HasLen, 0)
+		c.Assert(cc.getCalls(), qt.DeepEquals, []string{"unpause", "unpause"})
 		c.Assert(b.Prefs().LoggedOut, qt.IsTrue)
 		c.Assert(b.Prefs().WantRunning, qt.IsFalse)
 		c.Assert(ipn.NeedsLogin, qt.Equals, b.State())
@@ -627,7 +632,7 @@ func TestStateMachine(t *testing.T) {
 	// I guess, since that's supposed to be synchronous.
 	{
 		notifies.drain(0)
-		c.Assert([]string{"Logout"}, qt.DeepEquals, cc.getCalls())
+		c.Assert([]string{"Logout", "unpause"}, qt.DeepEquals, cc.getCalls())
 		c.Assert(cc.getCalls(), qt.HasLen, 0)
 		c.Assert(b.Prefs().LoggedOut, qt.IsTrue)
 		c.Assert(b.Prefs().WantRunning, qt.IsFalse)
@@ -641,8 +646,7 @@ func TestStateMachine(t *testing.T) {
 	cc.send(nil, "", false, nil)
 	{
 		notifies.drain(0)
-		c.Assert(cc.getCalls(), qt.HasLen, 0)
-		c.Assert(cc.getCalls(), qt.HasLen, 0)
+		c.Assert(cc.getCalls(), qt.DeepEquals, []string{"unpause", "unpause"})
 		c.Assert(b.Prefs().LoggedOut, qt.IsTrue)
 		c.Assert(b.Prefs().WantRunning, qt.IsFalse)
 		c.Assert(ipn.NeedsLogin, qt.Equals, b.State())
@@ -673,7 +677,7 @@ func TestStateMachine(t *testing.T) {
 		// BUG: We already called Shutdown(), no need to do it again.
 		// BUG: Way too soon for UpdateEndpoints.
 		// BUG: don't unpause because we're not logged in.
-		c.Assert([]string{"Shutdown", "New", "UpdateEndpoints", "unpause"}, qt.DeepEquals, cc.getCalls())
+		c.Assert([]string{"Shutdown", "unpause", "New", "UpdateEndpoints", "unpause"}, qt.DeepEquals, cc.getCalls())
 
 		nn := notifies.drain(2)
 		c.Assert(cc.getCalls(), qt.HasLen, 0)
@@ -698,7 +702,7 @@ func TestStateMachine(t *testing.T) {
 	})
 	{
 		nn := notifies.drain(3)
-		c.Assert([]string{"unpause"}, qt.DeepEquals, cc.getCalls())
+		c.Assert([]string{"unpause", "unpause"}, qt.DeepEquals, cc.getCalls())
 		c.Assert(nn[0].LoginFinished, qt.Not(qt.IsNil))
 		c.Assert(nn[1].Prefs, qt.Not(qt.IsNil))
 		c.Assert(nn[2].State, qt.Not(qt.IsNil))
@@ -738,9 +742,8 @@ func TestStateMachine(t *testing.T) {
 		//  on startup, otherwise UIs can't show the node list, login
 		//  name, etc when in state ipn.Stopped.
 		//  Arguably they shouldn't try. But they currently do.
-		c.Assert([]string{"Shutdown", "New", "UpdateEndpoints", "Login", "unpause"}, qt.DeepEquals, cc.getCalls())
-
 		nn := notifies.drain(2)
+		c.Assert([]string{"Shutdown", "unpause", "New", "UpdateEndpoints", "Login", "unpause"}, qt.DeepEquals, cc.getCalls())
 		c.Assert(cc.getCalls(), qt.HasLen, 0)
 		c.Assert(nn[0].Prefs, qt.Not(qt.IsNil))
 		c.Assert(nn[1].State, qt.Not(qt.IsNil))
@@ -749,6 +752,25 @@ func TestStateMachine(t *testing.T) {
 		c.Assert(ipn.Stopped, qt.Equals, *nn[1].State)
 	}
 
+	// When logged in but !WantRunning, ipn leaves us unpaused to retrieve
+	// the first netmap. Simulate that netmap being received, after which
+	// it should pause us, to avoid wasting CPU retrieving unnecessarily
+	// additional netmap updates.
+	//
+	// TODO: really the various GUIs and prefs should be refactored to
+	//  not require the netmap structure at all when starting while
+	//  !WantRunning. That would remove the need for this (or contacting
+	//  the control server at all when stopped).
+	t.Logf("\n\nStart4 -> netmap")
+	notifies.expect(0)
+	cc.send(nil, "", true, &netmap.NetworkMap{
+		MachineStatus: tailcfg.MachineAuthorized,
+	})
+	{
+		notifies.drain(0)
+		c.Assert([]string{"pause", "pause"}, qt.DeepEquals, cc.getCalls())
+	}
+
 	// Request connection.
 	// The state machine didn't call Login() earlier, so now it needs to.
 	t.Logf("\n\nWantRunning4 -> true")
@@ -775,7 +797,7 @@ func TestStateMachine(t *testing.T) {
 	})
 	{
 		nn := notifies.drain(2)
-		c.Assert([]string{"unpause"}, qt.DeepEquals, cc.getCalls())
+		c.Assert([]string{"pause"}, qt.DeepEquals, cc.getCalls())
 		// BUG: I would expect Prefs to change first, and state after.
 		c.Assert(nn[0].State, qt.Not(qt.IsNil))
 		c.Assert(nn[1].Prefs, qt.Not(qt.IsNil))
@@ -785,25 +807,27 @@ func TestStateMachine(t *testing.T) {
 	// We want to try logging in as a different user, while Stopped.
 	// First, start the login process (without logging out first).
 	t.Logf("\n\nLoginDifferent")
-	notifies.expect(2)
+	notifies.expect(1)
 	b.StartLoginInteractive()
 	url3 := "http://localhost:1/3"
 	cc.send(nil, url3, false, nil)
 	{
-		nn := notifies.drain(2)
+		nn := notifies.drain(1)
 		// It might seem like WantRunning should switch to true here,
 		// but that would be risky since we already have a valid
 		// user account. It might try to reconnect to the old account
 		// before the new one is ready. So no change yet.
-		c.Assert([]string{"Login", "unpause"}, qt.DeepEquals, cc.getCalls())
+		//
+		// Because the login hasn't yet completed, the old login
+		// is still valid, so it's correct that we stay paused.
+		c.Assert([]string{"Login", "pause"}, qt.DeepEquals, cc.getCalls())
 		c.Assert(nn[0].BrowseToURL, qt.Not(qt.IsNil))
-		c.Assert(nn[1].State, qt.Not(qt.IsNil))
 		c.Assert(*nn[0].BrowseToURL, qt.Equals, url3)
-		c.Assert(ipn.NeedsLogin, qt.Equals, *nn[1].State)
 	}
 
-	// Now, let's say the interactive login completed, using a different
-	// user account than before.
+	// Now, let's complete the interactive login, using a different
+	// user account than before. WantRunning changes to true after an
+	// interactive login, so we end up unpaused.
 	t.Logf("\n\nLoginDifferent URL visited")
 	notifies.expect(3)
 	cc.persist.LoginName = "user3"
@@ -812,7 +836,13 @@ func TestStateMachine(t *testing.T) {
 	})
 	{
 		nn := notifies.drain(3)
-		c.Assert([]string{"unpause"}, qt.DeepEquals, cc.getCalls())
+		// BUG: pause() being called here is a bad sign.
+		//  It means that either the state machine ran at least once
+		//  with the old netmap, or it ran with the new login+netmap
+		//  and !WantRunning. But since it's a fresh and successful
+		//  new login, WantRunning is true, so there was never a
+		//  reason to pause().
+		c.Assert([]string{"pause", "unpause"}, qt.DeepEquals, cc.getCalls())
 		c.Assert(nn[0].LoginFinished, qt.Not(qt.IsNil))
 		c.Assert(nn[1].Prefs, qt.Not(qt.IsNil))
 		c.Assert(nn[2].State, qt.Not(qt.IsNil))
@@ -831,7 +861,7 @@ func TestStateMachine(t *testing.T) {
 	{
 		// NOTE: cc.Shutdown() is correct here, since we didn't call
 		// b.Shutdown() ourselves.
-		c.Assert([]string{"Shutdown", "New", "UpdateEndpoints", "Login"}, qt.DeepEquals, cc.getCalls())
+		c.Assert([]string{"Shutdown", "unpause", "New", "UpdateEndpoints", "Login", "unpause"}, qt.DeepEquals, cc.getCalls())
 
 		nn := notifies.drain(1)
 		c.Assert(cc.getCalls(), qt.HasLen, 0)
@@ -850,7 +880,7 @@ func TestStateMachine(t *testing.T) {
 	})
 	{
 		nn := notifies.drain(1)
-		c.Assert([]string{"unpause"}, qt.DeepEquals, cc.getCalls())
+		c.Assert([]string{"unpause", "unpause"}, qt.DeepEquals, cc.getCalls())
 		// NOTE: No LoginFinished message since no interactive
 		// login was needed.
 		c.Assert(nn[0].State, qt.Not(qt.IsNil))
@@ -861,3 +891,29 @@ func TestStateMachine(t *testing.T) {
 		c.Assert(ipn.Starting, qt.Equals, b.State())
 	}
 }
+
+type testStateStorage struct {
+	mem     ipn.MemoryStore
+	written syncs.AtomicBool
+}
+
+func (s *testStateStorage) ReadState(id ipn.StateKey) ([]byte, error) {
+	return s.mem.ReadState(id)
+}
+
+func (s *testStateStorage) WriteState(id ipn.StateKey, bs []byte) error {
+	s.written.Set(true)
+	return s.mem.WriteState(id, bs)
+}
+
+// awaitWrite clears the "I've seen writes" bit, in prep for a future
+// call to sawWrite to see if a write arrived.
+func (s *testStateStorage) awaitWrite() { s.written.Set(false) }
+
+// sawWrite reports whether there's been a WriteState call since the most
+// recent awaitWrite call.
+func (s *testStateStorage) sawWrite() bool {
+	v := s.written.Get()
+	s.awaitWrite()
+	return v
+}

ipn/ipnstate/ipnstate.go

@@ -20,6 +20,7 @@ import (
 
 	"inet.af/netaddr"
 	"tailscale.com/tailcfg"
+	"tailscale.com/tstime/mono"
 	"tailscale.com/types/key"
 	"tailscale.com/util/dnsname"
 )
@@ -90,7 +91,7 @@ type PeerStatus struct {
 	RxBytes       int64
 	TxBytes       int64
 	Created       time.Time // time registered with tailcontrol
-	LastWrite     time.Time // time last packet sent
+	LastWrite     mono.Time // time last packet sent
 	LastSeen      time.Time // last seen to tailcontrol
 	LastHandshake time.Time // with local wireguard
 	KeepAlive     bool
@@ -320,7 +321,7 @@ table tbody tr:nth-child(even) td { background-color: #f5f5f5; }
 	f("<tr><th>Peer</th><th>OS</th><th>Node</th><th>Owner</th><th>Rx</th><th>Tx</th><th>Activity</th><th>Connection</th></tr>\n")
 	f("</thead>\n<tbody>\n")
 
-	now := time.Now()
+	now := mono.Now()
 
 	var peers []*PeerStatus
 	for _, peer := range st.Peers() {
@@ -378,7 +379,7 @@ table tbody tr:nth-child(even) td { background-color: #f5f5f5; }
 		f("<td>")
 
 		// TODO: let server report this active bool instead
-		active := !ps.LastWrite.IsZero() && time.Since(ps.LastWrite) < 2*time.Minute
+		active := !ps.LastWrite.IsZero() && mono.Since(ps.LastWrite) < 2*time.Minute
 		if active {
 			if ps.Relay != "" && ps.CurAddr == "" {
 				f("relay <b>%s</b>", html.EscapeString(ps.Relay))

ipn/localapi/localapi.go

@@ -102,6 +102,8 @@ func (h *Handler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
 		h.serveFileTargets(w, r)
 	case "/localapi/v0/set-dns":
 		h.serveSetDNS(w, r)
+	case "/localapi/v0/derpmap":
+		h.serveDERPMap(w, r)
 	case "/":
 		io.WriteString(w, "tailscaled\n")
 	default:
@@ -264,7 +266,7 @@ func (h *Handler) serveFiles(w http.ResponseWriter, r *http.Request) {
 		http.Error(w, "file access denied", http.StatusForbidden)
 		return
 	}
-	suffix := strings.TrimPrefix(r.URL.Path, "/localapi/v0/files/")
+	suffix := strings.TrimPrefix(r.URL.EscapedPath(), "/localapi/v0/files/")
 	if suffix == "" {
 		if r.Method != "GET" {
 			http.Error(w, "want GET to list files", 400)
@@ -403,6 +405,17 @@ func (h *Handler) serveSetDNS(w http.ResponseWriter, r *http.Request) {
 	json.NewEncoder(w).Encode(struct{}{})
 }
 
+func (h *Handler) serveDERPMap(w http.ResponseWriter, r *http.Request) {
+	if r.Method != "GET" {
+		http.Error(w, "want GET", 400)
+		return
+	}
+	w.Header().Set("Content-Type", "application/json")
+	e := json.NewEncoder(w)
+	e.SetIndent("", "\t")
+	e.Encode(h.b.DERPMap())
+}
+
 var dialPeerTransportOnce struct {
 	sync.Once
 	v *http.Transport

ipn/prefs.go

@@ -25,10 +25,16 @@ import (
 
 //go:generate go run tailscale.com/cmd/cloner -type=Prefs -output=prefs_clone.go
 
-// DefaultControlURL returns the URL base of the control plane
+// DefaultControlURL is the URL base of the control plane
 // ("coordination server") for use when no explicit one is configured.
 // The default control plane is the hosted version run by Tailscale.com.
-const DefaultControlURL = "https://login.tailscale.com"
+const DefaultControlURL = "https://controlplane.tailscale.com"
+
+// IsLoginServerSynonym reports whether a URL is a drop-in replacement
+// for the primary Tailscale login server.
+func IsLoginServerSynonym(val interface{}) bool {
+	return val == "https://login.tailscale.com" || val == "https://controlplane.tailscale.com"
+}
 
 // Prefs are the user modifiable settings of the Tailscale node agent.
 type Prefs struct {
@@ -405,6 +411,16 @@ func (p *Prefs) ControlURLOrDefault() string {
 	return DefaultControlURL
 }
 
+// AdminPageURL returns the admin web site URL for the current ControlURL.
+func (p *Prefs) AdminPageURL() string {
+	url := p.ControlURLOrDefault()
+	if IsLoginServerSynonym(url) {
+		// TODO(crawshaw): In future release, make this https://console.tailscale.com
+		url = "https://login.tailscale.com"
+	}
+	return url + "/admin/machines"
+}
+
 // PrefsFromBytes deserializes Prefs from a JSON blob. If
 // enforceDefaults is true, Prefs.RouteAll and Prefs.AllowSingleHosts
 // are forced on.

ipn/prefs_test.go

@@ -92,13 +92,13 @@ func TestPrefsEqual(t *testing.T) {
 		},
 
 		{
-			&Prefs{ControlURL: "https://login.tailscale.com"},
+			&Prefs{ControlURL: "https://controlplane.tailscale.com"},
 			&Prefs{ControlURL: "https://login.private.co"},
 			false,
 		},
 		{
-			&Prefs{ControlURL: "https://login.tailscale.com"},
-			&Prefs{ControlURL: "https://login.tailscale.com"},
+			&Prefs{ControlURL: "https://controlplane.tailscale.com"},
+			&Prefs{ControlURL: "https://controlplane.tailscale.com"},
 			true,
 		},
 
@@ -324,7 +324,7 @@ func TestBasicPrefs(t *testing.T) {
 	tstest.PanicOnLog()
 
 	p := Prefs{
-		ControlURL: "https://login.tailscale.com",
+		ControlURL: "https://controlplane.tailscale.com",
 	}
 	checkPrefs(t, p)
 }
@@ -336,7 +336,7 @@ func TestPrefsPersist(t *testing.T) {
 		LoginName: "test@example.com",
 	}
 	p := Prefs{
-		ControlURL: "https://login.tailscale.com",
+		ControlURL: "https://controlplane.tailscale.com",
 		CorpDNS:    true,
 		Persist:    &c,
 	}

ipn/store.go

@@ -67,9 +67,6 @@ func (s *MemoryStore) String() string { return "MemoryStore" }
 func (s *MemoryStore) ReadState(id StateKey) ([]byte, error) {
 	s.mu.Lock()
 	defer s.mu.Unlock()
-	if s.cache == nil {
-		s.cache = map[StateKey][]byte{}
-	}
 	bs, ok := s.cache[id]
 	if !ok {
 		return nil, ErrStateNotExist

logpolicy/logpolicy.go

@@ -128,6 +128,13 @@ func (l logWriter) Write(buf []byte) (int, error) {
 // logsDir returns the directory to use for log configuration and
 // buffer storage.
 func logsDir(logf logger.Logf) string {
+	if d := os.Getenv("TS_LOGS_DIR"); d != "" {
+		fi, err := os.Stat(d)
+		if err == nil && fi.IsDir() {
+			return d
+		}
+	}
+
 	// STATE_DIRECTORY is set by systemd 240+ but we support older
 	// systems-d. For example, Ubuntu 18.04 (Bionic Beaver) is 237.
 	systemdStateDir := os.Getenv("STATE_DIRECTORY")
@@ -180,6 +187,10 @@ func runningUnderSystemd() bool {
 	return false
 }
 
+func redirectStderrToLogPanics() bool {
+	return runningUnderSystemd() || os.Getenv("TS_PLEASE_PANIC") != ""
+}
+
 // tryFixLogStateLocation is a temporary fixup for
 // https://github.com/tailscale/tailscale/issues/247 . We accidentally
 // wrote logging state files to /, and then later to $CACHE_DIRECTORY
@@ -428,9 +439,14 @@ func New(collection string) *Policy {
 		c.HTTPC = &http.Client{Transport: newLogtailTransport(u.Host)}
 	}
 
-	filchBuf, filchErr := filch.New(filepath.Join(dir, cmdName), filch.Options{})
+	filchBuf, filchErr := filch.New(filepath.Join(dir, cmdName), filch.Options{
+		ReplaceStderr: redirectStderrToLogPanics(),
+	})
 	if filchBuf != nil {
 		c.Buffer = filchBuf
+		if filchBuf.OrigStderr != nil {
+			c.Stderr = filchBuf.OrigStderr
+		}
 	}
 	lw := logtail.NewLogger(c, log.Printf)
 	log.SetFlags(0) // other logflags are set on console, not here

logtail/logtail.go

@@ -118,9 +118,10 @@ type Logger struct {
 	bo             *backoff.Backoff
 	zstdEncoder    Encoder
 	uploadCancel   func()
+	explainedRaw   bool
 
 	shutdownStart chan struct{} // closed when shutdown begins
-	shutdownDone  chan struct{} // closd when shutdown complete
+	shutdownDone  chan struct{} // closed when shutdown complete
 }
 
 // SetVerbosityLevel controls the verbosity level that should be
@@ -230,6 +231,14 @@ func (l *Logger) drainPending() (res []byte) {
 			// outside of the logtail logger. Encode it.
 			// Do not add a client time, as it could have been
 			// been written a long time ago.
+			if !l.explainedRaw {
+				fmt.Fprintf(l.stderr, "RAW-STDERR: ***\n")
+				fmt.Fprintf(l.stderr, "RAW-STDERR: *** Lines prefixed with RAW-STDERR below bypassed logtail and probably come from a previous run of the program\n")
+				fmt.Fprintf(l.stderr, "RAW-STDERR: ***\n")
+				fmt.Fprintf(l.stderr, "RAW-STDERR:\n")
+				l.explainedRaw = true
+			}
+			fmt.Fprintf(l.stderr, "RAW-STDERR: %s", b)
 			b = l.encodeText(b, true)
 		}
 

net/dns/config.go

@@ -5,9 +5,12 @@
 package dns
 
 import (
+	"bufio"
+	"fmt"
 	"sort"
 
 	"inet.af/netaddr"
+	"tailscale.com/net/dns/resolver"
 	"tailscale.com/util/dnsname"
 )
 
@@ -38,6 +41,20 @@ type Config struct {
 	Hosts map[dnsname.FQDN][]netaddr.IP
 }
 
+// WriteToBufioWriter write a debug version of c for logs to w, omitting
+// spammy stuff like *.arpa entries and replacing it with a total count.
+func (c *Config) WriteToBufioWriter(w *bufio.Writer) {
+	w.WriteString("{DefaultResolvers:")
+	resolver.WriteIPPorts(w, c.DefaultResolvers)
+
+	w.WriteString(" Routes:")
+	resolver.WriteRoutes(w, c.Routes)
+
+	fmt.Fprintf(w, " SearchDomains:%v", c.SearchDomains)
+	fmt.Fprintf(w, " Hosts:%v", len(c.Hosts))
+	w.WriteString("}")
+}
+
 // needsAnyResolvers reports whether c requires a resolver to be set
 // at the OS level.
 func (c Config) needsOSResolver() bool {

net/dns/direct.go

@@ -2,23 +2,22 @@
 // Use of this source code is governed by a BSD-style
 // license that can be found in the LICENSE file.
 
-// +build linux freebsd openbsd
-
 package dns
 
 import (
 	"bufio"
 	"bytes"
+	"crypto/rand"
 	"fmt"
 	"io"
 	"io/ioutil"
 	"os"
 	"os/exec"
+	"path/filepath"
 	"runtime"
 	"strings"
 
 	"inet.af/netaddr"
-	"tailscale.com/atomicfile"
 	"tailscale.com/util/dnsname"
 )
 
@@ -77,21 +76,17 @@ func readResolv(r io.Reader) (config OSConfig, err error) {
 	return config, nil
 }
 
-func readResolvFile(path string) (OSConfig, error) {
-	var config OSConfig
-
-	f, err := os.Open(path)
+func (m directManager) readResolvFile(path string) (OSConfig, error) {
+	b, err := m.fs.ReadFile(path)
 	if err != nil {
-		return config, err
+		return OSConfig{}, err
 	}
-	defer f.Close()
-
-	return readResolv(f)
+	return readResolv(bytes.NewReader(b))
 }
 
 // readResolvConf reads DNS configuration from /etc/resolv.conf.
-func readResolvConf() (OSConfig, error) {
-	return readResolvFile(resolvConf)
+func (m directManager) readResolvConf() (OSConfig, error) {
+	return m.readResolvFile(resolvConf)
 }
 
 // resolvOwner returns the apparent owner of the resolv.conf
@@ -150,26 +145,32 @@ func isResolvedRunning() bool {
 // to the disappearance of the Tailscale interface.
 // The caller must call Down before program shutdown
 // or as cleanup if the program terminates unexpectedly.
-type directManager struct{}
+type directManager struct {
+	fs wholeFileFS
+}
 
-func newDirectManager() (directManager, error) {
-	return directManager{}, nil
+func newDirectManager() directManager {
+	return directManager{fs: directFS{}}
+}
+
+func newDirectManagerOnFS(fs wholeFileFS) directManager {
+	return directManager{fs: fs}
 }
 
 // ownedByTailscale reports whether /etc/resolv.conf seems to be a
 // tailscale-managed file.
 func (m directManager) ownedByTailscale() (bool, error) {
-	st, err := os.Stat(resolvConf)
+	isRegular, err := m.fs.Stat(resolvConf)
 	if err != nil {
 		if os.IsNotExist(err) {
 			return false, nil
 		}
 		return false, err
 	}
-	if !st.Mode().IsRegular() {
+	if !isRegular {
 		return false, nil
 	}
-	bs, err := ioutil.ReadFile(resolvConf)
+	bs, err := m.fs.ReadFile(resolvConf)
 	if err != nil {
 		return false, err
 	}
@@ -182,11 +183,11 @@ func (m directManager) ownedByTailscale() (bool, error) {
 // backupConfig creates or updates a backup of /etc/resolv.conf, if
 // resolv.conf does not currently contain a Tailscale-managed config.
 func (m directManager) backupConfig() error {
-	if _, err := os.Stat(resolvConf); err != nil {
+	if _, err := m.fs.Stat(resolvConf); err != nil {
 		if os.IsNotExist(err) {
 			// No resolv.conf, nothing to back up. Also get rid of any
 			// existing backup file, to avoid restoring something old.
-			os.Remove(backupConf)
+			m.fs.Remove(backupConf)
 			return nil
 		}
 		return err
@@ -200,11 +201,11 @@ func (m directManager) backupConfig() error {
 		return nil
 	}
 
-	return os.Rename(resolvConf, backupConf)
+	return m.fs.Rename(resolvConf, backupConf)
 }
 
 func (m directManager) restoreBackup() error {
-	if _, err := os.Stat(backupConf); err != nil {
+	if _, err := m.fs.Stat(backupConf); err != nil {
 		if os.IsNotExist(err) {
 			// No backup, nothing we can do.
 			return nil
@@ -215,7 +216,7 @@ func (m directManager) restoreBackup() error {
 	if err != nil {
 		return err
 	}
-	if _, err := os.Stat(resolvConf); err != nil && !os.IsNotExist(err) {
+	if _, err := m.fs.Stat(resolvConf); err != nil && !os.IsNotExist(err) {
 		return err
 	}
 	resolvConfExists := !os.IsNotExist(err)
@@ -223,12 +224,12 @@ func (m directManager) restoreBackup() error {
 	if resolvConfExists && !owned {
 		// There's already a non-tailscale config in place, get rid of
 		// our backup.
-		os.Remove(backupConf)
+		m.fs.Remove(backupConf)
 		return nil
 	}
 
 	// We own resolv.conf, and a backup exists.
-	if err := os.Rename(backupConf, resolvConf); err != nil {
+	if err := m.fs.Rename(backupConf, resolvConf); err != nil {
 		return err
 	}
 
@@ -247,7 +248,7 @@ func (m directManager) SetDNS(config OSConfig) error {
 
 		buf := new(bytes.Buffer)
 		writeResolvConf(buf, config.Nameservers, config.SearchDomains)
-		if err := atomicfile.WriteFile(resolvConf, buf.Bytes(), 0644); err != nil {
+		if err := atomicWriteFile(m.fs, resolvConf, buf.Bytes(), 0644); err != nil {
 			return err
 		}
 	}
@@ -279,7 +280,7 @@ func (m directManager) GetBaseConfig() (OSConfig, error) {
 		fileToRead = backupConf
 	}
 
-	return readResolvFile(fileToRead)
+	return m.readResolvFile(fileToRead)
 }
 
 func (m directManager) Close() error {
@@ -287,9 +288,9 @@ func (m directManager) Close() error {
 	// to it, but then we stopped because /etc/resolv.conf being a
 	// symlink to surprising places breaks snaps and other sandboxing
 	// things. Clean it up if it's still there.
-	os.Remove("/etc/resolv.tailscale.conf")
+	m.fs.Remove("/etc/resolv.tailscale.conf")
 
-	if _, err := os.Stat(backupConf); err != nil {
+	if _, err := m.fs.Stat(backupConf); err != nil {
 		if os.IsNotExist(err) {
 			// No backup, nothing we can do.
 			return nil
@@ -300,7 +301,7 @@ func (m directManager) Close() error {
 	if err != nil {
 		return err
 	}
-	_, err = os.Stat(resolvConf)
+	_, err = m.fs.Stat(resolvConf)
 	if err != nil && !os.IsNotExist(err) {
 		return err
 	}
@@ -309,12 +310,12 @@ func (m directManager) Close() error {
 	if resolvConfExists && !owned {
 		// There's already a non-tailscale config in place, get rid of
 		// our backup.
-		os.Remove(backupConf)
+		m.fs.Remove(backupConf)
 		return nil
 	}
 
 	// We own resolv.conf, and a backup exists.
-	if err := os.Rename(backupConf, resolvConf); err != nil {
+	if err := m.fs.Rename(backupConf, resolvConf); err != nil {
 		return err
 	}
 
@@ -324,3 +325,63 @@ func (m directManager) Close() error {
 
 	return nil
 }
+
+func atomicWriteFile(fs wholeFileFS, filename string, data []byte, perm os.FileMode) error {
+	var randBytes [12]byte
+	if _, err := rand.Read(randBytes[:]); err != nil {
+		return fmt.Errorf("atomicWriteFile: %w", err)
+	}
+
+	tmpName := fmt.Sprintf("%s.%x.tmp", filename, randBytes[:])
+	defer fs.Remove(tmpName)
+
+	if err := fs.WriteFile(tmpName, data, perm); err != nil {
+		return fmt.Errorf("atomicWriteFile: %w", err)
+	}
+	return fs.Rename(tmpName, filename)
+}
+
+// wholeFileFS is a high-level file system abstraction designed just for use
+// by directManager, with the goal that it is easy to implement over wsl.exe.
+//
+// All name parameters are absolute paths.
+type wholeFileFS interface {
+	Stat(name string) (isRegular bool, err error)
+	Rename(oldName, newName string) error
+	Remove(name string) error
+	ReadFile(name string) ([]byte, error)
+	WriteFile(name string, contents []byte, perm os.FileMode) error
+}
+
+// directFS is a wholeFileFS implemented directly on the OS.
+type directFS struct {
+	// prefix is file path prefix.
+	//
+	// All name parameters are absolute paths so this is typically a
+	// testing temporary directory like "/tmp".
+	prefix string
+}
+
+func (fs directFS) path(name string) string { return filepath.Join(fs.prefix, name) }
+
+func (fs directFS) Stat(name string) (isRegular bool, err error) {
+	fi, err := os.Stat(fs.path(name))
+	if err != nil {
+		return false, err
+	}
+	return fi.Mode().IsRegular(), nil
+}
+
+func (fs directFS) Rename(oldName, newName string) error {
+	return os.Rename(fs.path(oldName), fs.path(newName))
+}
+
+func (fs directFS) Remove(name string) error { return os.Remove(fs.path(name)) }
+
+func (fs directFS) ReadFile(name string) ([]byte, error) {
+	return ioutil.ReadFile(fs.path(name))
+}
+
+func (fs directFS) WriteFile(name string, contents []byte, perm os.FileMode) error {
+	return ioutil.WriteFile(fs.path(name), contents, perm)
+}

net/dns/direct_test.go

@@ -0,0 +1,83 @@
+// Copyright (c) 2021 Tailscale Inc & AUTHORS All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package dns
+
+import (
+	"io/ioutil"
+	"os"
+	"path/filepath"
+	"testing"
+
+	"inet.af/netaddr"
+	"tailscale.com/util/dnsname"
+)
+
+func TestSetDNS(t *testing.T) {
+	const orig = "nameserver 9.9.9.9 # orig"
+	tmp := t.TempDir()
+	resolvPath := filepath.Join(tmp, "etc", "resolv.conf")
+	backupPath := filepath.Join(tmp, "etc", "resolv.pre-tailscale-backup.conf")
+
+	if err := os.MkdirAll(filepath.Dir(resolvPath), 0777); err != nil {
+		t.Fatal(err)
+	}
+	if err := ioutil.WriteFile(resolvPath, []byte(orig), 0644); err != nil {
+		t.Fatal(err)
+	}
+
+	readFile := func(t *testing.T, path string) string {
+		t.Helper()
+		b, err := ioutil.ReadFile(path)
+		if err != nil {
+			t.Fatal(err)
+		}
+		return string(b)
+	}
+	assertBaseState := func(t *testing.T) {
+		if got := readFile(t, resolvPath); got != orig {
+			t.Fatalf("resolv.conf:\n%s, want:\n%s", got, orig)
+		}
+		if _, err := os.Stat(backupPath); !os.IsNotExist(err) {
+			t.Fatalf("resolv.conf backup: want it to be gone but: %v", err)
+		}
+	}
+
+	m := directManager{fs: directFS{prefix: tmp}}
+	if err := m.SetDNS(OSConfig{
+		Nameservers:   []netaddr.IP{netaddr.MustParseIP("8.8.8.8"), netaddr.MustParseIP("8.8.4.4")},
+		SearchDomains: []dnsname.FQDN{"ts.net.", "ts-dns.test."},
+		MatchDomains:  []dnsname.FQDN{"ignored."},
+	}); err != nil {
+		t.Fatal(err)
+	}
+	want := `# resolv.conf(5) file generated by tailscale
+# DO NOT EDIT THIS FILE BY HAND -- CHANGES WILL BE OVERWRITTEN
+
+nameserver 8.8.8.8
+nameserver 8.8.4.4
+search ts.net ts-dns.test
+`
+	if got := readFile(t, resolvPath); got != want {
+		t.Fatalf("resolv.conf:\n%s, want:\n%s", got, want)
+	}
+	if got := readFile(t, backupPath); got != orig {
+		t.Fatalf("resolv.conf backup:\n%s, want:\n%s", got, orig)
+	}
+
+	// Test that a nil OSConfig cleans up resolv.conf.
+	if err := m.SetDNS(OSConfig{}); err != nil {
+		t.Fatal(err)
+	}
+	assertBaseState(t)
+
+	// Test that Close cleans up resolv.conf.
+	if err := m.SetDNS(OSConfig{Nameservers: []netaddr.IP{netaddr.MustParseIP("8.8.8.8")}}); err != nil {
+		t.Fatal(err)
+	}
+	if err := m.Close(); err != nil {
+		t.Fatal(err)
+	}
+	assertBaseState(t)
+}

net/dns/ini.go

@@ -0,0 +1,31 @@
+// Copyright (c) 2021 Tailscale Inc & AUTHORS All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+// +build windows
+
+package dns
+
+import (
+	"regexp"
+	"strings"
+)
+
+// parseIni parses a basic .ini file, used for wsl.conf.
+func parseIni(data string) map[string]map[string]string {
+	sectionRE := regexp.MustCompile(`^\[([^]]+)\]`)
+	kvRE := regexp.MustCompile(`^\s*(\w+)\s*=\s*([^#]*)`)
+
+	ini := map[string]map[string]string{}
+	var section string
+	for _, line := range strings.Split(data, "\n") {
+		if res := sectionRE.FindStringSubmatch(line); len(res) > 1 {
+			section = res[1]
+			ini[section] = map[string]string{}
+		} else if res := kvRE.FindStringSubmatch(line); len(res) > 2 {
+			k, v := strings.TrimSpace(res[1]), strings.TrimSpace(res[2])
+			ini[section][k] = v
+		}
+	}
+	return ini
+}

net/dns/ini_test.go

@@ -0,0 +1,39 @@
+// Copyright (c) 2021 Tailscale Inc & AUTHORS All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+// +build windows
+
+package dns
+
+import (
+	"reflect"
+	"testing"
+)
+
+func TestParseIni(t *testing.T) {
+	var tests = []struct {
+		src  string
+		want map[string]map[string]string
+	}{
+		{
+			src: `# appended wsl.conf file
+[automount]
+	enabled = true
+	root=/mnt/
+# added by tailscale
+[network] # trailing comment
+generateResolvConf = false  # trailing comment`,
+			want: map[string]map[string]string{
+				"automount": map[string]string{"enabled": "true", "root": "/mnt/"},
+				"network":   map[string]string{"generateResolvConf": "false"},
+			},
+		},
+	}
+	for _, test := range tests {
+		got := parseIni(test.src)
+		if !reflect.DeepEqual(got, test.want) {
+			t.Errorf("for:\n%s\ngot:  %v\nwant: %v", test.src, got, test.want)
+		}
+	}
+}

net/dns/manager.go

@@ -5,6 +5,7 @@
 package dns
 
 import (
+	"bufio"
 	"runtime"
 	"time"
 
@@ -50,14 +51,18 @@ func NewManager(logf logger.Logf, oscfg OSConfigurator, linkMon *monitor.Mon, li
 }
 
 func (m *Manager) Set(cfg Config) error {
-	m.logf("Set: %+v", cfg)
+	m.logf("Set: %v", logger.ArgWriter(func(w *bufio.Writer) {
+		cfg.WriteToBufioWriter(w)
+	}))
 
 	rcfg, ocfg, err := m.compileConfig(cfg)
 	if err != nil {
 		return err
 	}
 
-	m.logf("Resolvercfg: %+v", rcfg)
+	m.logf("Resolvercfg: %v", logger.ArgWriter(func(w *bufio.Writer) {
+		rcfg.WriteToBufioWriter(w)
+	}))
 	m.logf("OScfg: %+v", ocfg)
 
 	if err := m.resolver.SetConfig(rcfg); err != nil {

net/dns/manager_freebsd.go

@@ -15,7 +15,7 @@ import (
 func NewOSConfigurator(logf logger.Logf, _ string) (OSConfigurator, error) {
 	bs, err := ioutil.ReadFile("/etc/resolv.conf")
 	if os.IsNotExist(err) {
-		return newDirectManager()
+		return newDirectManager(), nil
 	}
 	if err != nil {
 		return nil, fmt.Errorf("reading /etc/resolv.conf: %w", err)
@@ -25,6 +25,6 @@ func NewOSConfigurator(logf logger.Logf, _ string) (OSConfigurator, error) {
 	case "resolvconf":
 		return newResolvconfManager(logf)
 	default:
-		return newDirectManager()
+		return newDirectManager(), nil
 	}
 }

net/dns/manager_linux.go

@@ -42,7 +42,7 @@ func NewOSConfigurator(logf logger.Logf, interfaceName string) (ret OSConfigurat
 	bs, err := ioutil.ReadFile("/etc/resolv.conf")
 	if os.IsNotExist(err) {
 		dbg("rc", "missing")
-		return newDirectManager()
+		return newDirectManager(), nil
 	}
 	if err != nil {
 		return nil, fmt.Errorf("reading /etc/resolv.conf: %w", err)
@@ -58,11 +58,11 @@ func NewOSConfigurator(logf logger.Logf, interfaceName string) (ret OSConfigurat
 		// https://github.com/tailscale/tailscale/issues/2136
 		if err := resolvedIsActuallyResolver(); err != nil {
 			dbg("resolved", "not-in-use")
-			return newDirectManager()
+			return newDirectManager(), nil
 		}
 		if err := dbusPing("org.freedesktop.resolve1", "/org/freedesktop/resolve1"); err != nil {
 			dbg("resolved", "no")
-			return newDirectManager()
+			return newDirectManager(), nil
 		}
 		if err := dbusPing("org.freedesktop.NetworkManager", "/org/freedesktop/NetworkManager/DnsManager"); err != nil {
 			dbg("nm", "no")
@@ -125,7 +125,7 @@ func NewOSConfigurator(logf logger.Logf, interfaceName string) (ret OSConfigurat
 		dbg("rc", "resolvconf")
 		if _, err := exec.LookPath("resolvconf"); err != nil {
 			dbg("resolvconf", "no")
-			return newDirectManager()
+			return newDirectManager(), nil
 		}
 		dbg("resolvconf", "yes")
 		return newResolvconfManager(logf)
@@ -143,10 +143,10 @@ func NewOSConfigurator(logf logger.Logf, interfaceName string) (ret OSConfigurat
 		// anyway, so you still need a fallback path that uses
 		// directManager.
 		dbg("rc", "nm")
-		return newDirectManager()
+		return newDirectManager(), nil
 	default:
 		dbg("rc", "unknown")
-		return newDirectManager()
+		return newDirectManager(), nil
 	}
 }
 
@@ -195,7 +195,7 @@ func nmIsUsingResolved() error {
 }
 
 func resolvedIsActuallyResolver() error {
-	cfg, err := readResolvConf()
+	cfg, err := newDirectManager().readResolvConf()
 	if err != nil {
 		return err
 	}

net/dns/manager_openbsd.go

@@ -7,5 +7,5 @@ package dns
 import "tailscale.com/types/logger"
 
 func NewOSConfigurator(logger.Logf, string) (OSConfigurator, error) {
-	return newDirectManager()
+	return newDirectManager(), nil
 }

net/dns/manager_windows.go

@@ -35,16 +35,18 @@ const (
 )
 
 type windowsManager struct {
-	logf      logger.Logf
-	guid      string
-	nrptWorks bool
+	logf       logger.Logf
+	guid       string
+	nrptWorks  bool
+	wslManager *wslManager
 }
 
 func NewOSConfigurator(logf logger.Logf, interfaceName string) (OSConfigurator, error) {
 	ret := windowsManager{
-		logf:      logf,
-		guid:      interfaceName,
-		nrptWorks: isWindows10OrBetter(),
+		logf:       logf,
+		guid:       interfaceName,
+		nrptWorks:  isWindows10OrBetter(),
+		wslManager: newWSLManager(logf),
 	}
 
 	// Best-effort: if our NRPT rule exists, try to delete it. Unlike
@@ -57,6 +59,13 @@ func NewOSConfigurator(logf logger.Logf, interfaceName string) (OSConfigurator,
 		ret.delKey(nrptBase)
 	}
 
+	// Log WSL status once at startup.
+	if distros, err := wslDistros(); err != nil {
+		logf("WSL: could not list distributions: %v", err)
+	} else {
+		logf("WSL: found %d distributions", len(distros))
+	}
+
 	return ret, nil
 }
 
@@ -284,7 +293,7 @@ func (m windowsManager) SetDNS(cfg OSConfig) error {
 		}
 
 		t0 = time.Now()
-		m.logf("running ipconfig /registerdns ...")
+		m.logf("running ipconfig /flushdns ...")
 		cmd = exec.Command("ipconfig", "/flushdns")
 		cmd.SysProcAttr = &syscall.SysProcAttr{HideWindow: true}
 		err = cmd.Run()
@@ -296,6 +305,16 @@ func (m windowsManager) SetDNS(cfg OSConfig) error {
 		}
 	}()
 
+	// On initial setup of WSL, the restart caused by --shutdown is slow,
+	// so we do it out-of-line.
+	go func() {
+		if err := m.wslManager.SetDNS(cfg); err != nil {
+			m.logf("WSL SetDNS: %v", err) // continue
+		} else {
+			m.logf("WSL SetDNS: success")
+		}
+	}()
+
 	return nil
 }
 

net/dns/resolved.go

@@ -67,7 +67,7 @@ func isResolvedActive() bool {
 		return false
 	}
 
-	config, err := readResolvConf()
+	config, err := newDirectManager().readResolvConf()
 	if err != nil {
 		return false
 	}

net/dns/resolver/doh_test.go

@@ -0,0 +1,98 @@
+// Copyright (c) 2021 Tailscale Inc & AUTHORS All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package resolver
+
+import (
+	"context"
+	"flag"
+	"net/http"
+	"testing"
+
+	"golang.org/x/net/dns/dnsmessage"
+)
+
+var testDoH = flag.Bool("test-doh", false, "do real DoH tests against the network")
+
+const someDNSID = 123 // something non-zero as a test; in violation of spec's SHOULD of 0
+
+func someDNSQuestion(t testing.TB) []byte {
+	b := dnsmessage.NewBuilder(nil, dnsmessage.Header{
+		OpCode:           0, // query
+		RecursionDesired: true,
+		ID:               someDNSID,
+	})
+	b.StartQuestions() // err
+	b.Question(dnsmessage.Question{
+		Name:  dnsmessage.MustNewName("tailscale.com."),
+		Type:  dnsmessage.TypeA,
+		Class: dnsmessage.ClassINET,
+	})
+	msg, err := b.Finish()
+	if err != nil {
+		t.Fatal(err)
+	}
+	return msg
+}
+
+func TestDoH(t *testing.T) {
+	if !*testDoH {
+		t.Skip("skipping manual test without --test-doh flag")
+	}
+	if len(knownDoH) == 0 {
+		t.Fatal("no known DoH")
+	}
+
+	f := &forwarder{
+		dohSem: make(chan struct{}, 10),
+	}
+
+	for ip := range knownDoH {
+		t.Run(ip.String(), func(t *testing.T) {
+			urlBase, c, ok := f.getDoHClient(ip)
+			if !ok {
+				t.Fatal("expected DoH")
+			}
+			res, err := f.sendDoH(context.Background(), urlBase, c, someDNSQuestion(t))
+			if err != nil {
+				t.Fatal(err)
+			}
+			c.Transport.(*http.Transport).CloseIdleConnections()
+
+			var p dnsmessage.Parser
+			h, err := p.Start(res)
+			if err != nil {
+				t.Fatal(err)
+			}
+			if h.ID != someDNSID {
+				t.Errorf("response DNS ID = %v; want %v", h.ID, someDNSID)
+			}
+
+			p.SkipAllQuestions()
+			aa, err := p.AllAnswers()
+			if err != nil {
+				t.Fatal(err)
+			}
+			if len(aa) == 0 {
+				t.Fatal("no answers")
+			}
+			for _, r := range aa {
+				t.Logf("got: %v", r.GoString())
+			}
+		})
+	}
+}
+
+func TestDoHV6Fallback(t *testing.T) {
+	for ip, base := range knownDoH {
+		if ip.Is4() {
+			ip6, ok := dohV6(base)
+			if !ok {
+				t.Errorf("no v6 DoH known for %v", ip)
+			} else if !ip6.Is6() {
+				t.Errorf("dohV6(%q) returned non-v6 address %v", base, ip6)
+			}
+		}
+	}
+}

net/dns/resolver/forwarder.go

@@ -9,15 +9,22 @@ import (
 	"context"
 	"encoding/binary"
 	"errors"
+	"fmt"
 	"hash/crc32"
 	"io"
+	"io/ioutil"
 	"math/rand"
 	"net"
+	"net/http"
+	"runtime"
+	"sort"
+	"strings"
 	"sync"
 	"time"
 
 	dns "golang.org/x/net/dns/dnsmessage"
 	"inet.af/netaddr"
+	"tailscale.com/net/netns"
 	"tailscale.com/types/logger"
 	"tailscale.com/util/dnsname"
 	"tailscale.com/wgengine/monitor"
@@ -29,6 +36,16 @@ const headerBytes = 12
 const (
 	// responseTimeout is the maximal amount of time to wait for a DNS response.
 	responseTimeout = 5 * time.Second
+
+	// dohTransportTimeout is how long to keep idle HTTP
+	// connections open to DNS-over-HTTPs servers. This is pretty
+	// arbitrary.
+	dohTransportTimeout = 30 * time.Second
+
+	// wellKnownHostBackupDelay is how long to artificially delay upstream
+	// DNS queries to the "fallback" DNS server IP for a known provider
+	// (e.g. how long to wait to query Google's 8.8.4.4 after 8.8.8.8).
+	wellKnownHostBackupDelay = 200 * time.Millisecond
 )
 
 var errNoUpstreams = errors.New("upstream nameservers not set")
@@ -108,6 +125,7 @@ func clampEDNSSize(packet []byte, maxSize uint16) {
 		return
 	}
 
+	// https://datatracker.ietf.org/doc/html/rfc6891#section-6.1.2
 	opt := packet[len(packet)-optFixedBytes:]
 
 	if opt[0] != 0 {
@@ -124,8 +142,8 @@ func clampEDNSSize(packet []byte, maxSize uint16) {
 		// Be conservative and don't touch unknown versions.
 		return
 	}
-	// Ignore flags in opt[7:9]
-	if binary.BigEndian.Uint16(opt[10:12]) != 0 {
+	// Ignore flags in opt[6:9]
+	if binary.BigEndian.Uint16(opt[9:11]) != 0 {
 		// RDLEN must be 0 (no variable length data). We're at the end of the
 		// packet so this should be 0 anyway)..
 		return
@@ -141,7 +159,20 @@ func clampEDNSSize(packet []byte, maxSize uint16) {
 
 type route struct {
 	Suffix    dnsname.FQDN
-	Resolvers []netaddr.IPPort
+	Resolvers []resolverAndDelay
+}
+
+// resolverAndDelay is an upstream DNS resolver and a delay for how
+// long to wait before querying it.
+type resolverAndDelay struct {
+	// ipp is the upstream resolver.
+	ipp netaddr.IPPort
+
+	// startDelay is an amount to delay this resolver at
+	// start. It's used when, say, there are four Google or
+	// Cloudflare DNS IPs (two IPv4 + two IPv6) and we don't want
+	// to race all four at once.
+	startDelay time.Duration
 }
 
 // forwarder forwards DNS packets to a number of upstream nameservers.
@@ -149,6 +180,7 @@ type forwarder struct {
 	logf    logger.Logf
 	linkMon *monitor.Mon
 	linkSel ForwardLinkSelector
+	dohSem  chan struct{}
 
 	ctx       context.Context    // good until Close
 	ctxCancel context.CancelFunc // closes ctx
@@ -158,6 +190,8 @@ type forwarder struct {
 
 	mu sync.Mutex // guards following
 
+	dohClient map[netaddr.IP]*http.Client
+
 	// routes are per-suffix resolvers to use, with
 	// the most specific routes first.
 	routes []route
@@ -168,11 +202,18 @@ func init() {
 }
 
 func newForwarder(logf logger.Logf, responses chan packet, linkMon *monitor.Mon, linkSel ForwardLinkSelector) *forwarder {
+	maxDoHInFlight := 1000 // effectively unlimited
+	if runtime.GOOS == "ios" {
+		// No HTTP/2 on iOS yet (for size reasons), so DoH is
+		// pricier.
+		maxDoHInFlight = 10
+	}
 	f := &forwarder{
 		logf:      logger.WithPrefix(logf, "forward: "),
 		linkMon:   linkMon,
 		linkSel:   linkSel,
 		responses: responses,
+		dohSem:    make(chan struct{}, maxDoHInFlight),
 	}
 	f.ctx, f.ctxCancel = context.WithCancel(context.Background())
 	return f
@@ -183,7 +224,84 @@ func (f *forwarder) Close() error {
 	return nil
 }
 
-func (f *forwarder) setRoutes(routes []route) {
+// resolversWithDelays maps from a set of DNS server ip:ports (currently
+// the port is always 53) to a slice of a type that included a
+// startDelay. So if ipps contains e.g. four Google DNS IPs (two IPv4
+// + twoIPv6), this function partition adds delays to some.
+func resolversWithDelays(ipps []netaddr.IPPort) []resolverAndDelay {
+	type hostAndFam struct {
+		host string // some arbitrary string representing DNS host (currently the DoH base)
+		bits uint8  // either 32 or 128 for IPv4 vs IPv6s address family
+	}
+
+	// Track how many of each known resolver host are in the list,
+	// per address family.
+	total := map[hostAndFam]int{}
+
+	rr := make([]resolverAndDelay, len(ipps))
+	for _, ipp := range ipps {
+		ip := ipp.IP()
+		if host, ok := knownDoH[ip]; ok {
+			total[hostAndFam{host, ip.BitLen()}]++
+		}
+	}
+
+	done := map[hostAndFam]int{}
+	for i, ipp := range ipps {
+		ip := ipp.IP()
+		var startDelay time.Duration
+		if host, ok := knownDoH[ip]; ok {
+			key4 := hostAndFam{host, 32}
+			key6 := hostAndFam{host, 128}
+			switch {
+			case ip.Is4():
+				if done[key4] > 0 {
+					startDelay += wellKnownHostBackupDelay
+				}
+			case ip.Is6():
+				total4 := total[key4]
+				if total4 >= 2 {
+					// If we have two IPv4 IPs of the same provider
+					// already in the set, delay the IPv6 queries
+					// until halfway through the timeout (so wait
+					// 2.5 seconds). Even the network is IPv6-only,
+					// the DoH dialer will fallback to IPv6
+					// immediately anyway.
+					startDelay = responseTimeout / 2
+				} else if total4 == 1 {
+					startDelay += wellKnownHostBackupDelay
+				}
+				if done[key6] > 0 {
+					startDelay += wellKnownHostBackupDelay
+				}
+			}
+			done[hostAndFam{host, ip.BitLen()}]++
+		}
+		rr[i] = resolverAndDelay{
+			ipp:        ipp,
+			startDelay: startDelay,
+		}
+	}
+	return rr
+}
+
+// setRoutes sets the routes to use for DNS forwarding. It's called by
+// Resolver.SetConfig on reconfig.
+//
+// The memory referenced by routesBySuffix should not be modified.
+func (f *forwarder) setRoutes(routesBySuffix map[dnsname.FQDN][]netaddr.IPPort) {
+	routes := make([]route, 0, len(routesBySuffix))
+	for suffix, ipps := range routesBySuffix {
+		routes = append(routes, route{
+			Suffix:    suffix,
+			Resolvers: resolversWithDelays(ipps),
+		})
+	}
+	// Sort from longest prefix to shortest.
+	sort.Slice(routes, func(i, j int) bool {
+		return routes[i].Suffix.NumLabels() > routes[j].Suffix.NumLabels()
+	})
+
 	f.mu.Lock()
 	defer f.mu.Unlock()
 	f.routes = routes
@@ -210,21 +328,104 @@ func (f *forwarder) packetListener(ip netaddr.IP) (packetListener, error) {
 	return lc, nil
 }
 
+func (f *forwarder) getDoHClient(ip netaddr.IP) (urlBase string, c *http.Client, ok bool) {
+	urlBase, ok = knownDoH[ip]
+	if !ok {
+		return
+	}
+	f.mu.Lock()
+	defer f.mu.Unlock()
+	if c, ok := f.dohClient[ip]; ok {
+		return urlBase, c, true
+	}
+	if f.dohClient == nil {
+		f.dohClient = map[netaddr.IP]*http.Client{}
+	}
+	nsDialer := netns.NewDialer()
+	c = &http.Client{
+		Transport: &http.Transport{
+			IdleConnTimeout: dohTransportTimeout,
+			DialContext: func(ctx context.Context, netw, addr string) (net.Conn, error) {
+				if !strings.HasPrefix(netw, "tcp") {
+					return nil, fmt.Errorf("unexpected network %q", netw)
+				}
+				c, err := nsDialer.DialContext(ctx, "tcp", net.JoinHostPort(ip.String(), "443"))
+				// If v4 failed, try an equivalent v6 also in the time remaining.
+				if err != nil && ctx.Err() == nil {
+					if ip6, ok := dohV6(urlBase); ok && ip.Is4() {
+						if c6, err := nsDialer.DialContext(ctx, "tcp", net.JoinHostPort(ip6.String(), "443")); err == nil {
+							return c6, nil
+						}
+					}
+				}
+				return c, err
+			},
+		},
+	}
+	f.dohClient[ip] = c
+	return urlBase, c, true
+}
+
+const dohType = "application/dns-message"
+
+func (f *forwarder) releaseDoHSem() { <-f.dohSem }
+
+func (f *forwarder) sendDoH(ctx context.Context, urlBase string, c *http.Client, packet []byte) ([]byte, error) {
+	// Bound the number of HTTP requests in flight. This primarily
+	// matters for iOS where we're very memory constrained and
+	// HTTP requests are heavier on iOS where we don't include
+	// HTTP/2 for binary size reasons (as binaries on iOS linked
+	// with Go code cost memory proportional to the binary size,
+	// for reasons not fully understood).
+	select {
+	case f.dohSem <- struct{}{}:
+	case <-ctx.Done():
+		return nil, ctx.Err()
+	}
+	defer f.releaseDoHSem()
+
+	req, err := http.NewRequestWithContext(ctx, "POST", urlBase, bytes.NewReader(packet))
+	if err != nil {
+		return nil, err
+	}
+	req.Header.Set("Content-Type", dohType)
+	// Note: we don't currently set the Accept header (which is
+	// only a SHOULD in the spec) as iOS doesn't use HTTP/2 and
+	// we'd rather save a few bytes on outgoing requests when
+	// empirically no provider cares about the Accept header's
+	// absence.
+
+	hres, err := c.Do(req)
+	if err != nil {
+		return nil, err
+	}
+	defer hres.Body.Close()
+	if hres.StatusCode != 200 {
+		return nil, errors.New(hres.Status)
+	}
+	if ct := hres.Header.Get("Content-Type"); ct != dohType {
+		return nil, fmt.Errorf("unexpected response Content-Type %q", ct)
+	}
+	return ioutil.ReadAll(hres.Body)
+}
+
 // send sends packet to dst. It is best effort.
 //
 // send expects the reply to have the same txid as txidOut.
 //
-// The provided closeOnCtxDone lets send register values to Close if
-// the caller's ctx expires. This avoids send from allocating its own
-// waiting goroutine to interrupt the ReadFrom, as memory is tight on
-// iOS and we want the number of pending DNS lookups to be bursty
-// without too much associated goroutine/memory cost.
-func (f *forwarder) send(ctx context.Context, txidOut txid, closeOnCtxDone *closePool, packet []byte, dst netaddr.IPPort) ([]byte, error) {
-	// TODO(bradfitz): if dst.IP is 8.8.8.8 or 8.8.4.4 or 1.1.1.1, etc, or
-	// something dynamically probed earlier to support DoH or DoT,
-	// do that here instead.
+func (f *forwarder) send(ctx context.Context, fq *forwardQuery, dst netaddr.IPPort) ([]byte, error) {
+	ip := dst.IP()
 
-	ln, err := f.packetListener(dst.IP())
+	// Upgrade known DNS IPs to DoH (DNS-over-HTTPs).
+	if urlBase, dc, ok := f.getDoHClient(ip); ok {
+		res, err := f.sendDoH(ctx, urlBase, dc, fq.packet)
+		if err == nil || ctx.Err() != nil {
+			return res, err
+		}
+		f.logf("DoH error from %v: %v", ip, err)
+	}
+
+	ln, err := f.packetListener(ip)
 	if err != nil {
 		return nil, err
 	}
@@ -235,10 +436,10 @@ func (f *forwarder) send(ctx context.Context, txidOut txid, closeOnCtxDone *clos
 	}
 	defer conn.Close()
 
-	closeOnCtxDone.Add(conn)
-	defer closeOnCtxDone.Remove(conn)
+	fq.closeOnCtxDone.Add(conn)
+	defer fq.closeOnCtxDone.Remove(conn)
 
-	if _, err := conn.WriteTo(packet, dst.UDPAddr()); err != nil {
+	if _, err := conn.WriteTo(fq.packet, dst.UDPAddr()); err != nil {
 		if err := ctx.Err(); err != nil {
 			return nil, err
 		}
@@ -267,7 +468,7 @@ func (f *forwarder) send(ctx context.Context, txidOut txid, closeOnCtxDone *clos
 	}
 	out = out[:n]
 	txid := getTxID(out)
-	if txid != txidOut {
+	if txid != fq.txid {
 		return nil, errors.New("txid doesn't match")
 	}
 
@@ -290,7 +491,7 @@ func (f *forwarder) send(ctx context.Context, txidOut txid, closeOnCtxDone *clos
 }
 
 // resolvers returns the resolvers to use for domain.
-func (f *forwarder) resolvers(domain dnsname.FQDN) []netaddr.IPPort {
+func (f *forwarder) resolvers(domain dnsname.FQDN) []resolverAndDelay {
 	f.mu.Lock()
 	routes := f.routes
 	f.mu.Unlock()
@@ -302,6 +503,30 @@ func (f *forwarder) resolvers(domain dnsname.FQDN) []netaddr.IPPort {
 	return nil
 }
 
+// forwardQuery is information and state about a forwarded DNS query that's
+// being sent to 1 or more upstreams.
+//
+// In the case of racing against multiple equivalent upstreams
+// (e.g. Google or CloudFlare's 4 DNS IPs: 2 IPv4 + 2 IPv6), this type
+// handles racing them more intelligently than just blasting away 4
+// queries at once.
+type forwardQuery struct {
+	txid   txid
+	packet []byte
+
+	// closeOnCtxDone lets send register values to Close if the
+	// caller's ctx expires. This avoids send from allocating its
+	// own waiting goroutine to interrupt the ReadFrom, as memory
+	// is tight on iOS and we want the number of pending DNS
+	// lookups to be bursty without too much associated
+	// goroutine/memory cost.
+	closeOnCtxDone *closePool
+
+	// TODO(bradfitz): add race delay state:
+	// mu sync.Mutex
+	// ...
+}
+
 // forward forwards the query to all upstream nameservers and returns the first response.
 func (f *forwarder) forward(query packet) error {
 	domain, err := nameFromQuery(query.bs)
@@ -309,7 +534,6 @@ func (f *forwarder) forward(query packet) error {
 		return err
 	}
 
-	txid := getTxID(query.bs)
 	clampEDNSSize(query.bs, maxResponseBytes)
 
 	resolvers := f.resolvers(domain)
@@ -317,8 +541,12 @@ func (f *forwarder) forward(query packet) error {
 		return errNoUpstreams
 	}
 
-	closeOnCtxDone := new(closePool)
-	defer closeOnCtxDone.Close()
+	fq := &forwardQuery{
+		txid:           getTxID(query.bs),
+		packet:         query.bs,
+		closeOnCtxDone: new(closePool),
+	}
+	defer fq.closeOnCtxDone.Close()
 
 	ctx, cancel := context.WithTimeout(f.ctx, responseTimeout)
 	defer cancel()
@@ -329,9 +557,18 @@ func (f *forwarder) forward(query packet) error {
 		firstErr error
 	)
 
-	for _, ipp := range resolvers {
-		go func(ipp netaddr.IPPort) {
-			resb, err := f.send(ctx, txid, closeOnCtxDone, query.bs, ipp)
+	for _, rr := range resolvers {
+		go func(rr resolverAndDelay) {
+			if rr.startDelay > 0 {
+				timer := time.NewTimer(rr.startDelay)
+				select {
+				case <-timer.C:
+				case <-ctx.Done():
+					timer.Stop()
+					return
+				}
+			}
+			resb, err := f.send(ctx, fq, rr.ipp)
 			if err != nil {
 				mu.Lock()
 				defer mu.Unlock()
@@ -344,7 +581,7 @@ func (f *forwarder) forward(query packet) error {
 			case resc <- resb:
 			default:
 			}
-		}(ipp)
+		}(rr)
 	}
 
 	select {
@@ -432,3 +669,63 @@ func (p *closePool) Close() error {
 	}
 	return nil
 }
+
+var knownDoH = map[netaddr.IP]string{}
+
+var dohIPsOfBase = map[string][]netaddr.IP{}
+
+func addDoH(ipStr, base string) {
+	ip := netaddr.MustParseIP(ipStr)
+	knownDoH[ip] = base
+	dohIPsOfBase[base] = append(dohIPsOfBase[base], ip)
+}
+
+func dohV6(base string) (ip netaddr.IP, ok bool) {
+	for _, ip := range dohIPsOfBase[base] {
+		if ip.Is6() {
+			return ip, true
+		}
+	}
+	return ip, false
+}
+
+func init() {
+	// Cloudflare
+	addDoH("1.1.1.1", "https://cloudflare-dns.com/dns-query")
+	addDoH("1.0.0.1", "https://cloudflare-dns.com/dns-query")
+	addDoH("2606:4700:4700::1111", "https://cloudflare-dns.com/dns-query")
+	addDoH("2606:4700:4700::1001", "https://cloudflare-dns.com/dns-query")
+
+	// Cloudflare -Malware
+	addDoH("1.1.1.2", "https://security.cloudflare-dns.com/dns-query")
+	addDoH("1.0.0.2", "https://security.cloudflare-dns.com/dns-query")
+	addDoH("2606:4700:4700::1112", "https://security.cloudflare-dns.com/dns-query")
+	addDoH("2606:4700:4700::1002", "https://security.cloudflare-dns.com/dns-query")
+
+	// Cloudflare -Malware -Adult
+	addDoH("1.1.1.3", "https://family.cloudflare-dns.com/dns-query")
+	addDoH("1.0.0.3", "https://family.cloudflare-dns.com/dns-query")
+	addDoH("2606:4700:4700::1113", "https://family.cloudflare-dns.com/dns-query")
+	addDoH("2606:4700:4700::1003", "https://family.cloudflare-dns.com/dns-query")
+
+	// Google
+	addDoH("8.8.8.8", "https://dns.google/dns-query")
+	addDoH("8.8.4.4", "https://dns.google/dns-query")
+	addDoH("2001:4860:4860::8888", "https://dns.google/dns-query")
+	addDoH("2001:4860:4860::8844", "https://dns.google/dns-query")
+
+	// OpenDNS
+	// TODO(bradfitz): OpenDNS is unique amongst this current set in that
+	// its DoH DNS names resolve to different IPs than its normal DNS
+	// IPs. Support that later. For now we assume that they're the same.
+	// addDoH("208.67.222.222", "https://doh.opendns.com/dns-query")
+	// addDoH("208.67.220.220", "https://doh.opendns.com/dns-query")
+	// addDoH("208.67.222.123", "https://doh.familyshield.opendns.com/dns-query")
+	// addDoH("208.67.220.123", "https://doh.familyshield.opendns.com/dns-query")
+
+	// Quad9
+	addDoH("9.9.9.9", "https://dns.quad9.net/dns-query")
+	addDoH("149.112.112.112", "https://dns.quad9.net/dns-query")
+	addDoH("2620:fe::fe", "https://dns.quad9.net/dns-query")
+	addDoH("2620:fe::fe:9", "https://dns.quad9.net/dns-query")
+}

net/dns/resolver/forwarder_test.go

@@ -0,0 +1,90 @@
+// Copyright (c) 2021 Tailscale Inc & AUTHORS All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package resolver
+
+import (
+	"fmt"
+	"reflect"
+	"strings"
+	"testing"
+	"time"
+
+	"inet.af/netaddr"
+)
+
+func (rr resolverAndDelay) String() string {
+	return fmt.Sprintf("%v+%v", rr.ipp, rr.startDelay)
+}
+
+func TestResolversWithDelays(t *testing.T) {
+	// query
+	q := func(ss ...string) (ipps []netaddr.IPPort) {
+		for _, s := range ss {
+			ipps = append(ipps, netaddr.MustParseIPPort(s))
+		}
+		return
+	}
+	// output
+	o := func(ss ...string) (rr []resolverAndDelay) {
+		for _, s := range ss {
+			var d time.Duration
+			if i := strings.Index(s, "+"); i != -1 {
+				var err error
+				d, err = time.ParseDuration(s[i+1:])
+				if err != nil {
+					panic(fmt.Sprintf("parsing duration in %q: %v", s, err))
+				}
+				s = s[:i]
+			}
+			rr = append(rr, resolverAndDelay{
+				ipp:        netaddr.MustParseIPPort(s),
+				startDelay: d,
+			})
+		}
+		return
+	}
+
+	tests := []struct {
+		name string
+		in   []netaddr.IPPort
+		want []resolverAndDelay
+	}{
+		{
+			name: "unknown-no-delays",
+			in:   q("1.2.3.4:53", "2.3.4.5:53"),
+			want: o("1.2.3.4:53", "2.3.4.5:53"),
+		},
+		{
+			name: "google-all-ipv4",
+			in:   q("8.8.8.8:53", "8.8.4.4:53"),
+			want: o("8.8.8.8:53", "8.8.4.4:53+200ms"),
+		},
+		{
+			name: "google-only-ipv6",
+			in:   q("[2001:4860:4860::8888]:53", "[2001:4860:4860::8844]:53"),
+			want: o("[2001:4860:4860::8888]:53", "[2001:4860:4860::8844]:53+200ms"),
+		},
+		{
+			name: "google-all-four",
+			in:   q("8.8.8.8:53", "8.8.4.4:53", "[2001:4860:4860::8888]:53", "[2001:4860:4860::8844]:53"),
+			want: o("8.8.8.8:53", "8.8.4.4:53+200ms", "[2001:4860:4860::8888]:53+2.5s", "[2001:4860:4860::8844]:53+2.7s"),
+		},
+		{
+			name: "quad9-one-v4-one-v6",
+			in:   q("9.9.9.9:53", "[2620:fe::fe]:53"),
+			want: o("9.9.9.9:53", "[2620:fe::fe]:53+200ms"),
+		},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			got := resolversWithDelays(tt.in)
+			if !reflect.DeepEqual(got, tt.want) {
+				t.Errorf("got %v; want %v", got, tt.want)
+			}
+		})
+	}
+
+}

net/dns/resolver/tsdns.go

@@ -7,8 +7,10 @@
 package resolver
 
 import (
+	"bufio"
 	"encoding/hex"
 	"errors"
+	"fmt"
 	"runtime"
 	"sort"
 	"strings"
@@ -79,6 +81,74 @@ type Config struct {
 	LocalDomains []dnsname.FQDN
 }
 
+// WriteToBufioWriter write a debug version of c for logs to w, omitting
+// spammy stuff like *.arpa entries and replacing it with a total count.
+func (c *Config) WriteToBufioWriter(w *bufio.Writer) {
+	w.WriteString("{Routes:")
+	WriteRoutes(w, c.Routes)
+	fmt.Fprintf(w, " Hosts:%v LocalDomains:[", len(c.Hosts))
+	space := false
+	arpa := 0
+	for _, d := range c.LocalDomains {
+		if strings.HasSuffix(string(d), ".arpa.") {
+			arpa++
+			continue
+		}
+		if space {
+			w.WriteByte(' ')
+		}
+		w.WriteString(string(d))
+		space = true
+	}
+	w.WriteString("]")
+	if arpa > 0 {
+		fmt.Fprintf(w, "+%darpa", arpa)
+	}
+	w.WriteString("}")
+}
+
+// WriteIPPorts writes vv to w.
+func WriteIPPorts(w *bufio.Writer, vv []netaddr.IPPort) {
+	w.WriteByte('[')
+	var b []byte
+	for i, v := range vv {
+		if i > 0 {
+			w.WriteByte(' ')
+		}
+		b = v.AppendTo(b[:0])
+		w.Write(b)
+	}
+	w.WriteByte(']')
+}
+
+// WriteRoutes writes routes to w, omitting *.arpa routes and instead
+// summarizing how many of them there were.
+func WriteRoutes(w *bufio.Writer, routes map[dnsname.FQDN][]netaddr.IPPort) {
+	var kk []dnsname.FQDN
+	arpa := 0
+	for k := range routes {
+		if strings.HasSuffix(string(k), ".arpa.") {
+			arpa++
+			continue
+		}
+		kk = append(kk, k)
+	}
+	sort.Slice(kk, func(i, j int) bool { return kk[i] < kk[j] })
+	w.WriteByte('{')
+	for i, k := range kk {
+		if i > 0 {
+			w.WriteByte(' ')
+		}
+		w.WriteString(string(k))
+		w.WriteByte(':')
+		WriteIPPorts(w, routes[k])
+	}
+	w.WriteByte('}')
+	if arpa > 0 {
+		fmt.Fprintf(w, "+%darpa", arpa)
+	}
+}
+
 // Resolver is a DNS resolver for nodes on the Tailscale network,
 // associating them with domain names of the form <mynode>.<mydomain>.<root>.
 // If it is asked to resolve a domain that is not of that form,
@@ -138,7 +208,6 @@ func (r *Resolver) SetConfig(cfg Config) error {
 		r.saveConfigForTests(cfg)
 	}
 
-	routes := make([]route, 0, len(cfg.Routes))
 	reverse := make(map[netaddr.IP]dnsname.FQDN, len(cfg.Hosts))
 
 	for host, ips := range cfg.Hosts {
@@ -147,18 +216,7 @@ func (r *Resolver) SetConfig(cfg Config) error {
 		}
 	}
 
-	for suffix, ips := range cfg.Routes {
-		routes = append(routes, route{
-			Suffix:    suffix,
-			Resolvers: ips,
-		})
-	}
-	// Sort from longest prefix to shortest.
-	sort.Slice(routes, func(i, j int) bool {
-		return routes[i].Suffix.NumLabels() > routes[j].Suffix.NumLabels()
-	})
-
-	r.forwarder.setRoutes(routes)
+	r.forwarder.setRoutes(cfg.Routes)
 
 	r.mu.Lock()
 	defer r.mu.Unlock()

net/dns/resolver/tsdns_test.go

@@ -931,8 +931,11 @@ func TestAllocs(t *testing.T) {
 		query []byte
 		want  int
 	}{
-		// Name lowercasing and response slice created by dns.NewBuilder.
-		{"forward", dnspacket("test1.ipn.dev.", dns.TypeA, noEdns), 2},
+		// Name lowercasing, response slice created by dns.NewBuilder,
+		// and closure allocation from go call.
+		// (Closure allocation only happens when using new register ABI,
+		// which is amd64 with Go 1.17, and probably more platforms later.)
+		{"forward", dnspacket("test1.ipn.dev.", dns.TypeA, noEdns), 3},
 		// 3 extra allocs in rdnsNameToIPv4 and one in marshalPTRRecord (dns.NewName).
 		{"reverse", dnspacket("4.3.2.1.in-addr.arpa.", dns.TypePTR, noEdns), 5},
 	}

net/dns/wsl_windows.go

@@ -0,0 +1,254 @@
+// Copyright (c) 2021 Tailscale Inc & AUTHORS All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package dns
+
+import (
+	"bytes"
+	"fmt"
+	"os"
+	"os/exec"
+	"os/user"
+	"strings"
+	"syscall"
+	"unicode/utf16"
+
+	"golang.org/x/sys/windows"
+	"tailscale.com/types/logger"
+	"tailscale.com/util/winutil"
+)
+
+// wslDistros reports the names of the installed WSL2 linux distributions.
+func wslDistros() ([]string, error) {
+	b, err := wslCombinedOutput(exec.Command("wsl.exe", "-l"))
+	if err != nil {
+		return nil, fmt.Errorf("%v: %q", err, string(b))
+	}
+
+	// The first line of output is a WSL header. E.g.
+	//
+	//	C:\tsdev>wsl.exe -l
+	//	Windows Subsystem for Linux Distributions:
+	//	Ubuntu-20.04 (Default)
+	//
+	// We can skip it by passing '-q', but here we put it to work.
+	// It turns out wsl.exe -l is broken, and outputs UTF-16 names
+	// that nothing can read. (Try `wsl.exe -l | more`.)
+	// So we look at the header to see if it's UTF-16.
+	// If so, we run the rest through a UTF-16 parser.
+	//
+	// https://github.com/microsoft/WSL/issues/4607
+	var output string
+	if bytes.HasPrefix(b, []byte("W\x00i\x00n\x00d\x00o\x00w\x00s\x00")) {
+		output, err = decodeUTF16(b)
+		if err != nil {
+			return nil, fmt.Errorf("failed to decode wsl.exe -l output %q: %v", b, err)
+		}
+	} else {
+		output = string(b)
+	}
+	lines := strings.Split(output, "\n")
+	if len(lines) < 1 {
+		return nil, nil
+	}
+	lines = lines[1:] // drop "Windows Subsystem For Linux" header
+
+	var distros []string
+	for _, name := range lines {
+		name = strings.TrimSpace(name)
+		name = strings.TrimSuffix(name, " (Default)")
+		if name == "" {
+			continue
+		}
+		distros = append(distros, name)
+	}
+	return distros, nil
+}
+
+func decodeUTF16(b []byte) (string, error) {
+	if len(b) == 0 {
+		return "", nil
+	} else if len(b)%2 != 0 {
+		return "", fmt.Errorf("decodeUTF16: invalid length %d", len(b))
+	}
+	var u16 []uint16
+	for i := 0; i < len(b); i += 2 {
+		u16 = append(u16, uint16(b[i])+(uint16(b[i+1])<<8))
+	}
+	return string(utf16.Decode(u16)), nil
+}
+
+// wslManager is a DNS manager for WSL2 linux distributions.
+// It configures /etc/wsl.conf and /etc/resolv.conf.
+type wslManager struct {
+	logf logger.Logf
+}
+
+func newWSLManager(logf logger.Logf) *wslManager {
+	m := &wslManager{
+		logf: logf,
+	}
+	return m
+}
+
+func (wm *wslManager) SetDNS(cfg OSConfig) error {
+	distros, err := wslDistros()
+	if err != nil {
+		return err
+	} else if len(distros) == 0 {
+		return nil
+	}
+	managers := make(map[string]directManager)
+	for _, distro := range distros {
+		managers[distro] = newDirectManagerOnFS(wslFS{
+			user:   "root",
+			distro: distro,
+		})
+	}
+
+	if !cfg.IsZero() {
+		if wm.setWSLConf(managers) {
+			// What's this? So glad you asked.
+			//
+			// WSL2 writes the /etc/resolv.conf.
+			// It is aggressive about it. Every time you execute wsl.exe,
+			// it writes it. (Opening a terminal is done by running wsl.exe.)
+			// You can turn this off using /etc/wsl.conf! But: this wsl.conf
+			// file is only parsed when the VM boots up. To do that, we
+			// have to shut down WSL2.
+			//
+			// So we do it here, before we call wsl.exe to write resolv.conf.
+			if b, err := wslCombinedOutput(wslCommand("--shutdown")); err != nil {
+				wm.logf("WSL SetDNS shutdown: %v: %s", err, b)
+			}
+		}
+	}
+
+	for distro, m := range managers {
+		if err := m.SetDNS(cfg); err != nil {
+			wm.logf("WSL(%q) SetDNS: %v", distro, err)
+		}
+	}
+	return nil
+}
+
+const wslConf = "/etc/wsl.conf"
+const wslConfSection = `# added by tailscale
+[network]
+generateResolvConf = false
+`
+
+// setWSLConf attempts to disable generateResolvConf in each WSL2 linux.
+// If any are changed, it reports true.
+func (wm *wslManager) setWSLConf(managers map[string]directManager) (changed bool) {
+	for distro, m := range managers {
+		b, err := m.fs.ReadFile(wslConf)
+		if err != nil && !os.IsNotExist(err) {
+			wm.logf("WSL(%q) wsl.conf: read: %v", distro, err)
+			continue
+		}
+		ini := parseIni(string(b))
+		if v := ini["network"]["generateResolvConf"]; v == "" {
+			b = append(b, wslConfSection...)
+			if err := m.fs.WriteFile(wslConf, b, 0644); err != nil {
+				wm.logf("WSL(%q) wsl.conf: write: %v", distro, err)
+				continue
+			}
+			changed = true
+		}
+	}
+	return changed
+}
+
+func (m *wslManager) SupportsSplitDNS() bool { return false }
+func (m *wslManager) Close() error           { return m.SetDNS(OSConfig{}) }
+
+// wslFS is a pinholeFS implemented on top of wsl.exe.
+//
+// We access WSL2 file systems via wsl.exe instead of \\wsl$\ because
+// the netpath appears to operate as the standard user, not root.
+type wslFS struct {
+	user   string
+	distro string
+}
+
+func (fs wslFS) Stat(name string) (isRegular bool, err error) {
+	err = wslRun(fs.cmd("test", "-f", name))
+	if ee, _ := err.(*exec.ExitError); ee != nil {
+		if ee.ExitCode() == 1 {
+			return false, os.ErrNotExist
+		}
+		return false, err
+	}
+	return true, nil
+}
+
+func (fs wslFS) Rename(oldName, newName string) error {
+	return wslRun(fs.cmd("mv", "--", oldName, newName))
+}
+func (fs wslFS) Remove(name string) error { return wslRun(fs.cmd("rm", "--", name)) }
+
+func (fs wslFS) ReadFile(name string) ([]byte, error) {
+	b, err := wslCombinedOutput(fs.cmd("cat", "--", name))
+	if ee, _ := err.(*exec.ExitError); ee != nil && ee.ExitCode() == 1 {
+		return nil, os.ErrNotExist
+	}
+	return b, err
+}
+
+func (fs wslFS) WriteFile(name string, contents []byte, perm os.FileMode) error {
+	cmd := fs.cmd("tee", "--", name)
+	cmd.Stdin = bytes.NewReader(contents)
+	cmd.Stdout = nil
+	if err := wslRun(cmd); err != nil {
+		return err
+	}
+	return wslRun(fs.cmd("chmod", "--", fmt.Sprintf("%04o", perm), name))
+}
+
+func (fs wslFS) cmd(args ...string) *exec.Cmd {
+	cmd := wslCommand("-u", fs.user, "-d", fs.distro, "-e")
+	cmd.Args = append(cmd.Args, args...)
+	return cmd
+}
+
+func wslCommand(args ...string) *exec.Cmd {
+	cmd := exec.Command("wsl.exe", args...)
+	return cmd
+}
+
+func wslCombinedOutput(cmd *exec.Cmd) ([]byte, error) {
+	buf := new(bytes.Buffer)
+	cmd.Stdout = buf
+	cmd.Stderr = buf
+	err := wslRun(cmd)
+	return buf.Bytes(), err
+}
+
+func wslRun(cmd *exec.Cmd) (err error) {
+	defer func() {
+		if err != nil {
+			err = fmt.Errorf("wslRun(%v): %w", cmd.Args, err)
+		}
+	}()
+
+	var token windows.Token
+	if u, err := user.Current(); err == nil && u.Name == "SYSTEM" {
+		// We need to switch user to run wsl.exe.
+		// https://github.com/microsoft/WSL/issues/4803
+		sessionID := winutil.WTSGetActiveConsoleSessionId()
+		if sessionID != 0xFFFFFFFF {
+			if err := windows.WTSQueryUserToken(sessionID, &token); err != nil {
+				return err
+			}
+			defer token.Close()
+		}
+	}
+
+	cmd.SysProcAttr = &syscall.SysProcAttr{
+		Token:      syscall.Token(token),
+		HideWindow: true,
+	}
+	return cmd.Run()
+}

net/dnscache/dnscache.go

@@ -2,6 +2,8 @@
 // Use of this source code is governed by a BSD-style
 // license that can be found in the LICENSE file.
 
+// TODO(bradfitz): update this code to use netaddr more
+
 // Package dnscache contains a minimal DNS cache that makes a bunch of
 // assumptions that are only valid for us. Not recommended for general use.
 package dnscache
@@ -78,8 +80,9 @@ type Resolver struct {
 }
 
 type ipCacheEntry struct {
-	ip      net.IP // either v4 or v6
-	ip6     net.IP // nil if no v4 or no v6
+	ip      net.IP       // either v4 or v6
+	ip6     net.IP       // nil if no v4 or no v6
+	allIPs  []net.IPAddr // 1+ v4 and/or v6
 	expires time.Time
 }
 
@@ -105,81 +108,82 @@ var debug, _ = strconv.ParseBool(os.Getenv("TS_DEBUG_DNS_CACHE"))
 //
 // If err is nil, ip will be non-nil. The v6 address may be nil even
 // with a nil error.
-func (r *Resolver) LookupIP(ctx context.Context, host string) (ip, v6 net.IP, err error) {
+func (r *Resolver) LookupIP(ctx context.Context, host string) (ip, v6 net.IP, allIPs []net.IPAddr, err error) {
 	if ip := net.ParseIP(host); ip != nil {
 		if ip4 := ip.To4(); ip4 != nil {
-			return ip4, nil, nil
+			return ip4, nil, []net.IPAddr{{IP: ip4}}, nil
 		}
 		if debug {
 			log.Printf("dnscache: %q is an IP", host)
 		}
-		return ip, nil, nil
+		return ip, nil, []net.IPAddr{{IP: ip}}, nil
 	}
 
-	if ip, ip6, ok := r.lookupIPCache(host); ok {
+	if ip, ip6, allIPs, ok := r.lookupIPCache(host); ok {
 		if debug {
 			log.Printf("dnscache: %q = %v (cached)", host, ip)
 		}
-		return ip, ip6, nil
+		return ip, ip6, allIPs, nil
 	}
 
-	type ipPair struct {
+	type ipRes struct {
 		ip, ip6 net.IP
+		allIPs  []net.IPAddr
 	}
 	ch := r.sf.DoChan(host, func() (interface{}, error) {
-		ip, ip6, err := r.lookupIP(host)
+		ip, ip6, allIPs, err := r.lookupIP(host)
 		if err != nil {
 			return nil, err
 		}
-		return ipPair{ip, ip6}, nil
+		return ipRes{ip, ip6, allIPs}, nil
 	})
 	select {
 	case res := <-ch:
 		if res.Err != nil {
 			if r.UseLastGood {
-				if ip, ip6, ok := r.lookupIPCacheExpired(host); ok {
+				if ip, ip6, allIPs, ok := r.lookupIPCacheExpired(host); ok {
 					if debug {
 						log.Printf("dnscache: %q using %v after error", host, ip)
 					}
-					return ip, ip6, nil
+					return ip, ip6, allIPs, nil
 				}
 			}
 			if debug {
 				log.Printf("dnscache: error resolving %q: %v", host, res.Err)
 			}
-			return nil, nil, res.Err
+			return nil, nil, nil, res.Err
 		}
-		pair := res.Val.(ipPair)
-		return pair.ip, pair.ip6, nil
+		r := res.Val.(ipRes)
+		return r.ip, r.ip6, r.allIPs, nil
 	case <-ctx.Done():
 		if debug {
 			log.Printf("dnscache: context done while resolving %q: %v", host, ctx.Err())
 		}
-		return nil, nil, ctx.Err()
+		return nil, nil, nil, ctx.Err()
 	}
 }
 
-func (r *Resolver) lookupIPCache(host string) (ip, ip6 net.IP, ok bool) {
+func (r *Resolver) lookupIPCache(host string) (ip, ip6 net.IP, allIPs []net.IPAddr, ok bool) {
 	r.mu.Lock()
 	defer r.mu.Unlock()
 	if ent, ok := r.ipCache[host]; ok && ent.expires.After(time.Now()) {
-		return ent.ip, ent.ip6, true
+		return ent.ip, ent.ip6, ent.allIPs, true
 	}
-	return nil, nil, false
+	return nil, nil, nil, false
 }
 
-func (r *Resolver) lookupIPCacheExpired(host string) (ip, ip6 net.IP, ok bool) {
+func (r *Resolver) lookupIPCacheExpired(host string) (ip, ip6 net.IP, allIPs []net.IPAddr, ok bool) {
 	r.mu.Lock()
 	defer r.mu.Unlock()
 	if ent, ok := r.ipCache[host]; ok {
-		return ent.ip, ent.ip6, true
+		return ent.ip, ent.ip6, ent.allIPs, true
 	}
-	return nil, nil, false
+	return nil, nil, nil, false
 }
 
 func (r *Resolver) lookupTimeoutForHost(host string) time.Duration {
 	if r.UseLastGood {
-		if _, _, ok := r.lookupIPCacheExpired(host); ok {
+		if _, _, _, ok := r.lookupIPCacheExpired(host); ok {
 			// If we have some previous good value for this host,
 			// don't give this DNS lookup much time. If we're in a
 			// situation where the user's DNS server is unreachable
@@ -194,12 +198,12 @@ func (r *Resolver) lookupTimeoutForHost(host string) time.Duration {
 	return 10 * time.Second
 }
 
-func (r *Resolver) lookupIP(host string) (ip, ip6 net.IP, err error) {
-	if ip, ip6, ok := r.lookupIPCache(host); ok {
+func (r *Resolver) lookupIP(host string) (ip, ip6 net.IP, allIPs []net.IPAddr, err error) {
+	if ip, ip6, allIPs, ok := r.lookupIPCache(host); ok {
 		if debug {
 			log.Printf("dnscache: %q found in cache as %v", host, ip)
 		}
-		return ip, ip6, nil
+		return ip, ip6, allIPs, nil
 	}
 
 	ctx, cancel := context.WithTimeout(context.Background(), r.lookupTimeoutForHost(host))
@@ -218,10 +222,10 @@ func (r *Resolver) lookupIP(host string) (ip, ip6 net.IP, err error) {
 		}
 	}
 	if err != nil {
-		return nil, nil, err
+		return nil, nil, nil, err
 	}
 	if len(ips) == 0 {
-		return nil, nil, fmt.Errorf("no IPs for %q found", host)
+		return nil, nil, nil, fmt.Errorf("no IPs for %q found", host)
 	}
 
 	have4 := false
@@ -240,12 +244,12 @@ func (r *Resolver) lookupIP(host string) (ip, ip6 net.IP, err error) {
 			}
 		}
 	}
-	r.addIPCache(host, ip, ip6, r.ttl())
-	return ip, ip6, nil
+	r.addIPCache(host, ip, ip6, ips, r.ttl())
+	return ip, ip6, ips, nil
 }
 
-func (r *Resolver) addIPCache(host string, ip, ip6 net.IP, d time.Duration) {
-	if isPrivateIP(ip) {
+func (r *Resolver) addIPCache(host string, ip, ip6 net.IP, allIPs []net.IPAddr, d time.Duration) {
+	if naIP, _ := netaddr.FromStdIP(ip); naIP.IsPrivate() {
 		// Don't cache obviously wrong entries from captive portals.
 		// TODO: use DoH or DoT for the forwarding resolver?
 		if debug {
@@ -263,27 +267,14 @@ func (r *Resolver) addIPCache(host string, ip, ip6 net.IP, d time.Duration) {
 	if r.ipCache == nil {
 		r.ipCache = make(map[string]ipCacheEntry)
 	}
-	r.ipCache[host] = ipCacheEntry{ip: ip, ip6: ip6, expires: time.Now().Add(d)}
-}
-
-func mustCIDR(s string) *net.IPNet {
-	_, ipNet, err := net.ParseCIDR(s)
-	if err != nil {
-		panic(err)
+	r.ipCache[host] = ipCacheEntry{
+		ip:      ip,
+		ip6:     ip6,
+		allIPs:  allIPs,
+		expires: time.Now().Add(d),
 	}
-	return ipNet
 }
 
-func isPrivateIP(ip net.IP) bool {
-	return private1.Contains(ip) || private2.Contains(ip) || private3.Contains(ip)
-}
-
-var (
-	private1 = mustCIDR("10.0.0.0/8")
-	private2 = mustCIDR("172.16.0.0/12")
-	private3 = mustCIDR("192.168.0.0/16")
-)
-
 type DialContextFunc func(ctx context.Context, network, address string) (net.Conn, error)
 
 // Dialer returns a wrapped DialContext func that uses the provided dnsCache.
@@ -305,41 +296,131 @@ func Dialer(fwd DialContextFunc, dnsCache *Resolver) DialContextFunc {
 				// Return with original error
 				return
 			}
-			for _, ip := range ips {
-				dst := net.JoinHostPort(ip.String(), port)
-				if c, err := fwd(ctx, network, dst); err == nil {
-					retConn = c
-					ret = nil
-					return
-				}
+			if c, err := raceDial(ctx, fwd, network, ips, port); err == nil {
+				retConn = c
+				ret = nil
+				return
 			}
 		}()
 
-		ip, ip6, err := dnsCache.LookupIP(ctx, host)
+		ip, ip6, allIPs, err := dnsCache.LookupIP(ctx, host)
 		if err != nil {
 			return nil, fmt.Errorf("failed to resolve %q: %w", host, err)
 		}
-		dst := net.JoinHostPort(ip.String(), port)
-		if debug {
-			log.Printf("dnscache: dialing %s, %s for %s", network, dst, address)
+		i4s := v4addrs(allIPs)
+		if len(i4s) < 2 {
+			dst := net.JoinHostPort(ip.String(), port)
+			if debug {
+				log.Printf("dnscache: dialing %s, %s for %s", network, dst, address)
+			}
+			c, err := fwd(ctx, network, dst)
+			if err == nil || ctx.Err() != nil || ip6 == nil {
+				return c, err
+			}
+			// Fall back to trying IPv6.
+			dst = net.JoinHostPort(ip6.String(), port)
+			return fwd(ctx, network, dst)
 		}
-		c, err := fwd(ctx, network, dst)
-		if err == nil || ctx.Err() != nil || ip6 == nil {
-			return c, err
-		}
-		// Fall back to trying IPv6.
-		// TODO(bradfitz): this is a primarily for IPv6-only
-		// hosts; it's not supposed to be a real Happy
-		// Eyeballs implementation. We should use the net
-		// package's implementation of that by plumbing this
-		// dnscache impl into net.Dialer.Resolver.Dial and
-		// unmarshal/marshal DNS queries/responses to the net
-		// package. This works for v6-only hosts for now.
-		dst = net.JoinHostPort(ip6.String(), port)
-		return fwd(ctx, network, dst)
+
+		// Multiple IPv4 candidates, and 0+ IPv6.
+		ipsToTry := append(i4s, v6addrs(allIPs)...)
+		return raceDial(ctx, fwd, network, ipsToTry, port)
 	}
 }
 
+// fallbackDelay is how long to wait between trying subsequent
+// addresses when multiple options are available.
+// 300ms is the same as Go's Happy Eyeballs fallbackDelay value.
+const fallbackDelay = 300 * time.Millisecond
+
+// raceDial tries to dial port on each ip in ips, starting a new race
+// dial every fallbackDelay apart, returning whichever completes first.
+func raceDial(ctx context.Context, fwd DialContextFunc, network string, ips []netaddr.IP, port string) (net.Conn, error) {
+	ctx, cancel := context.WithCancel(ctx)
+	defer cancel()
+
+	type res struct {
+		c   net.Conn
+		err error
+	}
+	resc := make(chan res)           // must be unbuffered
+	failBoost := make(chan struct{}) // best effort send on dial failure
+
+	go func() {
+		for i, ip := range ips {
+			if i != 0 {
+				timer := time.NewTimer(fallbackDelay)
+				select {
+				case <-timer.C:
+				case <-failBoost:
+					timer.Stop()
+				case <-ctx.Done():
+					timer.Stop()
+					return
+				}
+			}
+			go func(ip netaddr.IP) {
+				c, err := fwd(ctx, network, net.JoinHostPort(ip.String(), port))
+				if err != nil {
+					// Best effort wake-up a pending dial.
+					// e.g. IPv4 dials failing quickly on an IPv6-only system.
+					// In that case we don't want to wait 300ms per IPv4 before
+					// we get to the IPv6 addresses.
+					select {
+					case failBoost <- struct{}{}:
+					default:
+					}
+				}
+				select {
+				case resc <- res{c, err}:
+				case <-ctx.Done():
+					if c != nil {
+						c.Close()
+					}
+				}
+			}(ip)
+		}
+	}()
+
+	var firstErr error
+	var fails int
+	for {
+		select {
+		case r := <-resc:
+			if r.c != nil {
+				return r.c, nil
+			}
+			fails++
+			if firstErr == nil {
+				firstErr = r.err
+			}
+			if fails == len(ips) {
+				return nil, firstErr
+			}
+		case <-ctx.Done():
+			return nil, ctx.Err()
+		}
+	}
+}
+
+func v4addrs(aa []net.IPAddr) (ret []netaddr.IP) {
+	for _, a := range aa {
+		if ip, ok := netaddr.FromStdIP(a.IP); ok && ip.Is4() {
+			ret = append(ret, ip)
+		}
+	}
+	return ret
+}
+
+func v6addrs(aa []net.IPAddr) (ret []netaddr.IP) {
+	for _, a := range aa {
+		if ip, ok := netaddr.FromStdIP(a.IP); ok && ip.Is6() {
+			ret = append(ret, ip)
+		}
+	}
+	return ret
+}
+
 var errTLSHandshakeTimeout = errors.New("timeout doing TLS handshake")
 
 // TLSDialer is like Dialer but returns a func suitable for using with net/http.Transport.DialTLSContext.

net/dnscache/dnscache_test.go

@@ -5,24 +5,29 @@
 package dnscache
 
 import (
+	"context"
+	"flag"
 	"net"
 	"testing"
+	"time"
 )
 
-func TestIsPrivateIP(t *testing.T) {
-	tests := []struct {
-		ip   string
-		want bool
-	}{
-		{"10.1.2.3", true},
-		{"172.16.1.100", true},
-		{"192.168.1.1", true},
-		{"1.2.3.4", false},
-	}
+var dialTest = flag.String("dial-test", "", "if non-empty, addr:port to test dial")
 
-	for _, test := range tests {
-		if got := isPrivateIP(net.ParseIP(test.ip)); got != test.want {
-			t.Errorf("isPrivateIP(%q)=%v, want %v", test.ip, got, test.want)
-		}
+func TestDialer(t *testing.T) {
+	if *dialTest == "" {
+		t.Skip("skipping; --dial-test is blank")
 	}
+	r := new(Resolver)
+	var std net.Dialer
+	dialer := Dialer(std.DialContext, r)
+	t0 := time.Now()
+	ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
+	defer cancel()
+	c, err := dialer(ctx, "tcp", *dialTest)
+	if err != nil {
+		t.Fatal(err)
+	}
+	t.Logf("dialed in %v", time.Since(t0))
+	c.Close()
 }

net/dnsfallback/dns-fallback-servers.json

@@ -90,18 +90,25 @@
 			"RegionName": "r4",
 			"Nodes": [
 				{
-					"Name": "4a",
+					"Name": "4c",
 					"RegionID": 4,
-					"HostName": "derp4.tailscale.com",
-					"IPv4": "167.172.182.26",
-					"IPv6": "2a03:b0c0:3:e0::36e:9001"
+					"HostName": "derp4c.tailscale.com",
+					"IPv4": "134.122.77.138",
+					"IPv6": "2a03:b0c0:3:d0::1501:6001"
 				},
 				{
-					"Name": "4b",
+					"Name": "4d",
 					"RegionID": 4,
-					"HostName": "derp4b.tailscale.com",
-					"IPv4": "157.230.25.0",
-					"IPv6": "2a03:b0c0:3:e0::58f:3001"
+					"HostName": "derp4d.tailscale.com",
+					"IPv4": "134.122.94.167",
+					"IPv6": "2a03:b0c0:3:d0::1501:b001"
+				},
+				{
+					"Name": "4e",
+					"RegionID": 4,
+					"HostName": "derp4e.tailscale.com",
+					"IPv4": "134.122.74.153",
+					"IPv6": "2a03:b0c0:3:d0::29:9001"
 				}
 			]
 		},
@@ -153,11 +160,25 @@
 			"RegionName": "r8",
 			"Nodes": [
 				{
-					"Name": "8a",
+					"Name": "8b",
 					"RegionID": 8,
-					"HostName": "derp8.tailscale.com",
-					"IPv4": "167.71.139.179",
-					"IPv6": "2a03:b0c0:1:e0::3cc:e001"
+					"HostName": "derp8b.tailscale.com",
+					"IPv4": "46.101.74.201",
+					"IPv6": "2a03:b0c0:1:d0::ec1:e001"
+				},
+				{
+					"Name": "8c",
+					"RegionID": 8,
+					"HostName": "derp8c.tailscale.com",
+					"IPv4": "206.189.16.32",
+					"IPv6": "2a03:b0c0:1:d0::e1f:4001"
+				},
+				{
+					"Name": "8d",
+					"RegionID": 8,
+					"HostName": "derp8d.tailscale.com",
+					"IPv4": "178.62.44.132",
+					"IPv6": "2a03:b0c0:1:d0::e08:e001"
 				}
 			]
 		},

net/dnsfallback/dnsfallback.go

@@ -87,7 +87,7 @@ func Lookup(ctx context.Context, host string) ([]netaddr.IP, error) {
 }
 
 // serverName and serverIP of are, say, "derpN.tailscale.com".
-// queryName is the name being sought (e.g. "login.tailscale.com"), passed as hint.
+// queryName is the name being sought (e.g. "controlplane.tailscale.com"), passed as hint.
 func bootstrapDNSMap(ctx context.Context, serverName string, serverIP netaddr.IP, queryName string) (dnsMap, error) {
 	dialer := netns.NewDialer()
 	tr := http.DefaultTransport.(*http.Transport).Clone()

net/interfaces/interfaces.go

@@ -22,7 +22,7 @@ import (
 
 // LoginEndpointForProxyDetermination is the URL used for testing
 // which HTTP proxy the system should use.
-var LoginEndpointForProxyDetermination = "https://login.tailscale.com/"
+var LoginEndpointForProxyDetermination = "https://controlplane.tailscale.com/"
 
 // Tailscale returns the current machine's Tailscale interface, if any.
 // If none is found, all zero values are returned.
@@ -134,7 +134,7 @@ func LocalAddresses() (regular, loopback []netaddr.IP, err error) {
 					// but their OS supports IPv6 so they have an fe80::
 					// address. We don't want to report all of those
 					// IPv6 LL to Control.
-				} else if ip.Is6() && tsaddr.IsULA(ip) {
+				} else if ip.Is6() && ip.IsPrivate() {
 					// Google Cloud Run uses NAT with IPv6 Unique
 					// Local Addresses to provide IPv6 connectivity.
 					ula6 = append(ula6, ip)
@@ -479,7 +479,7 @@ func HTTPOfListener(ln net.Listener) string {
 	var privateIP string
 	ForeachInterfaceAddress(func(i Interface, pfx netaddr.IPPrefix) {
 		ip := pfx.IP()
-		if isPrivateIP(ip) {
+		if ip.IsPrivate() {
 			if privateIP == "" {
 				privateIP = ip.String()
 			}
@@ -519,21 +519,15 @@ func LikelyHomeRouterIP() (gateway, myIP netaddr.IP, ok bool) {
 		if !i.IsUp() || ip.IsZero() || !myIP.IsZero() {
 			return
 		}
-		for _, prefix := range privatev4s {
-			if prefix.Contains(gateway) && prefix.Contains(ip) {
-				myIP = ip
-				ok = true
-				return
-			}
+		if gateway.IsPrivate() && ip.IsPrivate() {
+			myIP = ip
+			ok = true
+			return
 		}
 	})
 	return gateway, myIP, !myIP.IsZero()
 }
 
-func isPrivateIP(ip netaddr.IP) bool {
-	return private1.Contains(ip) || private2.Contains(ip) || private3.Contains(ip)
-}
-
 // isUsableV4 reports whether ip is a usable IPv4 address which could
 // conceivably be used to get Internet connectivity. Globally routable and
 // private IPv4 addresses are always Usable, and link local 169.254.x.x
@@ -554,23 +548,11 @@ func isUsableV4(ip netaddr.IP) bool {
 // (fc00::/7) are in some environments used with address translation.
 func isUsableV6(ip netaddr.IP) bool {
 	return v6Global1.Contains(ip) ||
-		(tsaddr.IsULA(ip) && !tsaddr.TailscaleULARange().Contains(ip))
-}
-
-func mustCIDR(s string) netaddr.IPPrefix {
-	prefix, err := netaddr.ParseIPPrefix(s)
-	if err != nil {
-		panic(err)
-	}
-	return prefix
+		(ip.Is6() && ip.IsPrivate() && !tsaddr.TailscaleULARange().Contains(ip))
 }
 
 var (
-	private1   = mustCIDR("10.0.0.0/8")
-	private2   = mustCIDR("172.16.0.0/12")
-	private3   = mustCIDR("192.168.0.0/16")
-	privatev4s = []netaddr.IPPrefix{private1, private2, private3}
-	v6Global1  = mustCIDR("2000::/3")
+	v6Global1 = netaddr.MustParseIPPrefix("2000::/3")
 )
 
 // anyInterestingIP reports whether pfxs contains any IP that matches

net/interfaces/interfaces_darwin_test.go

@@ -73,7 +73,7 @@ func likelyHomeRouterIPDarwinExec() (ret netaddr.IP, ok bool) {
 			return nil
 		}
 		ip, err := netaddr.ParseIP(string(mem.Append(nil, ipm)))
-		if err == nil && isPrivateIP(ip) {
+		if err == nil && ip.IsPrivate() {
 			ret = ip
 			// We've found what we're looking for.
 			return errStopReadingNetstatTable

net/interfaces/interfaces_linux.go

@@ -72,7 +72,7 @@ func likelyHomeRouterIPLinux() (ret netaddr.IP, ok bool) {
 			return nil // ignore error, skip line and keep going
 		}
 		ip := netaddr.IPv4(byte(ipu32), byte(ipu32>>8), byte(ipu32>>16), byte(ipu32>>24))
-		if isPrivateIP(ip) {
+		if ip.IsPrivate() {
 			ret = ip
 		}
 		return nil

net/interfaces/interfaces_linux_test.go

@@ -73,6 +73,32 @@ func TestExtremelyLongProcNetRoute(t *testing.T) {
 	}
 }
 
+// test the specific /proc/net/route path as found on AWS App Runner instances
+func TestAwsAppRunnerDefaultRouteInterface(t *testing.T) {
+	dir := t.TempDir()
+	savedProcNetRoutePath := procNetRoutePath
+	defer func() { procNetRoutePath = savedProcNetRoutePath }()
+	procNetRoutePath = filepath.Join(dir, "CloudRun")
+	buf := []byte("Iface\tDestination\tGateway\tFlags\tRefCnt\tUse\tMetric\tMask\tMTU\tWindow\tIRTT\n" +
+		"eth0\t00000000\tF9AFFEA9\t0003\t0\t0\t0\t00000000\t0\t0\t0\n" +
+		"*\tFEA9FEA9\t00000000\t0005\t0\t0\t0\tFFFFFFFF\t0\t0\t0\n" +
+		"ecs-eth0\t02AAFEA9\t01ACFEA9\t0007\t0\t0\t0\tFFFFFFFF\t0\t0\t0\n" +
+		"ecs-eth0\t00ACFEA9\t00000000\t0001\t0\t0\t0\t00FFFFFF\t0\t0\t0\n" +
+		"eth0\t00AFFEA9\t00000000\t0001\t0\t0\t0\t00FFFFFF\t0\t0\t0\n")
+	err := ioutil.WriteFile(procNetRoutePath, buf, 0644)
+	if err != nil {
+		t.Fatal(err)
+	}
+	got, err := DefaultRouteInterface()
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	if got != "eth0" {
+		t.Fatalf("got %s, want eth0", got)
+	}
+}
+
 func BenchmarkDefaultRouteInterface(b *testing.B) {
 	b.ReportAllocs()
 	for i := 0; i < b.N; i++ {

net/interfaces/interfaces_test.go

@@ -58,6 +58,8 @@ func TestIsUsableV6(t *testing.T) {
 		{"zeros", "0000:0000:0000:0000:0000:0000:0000:0000", false},
 		{"Link Local", "fe80::1", false},
 		{"Global", "2602::1", true},
+		{"IPv4 public", "192.0.2.1", false},
+		{"IPv4 private", "192.168.1.1", false},
 	}
 
 	for _, test := range tests {

net/interfaces/interfaces_windows.go

@@ -9,6 +9,7 @@ import (
 	"log"
 	"net"
 	"net/url"
+	"strings"
 	"syscall"
 	"unsafe"
 
@@ -92,7 +93,7 @@ func likelyHomeRouterIPWindows() (ret netaddr.IP, ok bool) {
 		}
 	}
 
-	if !ret.IsZero() && !isPrivateIP(ret) {
+	if !ret.IsZero() && !ret.IsPrivate() {
 		// Default route has a non-private gateway
 		return netaddr.IP{}, false
 	}
@@ -198,6 +199,10 @@ func getPACWindows() string {
 		}
 		defer globalFree.Call(uintptr(unsafe.Pointer(res)))
 		s := windows.UTF16PtrToString(res)
+		s = strings.TrimSpace(s)
+		if s == "" {
+			return "" // Issue 2357: invalid URL "\n" from winhttp; ignoring
+		}
 		if _, err := url.Parse(s); err != nil {
 			log.Printf("getPACWindows: invalid URL %q from winhttp; ignoring", s)
 			return ""

net/isoping/constants.go

@@ -1,24 +0,0 @@
-// Copyright (c) 2021 Tailscale Inc & AUTHORS All rights reserved.
-// Use of this source code is governed by a BSD-style
-// license that can be found in the LICENSE file.
-package isoping
-
-const (
-	MAGIC                           = 0x424c4950
-	SERVER_PORT                     = ":4948"
-	DEFAULT_PACKETS_PER_SEC float64 = 10.0
-	USEC_PER_CYCLE                  = (10 * 1000 * 1000)
-)
-
-// DIV takes two int64 divides the two and returns a float64
-func DIV(x int64, y int64) float64 {
-	if y == 0 {
-		return 0
-	}
-	return float64(x) / float64(y)
-}
-
-// DIFF takes the difference between two uint32s and returns int32
-func DIFF(x uint32, y uint32) int32 {
-	return int32(uint32(x) - uint32(y))
-}

net/isoping/isoping.go

@@ -1,475 +0,0 @@
-// Copyright (c) 2021 Tailscale Inc & AUTHORS All rights reserved.
-// Use of this source code is governed by a BSD-style
-// license that can be found in the LICENSE file.
-
-// package isoping implements isoping in Go.
-package isoping
-
-import (
-	"bytes"
-	"encoding/binary"
-	"log"
-	"math"
-	"net"
-	"reflect"
-	"time"
-)
-
-type Packet struct {
-	Magic        uint32 // Magic number to reject bogus packets
-	Id           uint32 // Id is a sequential packet id number
-	Txtime       uint32 // Txtime is the transmitter's monotonic time when pkt was sent
-	Clockdiff    uint32 // Clockdiff is an estimate of (transmitter's clk) - (receiver's clk)
-	Usec_per_pkt uint32 // Usec_per_pkt microseconds of delay between packets
-	Num_lost     uint32 // Num_lost is the number of pkts transmitter expected to get but didn't
-	First_ack    uint32 // First_ack is the starting index in acks[] circular buffer
-	Acks         [64]struct {
-		// txtime==0 for empty elements in this array.
-		Id     uint32 // Id field from a received packet
-		Rxtime uint32 // Rxtime is a receiver's monotonic time when pkt arrived
-	}
-}
-
-type Isoping struct {
-	ClockStartTime time.Time    // ClockStartTime is the time the program starts
-	IsServer       bool         // IsServer distinguishes if we are a server or client
-	Conn           *net.UDPConn // Conn is either the server or client's connection
-	Tx             Packet       // Tx is a Packet that will be sent
-	Rx             Packet       // Rx is a Packet that will be sent
-	LastAckInfo    string       // LastAckInfo human readable format of latest ack
-	ListenAddr     *net.UDPAddr // ListenAddr is the address of the listener
-	RemoteAddr     *net.UDPAddr // RemtoteAddr remote UDP address we send to.
-	RxAddr         *net.UDPAddr // RxAddr keeps track of what address we are sending to
-	LastRxAddr     *net.UDPAddr // LastRxAddr keeps track of what we last used
-	Quiet          bool
-
-	printsPerSec   float64
-	packetsPerSec  float64
-	usecPerPkt     int32
-	usecPerPrint   int32
-	nextTxId       uint32
-	nextRxId       uint32
-	nextRxackId    uint32
-	startRtxtime   uint32 // remote's txtime at startup
-	startRxtime    uint32 // local rxtime at startup
-	lastRxtime     uint32 // local rxtime of last received packet
-	minCycleRxdiff int32  // smallest packet delay seen this cycle
-	nextCycle      uint32 // time when next cycle begins
-	now            uint32 // current time
-	nextSend       uint32 // time when we'll send next pkt
-	numLost        uint32 // number of rx packets not received
-	nextTxackIndex int    // next array item to fill in tx.acks
-	lastPrint      uint32 // time of last packet printout
-	latTx          int64
-	latTxMin       int64
-	latTxMax       int64
-	latTxCount     int64
-	latTxSum       int64
-	latTxVarSum    int64
-
-	latRx       int64
-	latRxMin    int64
-	latRxMax    int64
-	latRxCount  int64
-	latRxSum    int64
-	latRxVarSum int64
-}
-
-// Incremental standard deviation calculation, without needing to know the
-// mean in advance.  See:
-// http://mathcentral.uregina.ca/QQ/database/QQ.09.02/carlos1.html
-func onepass_stddev(sumsq int64, sum int64, count int64) float64 {
-	numer := (count * sumsq) - (sum * sum)
-	denom := count * (count - 1)
-	return math.Sqrt(DIV(numer, denom))
-}
-
-// ustimenow subtracts the time since the program started and returns it
-func (srv *Isoping) ustimenow() uint64 {
-	tn := time.Since(srv.ClockStartTime)
-	return uint64(tn.Microseconds())
-}
-
-// Ustime casts the result of ustimenow to uint32 and returns it
-func (srv *Isoping) Ustime() uint32 {
-	return uint32(srv.ustimenow())
-}
-
-// initClock keeps track of when the server/client starts.
-// keeps the exact time and we can subtract from the time
-// to get monotonicClock values
-func (srv *Isoping) initClock() {
-	srv.ClockStartTime = time.Now()
-}
-
-// initClient sets the Isoping.Conn, to the address string otherwise
-// uses [::]:4948 as the default
-func (srv *Isoping) initClient(addressString string) {
-	srv.initClock()
-	srv.IsServer = false
-	udpaddr, err := net.ResolveUDPAddr("udp6", addressString)
-	if err != nil {
-		log.Println(err)
-		addr := "[::]" + SERVER_PORT
-		udpaddr, err = net.ResolveUDPAddr("udp6", addr)
-		if err != nil {
-			log.Println(err)
-			return
-		}
-		log.Printf("Address %v failed to resolve, using %v instead\n", addressString, udpaddr)
-	}
-
-	conn, err := net.DialUDP("udp6", nil, udpaddr)
-	if err != nil {
-		log.Println(err)
-		return
-	}
-
-	srv.RemoteAddr = udpaddr
-	srv.Conn = conn
-}
-
-// initServer sets the Conn field of Isoping, for the listener side.
-func (srv *Isoping) initServer() {
-	srv.initClock()
-	srv.IsServer = true
-	addr, err := net.ResolveUDPAddr("udp6", SERVER_PORT)
-	if err != nil {
-		log.Println(err)
-		return
-	}
-
-	srv.ListenAddr = addr
-	srv.Conn, err = net.ListenUDP("udp", addr)
-	if err != nil {
-		log.Printf("%v\n", err)
-		return
-	}
-}
-
-// initVars initializes a lot of the necessary variables for the calculation
-func (srv *Isoping) initVars() {
-	srv.packetsPerSec = DEFAULT_PACKETS_PER_SEC
-	srv.printsPerSec = -1
-
-	srv.usecPerPkt = int32(1e6 / srv.packetsPerSec)
-	srv.usecPerPrint = 0
-	if srv.usecPerPrint > 0 {
-		srv.usecPerPrint = int32(1e6 / srv.printsPerSec)
-	}
-	log.Println("UsecPerPkt : ", srv.usecPerPkt)
-	log.Println("UsecPerPrint : ", srv.usecPerPrint)
-
-	srv.nextTxId = 1
-	srv.nextRxId = 0
-
-	srv.nextRxackId = 0
-	srv.startRtxtime = 0
-	srv.startRxtime = 0
-	srv.lastRxtime = 0
-
-	srv.minCycleRxdiff = 0
-	srv.nextCycle = 0
-	srv.now = srv.Ustime()
-	srv.nextSend = srv.now + uint32(srv.usecPerPkt)
-	srv.numLost = 0
-	srv.nextTxackIndex = 0
-	srv.Tx = Packet{}
-	srv.Rx = Packet{}
-
-	srv.LastAckInfo = ""
-	srv.lastPrint = srv.now - uint32(srv.usecPerPkt)
-	srv.latTx, srv.latTxMin, srv.latTxMax = 0, 0x7fffffff, 0
-	srv.latTxCount, srv.latTxMin, srv.latTxVarSum = 0, 0, 0
-
-	srv.latRx, srv.latRxMin, srv.latRxMax = 0, 0x7fffffff, 0
-	srv.latRxCount, srv.latRxMin, srv.latRxVarSum = 0, 0, 0
-}
-
-// generateInitialPacket generates the inital packet Tx
-func (srv *Isoping) generateInitialPacket() (*bytes.Buffer, error) {
-	srv.Tx.Magic = MAGIC
-	srv.Tx.Id = srv.nextTxId
-	srv.nextTxId++
-	srv.Tx.Txtime = srv.nextSend
-	srv.Tx.Usec_per_pkt = uint32(srv.usecPerPkt)
-	srv.Tx.Clockdiff = 0
-	if srv.startRtxtime > 0 {
-		srv.Tx.Clockdiff = srv.startRtxtime - srv.startRxtime
-		log.Println("SETCLOCKDIFF to", srv.Rx.Clockdiff)
-	}
-	srv.Tx.Num_lost = srv.numLost
-	srv.Tx.First_ack = uint32(srv.nextTxackIndex)
-
-	// Setup the Tx to be sent from either server of client
-	buf := new(bytes.Buffer)
-	return buf, binary.Write(buf, binary.BigEndian, srv.Tx)
-}
-
-// Start starts the Isoping instance, if an address is passed a client is started
-// with that address, otherwise a server will start.
-func (srv *Isoping) Start(address ...string) {
-	if len(address) > 0 {
-		srv.initClient(address[0])
-	} else {
-		srv.initServer()
-	}
-	srv.initVars()
-}
-func (srv *Isoping) MainTest() {
-	for {
-		srv.initTimer()
-	}
-}
-
-func (srv *Isoping) initTimer() {
-	tv := time.Duration(0)
-	if DIFF(srv.nextSend, srv.now) < 0 {
-		log.Printf("Set to %v us\n", tv.Microseconds())
-	} else {
-		tv = time.Microsecond * time.Duration(DIFF(srv.nextSend, srv.now))
-		log.Printf("Set to %v us\n", tv.Microseconds())
-	}
-	log.Println(tv)
-
-	// emulate the select with timeout of tv.
-	if srv.RemoteAddr != nil {
-		deadline := time.Now().Add(time.Duration(tv.Microseconds()) * time.Microsecond)
-		log.Println("TIMEOUT DURATION : ", deadline)
-		err := srv.Conn.SetReadDeadline(deadline)
-		if err != nil {
-			log.Println(err)
-			return
-		}
-	} else {
-		// Reset the timeout, we will have no timeout in this case.
-		err := srv.Conn.SetReadDeadline(time.Time{})
-		if err != nil {
-			log.Println(err)
-			return
-		}
-	}
-	srv.now = srv.Ustime()
-}
-
-func (srv *Isoping) MainLoop() {
-	for {
-		srv.initTimer()
-		log.Printf("%d - %d = %d\n", srv.now, srv.nextSend, DIFF(srv.now, srv.nextSend))
-		if srv.RemoteAddr != nil && DIFF(srv.now, srv.nextSend) >= 0 {
-			// Check if it is a server or not
-			buf, err := srv.generateInitialPacket()
-			if err != nil {
-				log.Println(err)
-				continue
-			}
-			if srv.IsServer {
-				log.Println("Isoping Server Found.")
-				// Attempt to send the srv.Tx to the client
-				n, err := srv.Conn.WriteToUDP(buf.Bytes(), srv.RemoteAddr)
-				if err != nil {
-					log.Println(err)
-				}
-				log.Printf("Sent %v bytes to %v\n", n, srv.RemoteAddr)
-			} else {
-				log.Println("Isoping Client Found.")
-				n, err := srv.Conn.Write(buf.Bytes())
-				if err != nil {
-					log.Println(err)
-				}
-				log.Printf("Sent %v bytes to %v\n", n, srv.RemoteAddr)
-			}
-
-			if srv.IsServer && DIFF(srv.now, srv.lastRxtime) > 60*1000*1000 {
-				log.Println("client disconnected")
-				srv.RemoteAddr = nil
-			}
-			srv.nextSend += uint32(srv.usecPerPkt)
-		}
-
-		// handle incoming packet
-		log.Println("BEFORE READ")
-		p := make([]byte, binary.Size(srv.Rx))
-		got, rxaddr, err := srv.Conn.ReadFromUDP(p)
-		if err != nil {
-			// log.Println("READ error : ", err)
-			continue
-		} else {
-			log.Println("SUCC")
-		}
-		srv.RxAddr = rxaddr
-		log.Println("AFTER READ")
-		// Copy the data received into the Rx struct.
-		buffer := bytes.NewBuffer(p)
-		err = binary.Read(buffer, binary.BigEndian, &srv.Rx)
-		if err != nil {
-			log.Println("BINARY READ err: ", err)
-			continue
-		}
-
-		if got != binary.Size(srv.Rx) || srv.Rx.Magic != MAGIC {
-			log.Println("received Rx is not proper")
-			continue
-		}
-
-		log.Printf("%+v\n", srv.Rx)
-
-		if srv.IsServer {
-			if srv.RemoteAddr == nil || !reflect.DeepEqual(srv.RxAddr, srv.LastRxAddr) {
-				log.Printf("new client connected %v\n", srv.RxAddr)
-			}
-			srv.LastRxAddr = srv.RxAddr
-			log.Println(srv.LastRxAddr, srv.RxAddr)
-			srv.RemoteAddr = srv.LastRxAddr
-			srv.nextSend = srv.now + 10*1000
-			srv.nextTxId = 1
-
-			srv.nextRxId, srv.nextRxackId = 0, 0
-			srv.startRtxtime, srv.startRxtime = 0, 0
-			srv.numLost = 0
-			srv.nextTxackIndex = 0
-			srv.usecPerPkt = int32(srv.Rx.Usec_per_pkt)
-			// memset Tx to reset it.
-			srv.Tx = Packet{}
-			log.Println("Remote address for server : ", srv.RemoteAddr)
-		}
-
-		txtime := srv.Rx.Txtime
-		rxtime := srv.now
-		id := srv.Rx.Id
-
-		if srv.nextRxId == 0 {
-			log.Println(txtime, id)
-			srv.startRtxtime = txtime - id*uint32(srv.usecPerPkt)
-
-			srv.startRxtime = rxtime - id*uint32(srv.usecPerPkt)
-
-			srv.minCycleRxdiff = 0
-			srv.nextRxId = id
-			srv.nextCycle = srv.now + USEC_PER_CYCLE
-		}
-		log.Println("tmpdiff break")
-		tmpdiff := DIFF(id, srv.nextRxId)
-		if tmpdiff > 0 {
-			log.Printf("lost %v  expected=%v  got=%v\n",
-				int64(tmpdiff), int64(srv.nextRxId), int64(id))
-			srv.numLost += uint32(tmpdiff)
-			srv.nextRxId += uint32(tmpdiff) + 1
-		} else if tmpdiff == 0 {
-			log.Println("tmpdiff == 0 -> nextRxId ++")
-			srv.nextRxId++
-		} else if tmpdiff < 0 {
-			log.Println("out-of-order packets? %ld\n", int64(tmpdiff))
-		}
-
-		tmpdiff = DIFF(rxtime, srv.startRxtime+id*uint32(srv.usecPerPkt))
-		if tmpdiff < -20 {
-			log.Printf("time paradox: backsliding start by %v usec\n", tmpdiff)
-			srv.startRxtime = rxtime - id*uint32(srv.usecPerPkt)
-		}
-		rxdiff := DIFF(rxtime, srv.startRxtime+id*uint32(srv.usecPerPkt))
-
-		clockdiff := DIFF(srv.startRxtime, srv.startRtxtime)
-		rtt := clockdiff + int32(srv.Rx.Clockdiff)
-
-		// Casting issue here may exist
-		offset := DIFF(uint32(clockdiff), uint32(rtt/2))
-		if srv.Rx.Clockdiff == 0 {
-			srv.lastPrint = srv.now - uint32(srv.usecPerPrint) + 1
-		} else {
-			srv.latRxCount++
-			srv.latRx = int64(rxdiff) + int64(rtt/2)
-			// Take the mins and maxes.
-			if srv.latRxMin > srv.latRx {
-				srv.latRxMin = srv.latRx
-			}
-
-			if srv.latRxMax < srv.latRx {
-				srv.latRxMax = srv.latRx
-			}
-
-			srv.latRxSum += srv.latRx
-			srv.latRxVarSum += srv.latRx * srv.latRx
-		}
-		okToPrint := !srv.Quiet && DIFF(srv.now, srv.lastPrint) >= srv.usecPerPrint
-		log.Printf("OKTOPRINT : %v, QUIET : %v\n", okToPrint, srv.Quiet)
-		log.Printf("%v - %v = %v\n", srv.now, srv.lastPrint, DIFF(srv.now, srv.lastPrint))
-		log.Printf("usecPerPrint : %v", srv.usecPerPrint)
-
-		// Print the information.
-		if okToPrint {
-			ackinfo := srv.LastAckInfo
-			msRx := float64((rxdiff + rtt/2) / 1000.0)
-			min := float64((rtt / 2) / 1000.0)
-			rxNumLost := int64(srv.Rx.Num_lost)
-			nextTxId := srv.nextTxId - 1
-			numLost := int64(srv.numLost)
-			nextRxId := int64(srv.nextRxId - 1)
-			log.Printf("%12s  %6.1f ms rx  (min=%.1f)  loss: %v/%v tx  %v/%v rx\n",
-				ackinfo, msRx, min, rxNumLost, nextTxId, numLost, nextRxId)
-			srv.lastPrint = srv.now
-		}
-
-		if rxdiff < srv.minCycleRxdiff {
-			srv.minCycleRxdiff = rxdiff
-		}
-
-		if DIFF(srv.now, srv.nextCycle) >= 0 {
-			if srv.minCycleRxdiff > 0 {
-				log.Printf("clock skew: sliding start by %v usec\n", srv.minCycleRxdiff)
-				srv.startRxtime += uint32(srv.minCycleRxdiff)
-			}
-			srv.minCycleRxdiff = 0x7fffffff
-			srv.nextCycle += uint32(srv.minCycleRxdiff)
-		}
-		log.Println("SCHEDULING")
-
-		srv.Tx.Acks[srv.nextTxackIndex].Id = id
-		srv.Tx.Acks[srv.nextTxackIndex].Rxtime = rxtime
-		srv.nextTxackIndex = (srv.nextTxackIndex + 1) % len(srv.Tx.Acks)
-
-		first_ack := uint32(srv.Rx.First_ack)
-		log.Println("FOR START")
-		for i := uint32(0); i < uint32(len(srv.Rx.Acks)); i++ {
-			var acki uint32 = (first_ack + i) % uint32(len(srv.Rx.Acks))
-			var ackid uint32 = srv.Rx.Acks[acki].Id
-			if ackid == 0 {
-				continue
-			}
-			if DIFF(ackid, srv.nextRxackId) >= 0 {
-				log.Println("EXPECTED AN ACK")
-
-				startTxTime := srv.nextSend - srv.nextTxId*uint32(srv.usecPerPkt)
-				txtime := startTxTime + ackid*uint32(srv.usecPerPkt)
-				rrxtime := srv.Rx.Acks[acki].Rxtime
-				rxtime := rrxtime + uint32(offset)
-
-				txdiff := DIFF(rxtime, txtime)
-				if srv.usecPerPkt <= 0 && len(srv.LastAckInfo) > 0 {
-					log.Printf("%12s\n", srv.LastAckInfo)
-				}
-				if len(srv.LastAckInfo) == 0 {
-					//populate it
-				}
-				srv.nextRxackId = ackid + 1
-				srv.latTxCount++
-				srv.latTx = int64(txdiff)
-				if srv.latTxMin > srv.latTx {
-					srv.latTxMin = srv.latTx
-				}
-
-				if srv.latTxMax < srv.latTx {
-					srv.latTxMax = srv.latTx
-				}
-
-				srv.latTxSum += srv.latTx
-				srv.latTxVarSum += srv.latTx * srv.latTx
-			}
-		}
-		srv.lastRxtime = rxtime
-	}
-}
-
-func (srv *Isoping) printResult() {
-	log.Printf("\n---\n")
-}

net/isoping/isoping_test.go

@@ -1,105 +0,0 @@
-// Copyright (c) 2021 Tailscale Inc & AUTHORS All rights reserved.
-// Use of this source code is governed by a BSD-style
-// license that can be found in the LICENSE file.
-package isoping
-
-import (
-	"bytes"
-	"encoding/binary"
-	"math"
-	"testing"
-)
-
-// Tests if our stddev calculation is within reason
-// Must do some rounding to a certain significant digit
-// Currently only need 6 digits for the testing.
-func sigDigs(x float64, digs int) float64 {
-	return math.Round(x*math.Pow10(digs)) / math.Pow10(digs)
-}
-
-// TestOnepass_stddev tests if the function receives the same answer as in
-// the C implementation of this function.
-func TestOnepass_stddev(t *testing.T) {
-	t.Parallel()
-
-	answer := sigDigs(onepass_stddev(12, 2, 3), 6)
-	expected := 2.309401
-	answer2 := sigDigs(onepass_stddev(12023232232, 212, 321), 6)
-	expected2 := 6129.649279
-	if answer != expected {
-		t.Errorf("got %v, expected %v", answer, expected)
-	}
-	if answer2 != expected2 {
-		t.Errorf("got %v, expected %v", answer2, expected2)
-	}
-}
-
-// TestUstimeCast tests if casting was correct
-// sanity check, probably will be removed for redundancy
-func TestUstimeCast(t *testing.T) {
-	t.Parallel()
-
-	var num uint64 = 11471851221
-	var expected uint32 = 2881916629
-	if uint32(num) != expected {
-		t.Errorf("expected %v, got : %v", expected, uint32(num))
-	}
-}
-
-// TestValidInitialPacket will send a packet via UDP, and check if it matches
-// The size and the Magic number field that needs to be equal.
-// This mocks the initial packet sent in Isoping.
-func TestValidInitialPacket(t *testing.T) {
-	client := Isoping{IsServer: false}
-	client.Start("[::]:4948")
-
-	server := Isoping{IsServer: true}
-	server.Start()
-	defer server.Conn.Close()
-
-	buf, err := client.generateInitialPacket()
-	if err != nil {
-		t.Error(err)
-	}
-
-	// Client writes to the server, server tries to read it.
-	p := make([]byte, binary.Size(server.Rx))
-	if _, err := client.Conn.Write(buf.Bytes()); err != nil {
-		t.Error(err)
-	}
-
-	got, rxaddr, err := server.Conn.ReadFromUDP(p)
-	if err != nil {
-		t.Error(err)
-	}
-
-	buffer := bytes.NewBuffer(p)
-	defer buffer.Reset()
-
-	err = binary.Read(buffer, binary.BigEndian, &server.Rx)
-	if err != nil {
-		t.Error(err)
-	}
-
-	if got != binary.Size(server.Rx) || server.Rx.Magic != MAGIC {
-		t.Error("received Rx is not proper")
-	}
-
-	t.Logf("Proper Packet received from %v\n", rxaddr)
-}
-
-func TestMainLoop(t *testing.T) {
-	server := Isoping{}
-	server.Start()
-	defer server.Conn.Close()
-	server.MainLoop()
-	// server.MainTest()
-}
-
-func TestStartClient(t *testing.T) {
-	client := Isoping{}
-	client.Start("[::]:4948")
-	defer client.Conn.Close()
-	client.MainLoop()
-	// client.MainTest()
-}

net/netns/netns.go

@@ -17,6 +17,8 @@ package netns
 import (
 	"context"
 	"net"
+
+	"inet.af/netaddr"
 )
 
 // Listener returns a new net.Listener with its Control hook func
@@ -66,3 +68,19 @@ type Dialer interface {
 	Dial(network, address string) (net.Conn, error)
 	DialContext(ctx context.Context, network, address string) (net.Conn, error)
 }
+
+func isLocalhost(addr string) bool {
+	host, _, err := net.SplitHostPort(addr)
+	if err != nil {
+		// error means the string didn't contain a port number, so use the string directly
+		host = addr
+	}
+
+	// localhost6 == RedHat /etc/hosts for ::1, ip6-loopback & ip6-localhost == Debian /etc/hosts for ::1
+	if host == "localhost" || host == "localhost6" || host == "ip6-loopback" || host == "ip6-localhost" {
+		return true
+	}
+
+	ip, _ := netaddr.ParseIP(host)
+	return ip.IsLoopback()
+}

net/netns/netns_linux.go

@@ -7,8 +7,8 @@
 package netns
 
 import (
-	"flag"
 	"fmt"
+	"net"
 	"os"
 	"os/exec"
 	"sync"
@@ -26,32 +26,54 @@ import (
 // wgengine/router/router_linux.go.
 const tailscaleBypassMark = 0x80000
 
-// ipRuleOnce is the sync.Once & cached value for ipRuleAvailable.
-var ipRuleOnce struct {
+// socketMarkWorksOnce is the sync.Once & cached value for useSocketMark.
+var socketMarkWorksOnce struct {
 	sync.Once
 	v bool
 }
 
-// ipRuleAvailable reports whether the 'ip rule' command works.
-// If it doesn't, we have to use SO_BINDTODEVICE on our sockets instead.
-func ipRuleAvailable() bool {
-	ipRuleOnce.Do(func() {
-		ipRuleOnce.v = exec.Command("ip", "rule").Run() == nil
+// socketMarkWorks returns whether SO_MARK works.
+func socketMarkWorks() bool {
+	addr, err := net.ResolveUDPAddr("udp", "127.0.0.1:1")
+	if err != nil {
+		return true // unsure, returning true does the least harm.
+	}
+
+	sConn, err := net.DialUDP("udp", nil, addr)
+	if err != nil {
+		return true // unsure, return true
+	}
+	defer sConn.Close()
+
+	rConn, err := sConn.SyscallConn()
+	if err != nil {
+		return true // unsure, return true
+	}
+
+	var sockErr error
+	err = rConn.Control(func(fd uintptr) {
+		sockErr = setBypassMark(fd)
 	})
-	return ipRuleOnce.v
+	if err != nil || sockErr != nil {
+		return false
+	}
+
+	return true
+}
+
+// useSocketMark reports whether SO_MARK works.
+// If it doesn't, we have to use SO_BINDTODEVICE on our sockets instead.
+func useSocketMark() bool {
+	socketMarkWorksOnce.Do(func() {
+		ipRuleWorks := exec.Command("ip", "rule").Run() == nil
+		socketMarkWorksOnce.v = ipRuleWorks && socketMarkWorks()
+	})
+	return socketMarkWorksOnce.v
 }
 
 // ignoreErrors returns true if we should ignore setsocketopt errors in
 // this instance.
 func ignoreErrors() bool {
-	// If we're in a test, ignore errors. Assume the test knows
-	// what it's doing and will do its own skips or permission
-	// checks if it's setting up a world that needs netns to work.
-	// But by default, assume that tests don't need netns and it's
-	// harmless to ignore the sockopts failing.
-	if flag.CommandLine.Lookup("test.v") != nil {
-		return true
-	}
 	if os.Getuid() != 0 {
 		// only root can manipulate these socket flags
 		return true
@@ -64,9 +86,14 @@ func ignoreErrors() bool {
 // It's intentionally the same signature as net.Dialer.Control
 // and net.ListenConfig.Control.
 func control(network, address string, c syscall.RawConn) error {
+	if isLocalhost(address) {
+		// Don't bind to an interface for localhost connections.
+		return nil
+	}
+
 	var sockErr error
 	err := c.Control(func(fd uintptr) {
-		if ipRuleAvailable() {
+		if useSocketMark() {
 			sockErr = setBypassMark(fd)
 		} else {
 			sockErr = bindToDevice(fd)

net/netns/netns_linux_test.go

@@ -49,3 +49,9 @@ func TestBypassMarkInSync(t *testing.T) {
 	}
 	t.Errorf("tailscaleBypassMark not found in router_linux.go")
 }
+
+func TestSocketMarkWorks(t *testing.T) {
+	_ = socketMarkWorks()
+	// we cannot actually assert whether the test runner has SO_MARK available
+	// or not, as we don't know. We're just checking that it doesn't panic.
+}

net/netns/netns_test.go

@@ -40,3 +40,40 @@ func TestDial(t *testing.T) {
 	defer c.Close()
 	t.Logf("got addr %v", c.RemoteAddr())
 }
+
+func TestIsLocalhost(t *testing.T) {
+	tests := []struct {
+		name string
+		host string
+		want bool
+	}{
+		{"IPv4 loopback", "127.0.0.1", true},
+		{"IPv4 !loopback", "192.168.0.1", false},
+		{"IPv4 loopback with port", "127.0.0.1:1", true},
+		{"IPv4 !loopback with port", "192.168.0.1:1", false},
+		{"IPv4 unspecified", "0.0.0.0", false},
+		{"IPv4 unspecified with port", "0.0.0.0:1", false},
+		{"IPv6 loopback", "::1", true},
+		{"IPv6 !loopback", "2001:4860:4860::8888", false},
+		{"IPv6 loopback with port", "[::1]:1", true},
+		{"IPv6 !loopback with port", "[2001:4860:4860::8888]:1", false},
+		{"IPv6 unspecified", "::", false},
+		{"IPv6 unspecified with port", "[::]:1", false},
+		{"empty", "", false},
+		{"hostname", "example.com", false},
+		{"localhost", "localhost", true},
+		{"localhost6", "localhost6", true},
+		{"localhost with port", "localhost:1", true},
+		{"localhost6 with port", "localhost6:1", true},
+		{"ip6-localhost", "ip6-localhost", true},
+		{"ip6-localhost with port", "ip6-localhost:1", true},
+		{"ip6-loopback", "ip6-loopback", true},
+		{"ip6-loopback with port", "ip6-loopback:1", true},
+	}
+
+	for _, test := range tests {
+		if got := isLocalhost(test.host); got != test.want {
+			t.Errorf("isLocalhost(%q) = %v, want %v", test.name, got, test.want)
+		}
+	}
+}

net/portmapper/disabled_stubs.go

@@ -0,0 +1,25 @@
+// Copyright (c) 2021 Tailscale Inc & AUTHORS All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+// +build ios
+// (https://github.com/tailscale/tailscale/issues/2495)
+
+package portmapper
+
+import (
+	"context"
+
+	"inet.af/netaddr"
+)
+
+type upnpClient interface{}
+
+func (c *Client) getUPnPPortMapping(
+	ctx context.Context,
+	gw netaddr.IP,
+	internal netaddr.IPPort,
+	prevPort uint16,
+) (external netaddr.IPPort, ok bool) {
+	return netaddr.IPPort{}, false
+}

net/portmapper/portmapper.go

@@ -2,8 +2,8 @@
 // Use of this source code is governed by a BSD-style
 // license that can be found in the LICENSE file.
 
-// Package portmapper is a UDP port mapping client. It currently only does
-// NAT-PMP, but will likely do UPnP and perhaps PCP later.
+// Package portmapper is a UDP port mapping client. It currently allows for mapping over
+// NAT-PMP and UPnP, but will perhaps do PCP later.
 package portmapper
 
 import (
@@ -14,6 +14,7 @@ import (
 	"fmt"
 	"io"
 	"net"
+	"net/http"
 	"sync"
 	"time"
 
@@ -44,9 +45,15 @@ const trustServiceStillAvailableDuration = 10 * time.Minute
 type Client struct {
 	logf         logger.Logf
 	ipAndGateway func() (gw, ip netaddr.IP, ok bool)
+	onChange     func() // or nil
 
 	mu sync.Mutex // guards following, and all fields thereof
 
+	// runningCreate is whether we're currently working on creating
+	// a port mapping (whether GetCachedMappingOrStartCreatingOne kicked
+	// off a createMapping goroutine).
+	runningCreate bool
+
 	lastMyIP netaddr.IP
 	lastGW   netaddr.IP
 	closed   bool
@@ -57,29 +64,52 @@ type Client struct {
 	pmpPubIPTime time.Time  // time pmpPubIP last verified
 	pmpLastEpoch uint32
 
-	pcpSawTime  time.Time // time we last saw PCP was available
-	uPnPSawTime time.Time // time we last saw UPnP was available
+	pcpSawTime time.Time // time we last saw PCP was available
 
-	localPort  uint16
-	pmpMapping *pmpMapping // non-nil if we have a PMP mapping
+	uPnPSawTime    time.Time         // time we last saw UPnP was available
+	uPnPMeta       uPnPDiscoResponse // Location header from UPnP UDP discovery response
+	uPnPHTTPClient *http.Client      // nil until needed
+
+	localPort uint16
+
+	mapping mapping // non-nil if we have a mapping
+}
+
+// mapping represents a created port-mapping over some protocol.  It specifies a lease duration,
+// how to release the mapping, and whether the map is still valid.
+//
+// After a mapping is created, it should be immutable, and thus reads should be safe across
+// concurrent goroutines.
+type mapping interface {
+	// Release will attempt to unmap the established port mapping. It will block until completion,
+	// but can be called asynchronously. Release should be idempotent, and thus even if called
+	// multiple times should not cause additional side-effects.
+	Release(context.Context)
+	// goodUntil will return the lease time that the mapping is valid for.
+	GoodUntil() time.Time
+	// renewAfter returns the earliest time that the mapping should be renewed.
+	RenewAfter() time.Time
+	// externalIPPort indicates what port the mapping can be reached from on the outside.
+	External() netaddr.IPPort
 }
 
 // HaveMapping reports whether we have a current valid mapping.
 func (c *Client) HaveMapping() bool {
 	c.mu.Lock()
 	defer c.mu.Unlock()
-	return c.pmpMapping != nil && c.pmpMapping.useUntil.After(time.Now())
+	return c.mapping != nil && c.mapping.GoodUntil().After(time.Now())
 }
 
 // pmpMapping is an already-created PMP mapping.
 //
 // All fields are immutable once created.
 type pmpMapping struct {
-	gw       netaddr.IP
-	external netaddr.IPPort
-	internal netaddr.IPPort
-	useUntil time.Time // the mapping's lifetime minus renewal interval
-	epoch    uint32
+	gw         netaddr.IP
+	external   netaddr.IPPort
+	internal   netaddr.IPPort
+	renewAfter time.Time // the time at which we want to renew the mapping
+	goodUntil  time.Time // the mapping's total lifetime
+	epoch      uint32
 }
 
 // externalValid reports whether m.external is valid, with both its IP and Port populated.
@@ -87,9 +117,13 @@ func (m *pmpMapping) externalValid() bool {
 	return !m.external.IP().IsZero() && m.external.Port() != 0
 }
 
-// release does a best effort fire-and-forget release of the PMP mapping m.
-func (m *pmpMapping) release() {
-	uc, err := netns.Listener().ListenPacket(context.Background(), "udp4", ":0")
+func (p *pmpMapping) GoodUntil() time.Time     { return p.goodUntil }
+func (p *pmpMapping) RenewAfter() time.Time    { return p.renewAfter }
+func (p *pmpMapping) External() netaddr.IPPort { return p.external }
+
+// Release does a best effort fire-and-forget release of the PMP mapping m.
+func (m *pmpMapping) Release(ctx context.Context) {
+	uc, err := netns.Listener().ListenPacket(ctx, "udp4", ":0")
 	if err != nil {
 		return
 	}
@@ -99,10 +133,15 @@ func (m *pmpMapping) release() {
 }
 
 // NewClient returns a new portmapping client.
-func NewClient(logf logger.Logf) *Client {
+//
+// The optional onChange argument specifies a func to run in a new
+// goroutine whenever the port mapping status has changed. If nil,
+// it doesn't make a callback.
+func NewClient(logf logger.Logf, onChange func()) *Client {
 	return &Client{
 		logf:         logf,
 		ipAndGateway: interfaces.LikelyHomeRouterIP,
+		onChange:     onChange,
 	}
 }
 
@@ -154,7 +193,6 @@ func (c *Client) gatewayAndSelfIP() (gw, myIP netaddr.IP, ok bool) {
 		gw = netaddr.IP{}
 		myIP = netaddr.IP{}
 	}
-
 	c.mu.Lock()
 	defer c.mu.Unlock()
 
@@ -167,11 +205,11 @@ func (c *Client) gatewayAndSelfIP() (gw, myIP netaddr.IP, ok bool) {
 }
 
 func (c *Client) invalidateMappingsLocked(releaseOld bool) {
-	if c.pmpMapping != nil {
+	if c.mapping != nil {
 		if releaseOld {
-			c.pmpMapping.release()
+			c.mapping.Release(context.Background())
 		}
-		c.pmpMapping = nil
+		c.mapping = nil
 	}
 	c.pmpPubIP = netaddr.IP{}
 	c.pmpPubIPTime = time.Time{}
@@ -221,8 +259,7 @@ func closeCloserOnContextDone(ctx context.Context, c io.Closer) (stop func()) {
 	return func() { close(stopWaitDone) }
 }
 
-// NoMappingError is returned by CreateOrGetMapping when no NAT
-// mapping could be returned.
+// NoMappingError is returned when no NAT mapping could be done.
 type NoMappingError struct {
 	err error
 }
@@ -238,25 +275,76 @@ func IsNoMappingError(err error) bool {
 
 var (
 	ErrNoPortMappingServices = errors.New("no port mapping services were found")
-	ErrGatewayNotFound       = errors.New("failed to look up gateway address")
+	ErrGatewayRange          = errors.New("skipping portmap; gateway range likely lacks support")
 )
 
-// CreateOrGetMapping either creates a new mapping or returns a cached
+// GetCachedMappingOrStartCreatingOne quickly returns with our current cached portmapping, if any.
+// If there's not one, it starts up a background goroutine to create one.
+// If the background goroutine ends up creating one, the onChange hook registered with the
+// NewClient constructor (if any) will fire.
+func (c *Client) GetCachedMappingOrStartCreatingOne() (external netaddr.IPPort, ok bool) {
+	c.mu.Lock()
+	defer c.mu.Unlock()
+
+	// Do we have an existing mapping that's valid?
+	now := time.Now()
+	if m := c.mapping; m != nil {
+		if now.Before(m.GoodUntil()) {
+			if now.After(m.RenewAfter()) {
+				c.maybeStartMappingLocked()
+			}
+			return m.External(), true
+		}
+	}
+
+	c.maybeStartMappingLocked()
+	return netaddr.IPPort{}, false
+}
+
+// maybeStartMappingLocked starts a createMapping goroutine up, if one isn't already running.
+//
+// c.mu must be held.
+func (c *Client) maybeStartMappingLocked() {
+	if !c.runningCreate {
+		c.runningCreate = true
+		go c.createMapping()
+	}
+}
+
+func (c *Client) createMapping() {
+	ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
+	defer cancel()
+
+	defer func() {
+		c.mu.Lock()
+		defer c.mu.Unlock()
+		c.runningCreate = false
+	}()
+
+	if _, err := c.createOrGetMapping(ctx); err == nil && c.onChange != nil {
+		go c.onChange()
+	} else if err != nil && !IsNoMappingError(err) {
+		c.logf("createOrGetMapping: %v", err)
+	}
+}
+
+// createOrGetMapping either creates a new mapping or returns a cached
 // valid one.
 //
 // If no mapping is available, the error will be of type
 // NoMappingError; see IsNoMappingError.
-func (c *Client) CreateOrGetMapping(ctx context.Context) (external netaddr.IPPort, err error) {
+func (c *Client) createOrGetMapping(ctx context.Context) (external netaddr.IPPort, err error) {
 	gw, myIP, ok := c.gatewayAndSelfIP()
 	if !ok {
-		return netaddr.IPPort{}, NoMappingError{ErrGatewayNotFound}
+		return netaddr.IPPort{}, NoMappingError{ErrGatewayRange}
 	}
 
 	c.mu.Lock()
 	localPort := c.localPort
+	internalAddr := netaddr.IPPortFrom(myIP, localPort)
 	m := &pmpMapping{
 		gw:       gw,
-		internal: netaddr.IPPortFrom(myIP, localPort),
+		internal: internalAddr,
 	}
 
 	// prevPort is the port we had most previously, if any. We try
@@ -265,13 +353,13 @@ func (c *Client) CreateOrGetMapping(ctx context.Context) (external netaddr.IPPor
 
 	// Do we have an existing mapping that's valid?
 	now := time.Now()
-	if m := c.pmpMapping; m != nil {
-		if now.Before(m.useUntil) {
+	if m := c.mapping; m != nil {
+		if now.Before(m.RenewAfter()) {
 			defer c.mu.Unlock()
-			return m.external, nil
+			return m.External(), nil
 		}
 		// The mapping might still be valid, so just try to renew it.
-		prevPort = m.external.Port()
+		prevPort = m.External().Port()
 	}
 
 	// If we just did a Probe (e.g. via netchecker) but didn't
@@ -283,6 +371,10 @@ func (c *Client) CreateOrGetMapping(ctx context.Context) (external netaddr.IPPor
 	}
 	if c.lastProbe.After(now.Add(-5*time.Second)) && !haveRecentPMP {
 		c.mu.Unlock()
+		// fallback to UPnP portmapping
+		if mapping, ok := c.getUPnPPortMapping(ctx, gw, internalAddr, prevPort); ok {
+			return mapping, nil
+		}
 		return netaddr.IPPort{}, NoMappingError{ErrNoPortMappingServices}
 	}
 
@@ -320,6 +412,10 @@ func (c *Client) CreateOrGetMapping(ctx context.Context) (external netaddr.IPPor
 			if ctx.Err() == context.Canceled {
 				return netaddr.IPPort{}, err
 			}
+			// fallback to UPnP portmapping
+			if mapping, ok := c.getUPnPPortMapping(ctx, gw, internalAddr, prevPort); ok {
+				return mapping, nil
+			}
 			return netaddr.IPPort{}, NoMappingError{ErrNoPortMappingServices}
 		}
 		srcu := srci.(*net.UDPAddr)
@@ -342,8 +438,9 @@ func (c *Client) CreateOrGetMapping(ctx context.Context) (external netaddr.IPPor
 			if pres.OpCode == pmpOpReply|pmpOpMapUDP {
 				m.external = m.external.WithPort(pres.ExternalPort)
 				d := time.Duration(pres.MappingValidSeconds) * time.Second
-				d /= 2 // renew in half the time
-				m.useUntil = time.Now().Add(d)
+				now := time.Now()
+				m.goodUntil = now.Add(d)
+				m.renewAfter = now.Add(d / 2) // renew in half the time
 				m.epoch = pres.SecondsSinceEpoch
 			}
 		}
@@ -351,7 +448,7 @@ func (c *Client) CreateOrGetMapping(ctx context.Context) (external netaddr.IPPor
 		if m.externalValid() {
 			c.mu.Lock()
 			defer c.mu.Unlock()
-			c.pmpMapping = m
+			c.mapping = m
 			return m.external, nil
 		}
 	}
@@ -448,7 +545,7 @@ type ProbeResult struct {
 func (c *Client) Probe(ctx context.Context) (res ProbeResult, err error) {
 	gw, myIP, ok := c.gatewayAndSelfIP()
 	if !ok {
-		return res, ErrGatewayNotFound
+		return res, ErrGatewayRange
 	}
 	defer func() {
 		if err == nil {
@@ -509,9 +606,15 @@ func (c *Client) Probe(ctx context.Context) (res ProbeResult, err error) {
 		switch port {
 		case upnpPort:
 			if mem.Contains(mem.B(buf[:n]), mem.S(":InternetGatewayDevice:")) {
+				meta, err := parseUPnPDiscoResponse(buf[:n])
+				if err != nil {
+					c.logf("unrecognized UPnP discovery response; ignoring")
+				}
+				// log.Printf("UPnP reply %+v, %q", meta, buf[:n])
 				res.UPnP = true
 				c.mu.Lock()
 				c.uPnPSawTime = time.Now()
+				c.uPnPMeta = meta
 				c.mu.Unlock()
 			}
 		case pcpPort: // same as pmpPort
@@ -625,6 +728,8 @@ func parsePCPResponse(b []byte) (res pcpResponse, ok bool) {
 	return res, true
 }
 
+var pmpReqExternalAddrPacket = []byte{0, 0} // version 0, opcode 0 = "Public address request"
+
 const (
 	upnpPort = 1900
 )
@@ -634,5 +739,3 @@ var uPnPPacket = []byte("M-SEARCH * HTTP/1.1\r\n" +
 	"ST: ssdp:all\r\n" +
 	"MAN: \"ssdp:discover\"\r\n" +
 	"MX: 2\r\n\r\n")
-
-var pmpReqExternalAddrPacket = []byte{0, 0} // version 0, opcode 0 = "Public address request"

net/portmapper/portmapper_test.go

@@ -16,13 +16,14 @@ func TestCreateOrGetMapping(t *testing.T) {
 	if v, _ := strconv.ParseBool(os.Getenv("HIT_NETWORK")); !v {
 		t.Skip("skipping test without HIT_NETWORK=1")
 	}
-	c := NewClient(t.Logf)
+	c := NewClient(t.Logf, nil)
+	defer c.Close()
 	c.SetLocalPort(1234)
 	for i := 0; i < 2; i++ {
 		if i > 0 {
 			time.Sleep(100 * time.Millisecond)
 		}
-		ext, err := c.CreateOrGetMapping(context.Background())
+		ext, err := c.createOrGetMapping(context.Background())
 		t.Logf("Got: %v, %v", ext, err)
 	}
 }
@@ -31,13 +32,14 @@ func TestClientProbe(t *testing.T) {
 	if v, _ := strconv.ParseBool(os.Getenv("HIT_NETWORK")); !v {
 		t.Skip("skipping test without HIT_NETWORK=1")
 	}
-	c := NewClient(t.Logf)
-	for i := 0; i < 2; i++ {
+	c := NewClient(t.Logf, nil)
+	defer c.Close()
+	for i := 0; i < 3; i++ {
 		if i > 0 {
 			time.Sleep(100 * time.Millisecond)
 		}
 		res, err := c.Probe(context.Background())
-		t.Logf("Got: %+v, %v", res, err)
+		t.Logf("Got(t=%dms): %+v, %v", i*100, res, err)
 	}
 }
 
@@ -45,10 +47,11 @@ func TestClientProbeThenMap(t *testing.T) {
 	if v, _ := strconv.ParseBool(os.Getenv("HIT_NETWORK")); !v {
 		t.Skip("skipping test without HIT_NETWORK=1")
 	}
-	c := NewClient(t.Logf)
+	c := NewClient(t.Logf, nil)
+	defer c.Close()
 	c.SetLocalPort(1234)
 	res, err := c.Probe(context.Background())
 	t.Logf("Probe: %+v, %v", res, err)
-	ext, err := c.CreateOrGetMapping(context.Background())
-	t.Logf("CreateOrGetMapping: %v, %v", ext, err)
+	ext, err := c.createOrGetMapping(context.Background())
+	t.Logf("createOrGetMapping: %v, %v", ext, err)
 }

net/portmapper/upnp.go

@@ -0,0 +1,313 @@
+// Copyright (c) 2021 Tailscale Inc & AUTHORS All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+// +build !ios
+// (https://github.com/tailscale/tailscale/issues/2495)
+
+package portmapper
+
+import (
+	"bufio"
+	"bytes"
+	"context"
+	"fmt"
+	"log"
+	"math/rand"
+	"net/http"
+	"net/url"
+	"time"
+
+	"github.com/tailscale/goupnp"
+	"github.com/tailscale/goupnp/dcps/internetgateway2"
+	"inet.af/netaddr"
+	"tailscale.com/control/controlknobs"
+	"tailscale.com/net/netns"
+)
+
+// VerboseLogs controls verbose debug logging.
+// It exists for use by "tailscaled debug --portmap".
+var VerboseLogs bool
+
+// References:
+//
+// WANIP Connection v2: http://upnp.org/specs/gw/UPnP-gw-WANIPConnection-v2-Service.pdf
+
+// upnpMapping is a port mapping over the upnp protocol. After being created it is immutable,
+// but the client field may be shared across mapping instances.
+type upnpMapping struct {
+	gw         netaddr.IP
+	external   netaddr.IPPort
+	internal   netaddr.IPPort
+	goodUntil  time.Time
+	renewAfter time.Time
+
+	// client is a connection to a upnp device, and may be reused across different UPnP mappings.
+	client upnpClient
+}
+
+func (u *upnpMapping) GoodUntil() time.Time     { return u.goodUntil }
+func (u *upnpMapping) RenewAfter() time.Time    { return u.renewAfter }
+func (u *upnpMapping) External() netaddr.IPPort { return u.external }
+func (u *upnpMapping) Release(ctx context.Context) {
+	u.client.DeletePortMapping(ctx, "", u.external.Port(), "udp")
+}
+
+// upnpClient is an interface over the multiple different clients exported by goupnp,
+// exposing the functions we need for portmapping. Those clients are auto-generated from XML-specs,
+// which is why they're not very idiomatic.
+type upnpClient interface {
+	AddPortMapping(
+		ctx context.Context,
+
+		// remoteHost is the remote device sending packets to this device, in the format of x.x.x.x.
+		// The empty string, "", means any host out on the internet can send packets in.
+		remoteHost string,
+
+		// externalPort is the exposed port of this port mapping. Visible during NAT operations.
+		// 0 will let the router select the port, but there is an additional call,
+		// `AddAnyPortMapping`, which is available on 1 of the 3 possible protocols,
+		// which should be used if available. See `addAnyPortMapping` below, which calls this if
+		// `AddAnyPortMapping` is not supported.
+		externalPort uint16,
+
+		// protocol is whether this is over TCP or UDP. Either "tcp" or "udp".
+		protocol string,
+
+		// internalPort is the port that the gateway device forwards the traffic to.
+		internalPort uint16,
+		// internalClient is the IP address that packets will be forwarded to for this mapping.
+		// Internal client is of the form "x.x.x.x".
+		internalClient string,
+
+		// enabled is whether this portmapping should be enabled or disabled.
+		enabled bool,
+		// portMappingDescription is a user-readable description of this portmapping.
+		portMappingDescription string,
+		// leaseDurationSec is the duration of this portmapping. The value of this argument must be
+		// greater than 0. From the spec, it appears if it is set to 0, it will switch to using
+		// 604800 seconds, but not sure why this is desired. The recommended time is 3600 seconds.
+		leaseDurationSec uint32,
+	) error
+
+	DeletePortMapping(ctx context.Context, remoteHost string, externalPort uint16, protocol string) error
+	GetExternalIPAddress(ctx context.Context) (externalIPAddress string, err error)
+}
+
+// tsPortMappingDesc gets sent to UPnP clients as a human-readable label for the portmapping.
+// It is not used for anything other than labelling.
+const tsPortMappingDesc = "tailscale-portmap"
+
+// addAnyPortMapping abstracts over different UPnP client connections, calling the available
+// AddAnyPortMapping call if available for WAN IP connection v2, otherwise defaulting to the old
+// behavior of calling AddPortMapping with port = 0 to specify a wildcard port.
+// It returns the new external port (which may not be identical to the external port specified),
+// or an error.
+//
+// TODO(bradfitz): also returned the actual lease duration obtained. and check it regularly.
+func addAnyPortMapping(
+	ctx context.Context,
+	upnp upnpClient,
+	externalPort uint16,
+	internalPort uint16,
+	internalClient string,
+	leaseDuration time.Duration,
+) (newPort uint16, err error) {
+	if upnp, ok := upnp.(*internetgateway2.WANIPConnection2); ok {
+		return upnp.AddAnyPortMapping(
+			ctx,
+			"",
+			externalPort,
+			"udp",
+			internalPort,
+			internalClient,
+			true,
+			tsPortMappingDesc,
+			uint32(leaseDuration.Seconds()),
+		)
+	}
+	for externalPort == 0 {
+		externalPort = uint16(rand.Intn(65535))
+	}
+	err = upnp.AddPortMapping(
+		ctx,
+		"",
+		externalPort,
+		"udp",
+		internalPort,
+		internalClient,
+		true,
+		tsPortMappingDesc,
+		uint32(leaseDuration.Seconds()),
+	)
+	return externalPort, err
+}
+
+// getUPnPClient gets a client for interfacing with UPnP, ignoring the underlying protocol for
+// now.
+// Adapted from https://github.com/huin/goupnp/blob/master/GUIDE.md.
+//
+// The gw is the detected gateway.
+//
+// The meta is the most recently parsed UDP discovery packet response
+// from the Internet Gateway Device.
+//
+// The provided ctx is not retained in the returned upnpClient, but
+// its associated HTTP client is (if set via goupnp.WithHTTPClient).
+func getUPnPClient(ctx context.Context, gw netaddr.IP, meta uPnPDiscoResponse) (upnpClient, error) {
+	if controlknobs.DisableUPnP() {
+		return nil, nil
+	}
+
+	if meta.Location == "" {
+		return nil, nil
+	}
+
+	if VerboseLogs {
+		log.Printf("fetching %v", meta.Location)
+	}
+	u, err := url.Parse(meta.Location)
+	if err != nil {
+		return nil, err
+	}
+
+	ipp, err := netaddr.ParseIPPort(u.Host)
+	if err != nil {
+		return nil, fmt.Errorf("unexpected host %q in %q", u.Host, meta.Location)
+	}
+	if ipp.IP() != gw {
+		return nil, fmt.Errorf("UPnP discovered root %q does not match gateway IP %v; ignoring UPnP",
+			meta.Location, gw)
+	}
+
+	ctx, cancel := context.WithTimeout(ctx, 500*time.Millisecond)
+	defer cancel()
+
+	// This part does a network fetch.
+	root, err := goupnp.DeviceByURL(ctx, u)
+	if err != nil {
+		return nil, err
+	}
+
+	// These parts don't do a network fetch.
+	// Pick the best service type available.
+	if cc, _ := internetgateway2.NewWANIPConnection2ClientsFromRootDevice(ctx, root, u); len(cc) > 0 {
+		return cc[0], nil
+	}
+	if cc, _ := internetgateway2.NewWANIPConnection1ClientsFromRootDevice(ctx, root, u); len(cc) > 0 {
+		return cc[0], nil
+	}
+	if cc, _ := internetgateway2.NewWANPPPConnection1ClientsFromRootDevice(ctx, root, u); len(cc) > 0 {
+		return cc[0], nil
+	}
+	return nil, nil
+}
+
+func (c *Client) upnpHTTPClientLocked() *http.Client {
+	if c.uPnPHTTPClient == nil {
+		c.uPnPHTTPClient = &http.Client{
+			Transport: &http.Transport{
+				DialContext:     netns.NewDialer().DialContext,
+				IdleConnTimeout: 2 * time.Second, // LAN is cheap
+			},
+		}
+	}
+	return c.uPnPHTTPClient
+}
+
+// getUPnPPortMapping attempts to create a port-mapping over the UPnP protocol. On success,
+// it will return the externally exposed IP and port. Otherwise, it will return a zeroed IP and
+// port and an error.
+func (c *Client) getUPnPPortMapping(
+	ctx context.Context,
+	gw netaddr.IP,
+	internal netaddr.IPPort,
+	prevPort uint16,
+) (external netaddr.IPPort, ok bool) {
+	if controlknobs.DisableUPnP() {
+		return netaddr.IPPort{}, false
+	}
+	now := time.Now()
+	upnp := &upnpMapping{
+		gw:       gw,
+		internal: internal,
+	}
+
+	var client upnpClient
+	var err error
+	c.mu.Lock()
+	oldMapping, ok := c.mapping.(*upnpMapping)
+	meta := c.uPnPMeta
+	httpClient := c.upnpHTTPClientLocked()
+	c.mu.Unlock()
+	if ok && oldMapping != nil {
+		client = oldMapping.client
+	} else {
+		ctx := goupnp.WithHTTPClient(ctx, httpClient)
+		client, err = getUPnPClient(ctx, gw, meta)
+		if VerboseLogs {
+			log.Printf("getUPnPClient: %T, %v", client, err)
+		}
+		if err != nil {
+			return netaddr.IPPort{}, false
+		}
+	}
+	if client == nil {
+		return netaddr.IPPort{}, false
+	}
+
+	var newPort uint16
+	newPort, err = addAnyPortMapping(
+		ctx,
+		client,
+		prevPort,
+		internal.Port(),
+		internal.IP().String(),
+		time.Second*pmpMapLifetimeSec,
+	)
+	if VerboseLogs {
+		log.Printf("addAnyPortMapping: %v, %v", newPort, err)
+	}
+	if err != nil {
+		return netaddr.IPPort{}, false
+	}
+	// TODO cache this ip somewhere?
+	extIP, err := client.GetExternalIPAddress(ctx)
+	if VerboseLogs {
+		log.Printf("client.GetExternalIPAddress: %v, %v", extIP, err)
+	}
+	if err != nil {
+		// TODO this doesn't seem right
+		return netaddr.IPPort{}, false
+	}
+	externalIP, err := netaddr.ParseIP(extIP)
+	if err != nil {
+		return netaddr.IPPort{}, false
+	}
+
+	upnp.external = netaddr.IPPortFrom(externalIP, newPort)
+	d := time.Duration(pmpMapLifetimeSec) * time.Second
+	upnp.goodUntil = now.Add(d)
+	upnp.renewAfter = now.Add(d / 2)
+	upnp.client = client
+	c.mu.Lock()
+	defer c.mu.Unlock()
+	c.mapping = upnp
+	c.localPort = newPort
+	return upnp.external, true
+}
+
+type uPnPDiscoResponse struct {
+	Location string
+}
+
+// parseUPnPDiscoResponse parses a UPnP HTTP-over-UDP discovery response.
+func parseUPnPDiscoResponse(body []byte) (uPnPDiscoResponse, error) {
+	var r uPnPDiscoResponse
+	res, err := http.ReadResponse(bufio.NewReaderSize(bytes.NewReader(body), 128), nil)
+	if err != nil {
+		return r, err
+	}
+	r.Location = res.Header.Get("Location")
+	return r, nil
+}

net/portmapper/upnp_test.go

@@ -0,0 +1,95 @@
+// Copyright (c) 2021 Tailscale Inc & AUTHORS All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package portmapper
+
+import (
+	"context"
+	"fmt"
+	"io"
+	"net"
+	"net/http"
+	"net/http/httptest"
+	"reflect"
+	"testing"
+
+	"inet.af/netaddr"
+)
+
+// Google Wifi
+const (
+	googleWifiUPnPDisco = "HTTP/1.1 200 OK\r\nCACHE-CONTROL: max-age=120\r\nST: urn:schemas-upnp-org:device:InternetGatewayDevice:2\r\nUSN: uuid:a9708184-a6c0-413a-bbac-11bcf7e30ece::urn:schemas-upnp-org:device:InternetGatewayDevice:2\r\nEXT:\r\nSERVER: Linux/5.4.0-1034-gcp UPnP/1.1 MiniUPnPd/1.9\r\nLOCATION: http://192.168.86.1:5000/rootDesc.xml\r\nOPT: \"http://schemas.upnp.org/upnp/1/0/\"; ns=01\r\n01-NLS: 1\r\nBOOTID.UPNP.ORG: 1\r\nCONFIGID.UPNP.ORG: 1337\r\n\r\n"
+
+	googleWifiRootDescXML = `<?xml version="1.0"?>
+<root xmlns="urn:schemas-upnp-org:device-1-0"><specVersion><major>1</major><minor>0</minor></specVersion><device><deviceType>urn:schemas-upnp-org:device:InternetGatewayDevice:2</deviceType><friendlyName>OnHub</friendlyName><manufacturer>Google</manufacturer><manufacturerURL>http://google.com/</manufacturerURL><modelDescription>Wireless Router</modelDescription><modelName>OnHub</modelName><modelNumber>1</modelNumber><modelURL>https://on.google.com/hub/</modelURL><serialNumber>00000000</serialNumber><UDN>uuid:a9708184-a6c0-413a-bbac-11bcf7e30ece</UDN><serviceList><service><serviceType>urn:schemas-upnp-org:service:Layer3Forwarding:1</serviceType><serviceId>urn:upnp-org:serviceId:Layer3Forwarding1</serviceId><controlURL>/ctl/L3F</controlURL><eventSubURL>/evt/L3F</eventSubURL><SCPDURL>/L3F.xml</SCPDURL></service><service><serviceType>urn:schemas-upnp-org:service:DeviceProtection:1</serviceType><serviceId>urn:upnp-org:serviceId:DeviceProtection1</serviceId><controlURL>/ctl/DP</controlURL><eventSubURL>/evt/DP</eventSubURL><SCPDURL>/DP.xml</SCPDURL></service></serviceList><deviceList><device><deviceType>urn:schemas-upnp-org:device:WANDevice:2</deviceType><friendlyName>WANDevice</friendlyName><manufacturer>MiniUPnP</manufacturer><manufacturerURL>http://miniupnp.free.fr/</manufacturerURL><modelDescription>WAN Device</modelDescription><modelName>WAN Device</modelName><modelNumber>20210414</modelNumber><modelURL>http://miniupnp.free.fr/</modelURL><serialNumber>00000000</serialNumber><UDN>uuid:a9708184-a6c0-413a-bbac-11bcf7e30ecf</UDN><UPC>000000000000</UPC><serviceList><service><serviceType>urn:schemas-upnp-org:service:WANCommonInterfaceConfig:1</serviceType><serviceId>urn:upnp-org:serviceId:WANCommonIFC1</serviceId><controlURL>/ctl/CmnIfCfg</controlURL><eventSubURL>/evt/CmnIfCfg</eventSubURL><SCPDURL>/WANCfg.xml</SCPDURL></service></serviceList><deviceList><device><deviceType>urn:schemas-upnp-org:device:WANConnectionDevice:2</deviceType><friendlyName>WANConnectionDevice</friendlyName><manufacturer>MiniUPnP</manufacturer><manufacturerURL>http://miniupnp.free.fr/</manufacturerURL><modelDescription>MiniUPnP daemon</modelDescription><modelName>MiniUPnPd</modelName><modelNumber>20210414</modelNumber><modelURL>http://miniupnp.free.fr/</modelURL><serialNumber>00000000</serialNumber><UDN>uuid:a9708184-a6c0-413a-bbac-11bcf7e30ec0</UDN><UPC>000000000000</UPC><serviceList><service><serviceType>urn:schemas-upnp-org:service:WANIPConnection:2</serviceType><serviceId>urn:upnp-org:serviceId:WANIPConn1</serviceId><controlURL>/ctl/IPConn</controlURL><eventSubURL>/evt/IPConn</eventSubURL><SCPDURL>/WANIPCn.xml</SCPDURL></service></serviceList></device></deviceList></device></deviceList><presentationURL>http://testwifi.here/</presentationURL></device></root>`
+)
+
+// pfSense 2.5.0-RELEASE / FreeBSD 12.2-STABLE
+const (
+	pfSenseUPnPDisco = "HTTP/1.1 200 OK\r\nCACHE-CONTROL: max-age=120\r\nST: urn:schemas-upnp-org:device:InternetGatewayDevice:1\r\nUSN: uuid:bee7052b-49e8-3597-b545-55a1e38ac11::urn:schemas-upnp-org:device:InternetGatewayDevice:1\r\nEXT:\r\nSERVER: FreeBSD/12.2-STABLE UPnP/1.1 MiniUPnPd/2.2.1\r\nLOCATION: http://192.168.1.1:2189/rootDesc.xml\r\nOPT: \"http://schemas.upnp.org/upnp/1/0/\"; ns=01\r\n01-NLS: 1627958564\r\nBOOTID.UPNP.ORG: 1627958564\r\nCONFIGID.UPNP.ORG: 1337\r\n\r\n"
+
+	pfSenseRootDescXML = `<?xml version="1.0"?>
+<root xmlns="urn:schemas-upnp-org:device-1-0" configId="1337"><specVersion><major>1</major><minor>1</minor></specVersion><device><deviceType>urn:schemas-upnp-org:device:InternetGatewayDevice:1</deviceType><friendlyName>FreeBSD router</friendlyName><manufacturer>FreeBSD</manufacturer><manufacturerURL>http://www.freebsd.org/</manufacturerURL><modelDescription>FreeBSD router</modelDescription><modelName>FreeBSD router</modelName><modelNumber>2.5.0-RELEASE</modelNumber><modelURL>http://www.freebsd.org/</modelURL><serialNumber>BEE7052B</serialNumber><UDN>uuid:bee7052b-49e8-3597-b545-55a1e38ac11</UDN><serviceList><service><serviceType>urn:schemas-upnp-org:service:Layer3Forwarding:1</serviceType><serviceId>urn:upnp-org:serviceId:L3Forwarding1</serviceId><SCPDURL>/L3F.xml</SCPDURL><controlURL>/ctl/L3F</controlURL><eventSubURL>/evt/L3F</eventSubURL></service></serviceList><deviceList><device><deviceType>urn:schemas-upnp-org:device:WANDevice:1</deviceType><friendlyName>WANDevice</friendlyName><manufacturer>MiniUPnP</manufacturer><manufacturerURL>http://miniupnp.free.fr/</manufacturerURL><modelDescription>WAN Device</modelDescription><modelName>WAN Device</modelName><modelNumber>20210205</modelNumber><modelURL>http://miniupnp.free.fr/</modelURL><serialNumber>BEE7052B</serialNumber><UDN>uuid:bee7052b-49e8-3597-b545-55a1e38ac12</UDN><UPC>000000000000</UPC><serviceList><service><serviceType>urn:schemas-upnp-org:service:WANCommonInterfaceConfig:1</serviceType><serviceId>urn:upnp-org:serviceId:WANCommonIFC1</serviceId><SCPDURL>/WANCfg.xml</SCPDURL><controlURL>/ctl/CmnIfCfg</controlURL><eventSubURL>/evt/CmnIfCfg</eventSubURL></service></serviceList><deviceList><device><deviceType>urn:schemas-upnp-org:device:WANConnectionDevice:1</deviceType><friendlyName>WANConnectionDevice</friendlyName><manufacturer>MiniUPnP</manufacturer><manufacturerURL>http://miniupnp.free.fr/</manufacturerURL><modelDescription>MiniUPnP daemon</modelDescription><modelName>MiniUPnPd</modelName><modelNumber>20210205</modelNumber><modelURL>http://miniupnp.free.fr/</modelURL><serialNumber>BEE7052B</serialNumber><UDN>uuid:bee7052b-49e8-3597-b545-55a1e38ac13</UDN><UPC>000000000000</UPC><serviceList><service><serviceType>urn:schemas-upnp-org:service:WANIPConnection:1</serviceType><serviceId>urn:upnp-org:serviceId:WANIPConn1</serviceId><SCPDURL>/WANIPCn.xml</SCPDURL><controlURL>/ctl/IPConn</controlURL><eventSubURL>/evt/IPConn</eventSubURL></service></serviceList></device></deviceList></device></deviceList><presentationURL>https://192.168.1.1/</presentationURL></device></root>`
+)
+
+func TestParseUPnPDiscoResponse(t *testing.T) {
+	tests := []struct {
+		name    string
+		headers string
+		want    uPnPDiscoResponse
+	}{
+		{"google", googleWifiUPnPDisco, uPnPDiscoResponse{
+			Location: "http://192.168.86.1:5000/rootDesc.xml",
+		}},
+		{"pfsense", pfSenseUPnPDisco, uPnPDiscoResponse{
+			Location: "http://192.168.1.1:2189/rootDesc.xml",
+		}},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			got, err := parseUPnPDiscoResponse([]byte(tt.headers))
+			if err != nil {
+				t.Fatal(err)
+			}
+			if !reflect.DeepEqual(got, tt.want) {
+				t.Errorf("unexpected result:\n got: %+v\nwant: %+v\n", got, tt.want)
+			}
+		})
+	}
+}
+
+func TestGetUPnPClient(t *testing.T) {
+	tests := []struct {
+		name    string
+		xmlBody string
+		want    string
+	}{
+		{"google", googleWifiRootDescXML, "*internetgateway2.WANIPConnection2"},
+		{"pfsense", pfSenseRootDescXML, "*internetgateway2.WANIPConnection1"},
+		// TODO(bradfitz): find a PPP one in the wild
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+				if r.RequestURI == "/rootDesc.xml" {
+					io.WriteString(w, tt.xmlBody)
+					return
+				}
+				http.NotFound(w, r)
+			}))
+			defer ts.Close()
+			gw, _ := netaddr.FromStdIP(ts.Listener.Addr().(*net.TCPAddr).IP)
+			c, err := getUPnPClient(context.Background(), gw, uPnPDiscoResponse{
+				Location: ts.URL + "/rootDesc.xml",
+			})
+			if err != nil {
+				t.Fatal(err)
+			}
+			got := fmt.Sprintf("%T", c)
+			if got != tt.want {
+				t.Errorf("got %v; want %v", got, tt.want)
+			}
+		})
+	}
+}

net/socks5/socks5.go

@@ -2,8 +2,15 @@
 // Use of this source code is governed by a BSD-style
 // license that can be found in the LICENSE file.
 
-// Package socks5 is a SOCKS5 server implementation
-// for userspace networking in Tailscale.
+// Package socks5 is a SOCKS5 server implementation.
+//
+// This is used for userspace networking in Tailscale. Specifically,
+// this is used for dialing out of the machine to other nodes, without
+// the host kernel's involvement, so it doesn't proper routing tables,
+// TUN, IPv6, etc. This package is meant to only handle the SOCKS5 protocol
+// details and not any integration with Tailscale internals itself.
+//
+// The glue between this package and Tailscale is in net/socks5/tssocks.
 package socks5
 
 import (

net/socks5/tssocks/tssocks.go

@@ -0,0 +1,79 @@
+// Copyright (c) 2021 Tailscale Inc & AUTHORS All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+// Package tssocks is the glue between Tailscale and the net/socks5 package.
+package tssocks
+
+import (
+	"context"
+	"net"
+	"sync"
+
+	"inet.af/netaddr"
+	"tailscale.com/net/socks5"
+	"tailscale.com/net/tsaddr"
+	"tailscale.com/types/logger"
+	"tailscale.com/types/netmap"
+	"tailscale.com/wgengine"
+	"tailscale.com/wgengine/netstack"
+)
+
+// NewServer returns a new SOCKS5 server configured to dial out to
+// Tailscale addresses.
+//
+// The returned server is not yet listening. The caller must call
+// Serve with a listener.
+//
+// If ns is non-nil, it is used for dialing when needed.
+func NewServer(logf logger.Logf, e wgengine.Engine, ns *netstack.Impl) *socks5.Server {
+	d := &dialer{ns: ns}
+	e.AddNetworkMapCallback(d.onNewNetmap)
+	return &socks5.Server{
+		Logf:   logf,
+		Dialer: d.DialContext,
+	}
+}
+
+// dialer is the Tailscale SOCKS5 dialer.
+type dialer struct {
+	ns *netstack.Impl
+
+	mu  sync.Mutex
+	dns netstack.DNSMap
+}
+
+func (d *dialer) onNewNetmap(nm *netmap.NetworkMap) {
+	d.mu.Lock()
+	defer d.mu.Unlock()
+	d.dns = netstack.DNSMapFromNetworkMap(nm)
+}
+
+func (d *dialer) resolve(ctx context.Context, addr string) (netaddr.IPPort, error) {
+	d.mu.Lock()
+	dns := d.dns
+	d.mu.Unlock()
+	return dns.Resolve(ctx, addr)
+}
+
+func (d *dialer) DialContext(ctx context.Context, network, addr string) (net.Conn, error) {
+	ipp, err := d.resolve(ctx, addr)
+	if err != nil {
+		return nil, err
+	}
+	if d.ns != nil && d.useNetstackForIP(ipp.IP()) {
+		return d.ns.DialContextTCP(ctx, ipp.String())
+	}
+	var stdDialer net.Dialer
+	return stdDialer.DialContext(ctx, network, ipp.String())
+}
+
+func (d *dialer) useNetstackForIP(ip netaddr.IP) bool {
+	if d.ns == nil {
+		return false
+	}
+	// TODO(bradfitz): this isn't exactly right.
+	// We should also support subnets when the
+	// prefs are configured as such.
+	return tsaddr.IsTailscaleIP(ip)
+}

net/speedtest/speedtest.go

@@ -0,0 +1,88 @@
+// Copyright (c) 2021 Tailscale Inc & AUTHORS All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+// Package speedtest contains both server and client code for
+// running speedtests between tailscale nodes.
+package speedtest
+
+import (
+	"time"
+)
+
+const (
+	blockSize       = 32000                 // size of the block of data to send
+	MinDuration     = 5 * time.Second       // minimum duration for a test
+	DefaultDuration = MinDuration           // default duration for a test
+	MaxDuration     = 30 * time.Second      // maximum duration for a test
+	version         = 1                     // value used when comparing client and server versions
+	increment       = time.Second           // increment to display results for, in seconds
+	minInterval     = 10 * time.Millisecond // minimum interval length for a result to be included
+	DefaultPort     = 20333
+)
+
+// config is the initial message sent to the server, that contains information on how to
+// conduct the test.
+type config struct {
+	Version      int           `json:"version"`
+	TestDuration time.Duration `json:"time"`
+	Direction    Direction     `json:"direction"`
+}
+
+// configResponse is the response to the testConfig message. If the server has an
+// error with the config, the Error variable will hold that error value.
+type configResponse struct {
+	Error string `json:"error,omitempty"`
+}
+
+// This represents the Result of a speedtest within a specific interval
+type Result struct {
+	Bytes         int           // number of bytes sent/received during the interval
+	IntervalStart time.Duration // duration between the start of the interval and the start of the test
+	IntervalEnd   time.Duration // duration between the end of the interval and the start of the test
+	Total         bool          // if true, this result struct represents the entire test, rather than a segment of the test
+}
+
+func (r Result) MBitsPerSecond() float64 {
+	return r.MegaBits() / (r.IntervalEnd - r.IntervalStart).Seconds()
+}
+
+func (r Result) MegaBytes() float64 {
+	return float64(r.Bytes) / 1000000.0
+}
+
+func (r Result) MegaBits() float64 {
+	return r.MegaBytes() * 8.0
+}
+
+func (r Result) Interval() time.Duration {
+	return r.IntervalEnd - r.IntervalStart
+}
+
+type Direction int
+
+const (
+	Download Direction = iota
+	Upload
+)
+
+func (d Direction) String() string {
+	switch d {
+	case Upload:
+		return "upload"
+	case Download:
+		return "download"
+	default:
+		return ""
+	}
+}
+
+func (d *Direction) Reverse() {
+	switch *d {
+	case Upload:
+		*d = Download
+	case Download:
+		*d = Upload
+	default:
+	}
+}

net/speedtest/speedtest_client.go

@@ -0,0 +1,42 @@
+// Copyright (c) 2021 Tailscale Inc & AUTHORS All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package speedtest
+
+import (
+	"encoding/json"
+	"errors"
+	"net"
+	"time"
+)
+
+// RunClient dials the given address and starts a speedtest.
+// It returns any errors that come up in the tests.
+// If there are no errors in the test, it returns a slice of results.
+func RunClient(direction Direction, duration time.Duration, host string) ([]Result, error) {
+	conn, err := net.Dial("tcp", host)
+	if err != nil {
+		return nil, err
+	}
+
+	conf := config{TestDuration: duration, Version: version, Direction: direction}
+
+	defer conn.Close()
+	encoder := json.NewEncoder(conn)
+
+	if err = encoder.Encode(conf); err != nil {
+		return nil, err
+	}
+
+	var response configResponse
+	decoder := json.NewDecoder(conn)
+	if err = decoder.Decode(&response); err != nil {
+		return nil, err
+	}
+	if response.Error != "" {
+		return nil, errors.New(response.Error)
+	}
+
+	return doTest(conn, conf)
+}

net/speedtest/speedtest_server.go

@@ -0,0 +1,158 @@
+// Copyright (c) 2021 Tailscale Inc & AUTHORS All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package speedtest
+
+import (
+	"crypto/rand"
+	"encoding/json"
+	"errors"
+	"fmt"
+	"io"
+	"net"
+	"time"
+)
+
+// Serve starts up the server on a given host and port pair. It starts to listen for
+// connections and handles each one in a goroutine. Because it runs in an infinite loop,
+// this function only returns if any of the speedtests return with errors, or if the
+// listener is closed.
+func Serve(l net.Listener) error {
+	for {
+		conn, err := l.Accept()
+		if errors.Is(err, net.ErrClosed) {
+			return nil
+		}
+		if err != nil {
+			return err
+		}
+		err = handleConnection(conn)
+		if err != nil {
+			return err
+		}
+	}
+}
+
+// handleConnection handles the initial exchange between the server and the client.
+// It reads the testconfig message into a config struct. If any errors occur with
+// the testconfig (specifically, if there is a version mismatch), it will return those
+// errors to the client with a configResponse. After the exchange, it will start
+// the speed test.
+func handleConnection(conn net.Conn) error {
+	defer conn.Close()
+	var conf config
+
+	decoder := json.NewDecoder(conn)
+	err := decoder.Decode(&conf)
+	encoder := json.NewEncoder(conn)
+
+	// Both return and encode errors that occurred before the test started.
+	if err != nil {
+		encoder.Encode(configResponse{Error: err.Error()})
+		return err
+	}
+
+	// The server should always be doing the opposite of what the client is doing.
+	conf.Direction.Reverse()
+
+	if conf.Version != version {
+		err = fmt.Errorf("version mismatch! Server is version %d, client is version %d", version, conf.Version)
+		encoder.Encode(configResponse{Error: err.Error()})
+		return err
+	}
+
+	// Start the test
+	encoder.Encode(configResponse{})
+	_, err = doTest(conn, conf)
+	return err
+}
+
+// TODO include code to detect whether the code is direct vs DERP
+
+// doTest contains the code to run both the upload and download speedtest.
+// the direction value in the config parameter determines which test to run.
+func doTest(conn net.Conn, conf config) ([]Result, error) {
+	bufferData := make([]byte, blockSize)
+
+	intervalBytes := 0
+	totalBytes := 0
+
+	var currentTime time.Time
+	var results []Result
+
+	startTime := time.Now()
+	lastCalculated := startTime
+
+	if conf.Direction == Download {
+		conn.SetReadDeadline(time.Now().Add(conf.TestDuration).Add(5 * time.Second))
+	} else {
+		_, err := rand.Read(bufferData)
+		if err != nil {
+			return nil, err
+		}
+
+	}
+
+SpeedTestLoop:
+	for {
+		var n int
+		var err error
+
+		if conf.Direction == Download {
+			n, err = io.ReadFull(conn, bufferData)
+			switch err {
+			case io.EOF, io.ErrUnexpectedEOF:
+				break SpeedTestLoop
+			case nil:
+				// successful read
+			default:
+				return nil, fmt.Errorf("unexpected error has occured: %w", err)
+			}
+		} else {
+			// Need to change the data a little bit, to avoid any compression.
+			for i := range bufferData {
+				bufferData[i]++
+			}
+			n, err = conn.Write(bufferData)
+			if err != nil {
+				// If the write failed, there is most likely something wrong with the connection.
+				return nil, fmt.Errorf("upload failed: %w", err)
+			}
+		}
+		currentTime = time.Now()
+		intervalBytes += n
+
+		// checks if the current time is more or equal to the lastCalculated time plus the increment
+		if currentTime.After(lastCalculated.Add(increment)) {
+			intervalStart := lastCalculated.Sub(startTime)
+			intervalEnd := currentTime.Sub(startTime)
+			if (intervalEnd - intervalStart) > minInterval {
+				results = append(results, Result{Bytes: intervalBytes, IntervalStart: intervalStart, IntervalEnd: intervalEnd, Total: false})
+			}
+			lastCalculated = currentTime
+			totalBytes += intervalBytes
+			intervalBytes = 0
+		}
+
+		if conf.Direction == Upload && time.Since(startTime) > conf.TestDuration {
+			break SpeedTestLoop
+		}
+	}
+
+	// get last segment
+	intervalStart := lastCalculated.Sub(startTime)
+	intervalEnd := currentTime.Sub(startTime)
+	if (intervalEnd - intervalStart) > minInterval {
+		results = append(results, Result{Bytes: intervalBytes, IntervalStart: intervalStart, IntervalEnd: intervalEnd, Total: false})
+	}
+
+	// get total
+	totalBytes += intervalBytes
+	intervalEnd = currentTime.Sub(startTime)
+	if intervalEnd > minInterval {
+		results = append(results, Result{Bytes: totalBytes, IntervalStart: 0, IntervalEnd: intervalEnd, Total: true})
+	}
+
+	return results, nil
+}

net/speedtest/speedtest_test.go

@@ -0,0 +1,81 @@
+// Copyright (c) 2021 Tailscale Inc & AUTHORS All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package speedtest
+
+import (
+	"net"
+	"testing"
+)
+
+func TestDownload(t *testing.T) {
+	// start a listener and find the port where the server will be listening.
+	l, err := net.Listen("tcp", ":0")
+	if err != nil {
+		t.Fatal(err)
+	}
+	t.Cleanup(func() { l.Close() })
+
+	serverIP := l.Addr().String()
+	t.Log("server IP found:", serverIP)
+
+	type state struct {
+		err error
+	}
+	displayResult := func(t *testing.T, r Result) {
+		t.Helper()
+		t.Logf("{ Megabytes: %.2f, Start: %.1f, End: %.1f, Total: %t }", r.MegaBytes(), r.IntervalStart.Seconds(), r.IntervalEnd.Seconds(), r.Total)
+	}
+	stateChan := make(chan state, 1)
+
+	go func() {
+		err := Serve(l)
+		stateChan <- state{err: err}
+	}()
+
+	// ensure that the test returns an appropriate number of Result structs
+	expectedLen := int(DefaultDuration.Seconds()) + 1
+
+	t.Run("download test", func(t *testing.T) {
+		// conduct a download test
+		results, err := RunClient(Download, DefaultDuration, serverIP)
+
+		if err != nil {
+			t.Fatal("download test failed:", err)
+		}
+
+		if len(results) < expectedLen {
+			t.Fatalf("download results: expected length: %d, actual length: %d", expectedLen, len(results))
+		}
+
+		for _, result := range results {
+			displayResult(t, result)
+		}
+	})
+
+	t.Run("upload test", func(t *testing.T) {
+		// conduct an upload test
+		results, err := RunClient(Upload, DefaultDuration, serverIP)
+
+		if err != nil {
+			t.Fatal("upload test failed:", err)
+		}
+
+		if len(results) < expectedLen {
+			t.Fatalf("upload results: expected length: %d, actual length: %d", expectedLen, len(results))
+		}
+
+		for _, result := range results {
+			displayResult(t, result)
+		}
+	})
+
+	// causes the server goroutine to finish
+	l.Close()
+
+	testState := <-stateChan
+	if testState.err != nil {
+		t.Error("server error:", err)
+	}
+}