More logging :(

The cyolosecurity fork of certstore did not update its module name and
thus can only be used with a replace directive. This interferes with
installing using `go install` so I created a tailscale fork with an
updated module name.

Signed-off-by: Adrian Dewhurst <adrian@tailscale.com>

.github/ISSUE_TEMPLATE/bug_report.md

@@ -2,36 +2,7 @@
 name: Bug report
 about: Create a bug report
 title: ''
-labels: ''
+labels: 'needs-triage'
 assignees: ''
 
 ---
-
-<!-- Please note, this template is for definite bugs, not requests for
-support. If you need help with Tailscale, please email
-support@tailscale.com. We don't provide support via Github issues. -->
-
-**Describe the bug**
-A clear and concise description of what the bug is.
-
-**To Reproduce**
-Steps to reproduce the behavior:
-1. Go to '...'
-2. Click on '....'
-3. Scroll down to '....'
-4. See error
-
-**Expected behavior**
-A clear and concise description of what you expected to happen.
-
-**Screenshots**
-If applicable, add screenshots to help explain your problem.
-
-**Version information:**
- - Device: [e.g. iPhone X, laptop]
- - OS: [e.g. Windows, MacOS]
- - OS version: [e.g. Windows 10, Ubuntu 18.04]
- - Tailscale version: [e.g. 0.95-0]
-
-**Additional context**
-Add any other context about the problem here.

.github/ISSUE_TEMPLATE/feature_request.md

@@ -2,25 +2,6 @@
 name: Feature request
 about: Suggest an idea for this project
 title: ''
-labels: ''
+labels: 'needs-triage'
 assignees: ''
-
 ---
-
-**Is your feature request related to a problem? Please describe.**
-
-A clear and concise description of what the problem is. Ex. I'm always
-frustrated when [...]
-
-**Describe the solution you'd like**
-
-A clear and concise description of what you want to happen.
-
-**Describe alternatives you've considered**
-
-A clear and concise description of any alternative solutions or
-features you've considered.
-
-**Additional context**
-
-Add any other context or screenshots about the feature request here.

.github/workflows/coverage.yml

@@ -1,48 +0,0 @@
-name: Code Coverage
-
-on:
-  push:
-    branches:
-      - main
-  pull_request:
-    branches:
-      - '*'
-
-jobs:
-  build:
-    runs-on: ubuntu-latest
-
-    if: "!contains(github.event.head_commit.message, '[ci skip]')"
-
-    steps:
-
-    - name: Set up Go
-      uses: actions/setup-go@v1
-      with:
-        go-version: 1.16
-      id: go
-
-    - name: Check out code into the Go module directory
-      uses: actions/checkout@v1
-
-      # https://markphelps.me/2019/11/speed-up-your-go-builds-with-actions-cache/
-    - name: Restore Cache
-      uses: actions/cache@preview
-      id: cache
-      with:
-        path: ~/go/pkg/mod
-        key: ${{ runner.os }}-${{ hashFiles('**/go.sum') }}
-
-    - name: Basic build
-      run: go build ./cmd/...
-
-    - name: Run tests on linux with coverage data
-      run: go test -race -coverprofile=coverage.txt -bench=. -benchtime=1x ./...
-
-    - name: coveralls.io
-      uses: shogo82148/actions-goveralls@v1
-      env:
-        COVERALLS_TOKEN: ${{ secrets.COVERALLS_TOKEN }}
-        GITHUB_TOKEN: ${{ secrets.COVERALLS_BOT_PUBLIC_REPO_TOKEN }}
-      with:
-        path-to-profile: ./coverage.txt

.github/workflows/linux-race.yml

@@ -0,0 +1,48 @@
+name: Linux race
+
+on:
+  push:
+    branches:
+      - main
+  pull_request:
+    branches:
+      - '*'
+
+jobs:
+  build:
+    runs-on: ubuntu-latest
+
+    if: "!contains(github.event.head_commit.message, '[ci skip]')"
+
+    steps:
+
+    - name: Set up Go
+      uses: actions/setup-go@v1
+      with:
+        go-version: 1.16
+      id: go
+
+    - name: Check out code into the Go module directory
+      uses: actions/checkout@v1
+
+    - name: Basic build
+      run: go build ./cmd/...
+
+    - name: Run tests and benchmarks with -race flag on linux
+      run: go test -race -bench=. -benchtime=1x ./...
+
+    - uses: k0kubun/action-slack@v2.0.0
+      with:
+        payload: |
+          {
+            "attachments": [{
+              "text": "${{ job.status }}: ${{ github.workflow }} <https://github.com/${{ github.repository }}/commit/${{ github.sha }}/checks|${{ env.COMMIT_DATE }} #${{ env.COMMIT_NUMBER_OF_DAY }}> " +
+                      "(<https://github.com/${{ github.repository }}/commit/${{ github.sha }}|" + "${{ github.sha }}".substring(0, 10) + ">) " +
+                      "of ${{ github.repository }}@" + "${{ github.ref }}".split('/').reverse()[0] + " by ${{ github.event.head_commit.committer.name }}",
+              "color": "danger"
+            }]
+          }
+      env:
+        SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK_URL }}
+      if: failure() && github.event_name == 'push'
+

.github/workflows/linux.yml

@@ -29,7 +29,7 @@ jobs:
       run: go build ./cmd/...
 
     - name: Run tests on linux
-      run: go test ./...
+      run: go test -bench=. -benchtime=1x ./...
 
     - uses: k0kubun/action-slack@v2.0.0
       with:

.github/workflows/linux32.yml

@@ -29,7 +29,7 @@ jobs:
       run: GOARCH=386 go build ./cmd/...
 
     - name: Run tests on linux
-      run: GOARCH=386 go test ./...
+      run: GOARCH=386 go test -bench=. -benchtime=1x ./...
 
     - uses: k0kubun/action-slack@v2.0.0
       with:

.github/workflows/staticcheck.yml

@@ -24,11 +24,23 @@ jobs:
     - name: Run go vet
       run: go vet ./...
 
-    - name: Print staticcheck version
-      run: go run honnef.co/go/tools/cmd/staticcheck -version
+    - name: Install staticcheck
+      run: "GOBIN=~/.local/bin go install honnef.co/go/tools/cmd/staticcheck"
 
-    - name: Run staticcheck
-      run: "go run honnef.co/go/tools/cmd/staticcheck -- $(go list ./... | grep -v tempfork)"
+    - name: Print staticcheck version
+      run: "staticcheck -version"
+
+    - name: Run staticcheck (linux/amd64)
+      run: "GOOS=linux GOARCH=amd64 staticcheck -- $(go list ./... | grep -v tempfork)"
+
+    - name: Run staticcheck (darwin/amd64)
+      run: "GOOS=darwin GOARCH=amd64 staticcheck -- $(go list ./... | grep -v tempfork)"
+
+    - name: Run staticcheck (windows/amd64)
+      run: "GOOS=windows GOARCH=amd64 staticcheck -- $(go list ./... | grep -v tempfork)"
+
+    - name: Run staticcheck (windows/386)
+      run: "GOOS=windows GOARCH=386 staticcheck -- $(go list ./... | grep -v tempfork)"
 
     - uses: k0kubun/action-slack@v2.0.0
       with:

.github/workflows/windows-race.yml

@@ -0,0 +1,55 @@
+name: Windows race
+
+on:
+  push:
+    branches:
+      - main
+  pull_request:
+    branches:
+      - '*'
+
+jobs:
+  test:
+    runs-on: windows-latest
+
+    if: "!contains(github.event.head_commit.message, '[ci skip]')"
+
+    steps:
+
+    - name: Install Go
+      uses: actions/setup-go@v2
+      with:
+        go-version: 1.16.x
+
+    - name: Checkout code
+      uses: actions/checkout@v2
+
+    - name: Restore Cache
+      uses: actions/cache@v2
+      with:
+        path: ~/go/pkg/mod
+        key: ${{ runner.os }}-go-${{ hashFiles('**/go.sum') }}
+        restore-keys: |
+          ${{ runner.os }}-go-
+
+    - name: Test with -race flag
+      # Don't use -bench=. -benchtime=1x.
+      # Somewhere in the layers (powershell?)
+      # the equals signs cause great confusion.
+      run: go test -race -bench . -benchtime 1x ./...
+
+    - uses: k0kubun/action-slack@v2.0.0
+      with:
+        payload: |
+          {
+            "attachments": [{
+              "text": "${{ job.status }}: ${{ github.workflow }} <https://github.com/${{ github.repository }}/commit/${{ github.sha }}/checks|${{ env.COMMIT_DATE }} #${{ env.COMMIT_NUMBER_OF_DAY }}> " +
+                      "(<https://github.com/${{ github.repository }}/commit/${{ github.sha }}|" + "${{ github.sha }}".substring(0, 10) + ">) " +
+                      "of ${{ github.repository }}@" + "${{ github.ref }}".split('/').reverse()[0] + " by ${{ github.event.head_commit.committer.name }}",
+              "color": "danger"
+            }]
+          }
+      env:
+        SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK_URL }}
+      if: failure() && github.event_name == 'push'
+

.github/workflows/windows.yml

@@ -33,7 +33,10 @@ jobs:
           ${{ runner.os }}-go-
 
     - name: Test
-      run: go test ./...
+      # Don't use -bench=. -benchtime=1x.
+      # Somewhere in the layers (powershell?)
+      # the equals signs cause great confusion.
+      run: go test -bench . -benchtime 1x ./...
 
     - uses: k0kubun/action-slack@v2.0.0
       with:

VERSION.txt

@@ -1 +1 @@
-1.7.0
+1.9.0

api.md

@@ -367,10 +367,11 @@ Etag: "e0b2816b418b3f266309d94426ac7668ab3c1fa87798785bf82f1085cc2f6d9c"
 
 #### `POST /api/v2/tailnet/:tailnet/acl` - set ACL for a tailnet
 
-Sets the ACL for the given tailnet. HuJSON and JSON are both accepted inputs. An `If-Match` header can be set to avoid missed updates.
+Sets the ACL for the given domain.
+HuJSON and JSON are both accepted inputs.
+An `If-Match` header can be set to avoid missed updates.
 
-Returns error for invalid ACLs.
-Returns error if using an `If-Match` header and the ETag does not match.
+Returns the updated ACL in JSON or HuJSON according to the `Accept` header on success. Otherwise, errors are returned for incorrectly defined ACLs, ACLs with failing tests on attempted updates, and mismatched `If-Match` header and ETag.
 
 ##### Parameters
 
@@ -380,7 +381,17 @@ Returns error if using an `If-Match` header and the ETag does not match.
 `Accept` - Sets the return type of the updated ACL. Response is parsed `JSON` if `application/json` is explicitly named, otherwise HuJSON will be returned.
 
 ###### POST Body
-ACL JSON or HuJSON (see https://tailscale.com/kb/1018/acls)
+
+The POST body should be a JSON or [HuJSON](https://github.com/tailscale/hujson#hujson---human-json) formatted JSON object.
+An ACL policy may contain the following top-level properties:
+
+* `Groups` - Static groups of users which can be used for ACL rules.
+* `Hosts` - Hostname aliases to use in place of IP addresses or subnets.
+* `ACLs` - Access control lists.
+* `TagOwners` - Defines who is allowed to use which tags.
+* `Tests` - Run on ACL updates to check correct functionality of defined ACLs.
+
+See https://tailscale.com/kb/1018/acls for more information on those properties.
 
 ##### Example
 ```
@@ -411,7 +422,7 @@ curl 'https://api.tailscale.com/api/v2/tailnet/example.com/acl' \
 }'
 ```
 
-Response
+Response:
 ```
 // Example/default ACLs for unrestricted connections.
 {
@@ -436,9 +447,25 @@ Response
 }
 ```
 
+Failed test error response:
+```
+{
+    "message": "test(s) failed",
+    "data": [
+        {
+            "user": "user1@example.com",
+            "errors": [
+                "address \"user2@example.com:400\": want: Accept, got: Drop"
+            ]
+        }
+    ]
+}
+```
+
 <a name=tailnet-acl-preview-post></a>
 
 #### `POST /api/v2/tailnet/:tailnet/acl/preview` - preview rule matches on an ACL for a resource
+
 Determines what rules match for a user on an ACL without saving the ACL to the server.
 
 ##### Parameters
@@ -477,7 +504,7 @@ curl 'https://api.tailscale.com/api/v2/tailnet/example.com/acl?user=user1@exampl
 }'
 ```
 
-Response
+Response:
 ```
 {"matches":[{"users":["*"],"ports":["*:*"],"lineNumber":19}],"user":"user1@example.com"}
 ```

client/tailscale/apitype/apitype.go

@@ -0,0 +1,29 @@
+// Copyright (c) 2021 Tailscale Inc & AUTHORS All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+// Package apitype contains types for the Tailscale local API.
+package apitype
+
+import "tailscale.com/tailcfg"
+
+// WhoIsResponse is the JSON type returned by tailscaled debug server's /whois?ip=$IP handler.
+type WhoIsResponse struct {
+	Node        *tailcfg.Node
+	UserProfile *tailcfg.UserProfile
+}
+
+// FileTarget is a node to which files can be sent, and the PeerAPI
+// URL base to do so via.
+type FileTarget struct {
+	Node *tailcfg.Node
+
+	// PeerAPI is the http://ip:port URL base of the node's peer API,
+	// without any path (not even a single slash).
+	PeerAPIURL string
+}
+
+type WaitingFile struct {
+	Name string
+	Size int64
+}

client/tailscale/tailscale.go

@@ -6,6 +6,7 @@
 package tailscale
 
 import (
+	"bytes"
 	"context"
 	"encoding/json"
 	"errors"
@@ -18,10 +19,11 @@ import (
 	"strconv"
 	"strings"
 
+	"tailscale.com/client/tailscale/apitype"
+	"tailscale.com/ipn"
 	"tailscale.com/ipn/ipnstate"
 	"tailscale.com/paths"
 	"tailscale.com/safesocket"
-	"tailscale.com/tailcfg"
 )
 
 // TailscaledSocket is the tailscaled Unix socket.
@@ -64,9 +66,22 @@ func DoLocalRequest(req *http.Request) (*http.Response, error) {
 	return tsClient.Do(req)
 }
 
-// WhoIs returns the owner of the remoteAddr, which must be an IP or IP:port.
-func WhoIs(ctx context.Context, remoteAddr string) (*tailcfg.WhoIsResponse, error) {
-	req, err := http.NewRequestWithContext(ctx, "GET", "http://local-tailscaled.sock/localapi/v0/whois?addr="+url.QueryEscape(remoteAddr), nil)
+type errorJSON struct {
+	Error string
+}
+
+// bestError returns either err, or if body contains a valid JSON
+// object of type errorJSON, its non-empty error body.
+func bestError(err error, body []byte) error {
+	var j errorJSON
+	if err := json.Unmarshal(body, &j); err == nil && j.Error != "" {
+		return errors.New(j.Error)
+	}
+	return err
+}
+
+func send(ctx context.Context, method, path string, wantStatus int, body io.Reader) ([]byte, error) {
+	req, err := http.NewRequestWithContext(ctx, method, "http://local-tailscaled.sock"+path, body)
 	if err != nil {
 		return nil, err
 	}
@@ -75,60 +90,48 @@ func WhoIs(ctx context.Context, remoteAddr string) (*tailcfg.WhoIsResponse, erro
 		return nil, err
 	}
 	defer res.Body.Close()
-	slurp, _ := ioutil.ReadAll(res.Body)
-	if res.StatusCode != 200 {
-		return nil, fmt.Errorf("HTTP %s: %s", res.Status, slurp)
+	slurp, err := ioutil.ReadAll(res.Body)
+	if err != nil {
+		return nil, err
 	}
-	r := new(tailcfg.WhoIsResponse)
-	if err := json.Unmarshal(slurp, r); err != nil {
-		if max := 200; len(slurp) > max {
-			slurp = slurp[:max]
+	if res.StatusCode != wantStatus {
+		err := fmt.Errorf("HTTP %s: %s (expected %v)", res.Status, slurp, wantStatus)
+		return nil, bestError(err, slurp)
+	}
+	return slurp, nil
+}
+
+func get200(ctx context.Context, path string) ([]byte, error) {
+	return send(ctx, "GET", path, 200, nil)
+}
+
+// WhoIs returns the owner of the remoteAddr, which must be an IP or IP:port.
+func WhoIs(ctx context.Context, remoteAddr string) (*apitype.WhoIsResponse, error) {
+	body, err := get200(ctx, "/localapi/v0/whois?addr="+url.QueryEscape(remoteAddr))
+	if err != nil {
+		return nil, err
+	}
+	r := new(apitype.WhoIsResponse)
+	if err := json.Unmarshal(body, r); err != nil {
+		if max := 200; len(body) > max {
+			body = append(body[:max], "..."...)
 		}
-		return nil, fmt.Errorf("failed to parse JSON WhoIsResponse from %q", slurp)
+		return nil, fmt.Errorf("failed to parse JSON WhoIsResponse from %q", body)
 	}
 	return r, nil
 }
 
 // Goroutines returns a dump of the Tailscale daemon's current goroutines.
 func Goroutines(ctx context.Context) ([]byte, error) {
-	req, err := http.NewRequestWithContext(ctx, "GET", "http://local-tailscaled.sock/localapi/v0/goroutines", nil)
-	if err != nil {
-		return nil, err
-	}
-	res, err := DoLocalRequest(req)
-	if err != nil {
-		return nil, err
-	}
-	defer res.Body.Close()
-	body, err := ioutil.ReadAll(res.Body)
-	if err != nil {
-		return nil, err
-	}
-	if res.StatusCode != 200 {
-		return nil, fmt.Errorf("HTTP %s: %s", res.Status, body)
-	}
-	return body, nil
+	return get200(ctx, "/localapi/v0/goroutines")
 }
 
 // BugReport logs and returns a log marker that can be shared by the user with support.
 func BugReport(ctx context.Context, note string) (string, error) {
-	u := fmt.Sprintf("http://local-tailscaled.sock/localapi/v0/bugreport?note=%s", url.QueryEscape(note))
-	req, err := http.NewRequestWithContext(ctx, "POST", u, nil)
+	body, err := send(ctx, "POST", "/localapi/v0/bugreport?note="+url.QueryEscape(note), 200, nil)
 	if err != nil {
 		return "", err
 	}
-	res, err := DoLocalRequest(req)
-	if err != nil {
-		return "", err
-	}
-	defer res.Body.Close()
-	body, err := ioutil.ReadAll(res.Body)
-	if err != nil {
-		return "", err
-	}
-	if res.StatusCode != 200 {
-		return "", fmt.Errorf("HTTP %s: %s", res.Status, body)
-	}
 	return strings.TrimSpace(string(body)), nil
 }
 
@@ -143,67 +146,32 @@ func StatusWithoutPeers(ctx context.Context) (*ipnstate.Status, error) {
 }
 
 func status(ctx context.Context, queryString string) (*ipnstate.Status, error) {
-	req, err := http.NewRequestWithContext(ctx, "GET", "http://local-tailscaled.sock/localapi/v0/status"+queryString, nil)
+	body, err := get200(ctx, "/localapi/v0/status"+queryString)
 	if err != nil {
 		return nil, err
 	}
-	res, err := DoLocalRequest(req)
-	if err != nil {
-		return nil, err
-	}
-	defer res.Body.Close()
-	if res.StatusCode != 200 {
-		body, _ := ioutil.ReadAll(res.Body)
-		return nil, fmt.Errorf("HTTP %s: %s", res.Status, body)
-	}
 	st := new(ipnstate.Status)
-	if err := json.NewDecoder(res.Body).Decode(st); err != nil {
+	if err := json.Unmarshal(body, st); err != nil {
 		return nil, err
 	}
 	return st, nil
 }
 
-type WaitingFile struct {
-	Name string
-	Size int64
-}
-
-func WaitingFiles(ctx context.Context) ([]WaitingFile, error) {
-	req, err := http.NewRequestWithContext(ctx, "GET", "http://local-tailscaled.sock/localapi/v0/files/", nil)
+func WaitingFiles(ctx context.Context) ([]apitype.WaitingFile, error) {
+	body, err := get200(ctx, "/localapi/v0/files/")
 	if err != nil {
 		return nil, err
 	}
-	res, err := DoLocalRequest(req)
-	if err != nil {
-		return nil, err
-	}
-	defer res.Body.Close()
-	if res.StatusCode != 200 {
-		body, _ := ioutil.ReadAll(res.Body)
-		return nil, fmt.Errorf("HTTP %s: %s", res.Status, body)
-	}
-	var wfs []WaitingFile
-	if err := json.NewDecoder(res.Body).Decode(&wfs); err != nil {
+	var wfs []apitype.WaitingFile
+	if err := json.Unmarshal(body, &wfs); err != nil {
 		return nil, err
 	}
 	return wfs, nil
 }
 
 func DeleteWaitingFile(ctx context.Context, baseName string) error {
-	req, err := http.NewRequestWithContext(ctx, "DELETE", "http://local-tailscaled.sock/localapi/v0/files/"+url.PathEscape(baseName), nil)
-	if err != nil {
-		return err
-	}
-	res, err := DoLocalRequest(req)
-	if err != nil {
-		return err
-	}
-	defer res.Body.Close()
-	if res.StatusCode != http.StatusNoContent {
-		body, _ := ioutil.ReadAll(res.Body)
-		return fmt.Errorf("expected 204 No Content; got HTTP %s: %s", res.Status, body)
-	}
-	return nil
+	_, err := send(ctx, "DELETE", "/localapi/v0/files/"+url.PathEscape(baseName), http.StatusNoContent, nil)
+	return err
 }
 
 func GetWaitingFile(ctx context.Context, baseName string) (rc io.ReadCloser, size int64, err error) {
@@ -227,25 +195,27 @@ func GetWaitingFile(ctx context.Context, baseName string) (rc io.ReadCloser, siz
 	return res.Body, res.ContentLength, nil
 }
 
+func FileTargets(ctx context.Context) ([]apitype.FileTarget, error) {
+	body, err := get200(ctx, "/localapi/v0/file-targets")
+	if err != nil {
+		return nil, err
+	}
+	var fts []apitype.FileTarget
+	if err := json.Unmarshal(body, &fts); err != nil {
+		return nil, fmt.Errorf("invalid JSON: %w", err)
+	}
+	return fts, nil
+}
+
 func CheckIPForwarding(ctx context.Context) error {
-	req, err := http.NewRequestWithContext(ctx, "GET", "http://local-tailscaled.sock/localapi/v0/check-ip-forwarding", nil)
+	body, err := get200(ctx, "/localapi/v0/check-ip-forwarding")
 	if err != nil {
 		return err
 	}
-	res, err := DoLocalRequest(req)
-	if err != nil {
-		return err
-	}
-	defer res.Body.Close()
-	if res.StatusCode != 200 {
-		body, _ := ioutil.ReadAll(res.Body)
-		res.Body.Close()
-		return fmt.Errorf("HTTP %s: %s", res.Status, body)
-	}
 	var jres struct {
 		Warning string
 	}
-	if err := json.NewDecoder(res.Body).Decode(&jres); err != nil {
+	if err := json.Unmarshal(body, &jres); err != nil {
 		return fmt.Errorf("invalid JSON from check-ip-forwarding: %w", err)
 	}
 	if jres.Warning != "" {
@@ -253,3 +223,36 @@ func CheckIPForwarding(ctx context.Context) error {
 	}
 	return nil
 }
+
+func GetPrefs(ctx context.Context) (*ipn.Prefs, error) {
+	body, err := get200(ctx, "/localapi/v0/prefs")
+	if err != nil {
+		return nil, err
+	}
+	var p ipn.Prefs
+	if err := json.Unmarshal(body, &p); err != nil {
+		return nil, fmt.Errorf("invalid prefs JSON: %w", err)
+	}
+	return &p, nil
+}
+
+func EditPrefs(ctx context.Context, mp *ipn.MaskedPrefs) (*ipn.Prefs, error) {
+	mpj, err := json.Marshal(mp)
+	if err != nil {
+		return nil, err
+	}
+	body, err := send(ctx, "PATCH", "/localapi/v0/prefs", http.StatusOK, bytes.NewReader(mpj))
+	if err != nil {
+		return nil, err
+	}
+	var p ipn.Prefs
+	if err := json.Unmarshal(body, &p); err != nil {
+		return nil, fmt.Errorf("invalid prefs JSON: %w", err)
+	}
+	return &p, nil
+}
+
+func Logout(ctx context.Context) error {
+	_, err := send(ctx, "POST", "/localapi/v0/logout", http.StatusNoContent, nil)
+	return err
+}

cmd/hello/hello.go

@@ -18,7 +18,7 @@ import (
 	"strings"
 
 	"tailscale.com/client/tailscale"
-	"tailscale.com/tailcfg"
+	"tailscale.com/client/tailscale/apitype"
 )
 
 var (
@@ -107,18 +107,18 @@ type tmplData struct {
 	IP            string // "100.2.3.4"
 }
 
-func tailscaleIP(who *tailcfg.WhoIsResponse) string {
+func tailscaleIP(who *apitype.WhoIsResponse) string {
 	if who == nil {
 		return ""
 	}
 	for _, nodeIP := range who.Node.Addresses {
-		if nodeIP.IP.Is4() && nodeIP.IsSingleIP() {
-			return nodeIP.IP.String()
+		if nodeIP.IP().Is4() && nodeIP.IsSingleIP() {
+			return nodeIP.IP().String()
 		}
 	}
 	for _, nodeIP := range who.Node.Addresses {
 		if nodeIP.IsSingleIP() {
-			return nodeIP.IP.String()
+			return nodeIP.IP().String()
 		}
 	}
 	return ""

cmd/tailscale/cli/cli.go

@@ -8,13 +8,16 @@ package cli
 
 import (
 	"context"
+	"errors"
 	"flag"
 	"fmt"
+	"io"
 	"log"
 	"net"
 	"os"
 	"os/signal"
 	"runtime"
+	"strconv"
 	"strings"
 	"syscall"
 	"text/tabwriter"
@@ -24,21 +27,52 @@ import (
 	"tailscale.com/ipn"
 	"tailscale.com/paths"
 	"tailscale.com/safesocket"
+	"tailscale.com/syncs"
 )
 
 // ActLikeCLI reports whether a GUI application should act like the
 // CLI based on os.Args, GOOS, the context the process is running in
 // (pty, parent PID), etc.
 func ActLikeCLI() bool {
-	if len(os.Args) < 2 {
+	// This function is only used on macOS.
+	if runtime.GOOS != "darwin" {
 		return false
 	}
-	switch os.Args[1] {
-	case "up", "down", "status", "netcheck", "ping", "version",
-		"debug",
-		"-V", "--version", "-h", "--help":
+
+	// Escape hatch to let people force running the macOS
+	// GUI Tailscale binary as the CLI.
+	if v, _ := strconv.ParseBool(os.Getenv("TAILSCALE_BE_CLI")); v {
 		return true
 	}
+
+	// If our parent is launchd, we're definitely not
+	// being run as a CLI.
+	if os.Getppid() == 1 {
+		return false
+	}
+
+	// Xcode adds the -NSDocumentRevisionsDebugMode flag on execution.
+	// If present, we are almost certainly being run as a GUI.
+	for _, arg := range os.Args {
+		if arg == "-NSDocumentRevisionsDebugMode" {
+			return false
+		}
+	}
+
+	// Looking at the environment of the GUI Tailscale app (ps eww
+	// $PID), empirically none of these environment variables are
+	// present. But all or some of these should be present with
+	// Terminal.all and bash or zsh.
+	for _, e := range []string{
+		"SHLVL",
+		"TERM",
+		"TERM_PROGRAM",
+		"PS1",
+	} {
+		if os.Getenv(e) != "" {
+			return true
+		}
+	}
 	return false
 }
 
@@ -64,13 +98,14 @@ change in the future.
 		Subcommands: []*ffcli.Command{
 			upCmd,
 			downCmd,
+			logoutCmd,
 			netcheckCmd,
 			ipCmd,
 			statusCmd,
 			pingCmd,
 			versionCmd,
 			webCmd,
-			pushCmd,
+			fileCmd,
 			bugReportCmd,
 		},
 		FlagSet:   rootfs,
@@ -108,6 +143,8 @@ var rootArgs struct {
 	socket string
 }
 
+var gotSignal syncs.AtomicBool
+
 func connect(ctx context.Context) (net.Conn, *ipn.BackendClient, context.Context, context.CancelFunc) {
 	c, err := safesocket.Connect(rootArgs.socket, 41112)
 	if err != nil {
@@ -125,7 +162,14 @@ func connect(ctx context.Context) (net.Conn, *ipn.BackendClient, context.Context
 	go func() {
 		interrupt := make(chan os.Signal, 1)
 		signal.Notify(interrupt, syscall.SIGINT, syscall.SIGTERM)
-		<-interrupt
+		select {
+		case <-interrupt:
+		case <-ctx.Done():
+			// Context canceled elsewhere.
+			signal.Reset(syscall.SIGINT, syscall.SIGTERM)
+			return
+		}
+		gotSignal.Set(true)
 		c.Close()
 		cancel()
 	}()
@@ -135,19 +179,22 @@ func connect(ctx context.Context) (net.Conn, *ipn.BackendClient, context.Context
 }
 
 // pump receives backend messages on conn and pushes them into bc.
-func pump(ctx context.Context, bc *ipn.BackendClient, conn net.Conn) {
+func pump(ctx context.Context, bc *ipn.BackendClient, conn net.Conn) error {
 	defer conn.Close()
 	for ctx.Err() == nil {
 		msg, err := ipn.ReadMsg(conn)
 		if err != nil {
 			if ctx.Err() != nil {
-				return
+				return ctx.Err()
 			}
-			log.Printf("ReadMsg: %v\n", err)
-			break
+			if errors.Is(err, io.EOF) || errors.Is(err, net.ErrClosed) {
+				return fmt.Errorf("%w (tailscaled stopped running?)", err)
+			}
+			return err
 		}
 		bc.GotNotifyMsg(msg)
 	}
+	return ctx.Err()
 }
 
 func strSliceContains(ss []string, s string) bool {

cmd/tailscale/cli/cli_test.go

@@ -0,0 +1,668 @@
+// Copyright (c) 2021 Tailscale Inc & AUTHORS All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package cli
+
+import (
+	"bytes"
+	"encoding/json"
+	"flag"
+	"fmt"
+	"reflect"
+	"strings"
+	"testing"
+
+	"inet.af/netaddr"
+	"tailscale.com/ipn"
+	"tailscale.com/ipn/ipnstate"
+	"tailscale.com/types/preftype"
+)
+
+// geese is a collection of gooses. It need not be complete.
+// But it should include anything handled specially (e.g. linux, windows)
+// and at least one thing that's not (darwin, freebsd).
+var geese = []string{"linux", "darwin", "windows", "freebsd"}
+
+// Test that checkForAccidentalSettingReverts's updateMaskedPrefsFromUpFlag can handle
+// all flags. This will panic if a new flag creeps in that's unhandled.
+//
+// Also, issue 1880: advertise-exit-node was being ignored. Verify that all flags cause an edit.
+func TestUpdateMaskedPrefsFromUpFlag(t *testing.T) {
+	for _, goos := range geese {
+		var upArgs upArgsT
+		fs := newUpFlagSet(goos, &upArgs)
+		fs.VisitAll(func(f *flag.Flag) {
+			mp := new(ipn.MaskedPrefs)
+			updateMaskedPrefsFromUpFlag(mp, f.Name)
+			got := mp.Pretty()
+			wantEmpty := preflessFlag(f.Name)
+			isEmpty := got == "MaskedPrefs{}"
+			if isEmpty != wantEmpty {
+				t.Errorf("flag %q created MaskedPrefs %s; want empty=%v", f.Name, got, wantEmpty)
+			}
+		})
+	}
+}
+
+func TestCheckForAccidentalSettingReverts(t *testing.T) {
+	tests := []struct {
+		name     string
+		flags    []string // argv to be parsed by FlagSet
+		curPrefs *ipn.Prefs
+
+		curExitNodeIP netaddr.IP
+		curUser       string // os.Getenv("USER") on the client side
+		goos          string // empty means "linux"
+
+		want string
+	}{
+		{
+			name:  "bare_up_means_up",
+			flags: []string{},
+			curPrefs: &ipn.Prefs{
+				ControlURL:  ipn.DefaultControlURL,
+				WantRunning: false,
+				Hostname:    "foo",
+			},
+			want: "",
+		},
+		{
+			name:  "losing_hostname",
+			flags: []string{"--accept-dns"},
+			curPrefs: &ipn.Prefs{
+				ControlURL:       ipn.DefaultControlURL,
+				WantRunning:      false,
+				Hostname:         "foo",
+				CorpDNS:          true,
+				NetfilterMode:    preftype.NetfilterOn,
+				AllowSingleHosts: true,
+			},
+			want: accidentalUpPrefix + " --accept-dns --hostname=foo",
+		},
+		{
+			name:  "hostname_changing_explicitly",
+			flags: []string{"--hostname=bar"},
+			curPrefs: &ipn.Prefs{
+				ControlURL:       ipn.DefaultControlURL,
+				CorpDNS:          true,
+				NetfilterMode:    preftype.NetfilterOn,
+				AllowSingleHosts: true,
+				Hostname:         "foo",
+			},
+			want: "",
+		},
+		{
+			name:  "hostname_changing_empty_explicitly",
+			flags: []string{"--hostname="},
+			curPrefs: &ipn.Prefs{
+				ControlURL:       ipn.DefaultControlURL,
+				CorpDNS:          true,
+				NetfilterMode:    preftype.NetfilterOn,
+				AllowSingleHosts: true,
+				Hostname:         "foo",
+			},
+			want: "",
+		},
+		{
+			// Issue 1725: "tailscale up --authkey=..." (or other non-empty flags) works from
+			// a fresh server's initial prefs.
+			name:     "up_with_default_prefs",
+			flags:    []string{"--authkey=foosdlkfjskdljf"},
+			curPrefs: ipn.NewPrefs(),
+			want:     "",
+		},
+		{
+			name:  "implicit_operator_change",
+			flags: []string{"--hostname=foo"},
+			curPrefs: &ipn.Prefs{
+				ControlURL:       ipn.DefaultControlURL,
+				OperatorUser:     "alice",
+				AllowSingleHosts: true,
+				CorpDNS:          true,
+				NetfilterMode:    preftype.NetfilterOn,
+			},
+			curUser: "eve",
+			want:    accidentalUpPrefix + " --hostname=foo --operator=alice",
+		},
+		{
+			name:  "implicit_operator_matches_shell_user",
+			flags: []string{"--hostname=foo"},
+			curPrefs: &ipn.Prefs{
+				ControlURL:       ipn.DefaultControlURL,
+				AllowSingleHosts: true,
+				CorpDNS:          true,
+				NetfilterMode:    preftype.NetfilterOn,
+				OperatorUser:     "alice",
+			},
+			curUser: "alice",
+			want:    "",
+		},
+		{
+			name:  "error_advertised_routes_exit_node_removed",
+			flags: []string{"--advertise-routes=10.0.42.0/24"},
+			curPrefs: &ipn.Prefs{
+				ControlURL:       ipn.DefaultControlURL,
+				AllowSingleHosts: true,
+				CorpDNS:          true,
+				NetfilterMode:    preftype.NetfilterOn,
+				AdvertiseRoutes: []netaddr.IPPrefix{
+					netaddr.MustParseIPPrefix("10.0.42.0/24"),
+					netaddr.MustParseIPPrefix("0.0.0.0/0"),
+					netaddr.MustParseIPPrefix("::/0"),
+				},
+			},
+			want: accidentalUpPrefix + " --advertise-routes=10.0.42.0/24 --advertise-exit-node",
+		},
+		{
+			name:  "advertised_routes_exit_node_removed_explicit",
+			flags: []string{"--advertise-routes=10.0.42.0/24", "--advertise-exit-node=false"},
+			curPrefs: &ipn.Prefs{
+				ControlURL:       ipn.DefaultControlURL,
+				AllowSingleHosts: true,
+				CorpDNS:          true,
+				NetfilterMode:    preftype.NetfilterOn,
+				AdvertiseRoutes: []netaddr.IPPrefix{
+					netaddr.MustParseIPPrefix("10.0.42.0/24"),
+					netaddr.MustParseIPPrefix("0.0.0.0/0"),
+					netaddr.MustParseIPPrefix("::/0"),
+				},
+			},
+			want: "",
+		},
+		{
+			name:  "advertised_routes_includes_the_0_routes", // but no --advertise-exit-node
+			flags: []string{"--advertise-routes=11.1.43.0/24,0.0.0.0/0,::/0"},
+			curPrefs: &ipn.Prefs{
+				ControlURL:       ipn.DefaultControlURL,
+				AllowSingleHosts: true,
+				CorpDNS:          true,
+				NetfilterMode:    preftype.NetfilterOn,
+				AdvertiseRoutes: []netaddr.IPPrefix{
+					netaddr.MustParseIPPrefix("10.0.42.0/24"),
+					netaddr.MustParseIPPrefix("0.0.0.0/0"),
+					netaddr.MustParseIPPrefix("::/0"),
+				},
+			},
+			want: "",
+		},
+		{
+			name:  "advertise_exit_node", // Issue 1859
+			flags: []string{"--advertise-exit-node"},
+			curPrefs: &ipn.Prefs{
+				ControlURL:       ipn.DefaultControlURL,
+				AllowSingleHosts: true,
+				CorpDNS:          true,
+				NetfilterMode:    preftype.NetfilterOn,
+			},
+			want: "",
+		},
+		{
+			name:  "advertise_exit_node_over_existing_routes",
+			flags: []string{"--advertise-exit-node"},
+			curPrefs: &ipn.Prefs{
+				ControlURL:       ipn.DefaultControlURL,
+				AllowSingleHosts: true,
+				CorpDNS:          true,
+				NetfilterMode:    preftype.NetfilterOn,
+
+				AdvertiseRoutes: []netaddr.IPPrefix{
+					netaddr.MustParseIPPrefix("1.2.0.0/16"),
+				},
+			},
+			want: accidentalUpPrefix + " --advertise-exit-node --advertise-routes=1.2.0.0/16",
+		},
+		{
+			name:  "advertise_exit_node_over_existing_routes_and_exit_node",
+			flags: []string{"--advertise-exit-node"},
+			curPrefs: &ipn.Prefs{
+				ControlURL:       ipn.DefaultControlURL,
+				AllowSingleHosts: true,
+				CorpDNS:          true,
+				NetfilterMode:    preftype.NetfilterOn,
+				AdvertiseRoutes: []netaddr.IPPrefix{
+					netaddr.MustParseIPPrefix("0.0.0.0/0"),
+					netaddr.MustParseIPPrefix("::/0"),
+					netaddr.MustParseIPPrefix("1.2.0.0/16"),
+				},
+			},
+			want: accidentalUpPrefix + " --advertise-exit-node --advertise-routes=1.2.0.0/16",
+		},
+		{
+			name:  "exit_node_clearing", // Issue 1777
+			flags: []string{"--exit-node="},
+			curPrefs: &ipn.Prefs{
+				ControlURL:       ipn.DefaultControlURL,
+				AllowSingleHosts: true,
+				CorpDNS:          true,
+				NetfilterMode:    preftype.NetfilterOn,
+
+				ExitNodeID: "fooID",
+			},
+			want: "",
+		},
+		{
+			name:  "remove_all_implicit",
+			flags: []string{"--force-reauth"},
+			curPrefs: &ipn.Prefs{
+				WantRunning:      true,
+				ControlURL:       ipn.DefaultControlURL,
+				RouteAll:         true,
+				AllowSingleHosts: false,
+				ExitNodeIP:       netaddr.MustParseIP("100.64.5.6"),
+				CorpDNS:          false,
+				ShieldsUp:        true,
+				AdvertiseTags:    []string{"tag:foo", "tag:bar"},
+				Hostname:         "myhostname",
+				ForceDaemon:      true,
+				AdvertiseRoutes: []netaddr.IPPrefix{
+					netaddr.MustParseIPPrefix("10.0.0.0/16"),
+					netaddr.MustParseIPPrefix("0.0.0.0/0"),
+					netaddr.MustParseIPPrefix("::/0"),
+				},
+				NetfilterMode: preftype.NetfilterNoDivert,
+				OperatorUser:  "alice",
+			},
+			curUser: "eve",
+			want:    accidentalUpPrefix + " --force-reauth --accept-dns=false --accept-routes --advertise-exit-node --advertise-routes=10.0.0.0/16 --advertise-tags=tag:foo,tag:bar --exit-node=100.64.5.6 --host-routes=false --hostname=myhostname --netfilter-mode=nodivert --operator=alice --shields-up",
+		},
+		{
+			name:  "remove_all_implicit_except_hostname",
+			flags: []string{"--hostname=newhostname"},
+			curPrefs: &ipn.Prefs{
+				WantRunning:      true,
+				ControlURL:       ipn.DefaultControlURL,
+				RouteAll:         true,
+				AllowSingleHosts: false,
+				ExitNodeIP:       netaddr.MustParseIP("100.64.5.6"),
+				CorpDNS:          false,
+				ShieldsUp:        true,
+				AdvertiseTags:    []string{"tag:foo", "tag:bar"},
+				Hostname:         "myhostname",
+				ForceDaemon:      true,
+				AdvertiseRoutes: []netaddr.IPPrefix{
+					netaddr.MustParseIPPrefix("10.0.0.0/16"),
+				},
+				NetfilterMode: preftype.NetfilterNoDivert,
+				OperatorUser:  "alice",
+			},
+			curUser: "eve",
+			want:    accidentalUpPrefix + " --hostname=newhostname --accept-dns=false --accept-routes --advertise-routes=10.0.0.0/16 --advertise-tags=tag:foo,tag:bar --exit-node=100.64.5.6 --host-routes=false --netfilter-mode=nodivert --operator=alice --shields-up",
+		},
+		{
+			name:  "loggedout_is_implicit",
+			flags: []string{"--hostname=foo"},
+			curPrefs: &ipn.Prefs{
+				ControlURL:       ipn.DefaultControlURL,
+				LoggedOut:        true,
+				AllowSingleHosts: true,
+				CorpDNS:          true,
+				NetfilterMode:    preftype.NetfilterOn,
+			},
+			want: "", // not an error. LoggedOut is implicit.
+		},
+		{
+			// Test that a pre-1.8 version of Tailscale with bogus NoSNAT pref
+			// values is able to enable exit nodes without warnings.
+			name:  "make_windows_exit_node",
+			flags: []string{"--advertise-exit-node"},
+			curPrefs: &ipn.Prefs{
+				ControlURL:       ipn.DefaultControlURL,
+				AllowSingleHosts: true,
+				CorpDNS:          true,
+
+				// And assume this no-op accidental pre-1.8 value:
+				NoSNAT: true,
+			},
+			goos: "windows",
+			want: "", // not an error
+		},
+		{
+			name:  "ignore_netfilter_change_non_linux",
+			flags: []string{"--accept-dns"},
+			curPrefs: &ipn.Prefs{
+				ControlURL:       ipn.DefaultControlURL,
+				AllowSingleHosts: true,
+
+				NetfilterMode: preftype.NetfilterNoDivert, // we never had this bug, but pretend it got set non-zero on Windows somehow
+			},
+			goos: "windows",
+			want: "", // not an error
+		},
+		{
+			name:  "operator_losing_routes_step1", // https://twitter.com/EXPbits/status/1390418145047887877
+			flags: []string{"--operator=expbits"},
+			curPrefs: &ipn.Prefs{
+				ControlURL:       ipn.DefaultControlURL,
+				AllowSingleHosts: true,
+				CorpDNS:          true,
+				NetfilterMode:    preftype.NetfilterOn,
+				AdvertiseRoutes: []netaddr.IPPrefix{
+					netaddr.MustParseIPPrefix("0.0.0.0/0"),
+					netaddr.MustParseIPPrefix("::/0"),
+					netaddr.MustParseIPPrefix("1.2.0.0/16"),
+				},
+			},
+			want: accidentalUpPrefix + " --operator=expbits --advertise-exit-node --advertise-routes=1.2.0.0/16",
+		},
+		{
+			name:  "operator_losing_routes_step2", // https://twitter.com/EXPbits/status/1390418145047887877
+			flags: []string{"--operator=expbits", "--advertise-routes=1.2.0.0/16"},
+			curPrefs: &ipn.Prefs{
+				ControlURL:       ipn.DefaultControlURL,
+				AllowSingleHosts: true,
+				CorpDNS:          true,
+				NetfilterMode:    preftype.NetfilterOn,
+				AdvertiseRoutes: []netaddr.IPPrefix{
+					netaddr.MustParseIPPrefix("0.0.0.0/0"),
+					netaddr.MustParseIPPrefix("::/0"),
+					netaddr.MustParseIPPrefix("1.2.0.0/16"),
+				},
+			},
+			want: accidentalUpPrefix + " --advertise-routes=1.2.0.0/16 --operator=expbits --advertise-exit-node",
+		},
+		{
+			name:  "errors_preserve_explicit_flags",
+			flags: []string{"--reset", "--force-reauth=false", "--authkey=secretrand"},
+			curPrefs: &ipn.Prefs{
+				ControlURL:       ipn.DefaultControlURL,
+				WantRunning:      false,
+				CorpDNS:          true,
+				NetfilterMode:    preftype.NetfilterOn,
+				AllowSingleHosts: true,
+
+				Hostname: "foo",
+			},
+			want: accidentalUpPrefix + " --authkey=secretrand --force-reauth=false --reset --hostname=foo",
+		},
+		{
+			name:  "error_exit_node_omit_with_ip_pref",
+			flags: []string{"--hostname=foo"},
+			curPrefs: &ipn.Prefs{
+				ControlURL:       ipn.DefaultControlURL,
+				AllowSingleHosts: true,
+				CorpDNS:          true,
+				NetfilterMode:    preftype.NetfilterOn,
+
+				ExitNodeIP: netaddr.MustParseIP("100.64.5.4"),
+			},
+			want: accidentalUpPrefix + " --hostname=foo --exit-node=100.64.5.4",
+		},
+		{
+			name:          "error_exit_node_omit_with_id_pref",
+			flags:         []string{"--hostname=foo"},
+			curExitNodeIP: netaddr.MustParseIP("100.64.5.7"),
+			curPrefs: &ipn.Prefs{
+				ControlURL:       ipn.DefaultControlURL,
+				AllowSingleHosts: true,
+				CorpDNS:          true,
+				NetfilterMode:    preftype.NetfilterOn,
+
+				ExitNodeID: "some_stable_id",
+			},
+			want: accidentalUpPrefix + " --hostname=foo --exit-node=100.64.5.7",
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			goos := "linux"
+			if tt.goos != "" {
+				goos = tt.goos
+			}
+			var upArgs upArgsT
+			flagSet := newUpFlagSet(goos, &upArgs)
+			flagSet.Parse(tt.flags)
+			newPrefs, err := prefsFromUpArgs(upArgs, t.Logf, new(ipnstate.Status), goos)
+			if err != nil {
+				t.Fatal(err)
+			}
+			applyImplicitPrefs(newPrefs, tt.curPrefs, tt.curUser)
+			var got string
+			if err := checkForAccidentalSettingReverts(flagSet, tt.curPrefs, newPrefs, upCheckEnv{
+				goos:          goos,
+				curExitNodeIP: tt.curExitNodeIP,
+			}); err != nil {
+				got = err.Error()
+			}
+			if strings.TrimSpace(got) != tt.want {
+				t.Errorf("unexpected result\n got: %s\nwant: %s\n", got, tt.want)
+			}
+		})
+	}
+}
+
+func upArgsFromOSArgs(goos string, flagArgs ...string) (args upArgsT) {
+	fs := newUpFlagSet(goos, &args)
+	fs.Parse(flagArgs) // populates args
+	return
+}
+
+func TestPrefsFromUpArgs(t *testing.T) {
+	tests := []struct {
+		name     string
+		args     upArgsT
+		goos     string           // runtime.GOOS; empty means linux
+		st       *ipnstate.Status // or nil
+		want     *ipn.Prefs
+		wantErr  string
+		wantWarn string
+	}{
+		{
+			name: "default_linux",
+			goos: "linux",
+			args: upArgsFromOSArgs("linux"),
+			want: &ipn.Prefs{
+				ControlURL:       ipn.DefaultControlURL,
+				WantRunning:      true,
+				NoSNAT:           false,
+				NetfilterMode:    preftype.NetfilterOn,
+				CorpDNS:          true,
+				AllowSingleHosts: true,
+			},
+		},
+		{
+			name: "default_windows",
+			goos: "windows",
+			args: upArgsFromOSArgs("windows"),
+			want: &ipn.Prefs{
+				ControlURL:       ipn.DefaultControlURL,
+				WantRunning:      true,
+				CorpDNS:          true,
+				AllowSingleHosts: true,
+				NetfilterMode:    preftype.NetfilterOn,
+			},
+		},
+		{
+			name: "advertise_default_route",
+			args: upArgsFromOSArgs("linux", "--advertise-exit-node"),
+			want: &ipn.Prefs{
+				ControlURL:       ipn.DefaultControlURL,
+				WantRunning:      true,
+				AllowSingleHosts: true,
+				CorpDNS:          true,
+				AdvertiseRoutes: []netaddr.IPPrefix{
+					netaddr.MustParseIPPrefix("0.0.0.0/0"),
+					netaddr.MustParseIPPrefix("::/0"),
+				},
+				NetfilterMode: preftype.NetfilterOn,
+			},
+		},
+		{
+			name: "error_advertise_route_invalid_ip",
+			args: upArgsT{
+				advertiseRoutes: "foo",
+			},
+			wantErr: `"foo" is not a valid IP address or CIDR prefix`,
+		},
+		{
+			name: "error_advertise_route_unmasked_bits",
+			args: upArgsT{
+				advertiseRoutes: "1.2.3.4/16",
+			},
+			wantErr: `1.2.3.4/16 has non-address bits set; expected 1.2.0.0/16`,
+		},
+		{
+			name: "error_exit_node_bad_ip",
+			args: upArgsT{
+				exitNodeIP: "foo",
+			},
+			wantErr: `invalid IP address "foo" for --exit-node: ParseIP("foo"): unable to parse IP`,
+		},
+		{
+			name: "error_exit_node_allow_lan_without_exit_node",
+			args: upArgsT{
+				exitNodeAllowLANAccess: true,
+			},
+			wantErr: `--exit-node-allow-lan-access can only be used with --exit-node`,
+		},
+		{
+			name: "error_tag_prefix",
+			args: upArgsT{
+				advertiseTags: "foo",
+			},
+			wantErr: `tag: "foo": tags must start with 'tag:'`,
+		},
+		{
+			name: "error_long_hostname",
+			args: upArgsT{
+				hostname: strings.Repeat("a", 300),
+			},
+			wantErr: `hostname too long: 300 bytes (max 256)`,
+		},
+		{
+			name: "error_linux_netfilter_empty",
+			args: upArgsT{
+				netfilterMode: "",
+			},
+			wantErr: `invalid value --netfilter-mode=""`,
+		},
+		{
+			name: "error_linux_netfilter_bogus",
+			args: upArgsT{
+				netfilterMode: "bogus",
+			},
+			wantErr: `invalid value --netfilter-mode="bogus"`,
+		},
+		{
+			name: "error_exit_node_ip_is_self_ip",
+			args: upArgsT{
+				exitNodeIP: "100.105.106.107",
+			},
+			st: &ipnstate.Status{
+				TailscaleIPs: []netaddr.IP{netaddr.MustParseIP("100.105.106.107")},
+			},
+			wantErr: `cannot use 100.105.106.107 as the exit node as it is a local IP address to this machine, did you mean --advertise-exit-node?`,
+		},
+		{
+			name: "warn_linux_netfilter_nodivert",
+			goos: "linux",
+			args: upArgsT{
+				netfilterMode: "nodivert",
+			},
+			wantWarn: "netfilter=nodivert; add iptables calls to ts-* chains manually.",
+			want: &ipn.Prefs{
+				WantRunning:   true,
+				NetfilterMode: preftype.NetfilterNoDivert,
+				NoSNAT:        true,
+			},
+		},
+		{
+			name: "warn_linux_netfilter_off",
+			goos: "linux",
+			args: upArgsT{
+				netfilterMode: "off",
+			},
+			wantWarn: "netfilter=off; configure iptables yourself.",
+			want: &ipn.Prefs{
+				WantRunning:   true,
+				NetfilterMode: preftype.NetfilterOff,
+				NoSNAT:        true,
+			},
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			var warnBuf bytes.Buffer
+			warnf := func(format string, a ...interface{}) {
+				fmt.Fprintf(&warnBuf, format, a...)
+			}
+			goos := tt.goos
+			if goos == "" {
+				goos = "linux"
+			}
+			st := tt.st
+			if st == nil {
+				st = new(ipnstate.Status)
+			}
+			got, err := prefsFromUpArgs(tt.args, warnf, st, goos)
+			gotErr := fmt.Sprint(err)
+			if tt.wantErr != "" {
+				if tt.wantErr != gotErr {
+					t.Errorf("wrong error.\n got error: %v\nwant error: %v\n", gotErr, tt.wantErr)
+				}
+				return
+			}
+			if err != nil {
+				t.Fatal(err)
+			}
+			if tt.want == nil {
+				t.Fatal("tt.want is nil")
+			}
+			if !got.Equals(tt.want) {
+				jgot, _ := json.MarshalIndent(got, "", "\t")
+				jwant, _ := json.MarshalIndent(tt.want, "", "\t")
+				if bytes.Equal(jgot, jwant) {
+					t.Logf("prefs differ only in non-JSON-visible ways (nil/non-nil zero-length arrays)")
+				}
+				t.Errorf("wrong prefs\n got: %s\nwant: %s\n\ngot: %s\nwant: %s\n",
+					got.Pretty(), tt.want.Pretty(),
+					jgot, jwant,
+				)
+
+			}
+		})
+	}
+
+}
+
+func TestPrefFlagMapping(t *testing.T) {
+	prefHasFlag := map[string]bool{}
+	for _, pv := range prefsOfFlag {
+		for _, pref := range pv {
+			prefHasFlag[pref] = true
+		}
+	}
+
+	prefType := reflect.TypeOf(ipn.Prefs{})
+	for i := 0; i < prefType.NumField(); i++ {
+		prefName := prefType.Field(i).Name
+		if prefHasFlag[prefName] {
+			continue
+		}
+		switch prefName {
+		case "WantRunning", "Persist", "LoggedOut":
+			// All explicitly handled (ignored) by checkForAccidentalSettingReverts.
+			continue
+		case "OSVersion", "DeviceModel":
+			// Only used by Android, which doesn't have a CLI mode anyway, so
+			// fine to not map.
+			continue
+		case "NotepadURLs":
+			// TODO(bradfitz): https://github.com/tailscale/tailscale/issues/1830
+			continue
+		}
+		t.Errorf("unexpected new ipn.Pref field %q is not handled by up.go (see addPrefFlagMapping and checkForAccidentalSettingReverts)", prefName)
+	}
+}
+
+func TestFlagAppliesToOS(t *testing.T) {
+	for _, goos := range geese {
+		var upArgs upArgsT
+		fs := newUpFlagSet(goos, &upArgs)
+		fs.VisitAll(func(f *flag.Flag) {
+			if !flagAppliesToOS(f.Name, goos) {
+				t.Errorf("flagAppliesToOS(%q, %q) = false but found in %s set", f.Name, goos, goos)
+			}
+		})
+	}
+}

cmd/tailscale/cli/debug.go

@@ -13,11 +13,14 @@ import (
 	"io"
 	"log"
 	"os"
+	"runtime"
 	"strings"
 
 	"github.com/peterbourgon/ff/v2/ffcli"
 	"tailscale.com/client/tailscale"
 	"tailscale.com/ipn"
+	"tailscale.com/paths"
+	"tailscale.com/safesocket"
 )
 
 var debugCmd = &ffcli.Command{
@@ -27,23 +30,55 @@ var debugCmd = &ffcli.Command{
 		fs := flag.NewFlagSet("debug", flag.ExitOnError)
 		fs.BoolVar(&debugArgs.goroutines, "daemon-goroutines", false, "If true, dump the tailscaled daemon's goroutines")
 		fs.BoolVar(&debugArgs.ipn, "ipn", false, "If true, subscribe to IPN notifications")
+		fs.BoolVar(&debugArgs.prefs, "prefs", false, "If true, dump active prefs")
+		fs.BoolVar(&debugArgs.pretty, "pretty", false, "If true, pretty-print output (for --prefs)")
 		fs.BoolVar(&debugArgs.netMap, "netmap", true, "whether to include netmap in --ipn mode")
+		fs.BoolVar(&debugArgs.localCreds, "local-creds", false, "print how to connect to local tailscaled")
 		fs.StringVar(&debugArgs.file, "file", "", "get, delete:NAME, or NAME")
 		return fs
 	})(),
 }
 
 var debugArgs struct {
+	localCreds bool
 	goroutines bool
 	ipn        bool
 	netMap     bool
 	file       string
+	prefs      bool
+	pretty     bool
 }
 
 func runDebug(ctx context.Context, args []string) error {
 	if len(args) > 0 {
 		return errors.New("unknown arguments")
 	}
+	if debugArgs.localCreds {
+		port, token, err := safesocket.LocalTCPPortAndToken()
+		if err == nil {
+			fmt.Printf("curl -u:%s http://localhost:%d/localapi/v0/status\n", token, port)
+			return nil
+		}
+		if runtime.GOOS == "windows" {
+			fmt.Printf("curl http://localhost:41112/localapi/v0/status\n")
+			return nil
+		}
+		fmt.Printf("curl --unix-socket %s http://foo/localapi/v0/status\n", paths.DefaultTailscaledSocket())
+		return nil
+	}
+	if debugArgs.prefs {
+		prefs, err := tailscale.GetPrefs(ctx)
+		if err != nil {
+			return err
+		}
+		if debugArgs.pretty {
+			fmt.Println(prefs.Pretty())
+		} else {
+			j, _ := json.MarshalIndent(prefs, "", "\t")
+			fmt.Println(string(j))
+		}
+		return nil
+	}
 	if debugArgs.goroutines {
 		goroutines, err := tailscale.Goroutines(ctx)
 		if err != nil {

cmd/tailscale/cli/down.go

@@ -8,7 +8,7 @@ import (
 	"context"
 	"fmt"
 	"log"
-	"time"
+	"os"
 
 	"github.com/peterbourgon/ff/v2/ffcli"
 	"tailscale.com/client/tailscale"
@@ -33,39 +33,14 @@ func runDown(ctx context.Context, args []string) error {
 		return fmt.Errorf("error fetching current status: %w", err)
 	}
 	if st.BackendState == "Stopped" {
-		log.Printf("already stopped")
+		fmt.Fprintf(os.Stderr, "Tailscale was already stopped.\n")
 		return nil
 	}
-	log.Printf("was in state %q", st.BackendState)
-
-	c, bc, ctx, cancel := connect(ctx)
-	defer cancel()
-
-	timer := time.AfterFunc(5*time.Second, func() {
-		log.Fatalf("timeout running stop")
-	})
-	defer timer.Stop()
-
-	bc.SetNotifyCallback(func(n ipn.Notify) {
-		if n.ErrMessage != nil {
-			log.Fatal(*n.ErrMessage)
-		}
-		if n.State != nil {
-			log.Printf("now in state %q", *n.State)
-			if *n.State == ipn.Stopped {
-				cancel()
-			}
-			return
-		}
-	})
-
-	bc.EditPrefs(&ipn.MaskedPrefs{
+	_, err = tailscale.EditPrefs(ctx, &ipn.MaskedPrefs{
 		Prefs: ipn.Prefs{
 			WantRunning: false,
 		},
 		WantRunningSet: true,
 	})
-	pump(ctx, bc, c)
-
-	return nil
+	return err
 }

cmd/tailscale/cli/file.go

@@ -0,0 +1,444 @@
+// Copyright (c) 2021 Tailscale Inc & AUTHORS All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package cli
+
+import (
+	"bytes"
+	"context"
+	"errors"
+	"flag"
+	"fmt"
+	"io"
+	"io/ioutil"
+	"log"
+	"mime"
+	"net/http"
+	"net/url"
+	"os"
+	"path/filepath"
+	"strconv"
+	"strings"
+	"time"
+	"unicode/utf8"
+
+	"github.com/peterbourgon/ff/v2/ffcli"
+	"golang.org/x/time/rate"
+	"inet.af/netaddr"
+	"tailscale.com/client/tailscale"
+	"tailscale.com/client/tailscale/apitype"
+	"tailscale.com/ipn"
+	"tailscale.com/net/tsaddr"
+	"tailscale.com/version"
+)
+
+var fileCmd = &ffcli.Command{
+	Name:       "file",
+	ShortUsage: "file <cp|get> ...",
+	ShortHelp:  "Send or receive files",
+	Subcommands: []*ffcli.Command{
+		fileCpCmd,
+		fileGetCmd,
+	},
+	Exec: func(context.Context, []string) error {
+		// TODO(bradfitz): is there a better ffcli way to
+		// annotate subcommand-required commands that don't
+		// have an exec body of their own?
+		return errors.New("file subcommand required; run 'tailscale file -h' for details")
+	},
+}
+
+var fileCpCmd = &ffcli.Command{
+	Name:       "cp",
+	ShortUsage: "file cp <files...> <target>:",
+	ShortHelp:  "Copy file(s) to a host",
+	Exec:       runCp,
+	FlagSet: (func() *flag.FlagSet {
+		fs := flag.NewFlagSet("cp", flag.ExitOnError)
+		fs.StringVar(&cpArgs.name, "name", "", "alternate filename to use, especially useful when <file> is \"-\" (stdin)")
+		fs.BoolVar(&cpArgs.verbose, "verbose", false, "verbose output")
+		fs.BoolVar(&cpArgs.targets, "targets", false, "list possible file cp targets")
+		return fs
+	})(),
+}
+
+var cpArgs struct {
+	name    string
+	verbose bool
+	targets bool
+}
+
+func runCp(ctx context.Context, args []string) error {
+	if cpArgs.targets {
+		return runCpTargets(ctx, args)
+	}
+	if len(args) < 2 {
+		//lint:ignore ST1005 no sorry need that colon at the end
+		return errors.New("usage: tailscale file cp <files...> <target>:")
+	}
+	files, target := args[:len(args)-1], args[len(args)-1]
+	if !strings.HasSuffix(target, ":") {
+		return fmt.Errorf("final argument to 'tailscale file cp' must end in colon")
+	}
+	target = strings.TrimSuffix(target, ":")
+	hadBrackets := false
+	if strings.HasPrefix(target, "[") && strings.HasSuffix(target, "]") {
+		hadBrackets = true
+		target = strings.TrimSuffix(strings.TrimPrefix(target, "["), "]")
+	}
+	if ip, err := netaddr.ParseIP(target); err == nil && ip.Is6() && !hadBrackets {
+		return fmt.Errorf("an IPv6 literal must be written as [%s]", ip)
+	} else if hadBrackets && (err != nil || !ip.Is6()) {
+		return errors.New("unexpected brackets around target")
+	}
+	ip, err := tailscaleIPFromArg(ctx, target)
+	if err != nil {
+		return err
+	}
+
+	peerAPIBase, lastSeen, isOffline, err := discoverPeerAPIBase(ctx, ip)
+	if err != nil {
+		return fmt.Errorf("can't send to %s: %v", target, err)
+	}
+	if isOffline {
+		fmt.Fprintf(os.Stderr, "# warning: %s is offline\n", target)
+	} else if !lastSeen.IsZero() && time.Since(lastSeen) > lastSeenOld {
+		fmt.Fprintf(os.Stderr, "# warning: %s last seen %v ago\n", target, time.Since(lastSeen).Round(time.Minute))
+	}
+
+	if len(files) > 1 {
+		if cpArgs.name != "" {
+			return errors.New("can't use --name= with multiple files")
+		}
+		for _, fileArg := range files {
+			if fileArg == "-" {
+				return errors.New("can't use '-' as STDIN file when providing filename arguments")
+			}
+		}
+	}
+
+	for _, fileArg := range files {
+		var fileContents io.Reader
+		var name = cpArgs.name
+		var contentLength int64 = -1
+		if fileArg == "-" {
+			fileContents = os.Stdin
+			if name == "" {
+				name, fileContents, err = pickStdinFilename()
+				if err != nil {
+					return err
+				}
+			}
+		} else {
+			f, err := os.Open(fileArg)
+			if err != nil {
+				if version.IsSandboxedMacOS() {
+					return errors.New("the GUI version of Tailscale on macOS runs in a macOS sandbox that can't read files")
+				}
+				return err
+			}
+			defer f.Close()
+			fi, err := f.Stat()
+			if err != nil {
+				return err
+			}
+			if fi.IsDir() {
+				return errors.New("directories not supported")
+			}
+			contentLength = fi.Size()
+			fileContents = io.LimitReader(f, contentLength)
+			if name == "" {
+				name = filepath.Base(fileArg)
+			}
+
+			if slow, _ := strconv.ParseBool(os.Getenv("TS_DEBUG_SLOW_PUSH")); slow {
+				fileContents = &slowReader{r: fileContents}
+			}
+		}
+
+		dstURL := peerAPIBase + "/v0/put/" + url.PathEscape(name)
+		req, err := http.NewRequestWithContext(ctx, "PUT", dstURL, fileContents)
+		if err != nil {
+			return err
+		}
+		req.ContentLength = contentLength
+		if cpArgs.verbose {
+			log.Printf("sending to %v ...", dstURL)
+		}
+		res, err := http.DefaultClient.Do(req)
+		if err != nil {
+			return err
+		}
+		if res.StatusCode == 200 {
+			io.Copy(ioutil.Discard, res.Body)
+			res.Body.Close()
+			continue
+		}
+		io.Copy(os.Stdout, res.Body)
+		res.Body.Close()
+		return errors.New(res.Status)
+	}
+	return nil
+}
+
+func discoverPeerAPIBase(ctx context.Context, ipStr string) (base string, lastSeen time.Time, isOffline bool, err error) {
+	ip, err := netaddr.ParseIP(ipStr)
+	if err != nil {
+		return "", time.Time{}, false, err
+	}
+	fts, err := tailscale.FileTargets(ctx)
+	if err != nil {
+		return "", time.Time{}, false, err
+	}
+	for _, ft := range fts {
+		n := ft.Node
+		for _, a := range n.Addresses {
+			if a.IP() != ip {
+				continue
+			}
+			if n.LastSeen != nil {
+				lastSeen = *n.LastSeen
+			}
+			isOffline = n.Online != nil && !*n.Online
+			return ft.PeerAPIURL, lastSeen, isOffline, nil
+		}
+	}
+	return "", time.Time{}, false, fileTargetErrorDetail(ctx, ip)
+}
+
+// fileTargetErrorDetail returns a non-nil error saying why ip is an
+// invalid file sharing target.
+func fileTargetErrorDetail(ctx context.Context, ip netaddr.IP) error {
+	found := false
+	if st, err := tailscale.Status(ctx); err == nil && st.Self != nil {
+		for _, peer := range st.Peer {
+			for _, pip := range peer.TailscaleIPs {
+				if pip == ip {
+					found = true
+					if peer.UserID != st.Self.UserID {
+						return errors.New("owned by different user; can only send files to your own devices")
+					}
+				}
+			}
+		}
+	}
+	if found {
+		return errors.New("target seems to be running an old Tailscale version")
+	}
+	if !tsaddr.IsTailscaleIP(ip) {
+		return fmt.Errorf("unknown target; %v is not a Tailscale IP address", ip)
+	}
+	return errors.New("unknown target; not in your Tailnet")
+}
+
+const maxSniff = 4 << 20
+
+func ext(b []byte) string {
+	if len(b) < maxSniff && utf8.Valid(b) {
+		return ".txt"
+	}
+	if exts, _ := mime.ExtensionsByType(http.DetectContentType(b)); len(exts) > 0 {
+		return exts[0]
+	}
+	return ""
+}
+
+// pickStdinFilename reads a bit of stdin to return a good filename
+// for its contents. The returned Reader is the concatenation of the
+// read and unread bits.
+func pickStdinFilename() (name string, r io.Reader, err error) {
+	sniff, err := io.ReadAll(io.LimitReader(os.Stdin, maxSniff))
+	if err != nil {
+		return "", nil, err
+	}
+	return "stdin" + ext(sniff), io.MultiReader(bytes.NewReader(sniff), os.Stdin), nil
+}
+
+type slowReader struct {
+	r  io.Reader
+	rl *rate.Limiter
+}
+
+func (r *slowReader) Read(p []byte) (n int, err error) {
+	const burst = 4 << 10
+	plen := len(p)
+	if plen > burst {
+		plen = burst
+	}
+	if r.rl == nil {
+		r.rl = rate.NewLimiter(rate.Limit(1<<10), burst)
+	}
+	n, err = r.r.Read(p[:plen])
+	r.rl.WaitN(context.Background(), n)
+	return
+}
+
+const lastSeenOld = 20 * time.Minute
+
+func runCpTargets(ctx context.Context, args []string) error {
+	if len(args) > 0 {
+		return errors.New("invalid arguments with --targets")
+	}
+	fts, err := tailscale.FileTargets(ctx)
+	if err != nil {
+		return err
+	}
+	for _, ft := range fts {
+		n := ft.Node
+		var detail string
+		if n.Online != nil {
+			if !*n.Online {
+				detail = "offline"
+			}
+		} else {
+			detail = "unknown-status"
+		}
+		if detail != "" && n.LastSeen != nil {
+			d := time.Since(*n.LastSeen)
+			detail += fmt.Sprintf("; last seen %v ago", d.Round(time.Minute))
+		}
+		if detail != "" {
+			detail = "\t" + detail
+		}
+		fmt.Printf("%s\t%s%s\n", n.Addresses[0].IP(), n.ComputedName, detail)
+	}
+	return nil
+}
+
+var fileGetCmd = &ffcli.Command{
+	Name:       "get",
+	ShortUsage: "file get [--wait] [--verbose] <target-directory>",
+	ShortHelp:  "Move files out of the Tailscale file inbox",
+	Exec:       runFileGet,
+	FlagSet: (func() *flag.FlagSet {
+		fs := flag.NewFlagSet("get", flag.ExitOnError)
+		fs.BoolVar(&getArgs.wait, "wait", false, "wait for a file to arrive if inbox is empty")
+		fs.BoolVar(&getArgs.verbose, "verbose", false, "verbose output")
+		return fs
+	})(),
+}
+
+var getArgs struct {
+	wait    bool
+	verbose bool
+}
+
+func runFileGet(ctx context.Context, args []string) error {
+	if len(args) != 1 {
+		return errors.New("usage: file get <target-directory>")
+	}
+	log.SetFlags(0)
+
+	dir := args[0]
+	if dir == "/dev/null" {
+		return wipeInbox(ctx)
+	}
+
+	if fi, err := os.Stat(dir); err != nil || !fi.IsDir() {
+		return fmt.Errorf("%q is not a directory", dir)
+	}
+
+	var wfs []apitype.WaitingFile
+	var err error
+	for {
+		wfs, err = tailscale.WaitingFiles(ctx)
+		if err != nil {
+			return fmt.Errorf("getting WaitingFiles: %v", err)
+		}
+		if len(wfs) != 0 || !getArgs.wait {
+			break
+		}
+		if getArgs.verbose {
+			log.Printf("waiting for file...")
+		}
+		if err := waitForFile(ctx); err != nil {
+			return err
+		}
+	}
+
+	deleted := 0
+	for _, wf := range wfs {
+		rc, size, err := tailscale.GetWaitingFile(ctx, wf.Name)
+		if err != nil {
+			return fmt.Errorf("opening inbox file %q: %v", wf.Name, err)
+		}
+		targetFile := filepath.Join(dir, wf.Name)
+		of, err := os.OpenFile(targetFile, os.O_RDWR|os.O_CREATE|os.O_EXCL, 0644)
+		if err != nil {
+			if _, err := os.Stat(targetFile); err == nil {
+				return fmt.Errorf("refusing to overwrite %v", targetFile)
+			}
+			return err
+		}
+		_, err = io.Copy(of, rc)
+		rc.Close()
+		if err != nil {
+			return fmt.Errorf("failed to write %v: %v", targetFile, err)
+		}
+		if err := of.Close(); err != nil {
+			return err
+		}
+		if getArgs.verbose {
+			log.Printf("wrote %v (%d bytes)", wf.Name, size)
+		}
+		if err := tailscale.DeleteWaitingFile(ctx, wf.Name); err != nil {
+			return fmt.Errorf("deleting %q from inbox: %v", wf.Name, err)
+		}
+		deleted++
+	}
+	if getArgs.verbose {
+		log.Printf("moved %d files", deleted)
+	}
+	return nil
+}
+
+func wipeInbox(ctx context.Context) error {
+	if getArgs.wait {
+		return errors.New("can't use --wait with /dev/null target")
+	}
+	wfs, err := tailscale.WaitingFiles(ctx)
+	if err != nil {
+		return fmt.Errorf("getting WaitingFiles: %v", err)
+	}
+	deleted := 0
+	for _, wf := range wfs {
+		if getArgs.verbose {
+			log.Printf("deleting %v ...", wf.Name)
+		}
+		if err := tailscale.DeleteWaitingFile(ctx, wf.Name); err != nil {
+			return fmt.Errorf("deleting %q: %v", wf.Name, err)
+		}
+		deleted++
+	}
+	if getArgs.verbose {
+		log.Printf("deleted %d files", deleted)
+	}
+	return nil
+}
+
+func waitForFile(ctx context.Context) error {
+	c, bc, pumpCtx, cancel := connect(ctx)
+	defer cancel()
+	fileWaiting := make(chan bool, 1)
+	bc.SetNotifyCallback(func(n ipn.Notify) {
+		if n.ErrMessage != nil {
+			log.Fatal(*n.ErrMessage)
+		}
+		if n.FilesWaiting != nil {
+			select {
+			case fileWaiting <- true:
+			default:
+			}
+		}
+	})
+	go pump(pumpCtx, bc, c)
+	select {
+	case <-fileWaiting:
+		return nil
+	case <-pumpCtx.Done():
+		return pumpCtx.Err()
+	case <-ctx.Done():
+		return ctx.Err()
+	}
+}

cmd/tailscale/cli/ip.go

@@ -11,13 +11,16 @@ import (
 	"fmt"
 
 	"github.com/peterbourgon/ff/v2/ffcli"
+	"inet.af/netaddr"
 	"tailscale.com/client/tailscale"
+	"tailscale.com/ipn/ipnstate"
 )
 
 var ipCmd = &ffcli.Command{
 	Name:       "ip",
-	ShortUsage: "ip [-4] [-6]",
-	ShortHelp:  "Show this machine's current Tailscale IP address(es)",
+	ShortUsage: "ip [-4] [-6] [peername]",
+	ShortHelp:  "Show current Tailscale IP address(es)",
+	LongHelp:   "Shows the Tailscale IP address of the current machine without an argument. With an argument, it shows the IP of a named peer.",
 	Exec:       runIP,
 	FlagSet: (func() *flag.FlagSet {
 		fs := flag.NewFlagSet("ip", flag.ExitOnError)
@@ -33,9 +36,14 @@ var ipArgs struct {
 }
 
 func runIP(ctx context.Context, args []string) error {
-	if len(args) > 0 {
+	if len(args) > 1 {
 		return errors.New("unknown arguments")
 	}
+	var of string
+	if len(args) == 1 {
+		of = args[0]
+	}
+
 	v4, v6 := ipArgs.want4, ipArgs.want6
 	if v4 && v6 {
 		return errors.New("tailscale up -4 and -6 are mutually exclusive")
@@ -47,11 +55,24 @@ func runIP(ctx context.Context, args []string) error {
 	if err != nil {
 		return err
 	}
-	if len(st.TailscaleIPs) == 0 {
+	ips := st.TailscaleIPs
+	if of != "" {
+		ip, err := tailscaleIPFromArg(ctx, of)
+		if err != nil {
+			return err
+		}
+		peer, ok := peerMatchingIP(st, ip)
+		if !ok {
+			return fmt.Errorf("no peer found with IP %v", ip)
+		}
+		ips = peer.TailscaleIPs
+	}
+	if len(ips) == 0 {
 		return fmt.Errorf("no current Tailscale IPs; state: %v", st.BackendState)
 	}
+
 	match := false
-	for _, ip := range st.TailscaleIPs {
+	for _, ip := range ips {
 		if ip.Is4() && v4 || ip.Is6() && v6 {
 			match = true
 			fmt.Println(ip)
@@ -67,3 +88,18 @@ func runIP(ctx context.Context, args []string) error {
 	}
 	return nil
 }
+
+func peerMatchingIP(st *ipnstate.Status, ipStr string) (ps *ipnstate.PeerStatus, ok bool) {
+	ip, err := netaddr.ParseIP(ipStr)
+	if err != nil {
+		return
+	}
+	for _, ps = range st.Peer {
+		for _, pip := range ps.TailscaleIPs {
+			if ip == pip {
+				return ps, true
+			}
+		}
+	}
+	return nil, false
+}

cmd/tailscale/cli/logout.go

@@ -0,0 +1,34 @@
+// Copyright (c) 2020 Tailscale Inc & AUTHORS All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package cli
+
+import (
+	"context"
+	"log"
+	"strings"
+
+	"github.com/peterbourgon/ff/v2/ffcli"
+	"tailscale.com/client/tailscale"
+)
+
+var logoutCmd = &ffcli.Command{
+	Name:       "logout",
+	ShortUsage: "logout [flags]",
+	ShortHelp:  "Disconnect from Tailscale and expire current node key",
+
+	LongHelp: strings.TrimSpace(`
+"tailscale logout" brings the network down and invalidates
+the current node key, forcing a future use of it to cause
+a reauthentication.
+`),
+	Exec: runLogout,
+}
+
+func runLogout(ctx context.Context, args []string) error {
+	if len(args) > 0 {
+		log.Fatalf("too many non-flag arguments: %q", args)
+	}
+	return tailscale.Logout(ctx)
+}

cmd/tailscale/cli/ping.go

@@ -64,6 +64,7 @@ var pingArgs struct {
 }
 
 func runPing(ctx context.Context, args []string) error {
+	fmt.Println("runPing")
 	c, bc, ctx, cancel := connect(ctx)
 	defer cancel()
 
@@ -80,7 +81,8 @@ func runPing(ctx context.Context, args []string) error {
 			prc <- pr
 		}
 	})
-	go pump(ctx, bc, c)
+	pumpErr := make(chan error, 1)
+	go func() { pumpErr <- pump(ctx, bc, c) }()
 
 	hostOrIP := args[0]
 	ip, err := tailscaleIPFromArg(ctx, hostOrIP)
@@ -101,6 +103,8 @@ func runPing(ctx context.Context, args []string) error {
 		select {
 		case <-timer.C:
 			fmt.Printf("timeout waiting for ping reply\n")
+		case err := <-pumpErr:
+			return err
 		case pr := <-prc:
 			timer.Stop()
 			if pr.Err != "" {
@@ -136,6 +140,9 @@ func runPing(ctx context.Context, args []string) error {
 			if !anyPong {
 				return errors.New("no reply")
 			}
+			if pingArgs.untilDirect {
+				return errors.New("direct connection not established")
+			}
 			return nil
 		}
 	}
@@ -154,7 +161,10 @@ func tailscaleIPFromArg(ctx context.Context, hostOrIP string) (ip string, err er
 	}
 	for _, ps := range st.Peer {
 		if hostOrIP == dnsOrQuoteHostname(st, ps) || hostOrIP == ps.DNSName {
-			return ps.TailAddr, nil
+			if len(ps.TailscaleIPs) == 0 {
+				return "", errors.New("node found but lacks an IP")
+			}
+			return ps.TailscaleIPs[0].String(), nil
 		}
 	}
 

cmd/tailscale/cli/push.go

@@ -1,173 +0,0 @@
-// Copyright (c) 2021 Tailscale Inc & AUTHORS All rights reserved.
-// Use of this source code is governed by a BSD-style
-// license that can be found in the LICENSE file.
-
-package cli
-
-import (
-	"bytes"
-	"context"
-	"errors"
-	"flag"
-	"fmt"
-	"io"
-	"log"
-	"mime"
-	"net"
-	"net/http"
-	"net/url"
-	"os"
-	"time"
-	"unicode/utf8"
-
-	"github.com/peterbourgon/ff/v2/ffcli"
-	"tailscale.com/ipn"
-	"tailscale.com/ipn/ipnstate"
-)
-
-var pushCmd = &ffcli.Command{
-	Name:       "push",
-	ShortUsage: "push [--flags] <hostname-or-IP> <file>",
-	ShortHelp:  "Push a file to a host",
-	Exec:       runPush,
-	FlagSet: (func() *flag.FlagSet {
-		fs := flag.NewFlagSet("push", flag.ExitOnError)
-		fs.StringVar(&pushArgs.name, "name", "", "alternate filename to use, especially useful when <file> is \"-\" (stdin)")
-		fs.BoolVar(&pushArgs.verbose, "verbose", false, "verbose output")
-		return fs
-	})(),
-}
-
-var pushArgs struct {
-	name    string
-	verbose bool
-}
-
-func runPush(ctx context.Context, args []string) error {
-	if len(args) != 2 || args[0] == "" {
-		return errors.New("usage: push <hostname-or-IP> <file>")
-	}
-	var ip string
-
-	hostOrIP, fileArg := args[0], args[1]
-	ip, err := tailscaleIPFromArg(ctx, hostOrIP)
-	if err != nil {
-		return err
-	}
-
-	peerAPIPort, err := discoverPeerAPIPort(ctx, ip)
-	if err != nil {
-		return err
-	}
-
-	var fileContents io.Reader
-	var name = pushArgs.name
-	if fileArg == "-" {
-		fileContents = os.Stdin
-		if name == "" {
-			name, fileContents, err = pickStdinFilename()
-			if err != nil {
-				return err
-			}
-		}
-	} else {
-		f, err := os.Open(fileArg)
-		if err != nil {
-			return err
-		}
-		defer f.Close()
-		fileContents = f
-		if name == "" {
-			name = fileArg
-		}
-	}
-
-	dstURL := "http://" + net.JoinHostPort(ip, fmt.Sprint(peerAPIPort)) + "/v0/put/" + url.PathEscape(name)
-	req, err := http.NewRequestWithContext(ctx, "PUT", dstURL, fileContents)
-	if err != nil {
-		return err
-	}
-	if pushArgs.verbose {
-		log.Printf("sending to %v ...", dstURL)
-	}
-	res, err := http.DefaultClient.Do(req)
-	if err != nil {
-		return err
-	}
-	defer res.Body.Close()
-	if res.StatusCode == 200 {
-		return nil
-	}
-	io.Copy(os.Stdout, res.Body)
-	return errors.New(res.Status)
-}
-
-func discoverPeerAPIPort(ctx context.Context, ip string) (port uint16, err error) {
-	c, bc, ctx, cancel := connect(ctx)
-	defer cancel()
-
-	prc := make(chan *ipnstate.PingResult, 2)
-	bc.SetNotifyCallback(func(n ipn.Notify) {
-		if n.ErrMessage != nil {
-			log.Fatal(*n.ErrMessage)
-		}
-		if pr := n.PingResult; pr != nil && pr.IP == ip {
-			prc <- pr
-		}
-	})
-	go pump(ctx, bc, c)
-
-	ticker := time.NewTicker(time.Second)
-	defer ticker.Stop()
-
-	discoPings := 0
-	timer := time.NewTimer(10 * time.Second)
-	defer timer.Stop()
-
-	sendPings := func() {
-		bc.Ping(ip, false)
-		bc.Ping(ip, true)
-	}
-	sendPings()
-	for {
-		select {
-		case <-ticker.C:
-			sendPings()
-		case <-timer.C:
-			return 0, fmt.Errorf("timeout contacting %v; it offline?", ip)
-		case pr := <-prc:
-			if p := pr.PeerAPIPort; p != 0 {
-				return p, nil
-			}
-			discoPings++
-			if discoPings == 3 {
-				return 0, fmt.Errorf("%v is online, but seems to be running an old Tailscale version", ip)
-			}
-		case <-ctx.Done():
-			return 0, ctx.Err()
-		}
-	}
-}
-
-const maxSniff = 4 << 20
-
-func ext(b []byte) string {
-	if len(b) < maxSniff && utf8.Valid(b) {
-		return ".txt"
-	}
-	if exts, _ := mime.ExtensionsByType(http.DetectContentType(b)); len(exts) > 0 {
-		return exts[0]
-	}
-	return ""
-}
-
-// pickStdinFilename reads a bit of stdin to return a good filename
-// for its contents. The returned Reader is the concatenation of the
-// read and unread bits.
-func pickStdinFilename() (name string, r io.Reader, err error) {
-	sniff, err := io.ReadAll(io.LimitReader(os.Stdin, maxSniff))
-	if err != nil {
-		return "", nil, err
-	}
-	return "stdin" + ext(sniff), io.MultiReader(bytes.NewReader(sniff), os.Stdin), nil
-}

cmd/tailscale/cli/status.go

@@ -18,6 +18,7 @@ import (
 
 	"github.com/peterbourgon/ff/v2/ffcli"
 	"github.com/toqueteos/webbrowser"
+	"inet.af/netaddr"
 	"tailscale.com/client/tailscale"
 	"tailscale.com/ipn"
 	"tailscale.com/ipn/ipnstate"
@@ -106,9 +107,24 @@ func runStatus(ctx context.Context, args []string) error {
 		return err
 	}
 
-	if st.BackendState == ipn.Stopped.String() {
+	switch st.BackendState {
+	default:
+		fmt.Fprintf(os.Stderr, "unexpected state: %s\n", st.BackendState)
+		os.Exit(1)
+	case ipn.Stopped.String():
 		fmt.Println("Tailscale is stopped.")
 		os.Exit(1)
+	case ipn.NeedsLogin.String():
+		fmt.Println("Logged out.")
+		if st.AuthURL != "" {
+			fmt.Printf("\nLog in at: %s\n", st.AuthURL)
+		}
+		os.Exit(1)
+	case ipn.NeedsMachineAuth.String():
+		fmt.Println("Machine is not yet authorized by tailnet admin.")
+		os.Exit(1)
+	case ipn.Running.String():
+		// Run below.
 	}
 
 	var buf bytes.Buffer
@@ -116,7 +132,7 @@ func runStatus(ctx context.Context, args []string) error {
 	printPS := func(ps *ipnstate.PeerStatus) {
 		active := peerActive(ps)
 		f("%-15s %-20s %-12s %-7s ",
-			ps.TailAddr,
+			firstIPString(ps.TailscaleIPs),
 			dnsOrQuoteHostname(st, ps),
 			ownerLogin(st, ps),
 			ps.OS,
@@ -201,3 +217,10 @@ func ownerLogin(st *ipnstate.Status, ps *ipnstate.PeerStatus) string {
 	}
 	return u.LoginName
 }
+
+func firstIPString(v []netaddr.IP) string {
+	if len(v) == 0 {
+		return ""
+	}
+	return v[0].String()
+}

cmd/tailscale/cli/up.go

@@ -9,18 +9,22 @@ import (
 	"errors"
 	"flag"
 	"fmt"
-	"log"
 	"os"
+	"reflect"
 	"runtime"
 	"sort"
 	"strings"
 	"sync"
 
+	shellquote "github.com/kballard/go-shellquote"
 	"github.com/peterbourgon/ff/v2/ffcli"
 	"inet.af/netaddr"
 	"tailscale.com/client/tailscale"
 	"tailscale.com/ipn"
+	"tailscale.com/ipn/ipnstate"
+	"tailscale.com/safesocket"
 	"tailscale.com/tailcfg"
+	"tailscale.com/types/logger"
 	"tailscale.com/types/preftype"
 	"tailscale.com/version/distro"
 )
@@ -28,38 +32,56 @@ import (
 var upCmd = &ffcli.Command{
 	Name:       "up",
 	ShortUsage: "up [flags]",
-	ShortHelp:  "Connect to your Tailscale network",
+	ShortHelp:  "Connect to Tailscale, logging in if needed",
 
 	LongHelp: strings.TrimSpace(`
 "tailscale up" connects this machine to your Tailscale network,
 triggering authentication if necessary.
 
-The flags passed to this command are specific to this machine. If you don't
-specify any flags, options are reset to their default.
+With no flags, "tailscale up" brings the network online without
+changing any settings. (That is, it's the opposite of "tailscale
+down").
+
+If flags are specified, the flags must be the complete set of desired
+settings. An error is returned if any setting would be changed as a
+result of an unspecified flag's default value, unless the --reset
+flag is also used.
 `),
-	FlagSet: (func() *flag.FlagSet {
-		upf := flag.NewFlagSet("up", flag.ExitOnError)
-		upf.StringVar(&upArgs.server, "login-server", "https://login.tailscale.com", "base URL of control server")
-		upf.BoolVar(&upArgs.acceptRoutes, "accept-routes", false, "accept routes advertised by other Tailscale nodes")
-		upf.BoolVar(&upArgs.acceptDNS, "accept-dns", true, "accept DNS configuration from the admin panel")
-		upf.BoolVar(&upArgs.singleRoutes, "host-routes", true, "install host routes to other Tailscale nodes")
-		upf.StringVar(&upArgs.exitNodeIP, "exit-node", "", "Tailscale IP of the exit node for internet traffic")
-		upf.BoolVar(&upArgs.shieldsUp, "shields-up", false, "don't allow incoming connections")
-		upf.BoolVar(&upArgs.forceReauth, "force-reauth", false, "force reauthentication")
-		upf.StringVar(&upArgs.advertiseTags, "advertise-tags", "", "ACL tags to request (comma-separated, e.g. \"tag:eng,tag:montreal,tag:ssh\")")
-		upf.StringVar(&upArgs.authKey, "authkey", "", "node authorization key")
-		upf.StringVar(&upArgs.hostname, "hostname", "", "hostname to use instead of the one provided by the OS")
-		if runtime.GOOS == "linux" || isBSD(runtime.GOOS) {
-			upf.StringVar(&upArgs.advertiseRoutes, "advertise-routes", "", "routes to advertise to other nodes (comma-separated, e.g. \"10.0.0.0/8,192.168.0.0/24\")")
-			upf.BoolVar(&upArgs.advertiseDefaultRoute, "advertise-exit-node", false, "offer to be an exit node for internet traffic for the tailnet")
-		}
-		if runtime.GOOS == "linux" {
-			upf.BoolVar(&upArgs.snat, "snat-subnet-routes", true, "source NAT traffic to local routes advertised with --advertise-routes")
-			upf.StringVar(&upArgs.netfilterMode, "netfilter-mode", defaultNetfilterMode(), "netfilter mode (one of on, nodivert, off)")
-		}
-		return upf
-	})(),
-	Exec: runUp,
+	FlagSet: upFlagSet,
+	Exec:    runUp,
+}
+
+var upFlagSet = newUpFlagSet(runtime.GOOS, &upArgs)
+
+func newUpFlagSet(goos string, upArgs *upArgsT) *flag.FlagSet {
+	upf := flag.NewFlagSet("up", flag.ExitOnError)
+
+	upf.BoolVar(&upArgs.forceReauth, "force-reauth", false, "force reauthentication")
+	upf.BoolVar(&upArgs.reset, "reset", false, "reset unspecified settings to their default values")
+
+	upf.StringVar(&upArgs.server, "login-server", ipn.DefaultControlURL, "base URL of control server")
+	upf.BoolVar(&upArgs.acceptRoutes, "accept-routes", false, "accept routes advertised by other Tailscale nodes")
+	upf.BoolVar(&upArgs.acceptDNS, "accept-dns", true, "accept DNS configuration from the admin panel")
+	upf.BoolVar(&upArgs.singleRoutes, "host-routes", true, "install host routes to other Tailscale nodes")
+	upf.StringVar(&upArgs.exitNodeIP, "exit-node", "", "Tailscale IP of the exit node for internet traffic")
+	upf.BoolVar(&upArgs.exitNodeAllowLANAccess, "exit-node-allow-lan-access", false, "Allow direct access to the local network when routing traffic via an exit node")
+	upf.BoolVar(&upArgs.shieldsUp, "shields-up", false, "don't allow incoming connections")
+	upf.StringVar(&upArgs.advertiseTags, "advertise-tags", "", "comma-separated ACL tags to request; each must start with \"tag:\" (e.g. \"tag:eng,tag:montreal,tag:ssh\")")
+	upf.StringVar(&upArgs.authKey, "authkey", "", "node authorization key")
+	upf.StringVar(&upArgs.hostname, "hostname", "", "hostname to use instead of the one provided by the OS")
+	upf.StringVar(&upArgs.advertiseRoutes, "advertise-routes", "", "routes to advertise to other nodes (comma-separated, e.g. \"10.0.0.0/8,192.168.0.0/24\")")
+	upf.BoolVar(&upArgs.advertiseDefaultRoute, "advertise-exit-node", false, "offer to be an exit node for internet traffic for the tailnet")
+	if safesocket.GOOSUsesPeerCreds(goos) {
+		upf.StringVar(&upArgs.opUser, "operator", "", "Unix username to allow to operate on tailscaled without sudo")
+	}
+	switch goos {
+	case "linux":
+		upf.BoolVar(&upArgs.snat, "snat-subnet-routes", true, "source NAT traffic to local routes advertised with --advertise-routes")
+		upf.StringVar(&upArgs.netfilterMode, "netfilter-mode", defaultNetfilterMode(), "netfilter mode (one of on, nodivert, off)")
+	case "windows":
+		upf.BoolVar(&upArgs.forceDaemon, "unattended", false, "run in \"Unattended Mode\" where Tailscale keeps running even after the current GUI user logs out (Windows-only)")
+	}
+	return upf
 }
 
 func defaultNetfilterMode() string {
@@ -69,26 +91,28 @@ func defaultNetfilterMode() string {
 	return "on"
 }
 
-var upArgs struct {
-	server                string
-	acceptRoutes          bool
-	acceptDNS             bool
-	singleRoutes          bool
-	exitNodeIP            string
-	shieldsUp             bool
-	forceReauth           bool
-	advertiseRoutes       string
-	advertiseDefaultRoute bool
-	advertiseTags         string
-	snat                  bool
-	netfilterMode         string
-	authKey               string
-	hostname              string
+type upArgsT struct {
+	reset                  bool
+	server                 string
+	acceptRoutes           bool
+	acceptDNS              bool
+	singleRoutes           bool
+	exitNodeIP             string
+	exitNodeAllowLANAccess bool
+	shieldsUp              bool
+	forceReauth            bool
+	forceDaemon            bool
+	advertiseRoutes        string
+	advertiseDefaultRoute  bool
+	advertiseTags          string
+	snat                   bool
+	netfilterMode          string
+	authKey                string
+	hostname               string
+	opUser                 string
 }
 
-func isBSD(s string) bool {
-	return s == "dragonfly" || s == "freebsd" || s == "netbsd" || s == "openbsd"
-}
+var upArgs upArgsT
 
 func warnf(format string, args ...interface{}) {
 	fmt.Printf("Warning: "+format+"\n", args...)
@@ -99,16 +123,152 @@ var (
 	ipv6default = netaddr.MustParseIPPrefix("::/0")
 )
 
+// prefsFromUpArgs returns the ipn.Prefs for the provided args.
+//
+// Note that the parameters upArgs and warnf are named intentionally
+// to shadow the globals to prevent accidental misuse of them. This
+// function exists for testing and should have no side effects or
+// outside interactions (e.g. no making Tailscale local API calls).
+func prefsFromUpArgs(upArgs upArgsT, warnf logger.Logf, st *ipnstate.Status, goos string) (*ipn.Prefs, error) {
+	routeMap := map[netaddr.IPPrefix]bool{}
+	var default4, default6 bool
+	if upArgs.advertiseRoutes != "" {
+		advroutes := strings.Split(upArgs.advertiseRoutes, ",")
+		for _, s := range advroutes {
+			ipp, err := netaddr.ParseIPPrefix(s)
+			if err != nil {
+				return nil, fmt.Errorf("%q is not a valid IP address or CIDR prefix", s)
+			}
+			if ipp != ipp.Masked() {
+				return nil, fmt.Errorf("%s has non-address bits set; expected %s", ipp, ipp.Masked())
+			}
+			if ipp == ipv4default {
+				default4 = true
+			} else if ipp == ipv6default {
+				default6 = true
+			}
+			routeMap[ipp] = true
+		}
+		if default4 && !default6 {
+			return nil, fmt.Errorf("%s advertised without its IPv6 counterpart, please also advertise %s", ipv4default, ipv6default)
+		} else if default6 && !default4 {
+			return nil, fmt.Errorf("%s advertised without its IPv6 counterpart, please also advertise %s", ipv6default, ipv4default)
+		}
+	}
+	if upArgs.advertiseDefaultRoute {
+		routeMap[netaddr.MustParseIPPrefix("0.0.0.0/0")] = true
+		routeMap[netaddr.MustParseIPPrefix("::/0")] = true
+	}
+	routes := make([]netaddr.IPPrefix, 0, len(routeMap))
+	for r := range routeMap {
+		routes = append(routes, r)
+	}
+	sort.Slice(routes, func(i, j int) bool {
+		if routes[i].Bits() != routes[j].Bits() {
+			return routes[i].Bits() < routes[j].Bits()
+		}
+		return routes[i].IP().Less(routes[j].IP())
+	})
+
+	var exitNodeIP netaddr.IP
+	if upArgs.exitNodeIP != "" {
+		var err error
+		exitNodeIP, err = netaddr.ParseIP(upArgs.exitNodeIP)
+		if err != nil {
+			return nil, fmt.Errorf("invalid IP address %q for --exit-node: %v", upArgs.exitNodeIP, err)
+		}
+	} else if upArgs.exitNodeAllowLANAccess {
+		return nil, fmt.Errorf("--exit-node-allow-lan-access can only be used with --exit-node")
+	}
+
+	if upArgs.exitNodeIP != "" {
+		for _, ip := range st.TailscaleIPs {
+			if exitNodeIP == ip {
+				return nil, fmt.Errorf("cannot use %s as the exit node as it is a local IP address to this machine, did you mean --advertise-exit-node?", upArgs.exitNodeIP)
+			}
+		}
+	}
+
+	var tags []string
+	if upArgs.advertiseTags != "" {
+		tags = strings.Split(upArgs.advertiseTags, ",")
+		for _, tag := range tags {
+			err := tailcfg.CheckTag(tag)
+			if err != nil {
+				return nil, fmt.Errorf("tag: %q: %s", tag, err)
+			}
+		}
+	}
+
+	if len(upArgs.hostname) > 256 {
+		return nil, fmt.Errorf("hostname too long: %d bytes (max 256)", len(upArgs.hostname))
+	}
+
+	prefs := ipn.NewPrefs()
+	prefs.ControlURL = upArgs.server
+	prefs.WantRunning = true
+	prefs.RouteAll = upArgs.acceptRoutes
+	prefs.ExitNodeIP = exitNodeIP
+	prefs.ExitNodeAllowLANAccess = upArgs.exitNodeAllowLANAccess
+	prefs.CorpDNS = upArgs.acceptDNS
+	prefs.AllowSingleHosts = upArgs.singleRoutes
+	prefs.ShieldsUp = upArgs.shieldsUp
+	prefs.AdvertiseRoutes = routes
+	prefs.AdvertiseTags = tags
+	prefs.Hostname = upArgs.hostname
+	prefs.ForceDaemon = upArgs.forceDaemon
+	prefs.OperatorUser = upArgs.opUser
+
+	if goos == "linux" {
+		prefs.NoSNAT = !upArgs.snat
+
+		switch upArgs.netfilterMode {
+		case "on":
+			prefs.NetfilterMode = preftype.NetfilterOn
+		case "nodivert":
+			prefs.NetfilterMode = preftype.NetfilterNoDivert
+			warnf("netfilter=nodivert; add iptables calls to ts-* chains manually.")
+		case "off":
+			prefs.NetfilterMode = preftype.NetfilterOff
+			if defaultNetfilterMode() != "off" {
+				warnf("netfilter=off; configure iptables yourself.")
+			}
+		default:
+			return nil, fmt.Errorf("invalid value --netfilter-mode=%q", upArgs.netfilterMode)
+		}
+	}
+	return prefs, nil
+}
+
 func runUp(ctx context.Context, args []string) error {
 	if len(args) > 0 {
-		log.Fatalf("too many non-flag arguments: %q", args)
+		fatalf("too many non-flag arguments: %q", args)
+	}
+
+	st, err := tailscale.Status(ctx)
+	if err != nil {
+		fatalf("can't fetch status from tailscaled: %v", err)
+	}
+	origAuthURL := st.AuthURL
+
+	// printAuthURL reports whether we should print out the
+	// provided auth URL from an IPN notify.
+	printAuthURL := func(url string) bool {
+		if upArgs.authKey != "" {
+			// Issue 1755: when using an authkey, don't
+			// show an authURL that might still be pending
+			// from a previous non-completed interactive
+			// login.
+			return false
+		}
+		if upArgs.forceReauth && url == origAuthURL {
+			return false
+		}
+		return true
 	}
 
 	if distro.Get() == distro.Synology {
-		notSupported := "not yet supported on Synology; see https://github.com/tailscale/tailscale/issues/451"
-		if upArgs.advertiseRoutes != "" {
-			return errors.New("--advertise-routes is " + notSupported)
-		}
+		notSupported := "not supported on Synology; see https://github.com/tailscale/tailscale/issues/1995"
 		if upArgs.acceptRoutes {
 			return errors.New("--accept-routes is " + notSupported)
 		}
@@ -120,192 +280,498 @@ func runUp(ctx context.Context, args []string) error {
 		}
 	}
 
-	routeMap := map[netaddr.IPPrefix]bool{}
-	var default4, default6 bool
-	if upArgs.advertiseRoutes != "" {
-		advroutes := strings.Split(upArgs.advertiseRoutes, ",")
-		for _, s := range advroutes {
-			ipp, err := netaddr.ParseIPPrefix(s)
-			if err != nil {
-				fatalf("%q is not a valid IP address or CIDR prefix", s)
-			}
-			if ipp != ipp.Masked() {
-				fatalf("%s has non-address bits set; expected %s", ipp, ipp.Masked())
-			}
-			if ipp == ipv4default {
-				default4 = true
-			} else if ipp == ipv6default {
-				default6 = true
-			}
-			routeMap[ipp] = true
-		}
-		if default4 && !default6 {
-			fatalf("%s advertised without its IPv6 counterpart, please also advertise %s", ipv4default, ipv6default)
-		} else if default6 && !default4 {
-			fatalf("%s advertised without its IPv6 counterpart, please also advertise %s", ipv6default, ipv4default)
-		}
+	prefs, err := prefsFromUpArgs(upArgs, warnf, st, runtime.GOOS)
+	if err != nil {
+		fatalf("%s", err)
 	}
-	if upArgs.advertiseDefaultRoute {
-		routeMap[netaddr.MustParseIPPrefix("0.0.0.0/0")] = true
-		routeMap[netaddr.MustParseIPPrefix("::/0")] = true
-	}
-	if len(routeMap) > 0 {
+
+	if len(prefs.AdvertiseRoutes) > 0 {
 		if err := tailscale.CheckIPForwarding(context.Background()); err != nil {
 			warnf("%v", err)
 		}
 	}
-	routes := make([]netaddr.IPPrefix, 0, len(routeMap))
-	for r := range routeMap {
-		routes = append(routes, r)
-	}
-	sort.Slice(routes, func(i, j int) bool {
-		if routes[i].Bits != routes[j].Bits {
-			return routes[i].Bits < routes[j].Bits
-		}
-		return routes[i].IP.Less(routes[j].IP)
-	})
 
-	var exitNodeIP netaddr.IP
-	if upArgs.exitNodeIP != "" {
-		var err error
-		exitNodeIP, err = netaddr.ParseIP(upArgs.exitNodeIP)
-		if err != nil {
-			fatalf("invalid IP address %q for --exit-node: %v", upArgs.exitNodeIP, err)
+	curPrefs, err := tailscale.GetPrefs(ctx)
+	if err != nil {
+		return err
+	}
+
+	if !upArgs.reset {
+		applyImplicitPrefs(prefs, curPrefs, os.Getenv("USER"))
+
+		if err := checkForAccidentalSettingReverts(upFlagSet, curPrefs, prefs, upCheckEnv{
+			goos:          runtime.GOOS,
+			curExitNodeIP: exitNodeIP(prefs, st),
+		}); err != nil {
+			fatalf("%s", err)
 		}
 	}
 
-	var tags []string
-	if upArgs.advertiseTags != "" {
-		tags = strings.Split(upArgs.advertiseTags, ",")
-		for _, tag := range tags {
-			err := tailcfg.CheckTag(tag)
-			if err != nil {
-				fatalf("tag: %q: %s", tag, err)
-			}
-		}
+	controlURLChanged := curPrefs.ControlURL != prefs.ControlURL
+	if controlURLChanged && st.BackendState == ipn.Running.String() && !upArgs.forceReauth {
+		fatalf("can't change --login-server without --force-reauth")
 	}
 
-	if len(upArgs.hostname) > 256 {
-		fatalf("hostname too long: %d bytes (max 256)", len(upArgs.hostname))
+	// If we're already running and none of the flags require a
+	// restart, we can just do an EditPrefs call and change the
+	// prefs at runtime (e.g. changing hostname, changing
+	// advertised tags, routes, etc)
+	justEdit := st.BackendState == ipn.Running.String() &&
+		!upArgs.forceReauth &&
+		!upArgs.reset &&
+		upArgs.authKey == "" &&
+		!controlURLChanged
+	if justEdit {
+		mp := new(ipn.MaskedPrefs)
+		mp.WantRunningSet = true
+		mp.Prefs = *prefs
+		upFlagSet.Visit(func(f *flag.Flag) {
+			updateMaskedPrefsFromUpFlag(mp, f.Name)
+		})
+
+		_, err := tailscale.EditPrefs(ctx, mp)
+		return err
 	}
 
-	prefs := ipn.NewPrefs()
-	prefs.ControlURL = upArgs.server
-	prefs.WantRunning = true
-	prefs.RouteAll = upArgs.acceptRoutes
-	prefs.ExitNodeIP = exitNodeIP
-	prefs.CorpDNS = upArgs.acceptDNS
-	prefs.AllowSingleHosts = upArgs.singleRoutes
-	prefs.ShieldsUp = upArgs.shieldsUp
-	prefs.AdvertiseRoutes = routes
-	prefs.AdvertiseTags = tags
-	prefs.NoSNAT = !upArgs.snat
-	prefs.Hostname = upArgs.hostname
-	prefs.ForceDaemon = (runtime.GOOS == "windows")
+	// simpleUp is whether we're running a simple "tailscale up"
+	// to transition to running from a previously-logged-in but
+	// down state, without changing any settings.
+	simpleUp := upFlagSet.NFlag() == 0 &&
+		curPrefs.Persist != nil &&
+		curPrefs.Persist.LoginName != "" &&
+		st.BackendState != ipn.NeedsLogin.String()
 
-	if runtime.GOOS == "linux" {
-		switch upArgs.netfilterMode {
-		case "on":
-			prefs.NetfilterMode = preftype.NetfilterOn
-		case "nodivert":
-			prefs.NetfilterMode = preftype.NetfilterNoDivert
-			warnf("netfilter=nodivert; add iptables calls to ts-* chains manually.")
-		case "off":
-			prefs.NetfilterMode = preftype.NetfilterOff
-			warnf("netfilter=off; configure iptables yourself.")
-		default:
-			fatalf("invalid value --netfilter-mode: %q", upArgs.netfilterMode)
-		}
-	}
-
-	c, bc, ctx, cancel := connect(ctx)
+	// At this point we need to subscribe to the IPN bus to watch
+	// for state transitions and possible need to authenticate.
+	c, bc, pumpCtx, cancel := connect(ctx)
 	defer cancel()
 
-	if !prefs.ExitNodeIP.IsZero() {
-		st, err := tailscale.Status(ctx)
-		if err != nil {
-			fatalf("can't fetch status from tailscaled: %v", err)
-		}
-		for _, ip := range st.TailscaleIPs {
-			if prefs.ExitNodeIP == ip {
-				fatalf("cannot use %s as the exit node as it is a local IP address to this machine, did you mean --advertise-exit-node?", ip)
-			}
-		}
-	}
+	startingOrRunning := make(chan bool, 1) // gets value once starting or running
+	gotEngineUpdate := make(chan bool, 1)   // gets value upon an engine update
+	pumpErr := make(chan error, 1)
+	go func() { pumpErr <- pump(pumpCtx, bc, c) }()
 
-	var printed bool
+	printed := !simpleUp
 	var loginOnce sync.Once
 	startLoginInteractive := func() { loginOnce.Do(func() { bc.StartLoginInteractive() }) }
 
-	bc.SetPrefs(prefs)
-
-	opts := ipn.Options{
-		StateKey: ipn.GlobalDaemonStateKey,
-		AuthKey:  upArgs.authKey,
-		Notify: func(n ipn.Notify) {
-			if n.ErrMessage != nil {
-				msg := *n.ErrMessage
-				if msg == ipn.ErrMsgPermissionDenied {
-					switch runtime.GOOS {
-					case "windows":
-						msg += " (Tailscale service in use by other user?)"
-					default:
-						msg += " (try 'sudo tailscale up [...]')"
-					}
-				}
-				fatalf("backend error: %v\n", msg)
+	bc.SetNotifyCallback(func(n ipn.Notify) {
+		if n.Engine != nil {
+			select {
+			case gotEngineUpdate <- true:
+			default:
 			}
-			if s := n.State; s != nil {
-				switch *s {
-				case ipn.NeedsLogin:
-					printed = true
-					startLoginInteractive()
-				case ipn.NeedsMachineAuth:
-					printed = true
-					fmt.Fprintf(os.Stderr, "\nTo authorize your machine, visit (as admin):\n\n\t%s/admin/machines\n\n", upArgs.server)
-				case ipn.Starting, ipn.Running:
-					// Done full authentication process
-					if printed {
-						// Only need to print an update if we printed the "please click" message earlier.
-						fmt.Fprintf(os.Stderr, "Success.\n")
-					}
-					cancel()
+		}
+		if n.ErrMessage != nil {
+			msg := *n.ErrMessage
+			if msg == ipn.ErrMsgPermissionDenied {
+				switch runtime.GOOS {
+				case "windows":
+					msg += " (Tailscale service in use by other user?)"
+				default:
+					msg += " (try 'sudo tailscale up [...]')"
 				}
 			}
-			if url := n.BrowseToURL; url != nil {
-				fmt.Fprintf(os.Stderr, "\nTo authenticate, visit:\n\n\t%s\n\n", *url)
+			fatalf("backend error: %v\n", msg)
+		}
+		if s := n.State; s != nil {
+			switch *s {
+			case ipn.NeedsLogin:
+				printed = true
+				startLoginInteractive()
+			case ipn.NeedsMachineAuth:
+				printed = true
+				fmt.Fprintf(os.Stderr, "\nTo authorize your machine, visit (as admin):\n\n\t%s/admin/machines\n\n", upArgs.server)
+			case ipn.Starting, ipn.Running:
+				// Done full authentication process
+				if printed {
+					// Only need to print an update if we printed the "please click" message earlier.
+					fmt.Fprintf(os.Stderr, "Success.\n")
+				}
+				select {
+				case startingOrRunning <- true:
+				default:
+				}
+				cancel()
 			}
-		},
+		}
+		if url := n.BrowseToURL; url != nil && printAuthURL(*url) {
+			printed = true
+			fmt.Fprintf(os.Stderr, "\nTo authenticate, visit:\n\n\t%s\n\n", *url)
+		}
+	})
+	// Wait for backend client to be connected so we know
+	// we're subscribed to updates. Otherwise we can miss
+	// an update upon its transition to running. Do so by causing some traffic
+	// back to the bus that we then wait on.
+	bc.RequestEngineStatus()
+	select {
+	case <-gotEngineUpdate:
+	case <-pumpCtx.Done():
+		return pumpCtx.Err()
+	case err := <-pumpErr:
+		return err
 	}
 
-	// On Windows, we still run in mostly the "legacy" way that
-	// predated the server's StateStore. That is, we send an empty
-	// StateKey and send the prefs directly. Although the Windows
-	// supports server mode, though, the transition to StateStore
-	// is only half complete. Only server mode uses it, and the
-	// Windows service (~tailscaled) is the one that computes the
-	// StateKey based on the connection identity. So for now, just
-	// do as the Windows GUI's always done:
-	if runtime.GOOS == "windows" {
-		// The Windows service will set this as needed based
-		// on our connection's identity.
-		opts.StateKey = ""
-		opts.Prefs = prefs
+	// Special case: bare "tailscale up" means to just start
+	// running, if there's ever been a login.
+	if simpleUp {
+		_, err := tailscale.EditPrefs(ctx, &ipn.MaskedPrefs{
+			Prefs: ipn.Prefs{
+				WantRunning: true,
+			},
+			WantRunningSet: true,
+		})
+		if err != nil {
+			return err
+		}
+	} else {
+		opts := ipn.Options{
+			StateKey:    ipn.GlobalDaemonStateKey,
+			AuthKey:     upArgs.authKey,
+			UpdatePrefs: prefs,
+		}
+		// On Windows, we still run in mostly the "legacy" way that
+		// predated the server's StateStore. That is, we send an empty
+		// StateKey and send the prefs directly. Although the Windows
+		// supports server mode, though, the transition to StateStore
+		// is only half complete. Only server mode uses it, and the
+		// Windows service (~tailscaled) is the one that computes the
+		// StateKey based on the connection identity. So for now, just
+		// do as the Windows GUI's always done:
+		if runtime.GOOS == "windows" {
+			// The Windows service will set this as needed based
+			// on our connection's identity.
+			opts.StateKey = ""
+			opts.Prefs = prefs
+		}
+
+		bc.Start(opts)
+		if upArgs.forceReauth {
+			startLoginInteractive()
+		}
 	}
 
-	// We still have to Start right now because it's the only way to
-	// set up notifications and whatnot. This causes a bunch of churn
-	// every time the CLI touches anything.
-	//
-	// TODO(danderson): redo the frontend/backend API to assume
-	// ephemeral frontends that read/modify/write state, once
-	// Windows/Mac state is moved into backend.
-	bc.Start(opts)
-	if upArgs.forceReauth {
-		printed = true
-		startLoginInteractive()
+	select {
+	case <-startingOrRunning:
+		return nil
+	case <-pumpCtx.Done():
+		select {
+		case <-startingOrRunning:
+			return nil
+		default:
+		}
+		return pumpCtx.Err()
+	case err := <-pumpErr:
+		return err
 	}
-	pump(ctx, bc, c)
-
-	return nil
+}
+
+var (
+	prefsOfFlag = map[string][]string{} // "exit-node" => ExitNodeIP, ExitNodeID
+)
+
+func init() {
+	// Both these have the same ipn.Pref:
+	addPrefFlagMapping("advertise-exit-node", "AdvertiseRoutes")
+	addPrefFlagMapping("advertise-routes", "AdvertiseRoutes")
+
+	// And this flag has two ipn.Prefs:
+	addPrefFlagMapping("exit-node", "ExitNodeIP", "ExitNodeID")
+
+	// The rest are 1:1:
+	addPrefFlagMapping("accept-dns", "CorpDNS")
+	addPrefFlagMapping("accept-routes", "RouteAll")
+	addPrefFlagMapping("advertise-tags", "AdvertiseTags")
+	addPrefFlagMapping("host-routes", "AllowSingleHosts")
+	addPrefFlagMapping("hostname", "Hostname")
+	addPrefFlagMapping("login-server", "ControlURL")
+	addPrefFlagMapping("netfilter-mode", "NetfilterMode")
+	addPrefFlagMapping("shields-up", "ShieldsUp")
+	addPrefFlagMapping("snat-subnet-routes", "NoSNAT")
+	addPrefFlagMapping("exit-node-allow-lan-access", "ExitNodeAllowLANAccess")
+	addPrefFlagMapping("unattended", "ForceDaemon")
+	addPrefFlagMapping("operator", "OperatorUser")
+}
+
+func addPrefFlagMapping(flagName string, prefNames ...string) {
+	prefsOfFlag[flagName] = prefNames
+	prefType := reflect.TypeOf(ipn.Prefs{})
+	for _, pref := range prefNames {
+		// Crash at runtime if there's a typo in the prefName.
+		if _, ok := prefType.FieldByName(pref); !ok {
+			panic(fmt.Sprintf("invalid ipn.Prefs field %q", pref))
+		}
+	}
+}
+
+// preflessFlag reports whether flagName is a flag that doesn't
+// correspond to an ipn.Pref.
+func preflessFlag(flagName string) bool {
+	switch flagName {
+	case "authkey", "force-reauth", "reset":
+		return true
+	}
+	return false
+}
+
+func updateMaskedPrefsFromUpFlag(mp *ipn.MaskedPrefs, flagName string) {
+	if preflessFlag(flagName) {
+		return
+	}
+	if prefs, ok := prefsOfFlag[flagName]; ok {
+		for _, pref := range prefs {
+			reflect.ValueOf(mp).Elem().FieldByName(pref + "Set").SetBool(true)
+		}
+		return
+	}
+	panic(fmt.Sprintf("internal error: unhandled flag %q", flagName))
+}
+
+const accidentalUpPrefix = "Error: changing settings via 'tailscale up' requires mentioning all\n" +
+	"non-default flags. To proceed, either re-run your command with --reset or\n" +
+	"use the command below to explicitly mention the current value of\n" +
+	"all non-default settings:\n\n" +
+	"\ttailscale up"
+
+// upCheckEnv are extra parameters describing the environment as
+// needed by checkForAccidentalSettingReverts and friends.
+type upCheckEnv struct {
+	goos          string
+	curExitNodeIP netaddr.IP
+}
+
+// checkForAccidentalSettingReverts (the "up checker") checks for
+// people running "tailscale up" with a subset of the flags they
+// originally ran it with.
+//
+// For example, in Tailscale 1.6 and prior, a user might've advertised
+// a tag, but later tried to change just one other setting and forgot
+// to mention the tag later and silently wiped it out. We now
+// require --reset to change preferences to flag default values when
+// the flag is not mentioned on the command line.
+//
+// curPrefs is what's currently active on the server.
+//
+// mp is the mask of settings actually set, where mp.Prefs is the new
+// preferences to set, including any values set from implicit flags.
+func checkForAccidentalSettingReverts(flagSet *flag.FlagSet, curPrefs, newPrefs *ipn.Prefs, env upCheckEnv) error {
+	if curPrefs.ControlURL == "" {
+		// Don't validate things on initial "up" before a control URL has been set.
+		return nil
+	}
+
+	flagIsSet := map[string]bool{}
+	flagSet.Visit(func(f *flag.Flag) {
+		flagIsSet[f.Name] = true
+	})
+
+	if len(flagIsSet) == 0 {
+		// A bare "tailscale up" is a special case to just
+		// mean bringing the network up without any changes.
+		return nil
+	}
+
+	// flagsCur is what flags we'd need to use to keep the exact
+	// settings as-is.
+	flagsCur := prefsToFlags(env, curPrefs)
+	flagsNew := prefsToFlags(env, newPrefs)
+
+	var missing []string
+	for flagName := range flagsCur {
+		valCur, valNew := flagsCur[flagName], flagsNew[flagName]
+		if flagIsSet[flagName] {
+			continue
+		}
+		if reflect.DeepEqual(valCur, valNew) {
+			continue
+		}
+		missing = append(missing, fmtFlagValueArg(flagName, valCur))
+	}
+	if len(missing) == 0 {
+		return nil
+	}
+	sort.Strings(missing)
+
+	// Compute the stringification of the explicitly provided args in flagSet
+	// to prepend to the command to run.
+	var explicit []string
+	flagSet.Visit(func(f *flag.Flag) {
+		type isBool interface {
+			IsBoolFlag() bool
+		}
+		if ib, ok := f.Value.(isBool); ok && ib.IsBoolFlag() {
+			if f.Value.String() == "false" {
+				explicit = append(explicit, "--"+f.Name+"=false")
+			} else {
+				explicit = append(explicit, "--"+f.Name)
+			}
+		} else {
+			explicit = append(explicit, fmtFlagValueArg(f.Name, f.Value.String()))
+		}
+	})
+
+	var sb strings.Builder
+	sb.WriteString(accidentalUpPrefix)
+
+	for _, a := range append(explicit, missing...) {
+		fmt.Fprintf(&sb, " %s", a)
+	}
+	sb.WriteString("\n\n")
+	return errors.New(sb.String())
+}
+
+// applyImplicitPrefs mutates prefs to add implicit preferences. Currently
+// this is just the operator user, which only needs to be set if it doesn't
+// match the current user.
+//
+// curUser is os.Getenv("USER"). It's pulled out for testability.
+func applyImplicitPrefs(prefs, oldPrefs *ipn.Prefs, curUser string) {
+	if prefs.OperatorUser == "" && oldPrefs.OperatorUser == curUser {
+		prefs.OperatorUser = oldPrefs.OperatorUser
+	}
+}
+
+func flagAppliesToOS(flag, goos string) bool {
+	switch flag {
+	case "netfilter-mode", "snat-subnet-routes":
+		return goos == "linux"
+	case "unattended":
+		return goos == "windows"
+	}
+	return true
+}
+
+func prefsToFlags(env upCheckEnv, prefs *ipn.Prefs) (flagVal map[string]interface{}) {
+	ret := make(map[string]interface{})
+
+	exitNodeIPStr := func() string {
+		if !prefs.ExitNodeIP.IsZero() {
+			return prefs.ExitNodeIP.String()
+		}
+		if prefs.ExitNodeID.IsZero() || env.curExitNodeIP.IsZero() {
+			return ""
+		}
+		return env.curExitNodeIP.String()
+	}
+
+	fs := newUpFlagSet(env.goos, new(upArgsT) /* dummy */)
+	fs.VisitAll(func(f *flag.Flag) {
+		if preflessFlag(f.Name) {
+			return
+		}
+		set := func(v interface{}) {
+			if flagAppliesToOS(f.Name, env.goos) {
+				ret[f.Name] = v
+			} else {
+				ret[f.Name] = nil
+			}
+		}
+		switch f.Name {
+		default:
+			panic(fmt.Sprintf("unhandled flag %q", f.Name))
+		case "login-server":
+			set(prefs.ControlURL)
+		case "accept-routes":
+			set(prefs.RouteAll)
+		case "host-routes":
+			set(prefs.AllowSingleHosts)
+		case "accept-dns":
+			set(prefs.CorpDNS)
+		case "shields-up":
+			set(prefs.ShieldsUp)
+		case "exit-node":
+			set(exitNodeIPStr())
+		case "exit-node-allow-lan-access":
+			set(prefs.ExitNodeAllowLANAccess)
+		case "advertise-tags":
+			set(strings.Join(prefs.AdvertiseTags, ","))
+		case "hostname":
+			set(prefs.Hostname)
+		case "operator":
+			set(prefs.OperatorUser)
+		case "advertise-routes":
+			var sb strings.Builder
+			for i, r := range withoutExitNodes(prefs.AdvertiseRoutes) {
+				if i > 0 {
+					sb.WriteByte(',')
+				}
+				sb.WriteString(r.String())
+			}
+			set(sb.String())
+		case "advertise-exit-node":
+			set(hasExitNodeRoutes(prefs.AdvertiseRoutes))
+		case "snat-subnet-routes":
+			set(!prefs.NoSNAT)
+		case "netfilter-mode":
+			set(prefs.NetfilterMode.String())
+		case "unattended":
+			set(prefs.ForceDaemon)
+		}
+	})
+	return ret
+}
+
+func fmtFlagValueArg(flagName string, val interface{}) string {
+	if val == true {
+		return "--" + flagName
+	}
+	if val == "" {
+		return "--" + flagName + "="
+	}
+	return fmt.Sprintf("--%s=%v", flagName, shellquote.Join(fmt.Sprint(val)))
+}
+
+func hasExitNodeRoutes(rr []netaddr.IPPrefix) bool {
+	var v4, v6 bool
+	for _, r := range rr {
+		if r.Bits() == 0 {
+			if r.IP().Is4() {
+				v4 = true
+			} else if r.IP().Is6() {
+				v6 = true
+			}
+		}
+	}
+	return v4 && v6
+}
+
+// withoutExitNodes returns rr unchanged if it has only 1 or 0 /0
+// routes. If it has both IPv4 and IPv6 /0 routes, then it returns
+// a copy with all /0 routes removed.
+func withoutExitNodes(rr []netaddr.IPPrefix) []netaddr.IPPrefix {
+	if !hasExitNodeRoutes(rr) {
+		return rr
+	}
+	var out []netaddr.IPPrefix
+	for _, r := range rr {
+		if r.Bits() > 0 {
+			out = append(out, r)
+		}
+	}
+	return out
+}
+
+// exitNodeIP returns the exit node IP from p, using st to map
+// it from its ID form to an IP address if needed.
+func exitNodeIP(p *ipn.Prefs, st *ipnstate.Status) (ip netaddr.IP) {
+	if p == nil {
+		return
+	}
+	if !p.ExitNodeIP.IsZero() {
+		return p.ExitNodeIP
+	}
+	id := p.ExitNodeID
+	if id.IsZero() {
+		return
+	}
+	for _, p := range st.Peer {
+		if p.ID == id {
+			if len(p.TailscaleIPs) > 0 {
+				return p.TailscaleIPs[0]
+			}
+			break
+		}
+	}
+	return
 }

cmd/tailscale/cli/web.go

@@ -73,7 +73,11 @@ func runWeb(ctx context.Context, args []string) error {
 	}
 
 	if webArgs.cgi {
-		return cgi.Serve(http.HandlerFunc(webHandler))
+		if err := cgi.Serve(http.HandlerFunc(webHandler)); err != nil {
+			log.Printf("tailscale.cgi: %v", err)
+			return err
+		}
+		return nil
 	}
 	return http.ListenAndServe(webArgs.listen, http.HandlerFunc(webHandler))
 }
@@ -208,9 +212,10 @@ func webHandler(w http.ResponseWriter, r *http.Request) {
 	if r.Method == "POST" {
 		type mi map[string]interface{}
 		w.Header().Set("Content-Type", "application/json")
-		url, err := tailscaleUp(r.Context())
+		url, err := tailscaleUpForceReauth(r.Context())
 		if err != nil {
-			json.NewEncoder(w).Encode(mi{"error": err})
+			w.WriteHeader(500)
+			json.NewEncoder(w).Encode(mi{"error": err.Error()})
 			return
 		}
 		json.NewEncoder(w).Encode(mi{"url": url})
@@ -244,9 +249,9 @@ func webHandler(w http.ResponseWriter, r *http.Request) {
 }
 
 // TODO(crawshaw): some of this is very similar to the code in 'tailscale up', can we share anything?
-func tailscaleUp(ctx context.Context) (authURL string, retErr error) {
+func tailscaleUpForceReauth(ctx context.Context) (authURL string, retErr error) {
 	prefs := ipn.NewPrefs()
-	prefs.ControlURL = "https://login.tailscale.com"
+	prefs.ControlURL = ipn.DefaultControlURL
 	prefs.WantRunning = true
 	prefs.CorpDNS = true
 	prefs.AllowSingleHosts = true
@@ -256,36 +261,70 @@ func tailscaleUp(ctx context.Context) (authURL string, retErr error) {
 		prefs.NetfilterMode = preftype.NetfilterOff
 	}
 
-	c, bc, ctx, cancel := connect(ctx)
+	st, err := tailscale.Status(ctx)
+	if err != nil {
+		return "", fmt.Errorf("can't fetch status: %v", err)
+	}
+	origAuthURL := st.AuthURL
+
+	// printAuthURL reports whether we should print out the
+	// provided auth URL from an IPN notify.
+	printAuthURL := func(url string) bool {
+		return url != origAuthURL
+	}
+
+	c, bc, pumpCtx, cancel := connect(ctx)
 	defer cancel()
 
+	gotEngineUpdate := make(chan bool, 1) // gets value upon an engine update
+	go pump(pumpCtx, bc, c)
+
+	bc.SetNotifyCallback(func(n ipn.Notify) {
+		if n.Engine != nil {
+			select {
+			case gotEngineUpdate <- true:
+			default:
+			}
+		}
+		if n.ErrMessage != nil {
+			msg := *n.ErrMessage
+			if msg == ipn.ErrMsgPermissionDenied {
+				switch runtime.GOOS {
+				case "windows":
+					msg += " (Tailscale service in use by other user?)"
+				default:
+					msg += " (try 'sudo tailscale up [...]')"
+				}
+			}
+			retErr = fmt.Errorf("backend error: %v", msg)
+			cancel()
+		} else if url := n.BrowseToURL; url != nil && printAuthURL(*url) {
+			authURL = *url
+			cancel()
+		}
+	})
+	// Wait for backend client to be connected so we know
+	// we're subscribed to updates. Otherwise we can miss
+	// an update upon its transition to running. Do so by causing some traffic
+	// back to the bus that we then wait on.
+	bc.RequestEngineStatus()
+	select {
+	case <-gotEngineUpdate:
+	case <-pumpCtx.Done():
+		return authURL, pumpCtx.Err()
+	}
+
 	bc.SetPrefs(prefs)
 
-	opts := ipn.Options{
+	bc.Start(ipn.Options{
 		StateKey: ipn.GlobalDaemonStateKey,
-		Notify: func(n ipn.Notify) {
-			if n.ErrMessage != nil {
-				msg := *n.ErrMessage
-				if msg == ipn.ErrMsgPermissionDenied {
-					switch runtime.GOOS {
-					case "windows":
-						msg += " (Tailscale service in use by other user?)"
-					default:
-						msg += " (try 'sudo tailscale up [...]')"
-					}
-				}
-				retErr = fmt.Errorf("backend error: %v", msg)
-				cancel()
-			} else if url := n.BrowseToURL; url != nil {
-				authURL = *url
-				cancel()
-			}
-		},
-	}
-	bc.Start(opts)
+	})
 	bc.StartLoginInteractive()
-	pump(ctx, bc, c)
 
+	<-pumpCtx.Done() // wait for authURL or complete failure
+	if authURL == "" && retErr == nil {
+		retErr = pumpCtx.Err()
+	}
 	if authURL == "" && retErr == nil {
 		return "", fmt.Errorf("login failed with no backend error message")
 	}

cmd/tailscale/cli/web.html

@@ -11,140 +11,133 @@
 </head>
 
 <body class="py-14">
-	<main class="container max-w-lg mx-auto py-6 px-8 bg-white rounded-md shadow-2xl" style="width: 95%">
-		<header class="flex justify-between items-center min-width-0 py-2 mb-8">
-			<svg width="26" height="26" viewBox="0 0 23 23" title="Tailscale" fill="none" xmlns="http://www.w3.org/2000/svg"
-				class="flex-shrink-0 mr-4">
-				<circle opacity="0.2" cx="3.4" cy="3.25" r="2.7" fill="currentColor"></circle>
-				<circle cx="3.4" cy="11.3" r="2.7" fill="currentColor"></circle>
-				<circle opacity="0.2" cx="3.4" cy="19.5" r="2.7" fill="currentColor"></circle>
-				<circle cx="11.5" cy="11.3" r="2.7" fill="currentColor"></circle>
-				<circle cx="11.5" cy="19.5" r="2.7" fill="currentColor"></circle>
-				<circle opacity="0.2" cx="11.5" cy="3.25" r="2.7" fill="currentColor"></circle>
-				<circle opacity="0.2" cx="19.5" cy="3.25" r="2.7" fill="currentColor"></circle>
-				<circle cx="19.5" cy="11.3" r="2.7" fill="currentColor"></circle>
-				<circle opacity="0.2" cx="19.5" cy="19.5" r="2.7" fill="currentColor"></circle>
-			</svg>
-			<div class="flex items-center justify-end space-x-2 w-2/3">
-				{{ with .Profile.LoginName }}
-				<div class="text-right truncate leading-4">
-					<h4 class="truncate">{{.}}</h4>
-					<a href="#" class="text-xs text-gray-500 hover:text-gray-700 js-loginButton">Switch account</a>
-				</div>
+<main class="container max-w-lg mx-auto py-6 px-8 bg-white rounded-md shadow-2xl" style="width: 95%">
+	<header class="flex justify-between items-center min-width-0 py-2 mb-8">
+		<svg width="26" height="26" viewBox="0 0 23 23" title="Tailscale" fill="none" xmlns="http://www.w3.org/2000/svg"
+			class="flex-shrink-0 mr-4">
+			<circle opacity="0.2" cx="3.4" cy="3.25" r="2.7" fill="currentColor"></circle>
+			<circle cx="3.4" cy="11.3" r="2.7" fill="currentColor"></circle>
+			<circle opacity="0.2" cx="3.4" cy="19.5" r="2.7" fill="currentColor"></circle>
+			<circle cx="11.5" cy="11.3" r="2.7" fill="currentColor"></circle>
+			<circle cx="11.5" cy="19.5" r="2.7" fill="currentColor"></circle>
+			<circle opacity="0.2" cx="11.5" cy="3.25" r="2.7" fill="currentColor"></circle>
+			<circle opacity="0.2" cx="19.5" cy="3.25" r="2.7" fill="currentColor"></circle>
+			<circle cx="19.5" cy="11.3" r="2.7" fill="currentColor"></circle>
+			<circle opacity="0.2" cx="19.5" cy="19.5" r="2.7" fill="currentColor"></circle>
+		</svg>
+		<div class="flex items-center justify-end space-x-2 w-2/3">
+			{{ with .Profile.LoginName }}
+			<div class="text-right truncate leading-4">
+				<h4 class="truncate">{{.}}</h4>
+				<a href="#" class="text-xs text-gray-500 hover:text-gray-700 js-loginButton">Switch account</a>
+			</div>
+			{{ end }}
+			<div class="relative flex-shrink-0 w-8 h-8 rounded-full overflow-hidden">
+				{{ with .Profile.ProfilePicURL }}
+				<div class="w-8 h-8 flex pointer-events-none rounded-full bg-gray-200"
+					style="background-image: url('{{.}}'); background-size: cover;"></div>
+				{{ else }}
+				<div class="w-8 h-8 flex pointer-events-none rounded-full border border-gray-400 border-dashed"></div>
 				{{ end }}
-				<div class="relative flex-shrink-0 w-8 h-8 rounded-full overflow-hidden">
-					{{ with .Profile.ProfilePicURL }}
-					<div class="w-8 h-8 flex pointer-events-none rounded-full bg-gray-200"
-						style="background-image: url('{{.}}'); background-size: cover;"></div>
-					{{ else }}
-					<div class="w-8 h-8 flex pointer-events-none rounded-full border border-gray-400 border-dashed"></div>
-					{{ end }}
-				</div>
 			</div>
-		</header>
-		{{ if .IP }}
-		<div
-			class="border border-gray-200 bg-gray-0 rounded-lg p-2 pl-3 pr-3 mb-8 width-full flex items-center justify-between">
-			<div class="flex items-center min-width-0">
-				<svg class="flex-shrink-0 text-gray-600 mr-3 ml-1" xmlns="http://www.w3.org/2000/svg" width="20" height="20"
-					viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round"
-					stroke-linejoin="round">
-					<rect x="2" y="2" width="20" height="8" rx="2" ry="2"></rect>
-					<rect x="2" y="14" width="20" height="8" rx="2" ry="2"></rect>
-					<line x1="6" y1="6" x2="6.01" y2="6"></line>
-					<line x1="6" y1="18" x2="6.01" y2="18"></line>
-				</svg>
-				<h4 class="font-semibold truncate mr-2">{{.DeviceName}}</h4>
-			</div>
-			<h5>{{.IP}}</h5>
 		</div>
-		{{ end }}
-		{{ if or (eq .Status "NeedsLogin") (eq .Status "NoState") }}
-		{{ if .IP }}
-		<div class="mb-6">
-			<p class="text-gray-700">Your device's key has expired. Reauthenticate this device by logging in again, or <a
-					href="https://tailscale.com/kb/1028/key-expiry" class="link" target="_blank">learn more</a>.</p>
+	</header>
+	{{ if .IP }}
+	<div
+		class="border border-gray-200 bg-gray-0 rounded-lg p-2 pl-3 pr-3 mb-8 width-full flex items-center justify-between">
+		<div class="flex items-center min-width-0">
+			<svg class="flex-shrink-0 text-gray-600 mr-3 ml-1" xmlns="http://www.w3.org/2000/svg" width="20" height="20"
+				viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round"
+				stroke-linejoin="round">
+				<rect x="2" y="2" width="20" height="8" rx="2" ry="2"></rect>
+				<rect x="2" y="14" width="20" height="8" rx="2" ry="2"></rect>
+				<line x1="6" y1="6" x2="6.01" y2="6"></line>
+				<line x1="6" y1="18" x2="6.01" y2="18"></line>
+			</svg>
+			<h4 class="font-semibold truncate mr-2">{{.DeviceName}}</h4>
 		</div>
-		<a href="#" class="mb-4 js-loginButton" target="_blank">
-			<button class="button button-blue w-full">Reauthenticate</button>
-		</a>
-		{{ else }}
-		<div class="mb-6">
-			<h3 class="text-3xl font-semibold mb-3">Log in</h3>
-			<p class="text-gray-700">Get started by logging in to your Tailscale network. Or,&nbsp;learn&nbsp;more at <a
-					href="https://tailscale.com/" class="link" target="_blank">tailscale.com</a>.</p>
-		</div>
-		<a href="#" class="mb-4 js-loginButton" target="_blank">
-			<button class="button button-blue w-full">Log In</button>
-		</a>
-		{{ end }}
-		{{ else if eq .Status "NeedsMachineAuth" }}
-		<div class="mb-4">
-			This device is authorized, but needs approval from a network admin before it can connect to the network.
-		</div>
-		{{ else }}
-		<div class="mb-4">
-			<p>You are connected! Access this device over Tailscale using the device name or IP address above.</p>
-		</div>
-		<a href="#" class="mb-4 link font-medium js-loginButton" target="_blank">Reauthenticate</a>
-		{{ end }}
-	</main>
-	<script>
-		(function () {
-			let loginButtons = document.querySelectorAll(".js-loginButton");
-			let fetchingUrl = false;
+		<h5>{{.IP}}</h5>
+	</div>
+	{{ end }}
+	{{ if or (eq .Status "NeedsLogin") (eq .Status "NoState") }}
+	{{ if .IP }}
+	<div class="mb-6">
+		<p class="text-gray-700">Your device's key has expired. Reauthenticate this device by logging in again, or <a
+				href="https://tailscale.com/kb/1028/key-expiry" class="link" target="_blank">learn more</a>.</p>
+	</div>
+	<a href="#" class="mb-4 js-loginButton" target="_blank">
+		<button class="button button-blue w-full">Reauthenticate</button>
+	</a>
+	{{ else }}
+	<div class="mb-6">
+		<h3 class="text-3xl font-semibold mb-3">Log in</h3>
+		<p class="text-gray-700">Get started by logging in to your Tailscale network. Or,&nbsp;learn&nbsp;more at <a
+				href="https://tailscale.com/" class="link" target="_blank">tailscale.com</a>.</p>
+	</div>
+	<a href="#" class="mb-4 js-loginButton" target="_blank">
+		<button class="button button-blue w-full">Log In</button>
+	</a>
+	{{ end }}
+	{{ else if eq .Status "NeedsMachineAuth" }}
+	<div class="mb-4">
+		This device is authorized, but needs approval from a network admin before it can connect to the network.
+	</div>
+	{{ else }}
+	<div class="mb-4">
+		<p>You are connected! Access this device over Tailscale using the device name or IP address above.</p>
+	</div>
+	<a href="#" class="mb-4 link font-medium js-loginButton" target="_blank">Reauthenticate</a>
+	{{ end }}
+</main>
+<script>(function () {
+let loginButtons = document.querySelectorAll(".js-loginButton");
+let fetchingUrl = false;
 
-			function handleClick(e) {
-				e.preventDefault();
+function handleClick(e) {
+	e.preventDefault();
 
-				if (fetchingUrl) {
-					return;
-				}
+	if (fetchingUrl) {
+		return;
+	}
 
-				fetchingUrl = true;
-				const urlParams = new URLSearchParams(window.location.search);
-				const token = urlParams.get("SynoToken");
-				const nextParams = new URLSearchParams({ up: true });
-				if (token) {
-					nextParams.set("SynoToken", token)
-				}
-				const nextUrl = new URL(window.location);
-				nextUrl.search = nextParams.toString()
-				const url = nextUrl.toString();
+	fetchingUrl = true;
+	const urlParams = new URLSearchParams(window.location.search);
+	const token = urlParams.get("SynoToken");
+	const nextParams = new URLSearchParams({ up: true });
+	if (token) {
+		nextParams.set("SynoToken", token)
+	}
+	const nextUrl = new URL(window.location);
+	nextUrl.search = nextParams.toString()
+	const url = nextUrl.toString();
 
-				const tab = window.open("/redirect", "_blank");
+	fetch(url, {
+		method: "POST",
+		headers: {
+			"Accept": "application/json",
+			"Content-Type": "application/json",
+		}
+	}).then(res => res.json()).then(res => {
+		fetchingUrl = false;
+		const err = res["error"];
+		if (err) {
+			throw new Error(err);
+		}
+		const url = res["url"];
+		if (url) {
+			document.location.href = url;
+		} else {
+			location.reload();
+		}
+	}).catch(err => {
+		alert("Failed to log in: " + err.message);
+	});
+}
 
-				fetch(url, {
-					method: "POST",
-					headers: {
-						"Accept": "application/json",
-						"Content-Type": "application/json",
-					}
-				}).then(res => res.json()).then(res => {
-					fetchingUrl = false;
-					const err = res["error"];
-					if (err) {
-						throw new Error(err);
-					}
-					const url = res["url"];
-					if (url) {
-						authUrl = url;
-						tab.location = url;
-						tab.focus();
-					} else {
-						location.reload();
-					}
-				}).catch(err => {
-					tab.close();
-					alert("Failed to log in: " + err.message);
-				});
-			}
-
-			Array.from(loginButtons).forEach(el => {
-				el.addEventListener("click", handleClick);
-			})
-		})();
-	</script>
+Array.from(loginButtons).forEach(el => {
+	el.addEventListener("click", handleClick);
+})
+})();</script>
 </body>
 
 </html>

cmd/tailscale/depaware.txt

@@ -1,7 +1,9 @@
 tailscale.com/cmd/tailscale dependencies: (generated by github.com/tailscale/depaware)
 
-   W 💣 github.com/alexbrainman/sspi                                 from github.com/alexbrainman/sspi/negotiate
+   W 💣 github.com/alexbrainman/sspi                                 from github.com/alexbrainman/sspi/negotiate+
+   W    github.com/alexbrainman/sspi/internal/common                 from github.com/alexbrainman/sspi/negotiate
    W 💣 github.com/alexbrainman/sspi/negotiate                       from tailscale.com/net/tshttpproxy
+        github.com/kballard/go-shellquote                            from tailscale.com/cmd/tailscale/cli
         github.com/peterbourgon/ff/v2                                from github.com/peterbourgon/ff/v2/ffcli
         github.com/peterbourgon/ff/v2/ffcli                          from tailscale.com/cmd/tailscale/cli
         github.com/tcnksm/go-httpstat                                from tailscale.com/net/netcheck
@@ -14,12 +16,13 @@ tailscale.com/cmd/tailscale dependencies: (generated by github.com/tailscale/dep
         rsc.io/goversion/version                                     from tailscale.com/version
         tailscale.com/atomicfile                                     from tailscale.com/ipn
         tailscale.com/client/tailscale                               from tailscale.com/cmd/tailscale/cli
+        tailscale.com/client/tailscale/apitype                       from tailscale.com/client/tailscale+
         tailscale.com/cmd/tailscale/cli                              from tailscale.com/cmd/tailscale
         tailscale.com/derp                                           from tailscale.com/derp/derphttp
         tailscale.com/derp/derphttp                                  from tailscale.com/net/netcheck
         tailscale.com/derp/derpmap                                   from tailscale.com/cmd/tailscale/cli
         tailscale.com/disco                                          from tailscale.com/derp
-        tailscale.com/ipn                                            from tailscale.com/cmd/tailscale/cli
+        tailscale.com/ipn                                            from tailscale.com/cmd/tailscale/cli+
         tailscale.com/ipn/ipnstate                                   from tailscale.com/cmd/tailscale/cli+
         tailscale.com/metrics                                        from tailscale.com/derp
         tailscale.com/net/dnscache                                   from tailscale.com/derp/derphttp
@@ -31,7 +34,7 @@ tailscale.com/cmd/tailscale dependencies: (generated by github.com/tailscale/dep
         tailscale.com/net/portmapper                                 from tailscale.com/net/netcheck+
         tailscale.com/net/stun                                       from tailscale.com/net/netcheck
         tailscale.com/net/tlsdial                                    from tailscale.com/derp/derphttp
-        tailscale.com/net/tsaddr                                     from tailscale.com/net/interfaces
+        tailscale.com/net/tsaddr                                     from tailscale.com/net/interfaces+
      💣 tailscale.com/net/tshttpproxy                                from tailscale.com/derp/derphttp+
         tailscale.com/paths                                          from tailscale.com/cmd/tailscale/cli+
         tailscale.com/safesocket                                     from tailscale.com/cmd/tailscale/cli+
@@ -46,7 +49,6 @@ tailscale.com/cmd/tailscale dependencies: (generated by github.com/tailscale/dep
         tailscale.com/types/opt                                      from tailscale.com/net/netcheck+
         tailscale.com/types/persist                                  from tailscale.com/ipn
         tailscale.com/types/preftype                                 from tailscale.com/cmd/tailscale/cli+
-        tailscale.com/types/strbuilder                               from tailscale.com/net/packet
         tailscale.com/types/structs                                  from tailscale.com/ipn+
         tailscale.com/types/wgkey                                    from tailscale.com/types/netmap+
         tailscale.com/util/dnsname                                   from tailscale.com/cmd/tailscale/cli+
@@ -83,7 +85,7 @@ tailscale.com/cmd/tailscale dependencies: (generated by github.com/tailscale/dep
         golang.org/x/text/transform                                  from golang.org/x/text/secure/bidirule+
         golang.org/x/text/unicode/bidi                               from golang.org/x/net/idna+
         golang.org/x/text/unicode/norm                               from golang.org/x/net/idna
-        golang.org/x/time/rate                                       from tailscale.com/types/logger+
+        golang.org/x/time/rate                                       from tailscale.com/cmd/tailscale/cli+
         bufio                                                        from compress/flate+
         bytes                                                        from bufio+
         compress/flate                                               from compress/gzip+

cmd/tailscaled/depaware.txt

@@ -1,41 +1,42 @@
 tailscale.com/cmd/tailscaled dependencies: (generated by github.com/tailscale/depaware)
 
-   W 💣 github.com/alexbrainman/sspi                                 from github.com/alexbrainman/sspi/negotiate
+   W 💣 github.com/alexbrainman/sspi                                 from github.com/alexbrainman/sspi/negotiate+
+   W    github.com/alexbrainman/sspi/internal/common                 from github.com/alexbrainman/sspi/negotiate
    W 💣 github.com/alexbrainman/sspi/negotiate                       from tailscale.com/net/tshttpproxy
    L    github.com/coreos/go-iptables/iptables                       from tailscale.com/wgengine/router
-   W 💣 github.com/github/certstore                                  from tailscale.com/control/controlclient
         github.com/go-multierror/multierror                          from tailscale.com/wgengine/router+
    W 💣 github.com/go-ole/go-ole                                     from github.com/go-ole/go-ole/oleutil+
    W 💣 github.com/go-ole/go-ole/oleutil                             from tailscale.com/wgengine/winnet
    L 💣 github.com/godbus/dbus/v5                                    from tailscale.com/net/dns
+        github.com/golang/snappy                                     from github.com/klauspost/compress/zstd
         github.com/google/btree                                      from inet.af/netstack/tcpip/header+
    L    github.com/josharian/native                                  from github.com/mdlayher/netlink+
    L 💣 github.com/jsimonetti/rtnetlink                              from tailscale.com/wgengine/monitor
    L    github.com/jsimonetti/rtnetlink/internal/unix                from github.com/jsimonetti/rtnetlink
         github.com/klauspost/compress/fse                            from github.com/klauspost/compress/huff0
         github.com/klauspost/compress/huff0                          from github.com/klauspost/compress/zstd
-        github.com/klauspost/compress/snappy                         from github.com/klauspost/compress/zstd
         github.com/klauspost/compress/zstd                           from tailscale.com/smallzstd
         github.com/klauspost/compress/zstd/internal/xxhash           from github.com/klauspost/compress/zstd
    L 💣 github.com/mdlayher/netlink                                  from github.com/jsimonetti/rtnetlink+
    L 💣 github.com/mdlayher/netlink/nlenc                            from github.com/jsimonetti/rtnetlink+
    L    github.com/mdlayher/sdnotify                                 from tailscale.com/util/systemd
-   W    github.com/pkg/errors                                        from github.com/github/certstore
-     💣 github.com/tailscale/wireguard-go/conn                       from github.com/tailscale/wireguard-go/device+
-   W 💣 github.com/tailscale/wireguard-go/conn/winrio                from github.com/tailscale/wireguard-go/conn
-     💣 github.com/tailscale/wireguard-go/device                     from tailscale.com/wgengine+
-     💣 github.com/tailscale/wireguard-go/ipc                        from github.com/tailscale/wireguard-go/device
-   W 💣 github.com/tailscale/wireguard-go/ipc/winpipe                from github.com/tailscale/wireguard-go/ipc
-        github.com/tailscale/wireguard-go/ratelimiter                from github.com/tailscale/wireguard-go/device
-        github.com/tailscale/wireguard-go/replay                     from github.com/tailscale/wireguard-go/device
-        github.com/tailscale/wireguard-go/rwcancel                   from github.com/tailscale/wireguard-go/device+
-        github.com/tailscale/wireguard-go/tai64n                     from github.com/tailscale/wireguard-go/device+
-     💣 github.com/tailscale/wireguard-go/tun                        from github.com/tailscale/wireguard-go/device+
-   W 💣 github.com/tailscale/wireguard-go/tun/wintun                 from github.com/tailscale/wireguard-go/tun+
+   W    github.com/pkg/errors                                        from github.com/tailscale/certstore
+   W 💣 github.com/tailscale/certstore                               from tailscale.com/control/controlclient
         github.com/tcnksm/go-httpstat                                from tailscale.com/net/netcheck
      💣 go4.org/intern                                               from inet.af/netaddr
      💣 go4.org/mem                                                  from tailscale.com/control/controlclient+
         go4.org/unsafe/assume-no-moving-gc                           from go4.org/intern
+     💣 golang.zx2c4.com/wireguard/conn                              from golang.zx2c4.com/wireguard/device+
+   W 💣 golang.zx2c4.com/wireguard/conn/winrio                       from golang.zx2c4.com/wireguard/conn
+     💣 golang.zx2c4.com/wireguard/device                            from tailscale.com/net/tstun+
+     💣 golang.zx2c4.com/wireguard/ipc                               from golang.zx2c4.com/wireguard/device
+   W 💣 golang.zx2c4.com/wireguard/ipc/winpipe                       from golang.zx2c4.com/wireguard/ipc
+        golang.zx2c4.com/wireguard/ratelimiter                       from golang.zx2c4.com/wireguard/device
+        golang.zx2c4.com/wireguard/replay                            from golang.zx2c4.com/wireguard/device
+        golang.zx2c4.com/wireguard/rwcancel                          from golang.zx2c4.com/wireguard/device+
+        golang.zx2c4.com/wireguard/tai64n                            from golang.zx2c4.com/wireguard/device+
+     💣 golang.zx2c4.com/wireguard/tun                               from golang.zx2c4.com/wireguard/device+
+   W 💣 golang.zx2c4.com/wireguard/tun/wintun                        from golang.zx2c4.com/wireguard/tun+
    W 💣 golang.zx2c4.com/wireguard/windows/tunnel/winipcfg           from tailscale.com/net/interfaces+
         inet.af/netaddr                                              from tailscale.com/control/controlclient+
      💣 inet.af/netstack/gohacks                                     from inet.af/netstack/state/wire+
@@ -69,15 +70,17 @@ tailscale.com/cmd/tailscaled dependencies: (generated by github.com/tailscale/de
         inet.af/netstack/tcpip/transport/udp                         from inet.af/netstack/tcpip/adapters/gonet+
         inet.af/netstack/waiter                                      from inet.af/netstack/tcpip+
         inet.af/peercred                                             from tailscale.com/ipn/ipnserver
+   W 💣 inet.af/wf                                                   from tailscale.com/wf
         rsc.io/goversion/version                                     from tailscale.com/version
         tailscale.com/atomicfile                                     from tailscale.com/ipn+
+        tailscale.com/client/tailscale/apitype                       from tailscale.com/ipn/ipnlocal+
         tailscale.com/control/controlclient                          from tailscale.com/ipn/ipnlocal+
         tailscale.com/derp                                           from tailscale.com/derp/derphttp+
         tailscale.com/derp/derphttp                                  from tailscale.com/net/netcheck+
         tailscale.com/derp/derpmap                                   from tailscale.com/cmd/tailscaled+
         tailscale.com/disco                                          from tailscale.com/derp+
         tailscale.com/health                                         from tailscale.com/control/controlclient+
-        tailscale.com/internal/deepprint                             from tailscale.com/ipn/ipnlocal+
+        tailscale.com/internal/deephash                              from tailscale.com/ipn/ipnlocal+
         tailscale.com/ipn                                            from tailscale.com/ipn/ipnserver+
         tailscale.com/ipn/ipnlocal                                   from tailscale.com/ipn/ipnserver+
         tailscale.com/ipn/ipnserver                                  from tailscale.com/cmd/tailscaled
@@ -114,7 +117,6 @@ tailscale.com/cmd/tailscaled dependencies: (generated by github.com/tailscale/de
         tailscale.com/smallzstd                                      from tailscale.com/ipn/ipnserver+
         tailscale.com/syncs                                          from tailscale.com/net/interfaces+
         tailscale.com/tailcfg                                        from tailscale.com/control/controlclient+
-   W 💣 tailscale.com/tempfork/wireguard-windows/firewall            from tailscale.com/cmd/tailscaled
    W    tailscale.com/tsconst                                        from tailscale.com/net/interfaces
         tailscale.com/tstime                                         from tailscale.com/wgengine/magicsock
         tailscale.com/types/empty                                    from tailscale.com/control/controlclient+
@@ -127,21 +129,24 @@ tailscale.com/cmd/tailscaled dependencies: (generated by github.com/tailscale/de
         tailscale.com/types/opt                                      from tailscale.com/control/controlclient+
         tailscale.com/types/persist                                  from tailscale.com/control/controlclient+
         tailscale.com/types/preftype                                 from tailscale.com/ipn+
-        tailscale.com/types/strbuilder                               from tailscale.com/net/packet
         tailscale.com/types/structs                                  from tailscale.com/control/controlclient+
         tailscale.com/types/wgkey                                    from tailscale.com/control/controlclient+
+   L    tailscale.com/util/cmpver                                    from tailscale.com/net/dns
         tailscale.com/util/dnsname                                   from tailscale.com/ipn/ipnstate+
   LW    tailscale.com/util/endian                                    from tailscale.com/net/netns+
    L    tailscale.com/util/lineread                                  from tailscale.com/control/controlclient+
+        tailscale.com/util/osshare                                   from tailscale.com/cmd/tailscaled+
         tailscale.com/util/pidowner                                  from tailscale.com/ipn/ipnserver
         tailscale.com/util/racebuild                                 from tailscale.com/logpolicy
         tailscale.com/util/systemd                                   from tailscale.com/control/controlclient+
+        tailscale.com/util/uniq                                      from tailscale.com/wgengine/magicsock
         tailscale.com/util/winutil                                   from tailscale.com/logpolicy+
         tailscale.com/version                                        from tailscale.com/cmd/tailscaled+
         tailscale.com/version/distro                                 from tailscale.com/control/controlclient+
+   W    tailscale.com/wf                                             from tailscale.com/cmd/tailscaled
         tailscale.com/wgengine                                       from tailscale.com/cmd/tailscaled+
         tailscale.com/wgengine/filter                                from tailscale.com/control/controlclient+
-        tailscale.com/wgengine/magicsock                             from tailscale.com/cmd/tailscaled+
+        tailscale.com/wgengine/magicsock                             from tailscale.com/wgengine+
         tailscale.com/wgengine/monitor                               from tailscale.com/wgengine+
         tailscale.com/wgengine/netstack                              from tailscale.com/cmd/tailscaled
         tailscale.com/wgengine/router                                from tailscale.com/cmd/tailscaled+
@@ -150,7 +155,7 @@ tailscale.com/cmd/tailscaled dependencies: (generated by github.com/tailscale/de
         tailscale.com/wgengine/wglog                                 from tailscale.com/wgengine
    W 💣 tailscale.com/wgengine/winnet                                from tailscale.com/wgengine/router
         golang.org/x/crypto/blake2b                                  from golang.org/x/crypto/nacl/box
-        golang.org/x/crypto/blake2s                                  from github.com/tailscale/wireguard-go/device+
+        golang.org/x/crypto/blake2s                                  from golang.zx2c4.com/wireguard/device+
         golang.org/x/crypto/chacha20                                 from golang.org/x/crypto/chacha20poly1305
         golang.org/x/crypto/chacha20poly1305                         from crypto/tls+
         golang.org/x/crypto/cryptobyte                               from crypto/ecdsa+
@@ -159,31 +164,32 @@ tailscale.com/cmd/tailscaled dependencies: (generated by github.com/tailscale/de
         golang.org/x/crypto/hkdf                                     from crypto/tls
         golang.org/x/crypto/nacl/box                                 from tailscale.com/control/controlclient+
         golang.org/x/crypto/nacl/secretbox                           from golang.org/x/crypto/nacl/box
-        golang.org/x/crypto/poly1305                                 from github.com/tailscale/wireguard-go/device+
+        golang.org/x/crypto/poly1305                                 from golang.org/x/crypto/chacha20poly1305+
         golang.org/x/crypto/salsa20/salsa                            from golang.org/x/crypto/nacl/box+
         golang.org/x/net/bpf                                         from github.com/mdlayher/netlink+
         golang.org/x/net/dns/dnsmessage                              from net+
-        golang.org/x/net/http/httpguts                               from net/http
+        golang.org/x/net/http/httpguts                               from net/http+
         golang.org/x/net/http/httpproxy                              from net/http
         golang.org/x/net/http2/hpack                                 from net/http
         golang.org/x/net/idna                                        from golang.org/x/net/http/httpguts+
-        golang.org/x/net/ipv4                                        from github.com/tailscale/wireguard-go/device
-        golang.org/x/net/ipv6                                        from github.com/tailscale/wireguard-go/device+
+        golang.org/x/net/ipv4                                        from golang.zx2c4.com/wireguard/device
+        golang.org/x/net/ipv6                                        from golang.zx2c4.com/wireguard/device+
         golang.org/x/net/proxy                                       from tailscale.com/net/netns
    D    golang.org/x/net/route                                       from net+
         golang.org/x/sync/errgroup                                   from tailscale.com/derp
         golang.org/x/sync/singleflight                               from tailscale.com/net/dnscache
         golang.org/x/sys/cpu                                         from golang.org/x/crypto/blake2b+
   LD    golang.org/x/sys/unix                                        from github.com/jsimonetti/rtnetlink/internal/unix+
-   W    golang.org/x/sys/windows                                     from github.com/tailscale/wireguard-go/conn+
+   W    golang.org/x/sys/windows                                     from github.com/go-ole/go-ole+
    W    golang.org/x/sys/windows/registry                            from golang.zx2c4.com/wireguard/windows/tunnel/winipcfg+
-   W    golang.org/x/sys/windows/svc                                 from tailscale.com/cmd/tailscaled
+   W    golang.org/x/sys/windows/svc                                 from tailscale.com/cmd/tailscaled+
+   W    golang.org/x/sys/windows/svc/mgr                             from tailscale.com/cmd/tailscaled
         golang.org/x/term                                            from tailscale.com/logpolicy
         golang.org/x/text/secure/bidirule                            from golang.org/x/net/idna
         golang.org/x/text/transform                                  from golang.org/x/text/secure/bidirule+
         golang.org/x/text/unicode/bidi                               from golang.org/x/net/idna+
         golang.org/x/text/unicode/norm                               from golang.org/x/net/idna
-        golang.org/x/time/rate                                       from tailscale.com/types/logger+
+        golang.org/x/time/rate                                       from inet.af/netstack/tcpip/stack+
         bufio                                                        from compress/flate+
         bytes                                                        from bufio+
         compress/flate                                               from compress/gzip+
@@ -216,6 +222,7 @@ tailscale.com/cmd/tailscaled dependencies: (generated by github.com/tailscale/de
         debug/elf                                                    from rsc.io/goversion/version
         debug/macho                                                  from rsc.io/goversion/version
         debug/pe                                                     from rsc.io/goversion/version
+   L    embed                                                        from tailscale.com/net/dns
         encoding                                                     from encoding/json+
         encoding/asn1                                                from crypto/x509+
         encoding/base64                                              from encoding/json+
@@ -247,7 +254,8 @@ tailscale.com/cmd/tailscaled dependencies: (generated by github.com/tailscale/de
         net                                                          from crypto/tls+
         net/http                                                     from expvar+
         net/http/httptrace                                           from github.com/tcnksm/go-httpstat+
-        net/http/internal                                            from net/http
+        net/http/httputil                                            from tailscale.com/ipn/localapi
+        net/http/internal                                            from net/http+
         net/http/pprof                                               from tailscale.com/cmd/tailscaled
         net/textproto                                                from golang.org/x/net/http/httpguts+
         net/url                                                      from crypto/x509+

cmd/tailscaled/install_windows.go

@@ -0,0 +1,123 @@
+// Copyright (c) 2021 Tailscale Inc & AUTHORS All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package main
+
+import (
+	"context"
+	"errors"
+	"fmt"
+	"os"
+	"time"
+
+	"golang.org/x/sys/windows"
+	"golang.org/x/sys/windows/svc"
+	"golang.org/x/sys/windows/svc/mgr"
+	"tailscale.com/logtail/backoff"
+	"tailscale.com/types/logger"
+	"tailscale.com/util/osshare"
+)
+
+func init() {
+	installSystemDaemon = installSystemDaemonWindows
+	uninstallSystemDaemon = uninstallSystemDaemonWindows
+}
+
+func installSystemDaemonWindows(args []string) (err error) {
+	m, err := mgr.Connect()
+	if err != nil {
+		return fmt.Errorf("failed to connect to Windows service manager: %v", err)
+	}
+
+	service, err := m.OpenService(serviceName)
+	if err == nil {
+		service.Close()
+		return fmt.Errorf("service %q is already installed", serviceName)
+	}
+
+	// no such service; proceed to install the service.
+
+	exe, err := os.Executable()
+	if err != nil {
+		return err
+	}
+
+	c := mgr.Config{
+		ServiceType:  windows.SERVICE_WIN32_OWN_PROCESS,
+		StartType:    mgr.StartAutomatic,
+		ErrorControl: mgr.ErrorNormal,
+		DisplayName:  serviceName,
+		Description:  "Connects this computer to others on the Tailscale network.",
+	}
+
+	service, err = m.CreateService(serviceName, exe, c)
+	if err != nil {
+		return fmt.Errorf("failed to create %q service: %v", serviceName, err)
+	}
+	defer service.Close()
+
+	// Exponential backoff is often too aggressive, so use (mostly)
+	// squares instead.
+	ra := []mgr.RecoveryAction{
+		{mgr.ServiceRestart, 1 * time.Second},
+		{mgr.ServiceRestart, 2 * time.Second},
+		{mgr.ServiceRestart, 4 * time.Second},
+		{mgr.ServiceRestart, 9 * time.Second},
+		{mgr.ServiceRestart, 16 * time.Second},
+		{mgr.ServiceRestart, 25 * time.Second},
+		{mgr.ServiceRestart, 36 * time.Second},
+		{mgr.ServiceRestart, 49 * time.Second},
+		{mgr.ServiceRestart, 64 * time.Second},
+	}
+	const resetPeriodSecs = 60
+	err = service.SetRecoveryActions(ra, resetPeriodSecs)
+	if err != nil {
+		return fmt.Errorf("failed to set service recovery actions: %v", err)
+	}
+
+	return nil
+}
+
+func uninstallSystemDaemonWindows(args []string) (ret error) {
+	// Remove file sharing from Windows shell (noop in non-windows)
+	osshare.SetFileSharingEnabled(false, logger.Discard)
+
+	m, err := mgr.Connect()
+	if err != nil {
+		return fmt.Errorf("failed to connect to Windows service manager: %v", err)
+	}
+	defer m.Disconnect()
+
+	service, err := m.OpenService(serviceName)
+	if err != nil {
+		return fmt.Errorf("failed to open %q service: %v", serviceName, err)
+	}
+
+	st, err := service.Query()
+	if err != nil {
+		service.Close()
+		return fmt.Errorf("failed to query service state: %v", err)
+	}
+	if st.State != svc.Stopped {
+		service.Control(svc.Stop)
+	}
+	err = service.Delete()
+	service.Close()
+	if err != nil {
+		return fmt.Errorf("failed to delete service: %v", err)
+	}
+
+	bo := backoff.NewBackoff("uninstall", logger.Discard, 30*time.Second)
+	end := time.Now().Add(15 * time.Second)
+	for time.Until(end) > 0 {
+		service, err = m.OpenService(serviceName)
+		if err != nil {
+			// service is no longer openable; success!
+			break
+		}
+		service.Close()
+		bo.BackOff(context.Background(), errors.New("service not deleted"))
+	}
+	return nil
+}

cmd/tailscaled/tailscaled.go

@@ -29,19 +29,21 @@ import (
 	"time"
 
 	"github.com/go-multierror/multierror"
+	"inet.af/netaddr"
 	"tailscale.com/ipn/ipnserver"
 	"tailscale.com/logpolicy"
 	"tailscale.com/net/dns"
 	"tailscale.com/net/socks5"
+	"tailscale.com/net/tsaddr"
 	"tailscale.com/net/tstun"
 	"tailscale.com/paths"
 	"tailscale.com/types/flagtype"
 	"tailscale.com/types/logger"
 	"tailscale.com/types/netmap"
+	"tailscale.com/util/osshare"
 	"tailscale.com/version"
 	"tailscale.com/version/distro"
 	"tailscale.com/wgengine"
-	"tailscale.com/wgengine/magicsock"
 	"tailscale.com/wgengine/monitor"
 	"tailscale.com/wgengine/netstack"
 	"tailscale.com/wgengine/router"
@@ -114,7 +116,7 @@ func main() {
 	flag.StringVar(&args.debug, "debug", "", "listen address ([ip]:port) of optional debug server")
 	flag.StringVar(&args.socksAddr, "socks5-server", "", `optional [ip]:port to run a SOCK5 server (e.g. "localhost:1080")`)
 	flag.StringVar(&args.tunname, "tun", defaultTunName(), `tunnel interface name; use "userspace-networking" (beta) to not use TUN`)
-	flag.Var(flagtype.PortValue(&args.port, magicsock.DefaultPort), "port", "UDP port to listen on for WireGuard and peer-to-peer traffic; 0 means automatically select")
+	flag.Var(flagtype.PortValue(&args.port, 0), "port", "UDP port to listen on for WireGuard and peer-to-peer traffic; 0 means automatically select")
 	flag.StringVar(&args.statepath, "state", paths.DefaultTailscaledStateFile(), "path of state file")
 	flag.StringVar(&args.socketpath, "socket", paths.DefaultTailscaledSocket(), "path of the service unix socket")
 	flag.BoolVar(&printVersion, "version", false, "print version information and exit")
@@ -158,7 +160,12 @@ func main() {
 		log.Fatalf("--socket is required")
 	}
 
-	if err := run(); err != nil {
+	err := run()
+
+	// Remove file sharing from Windows shell (noop in non-windows)
+	osshare.SetFileSharingEnabled(false, logger.Discard)
+
+	if err != nil {
 		// No need to log; the func already did
 		os.Exit(1)
 	}
@@ -230,44 +237,40 @@ func run() error {
 	}
 
 	var ns *netstack.Impl
-	if useNetstack {
-		tunDev, magicConn, ok := e.(wgengine.InternalsGetter).GetInternals()
-		if !ok {
-			log.Fatalf("%T is not a wgengine.InternalsGetter", e)
-		}
-		ns, err = netstack.Create(logf, tunDev, e, magicConn)
-		if err != nil {
-			log.Fatalf("netstack.Create: %v", err)
-		}
-		if err := ns.Start(); err != nil {
-			log.Fatalf("failed to start netstack: %v", err)
-		}
+	if useNetstack || wrapNetstack {
+		onlySubnets := wrapNetstack && !useNetstack
+		ns = mustStartNetstack(logf, e, onlySubnets)
 	}
 
 	if socksListener != nil {
 		srv := &socks5.Server{
 			Logf: logger.WithPrefix(logf, "socks5: "),
 		}
-		if useNetstack {
-			srv.Dialer = func(ctx context.Context, network, addr string) (net.Conn, error) {
+		var (
+			mu  sync.Mutex // guards the following field
+			dns netstack.DNSMap
+		)
+		e.AddNetworkMapCallback(func(nm *netmap.NetworkMap) {
+			mu.Lock()
+			defer mu.Unlock()
+			dns = netstack.DNSMapFromNetworkMap(nm)
+		})
+		useNetstackForIP := func(ip netaddr.IP) bool {
+			// TODO(bradfitz): this isn't exactly right.
+			// We should also support subnets when the
+			// prefs are configured as such.
+			return tsaddr.IsTailscaleIP(ip)
+		}
+		srv.Dialer = func(ctx context.Context, network, addr string) (net.Conn, error) {
+			ipp, err := dns.Resolve(ctx, addr)
+			if err != nil {
+				return nil, err
+			}
+			if ns != nil && useNetstackForIP(ipp.IP()) {
 				return ns.DialContextTCP(ctx, addr)
 			}
-		} else {
-			var mu sync.Mutex
-			var dns netstack.DNSMap
-			e.AddNetworkMapCallback(func(nm *netmap.NetworkMap) {
-				mu.Lock()
-				defer mu.Unlock()
-				dns = netstack.DNSMapFromNetworkMap(nm)
-			})
-			srv.Dialer = func(ctx context.Context, network, addr string) (net.Conn, error) {
-				ipp, err := dns.Resolve(ctx, addr)
-				if err != nil {
-					return nil, err
-				}
-				var d net.Dialer
-				return d.DialContext(ctx, network, ipp.String())
-			}
+			var d net.Dialer
+			return d.DialContext(ctx, network, ipp.String())
 		}
 		go func() {
 			log.Fatalf("SOCKS5 server exited: %v", srv.Serve(socksListener))
@@ -300,8 +303,7 @@ func run() error {
 		Port:               41112,
 		StatePath:          args.statepath,
 		AutostartStateKey:  globalStateKey,
-		LegacyConfigPath:   paths.LegacyConfigPath(),
-		SurviveDisconnects: true,
+		SurviveDisconnects: runtime.GOOS != "windows",
 		DebugMux:           debugMux,
 	}
 	err = ipnserver.Run(ctx, logf, pol.PublicID.String(), ipnserver.FixedEngine(e), opts)
@@ -331,6 +333,28 @@ func createEngine(logf logger.Logf, linkMon *monitor.Mon) (e wgengine.Engine, us
 	return nil, false, multierror.New(errs)
 }
 
+var wrapNetstack = shouldWrapNetstack()
+
+func shouldWrapNetstack() bool {
+	if e := os.Getenv("TS_DEBUG_WRAP_NETSTACK"); e != "" {
+		v, err := strconv.ParseBool(e)
+		if err != nil {
+			log.Fatalf("invalid TS_DEBUG_WRAP_NETSTACK value: %v", err)
+		}
+		return v
+	}
+	if distro.Get() == distro.Synology {
+		return true
+	}
+	switch runtime.GOOS {
+	case "windows", "darwin":
+		// Enable on Windows and tailscaled-on-macOS (this doesn't
+		// affect the GUI clients).
+		return true
+	}
+	return false
+}
+
 func tryEngine(logf logger.Logf, linkMon *monitor.Mon, name string) (e wgengine.Engine, useNetstack bool, err error) {
 	conf := wgengine.Config{
 		ListenPort:  args.port,
@@ -338,7 +362,7 @@ func tryEngine(logf logger.Logf, linkMon *monitor.Mon, name string) (e wgengine.
 	}
 	useNetstack = name == "userspace-networking"
 	if !useNetstack {
-		dev, err := tstun.New(logf, name)
+		dev, devName, err := tstun.New(logf, name)
 		if err != nil {
 			tstun.Diagnose(logf, name)
 			return nil, false, err
@@ -349,14 +373,15 @@ func tryEngine(logf logger.Logf, linkMon *monitor.Mon, name string) (e wgengine.
 			dev.Close()
 			return nil, false, err
 		}
-		conf.Router = r
-		tunname, err := dev.Name()
+		d, err := dns.NewOSConfigurator(logf, devName)
 		if err != nil {
-			r.Close()
-			dev.Close()
 			return nil, false, err
 		}
-		conf.DNS = dns.NewOSConfigurator(logf, tunname)
+		conf.DNS = d
+		conf.Router = r
+		if wrapNetstack {
+			conf.Router = netstack.NewSubnetRouterWrapper(conf.Router)
+		}
 	}
 	e, err = wgengine.NewUserspaceEngine(logf, conf)
 	if err != nil {
@@ -384,3 +409,18 @@ func runDebugServer(mux *http.ServeMux, addr string) {
 		log.Fatal(err)
 	}
 }
+
+func mustStartNetstack(logf logger.Logf, e wgengine.Engine, onlySubnets bool) *netstack.Impl {
+	tunDev, magicConn, ok := e.(wgengine.InternalsGetter).GetInternals()
+	if !ok {
+		log.Fatalf("%T is not a wgengine.InternalsGetter", e)
+	}
+	ns, err := netstack.Create(logf, tunDev, e, magicConn, onlySubnets)
+	if err != nil {
+		log.Fatalf("netstack.Create: %v", err)
+	}
+	if err := ns.Start(); err != nil {
+		log.Fatalf("failed to start netstack: %v", err)
+	}
+	return ns
+}

cmd/tailscaled/tailscaled_windows.go

@@ -21,7 +21,6 @@ import (
 	"context"
 	"fmt"
 	"log"
-	"net"
 	"os"
 	"time"
 
@@ -30,11 +29,13 @@ import (
 	"golang.zx2c4.com/wireguard/windows/tunnel/winipcfg"
 	"tailscale.com/ipn/ipnserver"
 	"tailscale.com/logpolicy"
+	"tailscale.com/net/dns"
 	"tailscale.com/net/tstun"
-	"tailscale.com/tempfork/wireguard-windows/firewall"
 	"tailscale.com/types/logger"
 	"tailscale.com/version"
+	"tailscale.com/wf"
 	"tailscale.com/wgengine"
+	"tailscale.com/wgengine/netstack"
 	"tailscale.com/wgengine/router"
 )
 
@@ -142,13 +143,13 @@ func beFirewallKillswitch() bool {
 
 	luid, err := winipcfg.LUIDFromGUID(&guid)
 	if err != nil {
-		log.Fatalf("no interface with GUID %q", guid)
+		log.Fatalf("no interface with GUID %q: %v", guid, err)
 	}
 
-	noProtection := false
-	var dnsIPs []net.IP // unused in called code.
 	start := time.Now()
-	firewall.EnableFirewall(uint64(luid), noProtection, dnsIPs)
+	if _, err := wf.New(uint64(luid)); err != nil {
+		log.Fatalf("filewall creation failed: %v", err)
+	}
 	log.Printf("killswitch enabled, took %s", time.Since(start))
 
 	// Block until the monitor goroutine shuts us down.
@@ -157,86 +158,120 @@ func beFirewallKillswitch() bool {
 
 func startIPNServer(ctx context.Context, logid string) error {
 	var logf logger.Logf = log.Printf
-	var eng wgengine.Engine
-	var err error
 
-	getEngine := func() (wgengine.Engine, error) {
-		dev, err := tstun.New(logf, "Tailscale")
+	getEngineRaw := func() (wgengine.Engine, error) {
+		dev, devName, err := tstun.New(logf, "Tailscale")
 		if err != nil {
-			return nil, err
+			return nil, fmt.Errorf("TUN: %w", err)
 		}
 		r, err := router.New(logf, dev)
 		if err != nil {
 			dev.Close()
-			return nil, err
+			return nil, fmt.Errorf("router: %w", err)
+		}
+		if wrapNetstack {
+			r = netstack.NewSubnetRouterWrapper(r)
+		}
+		d, err := dns.NewOSConfigurator(logf, devName)
+		if err != nil {
+			r.Close()
+			dev.Close()
+			return nil, fmt.Errorf("DNS: %w", err)
 		}
 		eng, err := wgengine.NewUserspaceEngine(logf, wgengine.Config{
 			Tun:        dev,
 			Router:     r,
+			DNS:        d,
 			ListenPort: 41641,
 		})
 		if err != nil {
 			r.Close()
 			dev.Close()
-			return nil, err
+			return nil, fmt.Errorf("engine: %w", err)
+		}
+		onlySubnets := true
+		if wrapNetstack {
+			mustStartNetstack(logf, eng, onlySubnets)
 		}
 		return wgengine.NewWatchdog(eng), nil
 	}
 
-	if msg := os.Getenv("TS_DEBUG_WIN_FAIL"); msg != "" {
-		err = fmt.Errorf("pretending to be a service failure: %v", msg)
-	} else {
-		// We have a bunch of bug reports of wgengine.NewUserspaceEngine returning a few different errors,
-		// all intermittently. A few times I (Brad) have also seen sporadic failures that simply
-		// restarting fixed. So try a few times.
-		for try := 1; try <= 5; try++ {
-			if try > 1 {
-				// Only sleep a bit. Don't do some massive backoff because
-				// the frontend GUI has a 30 second timeout on connecting to us,
-				// but even 5 seconds is too long for them to get any results.
-				// 5 tries * 1 second each seems fine.
-				time.Sleep(time.Second)
-			}
-			eng, err = getEngine()
+	type engineOrError struct {
+		Engine wgengine.Engine
+		Err    error
+	}
+	engErrc := make(chan engineOrError)
+	t0 := time.Now()
+	go func() {
+		const ms = time.Millisecond
+		for try := 1; ; try++ {
+			logf("tailscaled: getting engine... (try %v)", try)
+			t1 := time.Now()
+			eng, err := getEngineRaw()
+			d, dt := time.Since(t1).Round(ms), time.Since(t1).Round(ms)
 			if err != nil {
-				logf("wgengine.NewUserspaceEngine: (try %v) %v", try, err)
-				continue
+				logf("tailscaled: engine fetch error (try %v) in %v (total %v, sysUptime %v): %v",
+					try, d, dt, windowsUptime().Round(time.Second), err)
+			} else {
+				if try > 1 {
+					logf("tailscaled: got engine on try %v in %v (total %v)", try, d, dt)
+				} else {
+					logf("tailscaled: got engine in %v", d)
+				}
 			}
-			if try > 1 {
-				logf("wgengine.NewUserspaceEngine: ended up working on try %v", try)
+			timer := time.NewTimer(5 * time.Second)
+			engErrc <- engineOrError{eng, err}
+			if err == nil {
+				timer.Stop()
+				return
 			}
-			break
+			<-timer.C
 		}
-	}
-	if err != nil {
-		// Log the error, but don't fatalf. We want to
-		// propagate the error message to the UI frontend. So
-		// we continue and tell the ipnserver to return that
-		// in a Notify message.
-		logf("wgengine.NewUserspaceEngine: %v", err)
-	}
+	}()
+
 	opts := ipnserver.Options{
 		Port:               41112,
 		SurviveDisconnects: false,
 		StatePath:          args.statepath,
 	}
-	if err != nil {
-		// Return nicer errors to users, annotated with logids, which helps
-		// when they file bugs.
-		rawGetEngine := getEngine // raw == without verbose logid-containing error
-		getEngine = func() (wgengine.Engine, error) {
-			eng, err := rawGetEngine()
-			if err != nil {
-				return nil, fmt.Errorf("wgengine.NewUserspaceEngine: %v\n\nlogid: %v", err, logid)
-			}
-			return eng, nil
+
+	// getEngine is called by ipnserver to get the engine. It's
+	// not called concurrently and is not called again once it
+	// successfully returns an engine.
+	getEngine := func() (wgengine.Engine, error) {
+		if msg := os.Getenv("TS_DEBUG_WIN_FAIL"); msg != "" {
+			return nil, fmt.Errorf("pretending to be a service failure: %v", msg)
+		}
+		for {
+			res := <-engErrc
+			if res.Engine != nil {
+				return res.Engine, nil
+			}
+			if time.Since(t0) < time.Minute || windowsUptime() < 10*time.Minute {
+				// Ignore errors during early boot. Windows 10 auto logs in the GUI
+				// way sooner than the networking stack components start up.
+				// So the network will fail for a bit (and require a few tries) while
+				// the GUI is still fine.
+				continue
+			}
+			// Return nicer errors to users, annotated with logids, which helps
+			// when they file bugs.
+			return nil, fmt.Errorf("%w\n\nlogid: %v", res.Err, logid)
 		}
-	} else {
-		getEngine = ipnserver.FixedEngine(eng)
 	}
-	err = ipnserver.Run(ctx, logf, logid, getEngine, opts)
+	err := ipnserver.Run(ctx, logf, logid, getEngine, opts)
 	if err != nil {
 		logf("ipnserver.Run: %v", err)
 	}
 	return err
 }
+
+var (
+	kernel32           = windows.NewLazySystemDLL("kernel32.dll")
+	getTickCount64Proc = kernel32.NewProc("GetTickCount64")
+)
+
+func windowsUptime() time.Duration {
+	r, _, _ := getTickCount64Proc.Call()
+	return time.Duration(int64(r)) * time.Millisecond
+}

cmd/tsshd/tsshd.go

@@ -59,11 +59,11 @@ func main() {
 
 	warned := false
 	for {
-		addr, iface, err := interfaces.Tailscale()
+		addrs, iface, err := interfaces.Tailscale()
 		if err != nil {
 			log.Fatalf("listing interfaces: %v", err)
 		}
-		if addr == nil {
+		if len(addrs) == 0 {
 			if !warned {
 				log.Printf("no tailscale interface found; polling until one is available")
 				warned = true
@@ -75,6 +75,13 @@ func main() {
 			continue
 		}
 		warned = false
+		var addr netaddr.IP
+		for _, a := range addrs {
+			if a.Is4() {
+				addr = a
+				break
+			}
+		}
 		listen := net.JoinHostPort(addr.String(), fmt.Sprint(*port))
 		log.Printf("tailscale ssh server listening on %v, %v", iface.Name, listen)
 		s := &ssh.Server{

control/controlclient/auto.go

@@ -2,18 +2,12 @@
 // Use of this source code is governed by a BSD-style
 // license that can be found in the LICENSE file.
 
-// Package controlclient implements the client for the Tailscale
-// control plane.
-//
-// It handles authentication, port picking, and collects the local
-// network configuration.
 package controlclient
 
 import (
 	"context"
-	"encoding/json"
 	"fmt"
-	"reflect"
+	"log"
 	"sync"
 	"time"
 
@@ -28,87 +22,28 @@ import (
 	"tailscale.com/types/wgkey"
 )
 
-// State is the high-level state of the client. It is used only in
-// unit tests for proper sequencing, don't depend on it anywhere else.
-// TODO(apenwarr): eliminate 'state', as it's now obsolete.
-type State int
-
-const (
-	StateNew = State(iota)
-	StateNotAuthenticated
-	StateAuthenticating
-	StateURLVisitRequired
-	StateAuthenticated
-	StateSynchronized // connected and received map update
-)
-
-func (s State) MarshalText() ([]byte, error) {
-	return []byte(s.String()), nil
-}
-
-func (s State) String() string {
-	switch s {
-	case StateNew:
-		return "state:new"
-	case StateNotAuthenticated:
-		return "state:not-authenticated"
-	case StateAuthenticating:
-		return "state:authenticating"
-	case StateURLVisitRequired:
-		return "state:url-visit-required"
-	case StateAuthenticated:
-		return "state:authenticated"
-	case StateSynchronized:
-		return "state:synchronized"
-	default:
-		return fmt.Sprintf("state:unknown:%d", int(s))
-	}
-}
-
-type Status struct {
-	_             structs.Incomparable
-	LoginFinished *empty.Message
-	Err           string
-	URL           string
-	Persist       *persist.Persist   // locally persisted configuration
-	NetMap        *netmap.NetworkMap // server-pushed configuration
-	Hostinfo      *tailcfg.Hostinfo  // current Hostinfo data
-	State         State
-}
-
-// Equal reports whether s and s2 are equal.
-func (s *Status) Equal(s2 *Status) bool {
-	if s == nil && s2 == nil {
-		return true
-	}
-	return s != nil && s2 != nil &&
-		(s.LoginFinished == nil) == (s2.LoginFinished == nil) &&
-		s.Err == s2.Err &&
-		s.URL == s2.URL &&
-		reflect.DeepEqual(s.Persist, s2.Persist) &&
-		reflect.DeepEqual(s.NetMap, s2.NetMap) &&
-		reflect.DeepEqual(s.Hostinfo, s2.Hostinfo) &&
-		s.State == s2.State
-}
-
-func (s Status) String() string {
-	b, err := json.MarshalIndent(s, "", "\t")
-	if err != nil {
-		panic(err)
-	}
-	return s.State.String() + " " + string(b)
-}
-
 type LoginGoal struct {
-	_            structs.Incomparable
-	wantLoggedIn bool                 // true if we *want* to be logged in
-	token        *tailcfg.Oauth2Token // oauth token to use when logging in
-	flags        LoginFlags           // flags to use when logging in
-	url          string               // auth url that needs to be visited
+	_               structs.Incomparable
+	wantLoggedIn    bool                 // true if we *want* to be logged in
+	token           *tailcfg.Oauth2Token // oauth token to use when logging in
+	flags           LoginFlags           // flags to use when logging in
+	url             string               // auth url that needs to be visited
+	loggedOutResult chan<- error
 }
 
-// Client connects to a tailcontrol server for a node.
-type Client struct {
+func (g *LoginGoal) sendLogoutError(err error) {
+	if g.loggedOutResult == nil {
+		return
+	}
+	select {
+	case g.loggedOutResult <- err:
+	default:
+	}
+}
+
+// Auto connects to a tailcontrol server for a node.
+// It's a concrete implementation of the Client interface.
+type Auto struct {
 	direct   *Direct // our interface to the server APIs
 	timeNow  func() time.Time
 	logf     logger.Logf
@@ -141,8 +76,8 @@ type Client struct {
 	mapDone    chan struct{}   // when closed, map goroutine is done
 }
 
-// New creates and starts a new Client.
-func New(opts Options) (*Client, error) {
+// New creates and starts a new Auto.
+func New(opts Options) (*Auto, error) {
 	c, err := NewNoStart(opts)
 	if c != nil {
 		c.Start()
@@ -150,8 +85,8 @@ func New(opts Options) (*Client, error) {
 	return c, err
 }
 
-// NewNoStart creates a new Client, but without calling Start on it.
-func NewNoStart(opts Options) (*Client, error) {
+// NewNoStart creates a new Auto, but without calling Start on it.
+func NewNoStart(opts Options) (*Auto, error) {
 	direct, err := NewDirect(opts)
 	if err != nil {
 		return nil, err
@@ -162,7 +97,7 @@ func NewNoStart(opts Options) (*Client, error) {
 	if opts.TimeNow == nil {
 		opts.TimeNow = time.Now
 	}
-	c := &Client{
+	c := &Auto{
 		direct:   direct,
 		timeNow:  opts.TimeNow,
 		logf:     opts.Logf,
@@ -178,7 +113,7 @@ func NewNoStart(opts Options) (*Client, error) {
 
 }
 
-func (c *Client) onHealthChange(sys health.Subsystem, err error) {
+func (c *Auto) onHealthChange(sys health.Subsystem, err error) {
 	if sys == health.SysOverall {
 		return
 	}
@@ -189,12 +124,13 @@ func (c *Client) onHealthChange(sys health.Subsystem, err error) {
 // SetPaused controls whether HTTP activity should be paused.
 //
 // The client can be paused and unpaused repeatedly, unlike Start and Shutdown, which can only be used once.
-func (c *Client) SetPaused(paused bool) {
+func (c *Auto) SetPaused(paused bool) {
 	c.mu.Lock()
 	defer c.mu.Unlock()
 	if paused == c.paused {
 		return
 	}
+	c.logf("setPaused(%v)", paused)
 	c.paused = paused
 	if paused {
 		// Only cancel the map routine. (The auth routine isn't expensive
@@ -211,7 +147,7 @@ func (c *Client) SetPaused(paused bool) {
 // Start starts the client's goroutines.
 //
 // It should only be called for clients created by NewNoStart.
-func (c *Client) Start() {
+func (c *Auto) Start() {
 	go c.authRoutine()
 	go c.mapRoutine()
 }
@@ -221,7 +157,8 @@ func (c *Client) Start() {
 // streaming response open), or start a new streaming one if necessary.
 //
 // It should be called whenever there's something new to tell the server.
-func (c *Client) sendNewMapRequest() {
+func (c *Auto) sendNewMapRequest() {
+	log.Println("sendNewMapRequest breakpoint")
 	c.mu.Lock()
 
 	// If we're not already streaming a netmap, or if we're already stuck
@@ -260,7 +197,7 @@ func (c *Client) sendNewMapRequest() {
 	}()
 }
 
-func (c *Client) cancelAuth() {
+func (c *Auto) cancelAuth() {
 	c.mu.Lock()
 	if c.authCancel != nil {
 		c.authCancel()
@@ -271,7 +208,7 @@ func (c *Client) cancelAuth() {
 	c.mu.Unlock()
 }
 
-func (c *Client) cancelMapLocked() {
+func (c *Auto) cancelMapLocked() {
 	if c.mapCancel != nil {
 		c.mapCancel()
 	}
@@ -280,13 +217,13 @@ func (c *Client) cancelMapLocked() {
 	}
 }
 
-func (c *Client) cancelMapUnsafely() {
+func (c *Auto) cancelMapUnsafely() {
 	c.mu.Lock()
 	c.cancelMapLocked()
 	c.mu.Unlock()
 }
 
-func (c *Client) cancelMapSafely() {
+func (c *Auto) cancelMapSafely() {
 	c.mu.Lock()
 	defer c.mu.Unlock()
 
@@ -322,7 +259,7 @@ func (c *Client) cancelMapSafely() {
 	}
 }
 
-func (c *Client) authRoutine() {
+func (c *Auto) authRoutine() {
 	defer close(c.authDone)
 	bo := backoff.NewBackoff("authRoutine", c.logf, 30*time.Second)
 
@@ -333,7 +270,7 @@ func (c *Client) authRoutine() {
 		if goal != nil {
 			c.logf("authRoutine: %s; wantLoggedIn=%v", c.state, goal.wantLoggedIn)
 		} else {
-			c.logf("authRoutine: %s; goal=nil", c.state)
+			c.logf("authRoutine: %s; goal=nil paused=%v", c.state, c.paused)
 		}
 		c.mu.Unlock()
 
@@ -363,6 +300,7 @@ func (c *Client) authRoutine() {
 
 		if !goal.wantLoggedIn {
 			err := c.direct.TryLogout(ctx)
+			goal.sendLogoutError(err)
 			if err != nil {
 				report(err, "TryLogout")
 				bo.BackOff(ctx, err)
@@ -402,7 +340,8 @@ func (c *Client) authRoutine() {
 				report(err, f)
 				bo.BackOff(ctx, err)
 				continue
-			} else if url != "" {
+			}
+			if url != "" {
 				if goal.url != "" {
 					err = fmt.Errorf("[unexpected] server required a new URL?")
 					report(err, "WaitLoginURL")
@@ -439,7 +378,7 @@ func (c *Client) authRoutine() {
 
 // Expiry returns the credential expiration time, or the zero time if
 // the expiration time isn't known. Used in tests only.
-func (c *Client) Expiry() *time.Time {
+func (c *Auto) Expiry() *time.Time {
 	c.mu.Lock()
 	defer c.mu.Unlock()
 	return c.expiry
@@ -447,21 +386,21 @@ func (c *Client) Expiry() *time.Time {
 
 // Direct returns the underlying direct client object. Used in tests
 // only.
-func (c *Client) Direct() *Direct {
+func (c *Auto) Direct() *Direct {
 	return c.direct
 }
 
 // unpausedChanLocked returns a new channel that is closed when the
-// current Client pause is unpaused.
+// current Auto pause is unpaused.
 //
 // c.mu must be held
-func (c *Client) unpausedChanLocked() <-chan struct{} {
+func (c *Auto) unpausedChanLocked() <-chan struct{} {
 	unpaused := make(chan struct{})
 	c.unpauseWaiters = append(c.unpauseWaiters, unpaused)
 	return unpaused
 }
 
-func (c *Client) mapRoutine() {
+func (c *Auto) mapRoutine() {
 	defer close(c.mapDone)
 	bo := backoff.NewBackoff("mapRoutine", c.logf, 30*time.Second)
 
@@ -583,7 +522,10 @@ func (c *Client) mapRoutine() {
 	}
 }
 
-func (c *Client) AuthCantContinue() bool {
+func (c *Auto) AuthCantContinue() bool {
+	if c == nil {
+		return true
+	}
 	c.mu.Lock()
 	defer c.mu.Unlock()
 
@@ -591,13 +533,13 @@ func (c *Client) AuthCantContinue() bool {
 }
 
 // SetStatusFunc sets fn as the callback to run on any status change.
-func (c *Client) SetStatusFunc(fn func(Status)) {
+func (c *Auto) SetStatusFunc(fn func(Status)) {
 	c.mu.Lock()
 	c.statusFunc = fn
 	c.mu.Unlock()
 }
 
-func (c *Client) SetHostinfo(hi *tailcfg.Hostinfo) {
+func (c *Auto) SetHostinfo(hi *tailcfg.Hostinfo) {
 	if hi == nil {
 		panic("nil Hostinfo")
 	}
@@ -610,7 +552,7 @@ func (c *Client) SetHostinfo(hi *tailcfg.Hostinfo) {
 	c.sendNewMapRequest()
 }
 
-func (c *Client) SetNetInfo(ni *tailcfg.NetInfo) {
+func (c *Auto) SetNetInfo(ni *tailcfg.NetInfo) {
 	if ni == nil {
 		panic("nil NetInfo")
 	}
@@ -623,7 +565,7 @@ func (c *Client) SetNetInfo(ni *tailcfg.NetInfo) {
 	c.sendNewMapRequest()
 }
 
-func (c *Client) sendStatus(who string, err error, url string, nm *netmap.NetworkMap) {
+func (c *Auto) sendStatus(who string, err error, url string, nm *netmap.NetworkMap) {
 	c.mu.Lock()
 	state := c.state
 	loggedIn := c.loggedIn
@@ -668,7 +610,7 @@ func (c *Client) sendStatus(who string, err error, url string, nm *netmap.Networ
 	c.mu.Unlock()
 }
 
-func (c *Client) Login(t *tailcfg.Oauth2Token, flags LoginFlags) {
+func (c *Auto) Login(t *tailcfg.Oauth2Token, flags LoginFlags) {
 	c.logf("client.Login(%v, %v)", t != nil, flags)
 
 	c.mu.Lock()
@@ -682,18 +624,42 @@ func (c *Client) Login(t *tailcfg.Oauth2Token, flags LoginFlags) {
 	c.cancelAuth()
 }
 
-func (c *Client) Logout() {
-	c.logf("client.Logout()")
+func (c *Auto) StartLogout() {
+	c.logf("client.StartLogout()")
 
 	c.mu.Lock()
 	c.loginGoal = &LoginGoal{
 		wantLoggedIn: false,
 	}
 	c.mu.Unlock()
-
 	c.cancelAuth()
 }
 
+func (c *Auto) Logout(ctx context.Context) error {
+	c.logf("client.Logout()")
+
+	errc := make(chan error, 1)
+
+	c.mu.Lock()
+	c.loginGoal = &LoginGoal{
+		wantLoggedIn:    false,
+		loggedOutResult: errc,
+	}
+	c.mu.Unlock()
+	c.cancelAuth()
+
+	timer := time.NewTimer(10 * time.Second)
+	defer timer.Stop()
+	select {
+	case err := <-errc:
+		return err
+	case <-ctx.Done():
+		return ctx.Err()
+	case <-timer.C:
+		return context.DeadlineExceeded
+	}
+}
+
 // UpdateEndpoints sets the client's discovered endpoints and sends
 // them to the control server if they've changed.
 //
@@ -701,14 +667,14 @@ func (c *Client) Logout() {
 //
 // The localPort field is unused except for integration tests in
 // another repo.
-func (c *Client) UpdateEndpoints(localPort uint16, endpoints []string) {
+func (c *Auto) UpdateEndpoints(localPort uint16, endpoints []tailcfg.Endpoint) {
 	changed := c.direct.SetEndpoints(localPort, endpoints)
 	if changed {
 		c.sendNewMapRequest()
 	}
 }
 
-func (c *Client) Shutdown() {
+func (c *Auto) Shutdown() {
 	c.logf("client.Shutdown()")
 
 	c.mu.Lock()
@@ -734,17 +700,17 @@ func (c *Client) Shutdown() {
 
 // NodePublicKey returns the node public key currently in use. This is
 // used exclusively in tests.
-func (c *Client) TestOnlyNodePublicKey() wgkey.Key {
+func (c *Auto) TestOnlyNodePublicKey() wgkey.Key {
 	priv := c.direct.GetPersist()
 	return priv.PrivateNodeKey.Public()
 }
 
-func (c *Client) TestOnlySetAuthKey(authkey string) {
+func (c *Auto) TestOnlySetAuthKey(authkey string) {
 	c.direct.mu.Lock()
 	defer c.direct.mu.Unlock()
 	c.direct.authKey = authkey
 }
 
-func (c *Client) TestOnlyTimeNow() time.Time {
+func (c *Auto) TestOnlyTimeNow() time.Time {
 	return c.timeNow()
 }

control/controlclient/client.go

@@ -0,0 +1,77 @@
+// Copyright (c) 2020 Tailscale Inc & AUTHORS All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+// Package controlclient implements the client for the Tailscale
+// control plane.
+//
+// It handles authentication, port picking, and collects the local
+// network configuration.
+package controlclient
+
+import (
+	"context"
+
+	"tailscale.com/tailcfg"
+)
+
+type LoginFlags int
+
+const (
+	LoginDefault     = LoginFlags(0)
+	LoginInteractive = LoginFlags(1 << iota) // force user login and key refresh
+)
+
+// Client represents a client connection to the control server.
+// Currently this is done through a pair of polling https requests in
+// the Auto client, but that might change eventually.
+type Client interface {
+	// SetStatusFunc provides a callback to call when control sends us
+	// a message.
+	SetStatusFunc(func(Status))
+	// Shutdown closes this session, which should not be used any further
+	// afterwards.
+	Shutdown()
+	// Login begins an interactive or non-interactive login process.
+	// Client will eventually call the Status callback with either a
+	// LoginFinished flag (on success) or an auth URL (if further
+	// interaction is needed).
+	Login(*tailcfg.Oauth2Token, LoginFlags)
+	// StartLogout starts an asynchronous logout process.
+	// When it finishes, the Status callback will be called while
+	// AuthCantContinue()==true.
+	StartLogout()
+	// Logout starts a synchronous logout process. It doesn't return
+	// until the logout operation has been completed.
+	Logout(context.Context) error
+	// SetPaused pauses or unpauses the controlclient activity as much
+	// as possible, without losing its internal state, to minimize
+	// unnecessary network activity.
+	// TODO: It might be better to simply shutdown the controlclient and
+	// make a new one when it's time to unpause.
+	SetPaused(bool)
+	// AuthCantContinue returns whether authentication is blocked. If it
+	// is, you either need to visit the auth URL (previously sent in a
+	// Status callback) or call the Login function appropriately.
+	// TODO: this probably belongs in the Status itself instead.
+	AuthCantContinue() bool
+	// SetHostinfo changes the Hostinfo structure that will be sent in
+	// subsequent node registration requests.
+	// TODO: a server-side change would let us simply upload this
+	// in a separate http request. It has nothing to do with the rest of
+	// the state machine.
+	SetHostinfo(*tailcfg.Hostinfo)
+	// SetNetinfo changes the NetIinfo structure that will be sent in
+	// subsequent node registration requests.
+	// TODO: a server-side change would let us simply upload this
+	// in a separate http request. It has nothing to do with the rest of
+	// the state machine.
+	SetNetInfo(*tailcfg.NetInfo)
+	// UpdateEndpoints changes the Endpoint structure that will be sent
+	// in subsequent node registration requests.
+	// TODO: localPort seems to be obsolete, remove it.
+	// TODO: a server-side change would let us simply upload this
+	// in a separate http request. It has nothing to do with the rest of
+	// the state machine.
+	UpdateEndpoints(localPort uint16, endpoints []tailcfg.Endpoint)
+}

control/controlclient/controlclient_test.go

@@ -22,7 +22,7 @@ func fieldsOf(t reflect.Type) (fields []string) {
 
 func TestStatusEqual(t *testing.T) {
 	// Verify that the Equal method stays in sync with reality
-	equalHandles := []string{"LoginFinished", "Err", "URL", "Persist", "NetMap", "Hostinfo", "State"}
+	equalHandles := []string{"LoginFinished", "Err", "URL", "NetMap", "State", "Persist", "Hostinfo"}
 	if have := fieldsOf(reflect.TypeOf(Status{})); !reflect.DeepEqual(have, equalHandles) {
 		t.Errorf("Status.Equal check might be out of sync\nfields: %q\nhandled: %q\n",
 			have, equalHandles)

control/controlclient/direct.go

@@ -23,7 +23,6 @@ import (
 	"path/filepath"
 	"reflect"
 	"runtime"
-	"sort"
 	"strconv"
 	"strings"
 	"sync"
@@ -33,6 +32,7 @@ import (
 	"golang.org/x/crypto/nacl/box"
 	"inet.af/netaddr"
 	"tailscale.com/health"
+	"tailscale.com/ipn/ipnstate"
 	"tailscale.com/log/logheap"
 	"tailscale.com/net/dnscache"
 	"tailscale.com/net/dnsfallback"
@@ -46,9 +46,9 @@ import (
 	"tailscale.com/types/opt"
 	"tailscale.com/types/persist"
 	"tailscale.com/types/wgkey"
+	"tailscale.com/util/dnsname"
 	"tailscale.com/util/systemd"
 	"tailscale.com/version"
-	"tailscale.com/wgengine/filter"
 	"tailscale.com/wgengine/monitor"
 )
 
@@ -67,6 +67,7 @@ type Direct struct {
 	debugFlags             []string
 	keepSharerAndUserSplit bool
 	skipIPForwardingCheck  bool
+	pinger                 Pinger
 
 	mu           sync.Mutex // mutex guards the following fields
 	serverKey    wgkey.Key
@@ -76,10 +77,15 @@ type Direct struct {
 	expiry       *time.Time
 	// hostinfo is mutated in-place while mu is held.
 	hostinfo      *tailcfg.Hostinfo // always non-nil
-	endpoints     []string
+	endpoints     []tailcfg.Endpoint
 	everEndpoints bool   // whether we've ever had non-empty endpoints
 	localPort     uint16 // or zero to mean auto
 }
+type Pinger interface {
+	// Ping is a request to start a discovery ping with the peer handling
+	// the given IP and then call cb with its ping latency & method.
+	Ping(ip netaddr.IP, useTSMP bool, cb func(*ipnstate.PingResult))
+}
 
 type Options struct {
 	Persist              persist.Persist               // initial persistent data
@@ -95,6 +101,7 @@ type Options struct {
 	HTTPTestClient       *http.Client // optional HTTP client to use (for tests only)
 	DebugFlags           []string     // debug settings to send to control
 	LinkMonitor          *monitor.Mon // optional link monitor
+	Pinger               Pinger
 
 	// KeepSharerAndUserSplit controls whether the client
 	// understands Node.Sharer. If false, the Sharer is mapped to the User.
@@ -144,7 +151,7 @@ func NewDirect(opts Options) (*Direct, error) {
 		tr := http.DefaultTransport.(*http.Transport).Clone()
 		tr.Proxy = tshttpproxy.ProxyFromEnvironment
 		tshttpproxy.SetTransportGetProxyConnectHeader(tr)
-		tr.TLSClientConfig = tlsdial.Config(serverURL.Host, tr.TLSClientConfig)
+		tr.TLSClientConfig = tlsdial.Config(serverURL.Hostname(), tr.TLSClientConfig)
 		tr.DialContext = dnscache.Dialer(dialer.DialContext, dnsCache)
 		tr.DialTLSContext = dnscache.TLSDialer(dialer.DialContext, dnsCache, tr.TLSClientConfig)
 		tr.ForceAttemptHTTP2 = true
@@ -166,6 +173,7 @@ func NewDirect(opts Options) (*Direct, error) {
 		keepSharerAndUserSplit: opts.KeepSharerAndUserSplit,
 		linkMon:                opts.LinkMonitor,
 		skipIPForwardingCheck:  opts.SkipIPForwardingCheck,
+		pinger:                 opts.Pinger,
 	}
 	if opts.Hostinfo == nil {
 		c.SetHostinfo(NewHostinfo())
@@ -179,6 +187,7 @@ var osVersion func() string // non-nil on some platforms
 
 func NewHostinfo() *tailcfg.Hostinfo {
 	hostname, _ := os.Hostname()
+	hostname = dnsname.FirstLabel(hostname)
 	var osv string
 	if osVersion != nil {
 		osv = osVersion()
@@ -251,30 +260,22 @@ func (c *Direct) GetPersist() persist.Persist {
 	return c.persist
 }
 
-type LoginFlags int
-
-const (
-	LoginDefault     = LoginFlags(0)
-	LoginInteractive = LoginFlags(1 << iota) // force user login and key refresh
-)
-
 func (c *Direct) TryLogout(ctx context.Context) error {
 	c.logf("direct.TryLogout()")
 
-	c.mu.Lock()
-	defer c.mu.Unlock()
+	mustRegen, newURL, err := c.doLogin(ctx, loginOpt{Logout: true})
+	c.logf("TryLogout control response: mustRegen=%v, newURL=%v, err=%v", mustRegen, newURL, err)
 
-	// TODO(crawshaw): Tell the server. This node key should be
-	// immediately invalidated.
-	//if !c.persist.PrivateNodeKey.IsZero() {
-	//}
+	c.mu.Lock()
 	c.persist = persist.Persist{}
-	return nil
+	c.mu.Unlock()
+
+	return err
 }
 
 func (c *Direct) TryLogin(ctx context.Context, t *tailcfg.Oauth2Token, flags LoginFlags) (url string, err error) {
 	c.logf("direct.TryLogin(token=%v, flags=%v)", t != nil, flags)
-	return c.doLoginOrRegen(ctx, t, flags, false, "")
+	return c.doLoginOrRegen(ctx, loginOpt{Token: t, Flags: flags})
 }
 
 // WaitLoginURL sits in a long poll waiting for the user to authenticate at url.
@@ -282,22 +283,30 @@ func (c *Direct) TryLogin(ctx context.Context, t *tailcfg.Oauth2Token, flags Log
 // On success, newURL and err will both be nil.
 func (c *Direct) WaitLoginURL(ctx context.Context, url string) (newURL string, err error) {
 	c.logf("direct.WaitLoginURL")
-	return c.doLoginOrRegen(ctx, nil, LoginDefault, false, url)
+	return c.doLoginOrRegen(ctx, loginOpt{URL: url})
 }
 
-func (c *Direct) doLoginOrRegen(ctx context.Context, t *tailcfg.Oauth2Token, flags LoginFlags, regen bool, url string) (newURL string, err error) {
-	mustregen, url, err := c.doLogin(ctx, t, flags, regen, url)
+func (c *Direct) doLoginOrRegen(ctx context.Context, opt loginOpt) (newURL string, err error) {
+	mustRegen, url, err := c.doLogin(ctx, opt)
 	if err != nil {
 		return url, err
 	}
-	if mustregen {
-		_, url, err = c.doLogin(ctx, t, flags, true, url)
+	if mustRegen {
+		opt.Regen = true
+		_, url, err = c.doLogin(ctx, opt)
 	}
-
 	return url, err
 }
 
-func (c *Direct) doLogin(ctx context.Context, t *tailcfg.Oauth2Token, flags LoginFlags, regen bool, url string) (mustregen bool, newurl string, err error) {
+type loginOpt struct {
+	Token  *tailcfg.Oauth2Token
+	Flags  LoginFlags
+	Regen  bool
+	URL    string
+	Logout bool
+}
+
+func (c *Direct) doLogin(ctx context.Context, opt loginOpt) (mustRegen bool, newURL string, err error) {
 	c.mu.Lock()
 	persist := c.persist
 	tryingNewKey := c.tryingNewKey
@@ -316,22 +325,27 @@ func (c *Direct) doLogin(ctx context.Context, t *tailcfg.Oauth2Token, flags Logi
 		return false, "", errors.New("getMachinePrivKey returned zero key")
 	}
 
-	if expired {
-		c.logf("Old key expired -> regen=true")
-		systemd.Status("key expired; run 'tailscale up' to authenticate")
-		regen = true
-	}
-	if (flags & LoginInteractive) != 0 {
-		c.logf("LoginInteractive -> regen=true")
-		regen = true
+	regen := opt.Regen
+	if opt.Logout {
+		c.logf("logging out...")
+	} else {
+		if expired {
+			c.logf("Old key expired -> regen=true")
+			systemd.Status("key expired; run 'tailscale up' to authenticate")
+			regen = true
+		}
+		if (opt.Flags & LoginInteractive) != 0 {
+			c.logf("LoginInteractive -> regen=true")
+			regen = true
+		}
 	}
 
-	c.logf("doLogin(regen=%v, hasUrl=%v)", regen, url != "")
+	c.logf("doLogin(regen=%v, hasUrl=%v)", regen, opt.URL != "")
 	if serverKey.IsZero() {
 		var err error
 		serverKey, err = loadServerKey(ctx, c.httpc, c.serverURL)
 		if err != nil {
-			return regen, url, err
+			return regen, opt.URL, err
 		}
 
 		c.mu.Lock()
@@ -340,17 +354,21 @@ func (c *Direct) doLogin(ctx context.Context, t *tailcfg.Oauth2Token, flags Logi
 	}
 
 	var oldNodeKey wgkey.Key
-	if url != "" {
-	} else if regen || persist.PrivateNodeKey.IsZero() {
+	switch {
+	case opt.Logout:
+		tryingNewKey = persist.PrivateNodeKey
+	case opt.URL != "":
+		// Nothing.
+	case regen || persist.PrivateNodeKey.IsZero():
 		c.logf("Generating a new nodekey.")
 		persist.OldPrivateNodeKey = persist.PrivateNodeKey
 		key, err := wgkey.NewPrivate()
 		if err != nil {
 			c.logf("login keygen: %v", err)
-			return regen, url, err
+			return regen, opt.URL, err
 		}
 		tryingNewKey = key
-	} else {
+	default:
 		// Try refreshing the current key first
 		tryingNewKey = persist.PrivateNodeKey
 	}
@@ -359,11 +377,14 @@ func (c *Direct) doLogin(ctx context.Context, t *tailcfg.Oauth2Token, flags Logi
 	}
 
 	if tryingNewKey.IsZero() {
+		if opt.Logout {
+			return false, "", errors.New("no nodekey to log out")
+		}
 		log.Fatalf("tryingNewKey is empty, give up")
 	}
 	if backendLogID == "" {
 		err = errors.New("hostinfo: BackendLogID missing")
-		return regen, url, err
+		return regen, opt.URL, err
 	}
 	now := time.Now().Round(time.Second)
 	request := tailcfg.RegisterRequest{
@@ -371,13 +392,16 @@ func (c *Direct) doLogin(ctx context.Context, t *tailcfg.Oauth2Token, flags Logi
 		OldNodeKey: tailcfg.NodeKey(oldNodeKey),
 		NodeKey:    tailcfg.NodeKey(tryingNewKey.Public()),
 		Hostinfo:   hostinfo,
-		Followup:   url,
+		Followup:   opt.URL,
 		Timestamp:  &now,
 	}
+	if opt.Logout {
+		request.Expiry = time.Unix(123, 0) // far in the past
+	}
 	c.logf("RegisterReq: onode=%v node=%v fup=%v",
 		request.OldNodeKey.ShortString(),
-		request.NodeKey.ShortString(), url != "")
-	request.Auth.Oauth2Token = t
+		request.NodeKey.ShortString(), opt.URL != "")
+	request.Auth.Oauth2Token = opt.Token
 	request.Auth.Provider = persist.Provider
 	request.Auth.LoginName = persist.LoginName
 	request.Auth.AuthKey = authKey
@@ -391,38 +415,48 @@ func (c *Direct) doLogin(ctx context.Context, t *tailcfg.Oauth2Token, flags Logi
 
 		// Don't log the common error types. Signatures are not usually enabled,
 		// so these are expected.
-		if err != errCertificateNotConfigured && err != errNoCertStore {
+		if !errors.Is(err, errCertificateNotConfigured) && !errors.Is(err, errNoCertStore) {
 			c.logf("RegisterReq sign error: %v", err)
 		}
 	}
+	if debugRegister {
+		j, _ := json.MarshalIndent(request, "", "\t")
+		c.logf("RegisterRequest: %s", j)
+	}
+
 	bodyData, err := encode(request, &serverKey, &machinePrivKey)
 	if err != nil {
-		return regen, url, err
+		return regen, opt.URL, err
 	}
 	body := bytes.NewReader(bodyData)
 
 	u := fmt.Sprintf("%s/machine/%s", c.serverURL, machinePrivKey.Public().HexString())
 	req, err := http.NewRequest("POST", u, body)
 	if err != nil {
-		return regen, url, err
+		return regen, opt.URL, err
 	}
 	req = req.WithContext(ctx)
 
 	res, err := c.httpc.Do(req)
 	if err != nil {
-		return regen, url, fmt.Errorf("register request: %v", err)
+		return regen, opt.URL, fmt.Errorf("register request: %v", err)
 	}
 	if res.StatusCode != 200 {
 		msg, _ := ioutil.ReadAll(res.Body)
 		res.Body.Close()
-		return regen, url, fmt.Errorf("register request: http %d: %.200s",
+		return regen, opt.URL, fmt.Errorf("register request: http %d: %.200s",
 			res.StatusCode, strings.TrimSpace(string(msg)))
 	}
 	resp := tailcfg.RegisterResponse{}
 	if err := decode(res, &resp, &serverKey, &machinePrivKey); err != nil {
 		c.logf("error decoding RegisterResponse with server key %s and machine key %s: %v", serverKey, machinePrivKey.Public(), err)
-		return regen, url, fmt.Errorf("register request: %v", err)
+		return regen, opt.URL, fmt.Errorf("register request: %v", err)
 	}
+	if debugRegister {
+		j, _ := json.MarshalIndent(resp, "", "\t")
+		c.logf("RegisterResponse: %s", j)
+	}
+
 	// Log without PII:
 	c.logf("RegisterReq: got response; nodeKeyExpired=%v, machineAuthorized=%v; authURL=%v",
 		resp.NodeKeyExpired, resp.MachineAuthorized, resp.AuthURL != "")
@@ -435,10 +469,10 @@ func (c *Direct) doLogin(ctx context.Context, t *tailcfg.Oauth2Token, flags Logi
 			request.NodeKey.ShortString())
 		return true, "", nil
 	}
-	if persist.Provider == "" {
+	if resp.Login.Provider != "" {
 		persist.Provider = resp.Login.Provider
 	}
-	if persist.LoginName == "" {
+	if resp.Login.LoginName != "" {
 		persist.LoginName = resp.Login.LoginName
 	}
 
@@ -474,7 +508,7 @@ func (c *Direct) doLogin(ctx context.Context, t *tailcfg.Oauth2Token, flags Logi
 	return false, resp.AuthURL, nil
 }
 
-func sameStrings(a, b []string) bool {
+func sameEndpoints(a, b []tailcfg.Endpoint) bool {
 	if len(a) != len(b) {
 		return false
 	}
@@ -490,15 +524,19 @@ func sameStrings(a, b []string) bool {
 // whether they've changed.
 //
 // It does not retain the provided slice.
-func (c *Direct) newEndpoints(localPort uint16, endpoints []string) (changed bool) {
+func (c *Direct) newEndpoints(localPort uint16, endpoints []tailcfg.Endpoint) (changed bool) {
 	c.mu.Lock()
 	defer c.mu.Unlock()
 
 	// Nothing new?
-	if c.localPort == localPort && sameStrings(c.endpoints, endpoints) {
+	if c.localPort == localPort && sameEndpoints(c.endpoints, endpoints) {
 		return false // unchanged
 	}
-	c.logf("client.newEndpoints(%v, %v)", localPort, endpoints)
+	var epStrs []string
+	for _, ep := range endpoints {
+		epStrs = append(epStrs, ep.Addr.String())
+	}
+	c.logf("client.newEndpoints(%v, %v)", localPort, epStrs)
 	c.localPort = localPort
 	c.endpoints = append(c.endpoints[:0], endpoints...)
 	if len(endpoints) > 0 {
@@ -510,7 +548,7 @@ func (c *Direct) newEndpoints(localPort uint16, endpoints []string) (changed boo
 // SetEndpoints updates the list of locally advertised endpoints.
 // It won't be replicated to the server until a *fresh* call to PollNetMap().
 // You don't need to restart PollNetMap if we return changed==false.
-func (c *Direct) SetEndpoints(localPort uint16, endpoints []string) (changed bool) {
+func (c *Direct) SetEndpoints(localPort uint16, endpoints []tailcfg.Endpoint) (changed bool) {
 	// (no log message on function entry, because it clutters the logs
 	//  if endpoints haven't changed. newEndpoints() will log it.)
 	return c.newEndpoints(localPort, endpoints)
@@ -524,6 +562,7 @@ func inTest() bool { return flag.Lookup("test.v") != nil }
 // maxPolls is how many network maps to download; common values are 1
 // or -1 (to keep a long-poll query open to the server).
 func (c *Direct) PollNetMap(ctx context.Context, maxPolls int, cb func(*netmap.NetworkMap)) error {
+	log.Println("POLLNETMAP BREAKPOINT")
 	return c.sendMapRequest(ctx, maxPolls, cb)
 }
 
@@ -531,19 +570,31 @@ func (c *Direct) PollNetMap(ctx context.Context, maxPolls int, cb func(*netmap.N
 // but does not fetch anything. It returns an error if the server did not return a
 // successful 200 OK response.
 func (c *Direct) SendLiteMapUpdate(ctx context.Context) error {
+	log.Println("SendLiteMapUpdate BREAKPOINT")
 	return c.sendMapRequest(ctx, 1, nil)
 }
 
+// If we go more than pollTimeout without hearing from the server,
+// end the long poll. We should be receiving a keep alive ping
+// every minute.
+const pollTimeout = 120 * time.Second
+
 // cb nil means to omit peers.
 func (c *Direct) sendMapRequest(ctx context.Context, maxPolls int, cb func(*netmap.NetworkMap)) error {
 	c.mu.Lock()
+	log.Println("sendMapRequest ENDPOINT")
 	persist := c.persist
 	serverURL := c.serverURL
 	serverKey := c.serverKey
 	hostinfo := c.hostinfo.Clone()
 	backendLogID := hostinfo.BackendLogID
 	localPort := c.localPort
-	ep := append([]string(nil), c.endpoints...)
+	var epStrs []string
+	var epTypes []tailcfg.EndpointType
+	for _, ep := range c.endpoints {
+		epStrs = append(epStrs, ep.Addr.String())
+		epTypes = append(epTypes, ep.Type)
+	}
 	everEndpoints := c.everEndpoints
 	c.mu.Unlock()
 
@@ -563,7 +614,7 @@ func (c *Direct) sendMapRequest(ctx context.Context, maxPolls int, cb func(*netm
 	}
 
 	allowStream := maxPolls != 1
-	c.logf("[v1] PollNetMap: stream=%v :%v ep=%v", allowStream, localPort, ep)
+	c.logf("[v1] PollNetMap: stream=%v :%v ep=%v", allowStream, localPort, epStrs)
 
 	vlogf := logger.Discard
 	if Debug.NetMap {
@@ -573,15 +624,16 @@ func (c *Direct) sendMapRequest(ctx context.Context, maxPolls int, cb func(*netm
 	}
 
 	request := &tailcfg.MapRequest{
-		Version:    tailcfg.CurrentMapRequestVersion,
-		KeepAlive:  c.keepAlive,
-		NodeKey:    tailcfg.NodeKey(persist.PrivateNodeKey.Public()),
-		DiscoKey:   c.discoPubKey,
-		Endpoints:  ep,
-		Stream:     allowStream,
-		Hostinfo:   hostinfo,
-		DebugFlags: c.debugFlags,
-		OmitPeers:  cb == nil,
+		Version:       tailcfg.CurrentMapRequestVersion,
+		KeepAlive:     c.keepAlive,
+		NodeKey:       tailcfg.NodeKey(persist.PrivateNodeKey.Public()),
+		DiscoKey:      c.discoPubKey,
+		Endpoints:     epStrs,
+		EndpointTypes: epTypes,
+		Stream:        allowStream,
+		Hostinfo:      hostinfo,
+		DebugFlags:    c.debugFlags,
+		OmitPeers:     cb == nil,
 	}
 	var extraDebugFlags []string
 	if hostinfo != nil && c.linkMon != nil && !c.skipIPForwardingCheck &&
@@ -609,7 +661,7 @@ func (c *Direct) sendMapRequest(ctx context.Context, maxPolls int, cb func(*netm
 	// TODO(bradfitz): we skip this optimization in tests, though,
 	// because the e2e tests are currently hyperspecific about the
 	// ordering of things. The e2e tests need love.
-	if len(ep) == 0 && !everEndpoints && !inTest() {
+	if len(epStrs) == 0 && !everEndpoints && !inTest() {
 		request.ReadOnly = true
 	}
 
@@ -652,10 +704,6 @@ func (c *Direct) sendMapRequest(ctx context.Context, maxPolls int, cb func(*netm
 		return nil
 	}
 
-	// If we go more than pollTimeout without hearing from the server,
-	// end the long poll. We should be receiving a keep alive ping
-	// every minute.
-	const pollTimeout = 120 * time.Second
 	timeout := time.NewTimer(pollTimeout)
 	timeoutReset := make(chan struct{})
 	pollDone := make(chan struct{})
@@ -685,10 +733,11 @@ func (c *Direct) sendMapRequest(ctx context.Context, maxPolls int, cb func(*netm
 		}
 	}()
 
-	var lastDERPMap *tailcfg.DERPMap
-	var lastUserProfile = map[tailcfg.UserID]tailcfg.UserProfile{}
-	var lastParsedPacketFilter []filter.Match
-	var collectServices bool
+	sess := newMapSession(persist.PrivateNodeKey)
+	sess.logf = c.logf
+	sess.vlogf = vlogf
+	sess.machinePubKey = machinePubKey
+	sess.keepSharerAndUserSplit = c.keepSharerAndUserSplit
 
 	// If allowStream, then the server will use an HTTP long poll to
 	// return incremental results. There is always one response right
@@ -697,7 +746,6 @@ func (c *Direct) sendMapRequest(ctx context.Context, maxPolls int, cb func(*netm
 	// the same format before just closing the connection.
 	// We can use this same read loop either way.
 	var msg []byte
-	var previousPeers []*tailcfg.Node // for delta-purposes
 	for i := 0; i < maxPolls || maxPolls < 0; i++ {
 		vlogf("netmap: starting size read after %v (poll %v)", time.Since(t0).Round(time.Millisecond), i)
 		var siz [4]byte
@@ -725,6 +773,11 @@ func (c *Direct) sendMapRequest(ctx context.Context, maxPolls int, cb func(*netm
 		}
 
 		if pr := resp.PingRequest; pr != nil {
+			// return err
+			log.Println("Ping Triggered")
+			for i := 0; i < 10; i++ {
+				c.CustomPing(&resp)
+			}
 			go answerPing(c.logf, c.httpc, pr)
 		}
 
@@ -744,16 +797,6 @@ func (c *Direct) sendMapRequest(ctx context.Context, maxPolls int, cb func(*netm
 			continue
 		}
 
-		undeltaPeers(&resp, previousPeers)
-		previousPeers = cloneNodes(resp.Peers) // defensive/lazy clone, since this escapes to who knows where
-		for _, up := range resp.UserProfiles {
-			lastUserProfile[up.ID] = up
-		}
-
-		if resp.DERPMap != nil {
-			vlogf("netmap: new map contains DERP map")
-			lastDERPMap = resp.DERPMap
-		}
 		if resp.Debug != nil {
 			if resp.Debug.LogHeapPprof {
 				go logheap.LogHeap(resp.Debug.LogHeapURL)
@@ -763,18 +806,40 @@ func (c *Direct) sendMapRequest(ctx context.Context, maxPolls int, cb func(*netm
 			}
 			setControlAtomic(&controlUseDERPRoute, resp.Debug.DERPRoute)
 			setControlAtomic(&controlTrimWGConfig, resp.Debug.TrimWGConfig)
+			if sleep := time.Duration(resp.Debug.SleepSeconds * float64(time.Second)); sleep > 0 {
+				if err := sleepAsRequested(ctx, c.logf, timeoutReset, sleep); err != nil {
+					return err
+				}
+			}
 		}
+
+		nm := sess.netmapForResponse(&resp)
+		if nm.SelfNode == nil {
+			c.logf("MapResponse lacked node")
+			return errors.New("MapResponse lacked node")
+		}
+
 		// Temporarily (2020-06-29) support removing all but
 		// discovery-supporting nodes during development, for
 		// less noise.
 		if Debug.OnlyDisco {
-			filtered := resp.Peers[:0]
-			for _, p := range resp.Peers {
-				if !p.DiscoKey.IsZero() {
-					filtered = append(filtered, p)
+			anyOld, numDisco := false, 0
+			for _, p := range nm.Peers {
+				if p.DiscoKey.IsZero() {
+					anyOld = true
+				} else {
+					numDisco++
 				}
 			}
-			resp.Peers = filtered
+			if anyOld {
+				filtered := make([]*tailcfg.Node, 0, numDisco)
+				for _, p := range nm.Peers {
+					if !p.DiscoKey.IsZero() {
+						filtered = append(filtered, p)
+					}
+				}
+				nm.Peers = filtered
+			}
 		}
 		if Debug.StripEndpoints {
 			for _, p := range resp.Peers {
@@ -784,13 +849,8 @@ func (c *Direct) sendMapRequest(ctx context.Context, maxPolls int, cb func(*netm
 				p.Endpoints = []string{"127.9.9.9:456"}
 			}
 		}
-
-		if pf := resp.PacketFilter; pf != nil {
-			lastParsedPacketFilter = c.parsePacketFilter(pf)
-		}
-
-		if v, ok := resp.CollectServices.Get(); ok {
-			collectServices = v
+		if Debug.StripCaps {
+			nm.SelfNode.Capabilities = nil
 		}
 
 		// Get latest localPort. This might've changed if
@@ -798,67 +858,9 @@ func (c *Direct) sendMapRequest(ctx context.Context, maxPolls int, cb func(*netm
 		// the end-to-end test.
 		// TODO(bradfitz): remove the NetworkMap.LocalPort field entirely.
 		c.mu.Lock()
-		localPort = c.localPort
+		nm.LocalPort = c.localPort
 		c.mu.Unlock()
 
-		nm := &netmap.NetworkMap{
-			SelfNode:        resp.Node,
-			NodeKey:         tailcfg.NodeKey(persist.PrivateNodeKey.Public()),
-			PrivateKey:      persist.PrivateNodeKey,
-			MachineKey:      machinePubKey,
-			Expiry:          resp.Node.KeyExpiry,
-			Name:            resp.Node.Name,
-			Addresses:       resp.Node.Addresses,
-			Peers:           resp.Peers,
-			LocalPort:       localPort,
-			User:            resp.Node.User,
-			UserProfiles:    make(map[tailcfg.UserID]tailcfg.UserProfile),
-			Domain:          resp.Domain,
-			DNS:             resp.DNSConfig,
-			Hostinfo:        resp.Node.Hostinfo,
-			PacketFilter:    lastParsedPacketFilter,
-			CollectServices: collectServices,
-			DERPMap:         lastDERPMap,
-			Debug:           resp.Debug,
-		}
-		addUserProfile := func(userID tailcfg.UserID) {
-			if _, dup := nm.UserProfiles[userID]; dup {
-				// Already populated it from a previous peer.
-				return
-			}
-			if up, ok := lastUserProfile[userID]; ok {
-				nm.UserProfiles[userID] = up
-			}
-		}
-		addUserProfile(nm.User)
-		magicDNSSuffix := nm.MagicDNSSuffix()
-		nm.SelfNode.InitDisplayNames(magicDNSSuffix)
-		for _, peer := range resp.Peers {
-			peer.InitDisplayNames(magicDNSSuffix)
-			if !peer.Sharer.IsZero() {
-				if c.keepSharerAndUserSplit {
-					addUserProfile(peer.Sharer)
-				} else {
-					peer.User = peer.Sharer
-				}
-			}
-			addUserProfile(peer.User)
-		}
-		if resp.Node.MachineAuthorized {
-			nm.MachineStatus = tailcfg.MachineAuthorized
-		} else {
-			nm.MachineStatus = tailcfg.MachineUnauthorized
-		}
-		if len(resp.DNS) > 0 {
-			nm.DNS.Nameservers = resp.DNS
-		}
-		if len(resp.SearchPaths) > 0 {
-			nm.DNS.Domains = resp.SearchPaths
-		}
-		if Debug.ProxyDNS {
-			nm.DNS.Proxied = true
-		}
-
 		// Printing the netmap can be extremely verbose, but is very
 		// handy for debugging. Let's limit how often we do it.
 		// Code elsewhere prints netmap diffs every time, so this
@@ -874,6 +876,8 @@ func (c *Direct) sendMapRequest(ctx context.Context, maxPolls int, cb func(*netm
 		c.expiry = &nm.Expiry
 		c.mu.Unlock()
 
+		// log.Println("MAPRESPONSE: ", resp.Node)
+		// c.logf("MAPRESPONSE: %v", resp.Node)
 		cb(nm)
 	}
 	if ctx.Err() != nil {
@@ -894,7 +898,10 @@ func decode(res *http.Response, v interface{}, serverKey *wgkey.Key, mkey *wgkey
 	return decodeMsg(msg, v, serverKey, mkey)
 }
 
-var debugMap, _ = strconv.ParseBool(os.Getenv("TS_DEBUG_MAP"))
+var (
+	debugMap, _      = strconv.ParseBool(os.Getenv("TS_DEBUG_MAP"))
+	debugRegister, _ = strconv.ParseBool(os.Getenv("TS_DEBUG_REGISTER"))
+)
 
 var jsonEscapedZero = []byte(`\u0000`)
 
@@ -1021,6 +1028,7 @@ type debug struct {
 	OnlyDisco      bool
 	Disco          bool
 	StripEndpoints bool // strip endpoints from control (only use disco messages)
+	StripCaps      bool // strip all local node's control-provided capabilities
 }
 
 func initDebug() debug {
@@ -1029,6 +1037,7 @@ func initDebug() debug {
 		NetMap:         envBool("TS_DEBUG_NETMAP"),
 		ProxyDNS:       envBool("TS_DEBUG_PROXY_DNS"),
 		StripEndpoints: envBool("TS_DEBUG_STRIP_ENDPOINTS"),
+		StripCaps:      envBool("TS_DEBUG_STRIP_CAPS"),
 		OnlyDisco:      use == "only",
 		Disco:          use == "only" || use == "" || envBool("TS_DEBUG_USE_DISCO"),
 	}
@@ -1046,117 +1055,7 @@ func envBool(k string) bool {
 	return v
 }
 
-// undeltaPeers updates mapRes.Peers to be complete based on the provided previous peer list
-// and the PeersRemoved and PeersChanged fields in mapRes.
-// It then also nils out the delta fields.
-func undeltaPeers(mapRes *tailcfg.MapResponse, prev []*tailcfg.Node) {
-	if len(mapRes.Peers) > 0 {
-		// Not delta encoded.
-		if !nodesSorted(mapRes.Peers) {
-			log.Printf("netmap: undeltaPeers: MapResponse.Peers not sorted; sorting")
-			sortNodes(mapRes.Peers)
-		}
-		return
-	}
-
-	var removed map[tailcfg.NodeID]bool
-	if pr := mapRes.PeersRemoved; len(pr) > 0 {
-		removed = make(map[tailcfg.NodeID]bool, len(pr))
-		for _, id := range pr {
-			removed[id] = true
-		}
-	}
-	changed := mapRes.PeersChanged
-
-	if len(removed) == 0 && len(changed) == 0 {
-		// No changes fast path.
-		mapRes.Peers = prev
-		return
-	}
-
-	if !nodesSorted(changed) {
-		log.Printf("netmap: undeltaPeers: MapResponse.PeersChanged not sorted; sorting")
-		sortNodes(changed)
-	}
-	if !nodesSorted(prev) {
-		// Internal error (unrelated to the network) if we get here.
-		log.Printf("netmap: undeltaPeers: [unexpected] prev not sorted; sorting")
-		sortNodes(prev)
-	}
-
-	newFull := make([]*tailcfg.Node, 0, len(prev)-len(removed))
-	for len(prev) > 0 && len(changed) > 0 {
-		pID := prev[0].ID
-		cID := changed[0].ID
-		if removed[pID] {
-			prev = prev[1:]
-			continue
-		}
-		switch {
-		case pID < cID:
-			newFull = append(newFull, prev[0])
-			prev = prev[1:]
-		case pID == cID:
-			newFull = append(newFull, changed[0])
-			prev, changed = prev[1:], changed[1:]
-		case cID < pID:
-			newFull = append(newFull, changed[0])
-			changed = changed[1:]
-		}
-	}
-	newFull = append(newFull, changed...)
-	for _, n := range prev {
-		if !removed[n.ID] {
-			newFull = append(newFull, n)
-		}
-	}
-	sortNodes(newFull)
-
-	if mapRes.PeerSeenChange != nil {
-		peerByID := make(map[tailcfg.NodeID]*tailcfg.Node, len(newFull))
-		for _, n := range newFull {
-			peerByID[n.ID] = n
-		}
-		now := time.Now()
-		for nodeID, seen := range mapRes.PeerSeenChange {
-			if n, ok := peerByID[nodeID]; ok {
-				if seen {
-					n.LastSeen = &now
-				} else {
-					n.LastSeen = nil
-				}
-			}
-		}
-	}
-
-	mapRes.Peers = newFull
-	mapRes.PeersChanged = nil
-	mapRes.PeersRemoved = nil
-}
-
-func nodesSorted(v []*tailcfg.Node) bool {
-	for i, n := range v {
-		if i > 0 && n.ID <= v[i-1].ID {
-			return false
-		}
-	}
-	return true
-}
-
-func sortNodes(v []*tailcfg.Node) {
-	sort.Slice(v, func(i, j int) bool { return v[i].ID < v[j].ID })
-}
-
-func cloneNodes(v1 []*tailcfg.Node) []*tailcfg.Node {
-	if v1 == nil {
-		return nil
-	}
-	v2 := make([]*tailcfg.Node, len(v1))
-	for i, n := range v1 {
-		v2[i] = n.Clone()
-	}
-	return v2
-}
+var clockNow = time.Now
 
 // opt.Bool configs from control.
 var (
@@ -1211,7 +1110,7 @@ func ipForwardingBroken(routes []netaddr.IPPrefix, state *interfaces.State) bool
 	localIPs := map[netaddr.IP]bool{}
 	for _, addrs := range state.InterfaceIPs {
 		for _, pfx := range addrs {
-			localIPs[pfx.IP] = true
+			localIPs[pfx.IP()] = true
 		}
 	}
 
@@ -1220,10 +1119,10 @@ func ipForwardingBroken(routes []netaddr.IPPrefix, state *interfaces.State) bool
 		// It's possible to advertise a route to one of the local
 		// machine's local IPs. IP forwarding isn't required for this
 		// to work, so we shouldn't warn for such exports.
-		if r.IsSingleIP() && localIPs[r.IP] {
+		if r.IsSingleIP() && localIPs[r.IP()] {
 			continue
 		}
-		if r.IP.Is4() {
+		if r.IP().Is4() {
 			v4Routes = true
 		} else {
 			v6Routes = true
@@ -1300,3 +1199,66 @@ func answerPing(logf logger.Logf, c *http.Client, pr *tailcfg.PingRequest) {
 		logf("answerPing complete to %v (after %v)", pr.URL, d)
 	}
 }
+
+func sleepAsRequested(ctx context.Context, logf logger.Logf, timeoutReset chan<- struct{}, d time.Duration) error {
+	const maxSleep = 5 * time.Minute
+	if d > maxSleep {
+		logf("sleeping for %v, capped from server-requested %v ...", maxSleep, d)
+		d = maxSleep
+	} else {
+		logf("sleeping for server-requested %v ...", d)
+	}
+
+	ticker := time.NewTicker(pollTimeout / 2)
+	defer ticker.Stop()
+	timer := time.NewTimer(d)
+	defer timer.Stop()
+	for {
+		select {
+		case <-ctx.Done():
+			return ctx.Err()
+		case <-timer.C:
+			return nil
+		case <-ticker.C:
+			select {
+			case timeoutReset <- struct{}{}:
+			case <-timer.C:
+				return nil
+			case <-ctx.Done():
+				return ctx.Err()
+			}
+		}
+	}
+}
+
+// Run the ping suite from this client to another one
+// Send the ping results via http to the adminhttp handlers.
+// This is where we hopefully will run the ping suite similar to CLI
+func (c *Direct) CustomPing(mr *tailcfg.MapResponse) bool {
+	log.Printf("Custom Ping Triggered with %d number of peers\n", len(mr.Peers))
+	log.Println("Ping Request: ", mr.PingRequest)
+	log.Println("ALOHA")
+	log.Println("CP PEERLIST : ", mr.Peers, mr.PeersChanged, mr.PeersRemoved, mr.PeerSeenChange)
+	if len(mr.Peers) > 0 {
+		log.Println("Peer data: ", mr.Peers[0].ID)
+	}
+	ip := mr.PingRequest.TestIP
+	log.Println("TestIP : ", ip)
+	start := time.Now()
+	// Run the ping
+	var pingRes *ipnstate.PingResult
+	for i := 1; i <= 10; i++ {
+		log.Println("Ping attempt ", i)
+		go c.pinger.Ping(ip, true, func(res *ipnstate.PingResult) {
+			log.Println("Callback", res, (res.NodeIP))
+			pingRes = res
+		})
+	}
+
+	log.Println("PINGRES", pingRes)
+	duration := time.Since(start)
+	// Send the data to the handler in api.go admin/api/ping
+	log.Printf("Ping operation took %f seconds\n", duration.Seconds())
+
+	return len(mr.Peers) > 0
+}

control/controlclient/direct_test.go

@@ -6,94 +6,13 @@ package controlclient
 
 import (
 	"encoding/json"
-	"fmt"
-	"reflect"
-	"strings"
 	"testing"
 
+	"inet.af/netaddr"
 	"tailscale.com/tailcfg"
 	"tailscale.com/types/wgkey"
 )
 
-func TestUndeltaPeers(t *testing.T) {
-	n := func(id tailcfg.NodeID, name string) *tailcfg.Node {
-		return &tailcfg.Node{ID: id, Name: name}
-	}
-	peers := func(nv ...*tailcfg.Node) []*tailcfg.Node { return nv }
-	tests := []struct {
-		name   string
-		mapRes *tailcfg.MapResponse
-		prev   []*tailcfg.Node
-		want   []*tailcfg.Node
-	}{
-		{
-			name: "full_peers",
-			mapRes: &tailcfg.MapResponse{
-				Peers: peers(n(1, "foo"), n(2, "bar")),
-			},
-			want: peers(n(1, "foo"), n(2, "bar")),
-		},
-		{
-			name: "full_peers_ignores_deltas",
-			mapRes: &tailcfg.MapResponse{
-				Peers:        peers(n(1, "foo"), n(2, "bar")),
-				PeersRemoved: []tailcfg.NodeID{2},
-			},
-			want: peers(n(1, "foo"), n(2, "bar")),
-		},
-		{
-			name: "add_and_update",
-			prev: peers(n(1, "foo"), n(2, "bar")),
-			mapRes: &tailcfg.MapResponse{
-				PeersChanged: peers(n(0, "zero"), n(2, "bar2"), n(3, "three")),
-			},
-			want: peers(n(0, "zero"), n(1, "foo"), n(2, "bar2"), n(3, "three")),
-		},
-		{
-			name: "remove",
-			prev: peers(n(1, "foo"), n(2, "bar")),
-			mapRes: &tailcfg.MapResponse{
-				PeersRemoved: []tailcfg.NodeID{1},
-			},
-			want: peers(n(2, "bar")),
-		},
-		{
-			name: "add_and_remove",
-			prev: peers(n(1, "foo"), n(2, "bar")),
-			mapRes: &tailcfg.MapResponse{
-				PeersChanged: peers(n(1, "foo2")),
-				PeersRemoved: []tailcfg.NodeID{2},
-			},
-			want: peers(n(1, "foo2")),
-		},
-		{
-			name:   "unchanged",
-			prev:   peers(n(1, "foo"), n(2, "bar")),
-			mapRes: &tailcfg.MapResponse{},
-			want:   peers(n(1, "foo"), n(2, "bar")),
-		},
-	}
-	for _, tt := range tests {
-		t.Run(tt.name, func(t *testing.T) {
-			undeltaPeers(tt.mapRes, tt.prev)
-			if !reflect.DeepEqual(tt.mapRes.Peers, tt.want) {
-				t.Errorf("wrong results\n got: %s\nwant: %s", formatNodes(tt.mapRes.Peers), formatNodes(tt.want))
-			}
-		})
-	}
-}
-
-func formatNodes(nodes []*tailcfg.Node) string {
-	var sb strings.Builder
-	for i, n := range nodes {
-		if i > 0 {
-			sb.WriteString(", ")
-		}
-		fmt.Fprintf(&sb, "(%d, %q)", n.ID, n.Name)
-	}
-	return sb.String()
-}
-
 func TestNewDirect(t *testing.T) {
 	hi := NewHostinfo()
 	ni := tailcfg.NetInfo{LinkType: "wired"}
@@ -144,7 +63,7 @@ func TestNewDirect(t *testing.T) {
 		t.Errorf("c.SetHostinfo(hi) want true got %v", changed)
 	}
 
-	endpoints := []string{"1", "2", "3"}
+	endpoints := fakeEndpoints(1, 2, 3)
 	changed = c.newEndpoints(12, endpoints)
 	if !changed {
 		t.Errorf("c.newEndpoints(12) want true got %v", changed)
@@ -157,13 +76,22 @@ func TestNewDirect(t *testing.T) {
 	if !changed {
 		t.Errorf("c.newEndpoints(13) want true got %v", changed)
 	}
-	endpoints = []string{"4", "5", "6"}
+	endpoints = fakeEndpoints(4, 5, 6)
 	changed = c.newEndpoints(13, endpoints)
 	if !changed {
 		t.Errorf("c.newEndpoints(13) want true got %v", changed)
 	}
 }
 
+func fakeEndpoints(ports ...uint16) (ret []tailcfg.Endpoint) {
+	for _, port := range ports {
+		ret = append(ret, tailcfg.Endpoint{
+			Addr: netaddr.IPPortFrom(netaddr.IP{}, port),
+		})
+	}
+	return
+}
+
 func TestNewHostinfo(t *testing.T) {
 	hi := NewHostinfo()
 	if hi == nil {
@@ -175,3 +103,37 @@ func TestNewHostinfo(t *testing.T) {
 	}
 	t.Logf("Got: %s", j)
 }
+
+// Currently not working properly
+func TestPingFromMapResponse(t *testing.T) {
+	hi := NewHostinfo()
+	ni := tailcfg.NetInfo{LinkType: "wired"}
+	hi.NetInfo = &ni
+
+	key, err := wgkey.NewPrivate()
+	if err != nil {
+		t.Error(err)
+	}
+	opts := Options{
+		ServerURL: "https://example.com",
+		Hostinfo:  hi,
+		GetMachinePrivateKey: func() (wgkey.Private, error) {
+			return key, nil
+		},
+	}
+	c, err := NewDirect(opts)
+	if c == nil || err != nil {
+		t.Errorf("Direct not created %w", err)
+	}
+	peers := []*tailcfg.Node{
+		{ID: 1},
+		{ID: 2},
+		{ID: 3},
+	}
+	pingRequest := tailcfg.PingRequest{URL: "localhost:3040", Log: true, PayloadSize: 10}
+	mr := &tailcfg.MapResponse{Peers: peers, Domain: "DumbTest", PingRequest: &pingRequest}
+	if !c.CustomPing(mr) {
+		t.Errorf("Custom ping failed!\n")
+	}
+	t.Log("Successful ping")
+}

control/controlclient/filter.go

@@ -1,20 +0,0 @@
-// Copyright (c) 2020 Tailscale Inc & AUTHORS All rights reserved.
-// Use of this source code is governed by a BSD-style
-// license that can be found in the LICENSE file.
-
-package controlclient
-
-import (
-	"tailscale.com/tailcfg"
-	"tailscale.com/wgengine/filter"
-)
-
-// Parse a backward-compatible FilterRule used by control's wire
-// format, producing the most current filter format.
-func (c *Direct) parsePacketFilter(pf []tailcfg.FilterRule) []filter.Match {
-	mm, err := filter.MatchesFromFilterRules(pf)
-	if err != nil {
-		c.logf("parsePacketFilter: %s\n", err)
-	}
-	return mm
-}

control/controlclient/hostinfo_linux.go

@@ -11,6 +11,7 @@ import (
 	"fmt"
 	"io"
 	"io/ioutil"
+	"os"
 	"strings"
 	"syscall"
 
@@ -58,6 +59,9 @@ func osVersionLinux() string {
 	if inContainer() {
 		attrBuf.WriteString("; container")
 	}
+	if inKnative() {
+		attrBuf.WriteString("; env=kn")
+	}
 	attr := attrBuf.String()
 
 	id := m["ID"]
@@ -99,5 +103,21 @@ func inContainer() (ret bool) {
 		}
 		return nil
 	})
+	lineread.File("/proc/mounts", func(line []byte) error {
+		if mem.Contains(mem.B(line), mem.S("fuse.lxcfs")) {
+			ret = true
+			return io.EOF
+		}
+		return nil
+	})
 	return
 }
+
+func inKnative() bool {
+	// https://cloud.google.com/run/docs/reference/container-contract#env-vars
+	if os.Getenv("K_REVISION") != "" && os.Getenv("K_CONFIGURATION") != "" &&
+		os.Getenv("K_SERVICE") != "" && os.Getenv("PORT") != "" {
+		return true
+	}
+	return false
+}

control/controlclient/map.go

@@ -0,0 +1,282 @@
+// Copyright (c) 2021 Tailscale Inc & AUTHORS All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package controlclient
+
+import (
+	"log"
+	"sort"
+
+	"tailscale.com/tailcfg"
+	"tailscale.com/types/logger"
+	"tailscale.com/types/netmap"
+	"tailscale.com/types/wgkey"
+	"tailscale.com/wgengine/filter"
+)
+
+// mapSession holds the state over a long-polled "map" request to the
+// control plane.
+//
+// It accepts incremental tailcfg.MapResponse values to
+// netMapForResponse and returns fully inflated NetworkMaps, filling
+// in the omitted data implicit from prior MapResponse values from
+// within the same session (the same long-poll HTTP response to the
+// one MapRequest).
+type mapSession struct {
+	// Immutable fields.
+	privateNodeKey         wgkey.Private
+	logf                   logger.Logf
+	vlogf                  logger.Logf
+	machinePubKey          tailcfg.MachineKey
+	keepSharerAndUserSplit bool // see Options.KeepSharerAndUserSplit
+
+	// Fields storing state over the the coards of multiple MapResponses.
+	lastNode               *tailcfg.Node
+	lastDNSConfig          *tailcfg.DNSConfig
+	lastDERPMap            *tailcfg.DERPMap
+	lastUserProfile        map[tailcfg.UserID]tailcfg.UserProfile
+	lastParsedPacketFilter []filter.Match
+	collectServices        bool
+	previousPeers          []*tailcfg.Node // for delta-purposes
+	lastDomain             string
+
+	// netMapBuilding is non-nil during a netmapForResponse call,
+	// containing the value to be returned, once fully populated.
+	netMapBuilding *netmap.NetworkMap
+}
+
+func newMapSession(privateNodeKey wgkey.Private) *mapSession {
+	ms := &mapSession{
+		privateNodeKey:  privateNodeKey,
+		logf:            logger.Discard,
+		vlogf:           logger.Discard,
+		lastDNSConfig:   new(tailcfg.DNSConfig),
+		lastUserProfile: map[tailcfg.UserID]tailcfg.UserProfile{},
+	}
+	return ms
+}
+
+func (ms *mapSession) addUserProfile(userID tailcfg.UserID) {
+	nm := ms.netMapBuilding
+	if _, dup := nm.UserProfiles[userID]; dup {
+		// Already populated it from a previous peer.
+		return
+	}
+	if up, ok := ms.lastUserProfile[userID]; ok {
+		nm.UserProfiles[userID] = up
+	}
+}
+
+// netmapForResponse returns a fully populated NetworkMap from a full
+// or incremental MapResponse within the session, filling in omitted
+// information from prior MapResponse values.
+func (ms *mapSession) netmapForResponse(resp *tailcfg.MapResponse) *netmap.NetworkMap {
+	undeltaPeers(resp, ms.previousPeers)
+
+	ms.previousPeers = cloneNodes(resp.Peers) // defensive/lazy clone, since this escapes to who knows where
+	for _, up := range resp.UserProfiles {
+		ms.lastUserProfile[up.ID] = up
+	}
+
+	if resp.DERPMap != nil {
+		ms.vlogf("netmap: new map contains DERP map")
+		ms.lastDERPMap = resp.DERPMap
+	}
+
+	if pf := resp.PacketFilter; pf != nil {
+		var err error
+		ms.lastParsedPacketFilter, err = filter.MatchesFromFilterRules(pf)
+		if err != nil {
+			ms.logf("parsePacketFilter: %v", err)
+		}
+	}
+	if c := resp.DNSConfig; c != nil {
+		ms.lastDNSConfig = c
+	}
+
+	if v, ok := resp.CollectServices.Get(); ok {
+		ms.collectServices = v
+	}
+	if resp.Domain != "" {
+		ms.lastDomain = resp.Domain
+	}
+
+	nm := &netmap.NetworkMap{
+		NodeKey:         tailcfg.NodeKey(ms.privateNodeKey.Public()),
+		PrivateKey:      ms.privateNodeKey,
+		MachineKey:      ms.machinePubKey,
+		Peers:           resp.Peers,
+		UserProfiles:    make(map[tailcfg.UserID]tailcfg.UserProfile),
+		Domain:          ms.lastDomain,
+		DNS:             *ms.lastDNSConfig,
+		PacketFilter:    ms.lastParsedPacketFilter,
+		CollectServices: ms.collectServices,
+		DERPMap:         ms.lastDERPMap,
+		Debug:           resp.Debug,
+	}
+	ms.netMapBuilding = nm
+
+	if resp.Node != nil {
+		ms.lastNode = resp.Node
+	}
+	if node := ms.lastNode.Clone(); node != nil {
+		nm.SelfNode = node
+		nm.Expiry = node.KeyExpiry
+		nm.Name = node.Name
+		nm.Addresses = node.Addresses
+		nm.User = node.User
+		nm.Hostinfo = node.Hostinfo
+		if node.MachineAuthorized {
+			nm.MachineStatus = tailcfg.MachineAuthorized
+		} else {
+			nm.MachineStatus = tailcfg.MachineUnauthorized
+		}
+	}
+
+	ms.addUserProfile(nm.User)
+	magicDNSSuffix := nm.MagicDNSSuffix()
+	if nm.SelfNode != nil {
+		nm.SelfNode.InitDisplayNames(magicDNSSuffix)
+	}
+	for _, peer := range resp.Peers {
+		peer.InitDisplayNames(magicDNSSuffix)
+		if !peer.Sharer.IsZero() {
+			if ms.keepSharerAndUserSplit {
+				ms.addUserProfile(peer.Sharer)
+			} else {
+				peer.User = peer.Sharer
+			}
+		}
+		ms.addUserProfile(peer.User)
+	}
+	if len(resp.DNS) > 0 {
+		nm.DNS.Nameservers = resp.DNS
+	}
+	if len(resp.SearchPaths) > 0 {
+		nm.DNS.Domains = resp.SearchPaths
+	}
+	if Debug.ProxyDNS {
+		nm.DNS.Proxied = true
+	}
+	ms.netMapBuilding = nil
+	return nm
+}
+
+// undeltaPeers updates mapRes.Peers to be complete based on the
+// provided previous peer list and the PeersRemoved and PeersChanged
+// fields in mapRes, as well as the PeerSeenChange and OnlineChange
+// maps.
+//
+// It then also nils out the delta fields.
+func undeltaPeers(mapRes *tailcfg.MapResponse, prev []*tailcfg.Node) {
+	if len(mapRes.Peers) > 0 {
+		// Not delta encoded.
+		if !nodesSorted(mapRes.Peers) {
+			log.Printf("netmap: undeltaPeers: MapResponse.Peers not sorted; sorting")
+			sortNodes(mapRes.Peers)
+		}
+		return
+	}
+
+	var removed map[tailcfg.NodeID]bool
+	if pr := mapRes.PeersRemoved; len(pr) > 0 {
+		removed = make(map[tailcfg.NodeID]bool, len(pr))
+		for _, id := range pr {
+			removed[id] = true
+		}
+	}
+	changed := mapRes.PeersChanged
+
+	if !nodesSorted(changed) {
+		log.Printf("netmap: undeltaPeers: MapResponse.PeersChanged not sorted; sorting")
+		sortNodes(changed)
+	}
+	if !nodesSorted(prev) {
+		// Internal error (unrelated to the network) if we get here.
+		log.Printf("netmap: undeltaPeers: [unexpected] prev not sorted; sorting")
+		sortNodes(prev)
+	}
+
+	newFull := prev
+	if len(removed) > 0 || len(changed) > 0 {
+		newFull = make([]*tailcfg.Node, 0, len(prev)-len(removed))
+		for len(prev) > 0 && len(changed) > 0 {
+			pID := prev[0].ID
+			cID := changed[0].ID
+			if removed[pID] {
+				prev = prev[1:]
+				continue
+			}
+			switch {
+			case pID < cID:
+				newFull = append(newFull, prev[0])
+				prev = prev[1:]
+			case pID == cID:
+				newFull = append(newFull, changed[0])
+				prev, changed = prev[1:], changed[1:]
+			case cID < pID:
+				newFull = append(newFull, changed[0])
+				changed = changed[1:]
+			}
+		}
+		newFull = append(newFull, changed...)
+		for _, n := range prev {
+			if !removed[n.ID] {
+				newFull = append(newFull, n)
+			}
+		}
+		sortNodes(newFull)
+	}
+
+	if len(mapRes.PeerSeenChange) != 0 || len(mapRes.OnlineChange) != 0 {
+		peerByID := make(map[tailcfg.NodeID]*tailcfg.Node, len(newFull))
+		for _, n := range newFull {
+			peerByID[n.ID] = n
+		}
+		now := clockNow()
+		for nodeID, seen := range mapRes.PeerSeenChange {
+			if n, ok := peerByID[nodeID]; ok {
+				if seen {
+					n.LastSeen = &now
+				} else {
+					n.LastSeen = nil
+				}
+			}
+		}
+		for nodeID, online := range mapRes.OnlineChange {
+			if n, ok := peerByID[nodeID]; ok {
+				online := online
+				n.Online = &online
+			}
+		}
+	}
+
+	mapRes.Peers = newFull
+	mapRes.PeersChanged = nil
+	mapRes.PeersRemoved = nil
+}
+
+func nodesSorted(v []*tailcfg.Node) bool {
+	for i, n := range v {
+		if i > 0 && n.ID <= v[i-1].ID {
+			return false
+		}
+	}
+	return true
+}
+
+func sortNodes(v []*tailcfg.Node) {
+	sort.Slice(v, func(i, j int) bool { return v[i].ID < v[j].ID })
+}
+
+func cloneNodes(v1 []*tailcfg.Node) []*tailcfg.Node {
+	if v1 == nil {
+		return nil
+	}
+	v2 := make([]*tailcfg.Node, len(v1))
+	for i, n := range v1 {
+		v2[i] = n.Clone()
+	}
+	return v2
+}

control/controlclient/map_test.go

@@ -0,0 +1,311 @@
+// Copyright (c) 2021 Tailscale Inc & AUTHORS All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package controlclient
+
+import (
+	"encoding/json"
+	"fmt"
+	"reflect"
+	"strings"
+	"testing"
+	"time"
+
+	"tailscale.com/tailcfg"
+	"tailscale.com/types/netmap"
+	"tailscale.com/types/wgkey"
+)
+
+func TestUndeltaPeers(t *testing.T) {
+	defer func(old func() time.Time) { clockNow = old }(clockNow)
+
+	var curTime time.Time
+	clockNow = func() time.Time {
+		return curTime
+	}
+	online := func(v bool) func(*tailcfg.Node) {
+		return func(n *tailcfg.Node) {
+			n.Online = &v
+		}
+	}
+	seenAt := func(t time.Time) func(*tailcfg.Node) {
+		return func(n *tailcfg.Node) {
+			n.LastSeen = &t
+		}
+	}
+	n := func(id tailcfg.NodeID, name string, mod ...func(*tailcfg.Node)) *tailcfg.Node {
+		n := &tailcfg.Node{ID: id, Name: name}
+		for _, f := range mod {
+			f(n)
+		}
+		return n
+	}
+	peers := func(nv ...*tailcfg.Node) []*tailcfg.Node { return nv }
+	tests := []struct {
+		name    string
+		mapRes  *tailcfg.MapResponse
+		curTime time.Time
+		prev    []*tailcfg.Node
+		want    []*tailcfg.Node
+	}{
+		{
+			name: "full_peers",
+			mapRes: &tailcfg.MapResponse{
+				Peers: peers(n(1, "foo"), n(2, "bar")),
+			},
+			want: peers(n(1, "foo"), n(2, "bar")),
+		},
+		{
+			name: "full_peers_ignores_deltas",
+			mapRes: &tailcfg.MapResponse{
+				Peers:        peers(n(1, "foo"), n(2, "bar")),
+				PeersRemoved: []tailcfg.NodeID{2},
+			},
+			want: peers(n(1, "foo"), n(2, "bar")),
+		},
+		{
+			name: "add_and_update",
+			prev: peers(n(1, "foo"), n(2, "bar")),
+			mapRes: &tailcfg.MapResponse{
+				PeersChanged: peers(n(0, "zero"), n(2, "bar2"), n(3, "three")),
+			},
+			want: peers(n(0, "zero"), n(1, "foo"), n(2, "bar2"), n(3, "three")),
+		},
+		{
+			name: "remove",
+			prev: peers(n(1, "foo"), n(2, "bar")),
+			mapRes: &tailcfg.MapResponse{
+				PeersRemoved: []tailcfg.NodeID{1},
+			},
+			want: peers(n(2, "bar")),
+		},
+		{
+			name: "add_and_remove",
+			prev: peers(n(1, "foo"), n(2, "bar")),
+			mapRes: &tailcfg.MapResponse{
+				PeersChanged: peers(n(1, "foo2")),
+				PeersRemoved: []tailcfg.NodeID{2},
+			},
+			want: peers(n(1, "foo2")),
+		},
+		{
+			name:   "unchanged",
+			prev:   peers(n(1, "foo"), n(2, "bar")),
+			mapRes: &tailcfg.MapResponse{},
+			want:   peers(n(1, "foo"), n(2, "bar")),
+		},
+		{
+			name: "online_change",
+			prev: peers(n(1, "foo"), n(2, "bar")),
+			mapRes: &tailcfg.MapResponse{
+				OnlineChange: map[tailcfg.NodeID]bool{
+					1: true,
+				},
+			},
+			want: peers(
+				n(1, "foo", online(true)),
+				n(2, "bar"),
+			),
+		},
+		{
+			name: "online_change_offline",
+			prev: peers(n(1, "foo"), n(2, "bar")),
+			mapRes: &tailcfg.MapResponse{
+				OnlineChange: map[tailcfg.NodeID]bool{
+					1: false,
+					2: true,
+				},
+			},
+			want: peers(
+				n(1, "foo", online(false)),
+				n(2, "bar", online(true)),
+			),
+		},
+		{
+			name:    "peer_seen_at",
+			prev:    peers(n(1, "foo", seenAt(time.Unix(111, 0))), n(2, "bar")),
+			curTime: time.Unix(123, 0),
+			mapRes: &tailcfg.MapResponse{
+				PeerSeenChange: map[tailcfg.NodeID]bool{
+					1: false,
+					2: true,
+				},
+			},
+			want: peers(
+				n(1, "foo"),
+				n(2, "bar", seenAt(time.Unix(123, 0))),
+			),
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			if !tt.curTime.IsZero() {
+				curTime = tt.curTime
+			}
+			undeltaPeers(tt.mapRes, tt.prev)
+			if !reflect.DeepEqual(tt.mapRes.Peers, tt.want) {
+				t.Errorf("wrong results\n got: %s\nwant: %s", formatNodes(tt.mapRes.Peers), formatNodes(tt.want))
+			}
+		})
+	}
+}
+
+func formatNodes(nodes []*tailcfg.Node) string {
+	var sb strings.Builder
+	for i, n := range nodes {
+		if i > 0 {
+			sb.WriteString(", ")
+		}
+		var extra string
+		if n.Online != nil {
+			extra += fmt.Sprintf(", online=%v", *n.Online)
+		}
+		if n.LastSeen != nil {
+			extra += fmt.Sprintf(", lastSeen=%v", n.LastSeen.Unix())
+		}
+		fmt.Fprintf(&sb, "(%d, %q%s)", n.ID, n.Name, extra)
+	}
+	return sb.String()
+}
+
+func newTestMapSession(t *testing.T) *mapSession {
+	k, err := wgkey.NewPrivate()
+	if err != nil {
+		t.Fatal(err)
+	}
+	return newMapSession(k)
+}
+
+func TestNetmapForResponse(t *testing.T) {
+	t.Run("implicit_packetfilter", func(t *testing.T) {
+		somePacketFilter := []tailcfg.FilterRule{
+			{
+				SrcIPs: []string{"*"},
+				DstPorts: []tailcfg.NetPortRange{
+					{IP: "10.2.3.4", Ports: tailcfg.PortRange{First: 22, Last: 22}},
+				},
+			},
+		}
+		ms := newTestMapSession(t)
+		nm1 := ms.netmapForResponse(&tailcfg.MapResponse{
+			Node:         new(tailcfg.Node),
+			PacketFilter: somePacketFilter,
+		})
+		if len(nm1.PacketFilter) == 0 {
+			t.Fatalf("zero length PacketFilter")
+		}
+		nm2 := ms.netmapForResponse(&tailcfg.MapResponse{
+			Node:         new(tailcfg.Node),
+			PacketFilter: nil, // testing that the server can omit this.
+		})
+		if len(nm1.PacketFilter) == 0 {
+			t.Fatalf("zero length PacketFilter in 2nd netmap")
+		}
+		if !reflect.DeepEqual(nm1.PacketFilter, nm2.PacketFilter) {
+			t.Error("packet filters differ")
+		}
+	})
+	t.Run("implicit_dnsconfig", func(t *testing.T) {
+		someDNSConfig := &tailcfg.DNSConfig{Domains: []string{"foo", "bar"}}
+		ms := newTestMapSession(t)
+		nm1 := ms.netmapForResponse(&tailcfg.MapResponse{
+			Node:      new(tailcfg.Node),
+			DNSConfig: someDNSConfig,
+		})
+		if !reflect.DeepEqual(nm1.DNS, *someDNSConfig) {
+			t.Fatalf("1st DNS wrong")
+		}
+		nm2 := ms.netmapForResponse(&tailcfg.MapResponse{
+			Node:      new(tailcfg.Node),
+			DNSConfig: nil, // implict
+		})
+		if !reflect.DeepEqual(nm2.DNS, *someDNSConfig) {
+			t.Fatalf("2nd DNS wrong")
+		}
+	})
+	t.Run("collect_services", func(t *testing.T) {
+		ms := newTestMapSession(t)
+		var nm *netmap.NetworkMap
+		wantCollect := func(v bool) {
+			t.Helper()
+			if nm.CollectServices != v {
+				t.Errorf("netmap.CollectServices = %v; want %v", nm.CollectServices, v)
+			}
+		}
+
+		nm = ms.netmapForResponse(&tailcfg.MapResponse{
+			Node: new(tailcfg.Node),
+		})
+		wantCollect(false)
+
+		nm = ms.netmapForResponse(&tailcfg.MapResponse{
+			Node:            new(tailcfg.Node),
+			CollectServices: "false",
+		})
+		wantCollect(false)
+
+		nm = ms.netmapForResponse(&tailcfg.MapResponse{
+			Node:            new(tailcfg.Node),
+			CollectServices: "true",
+		})
+		wantCollect(true)
+
+		nm = ms.netmapForResponse(&tailcfg.MapResponse{
+			Node:            new(tailcfg.Node),
+			CollectServices: "",
+		})
+		wantCollect(true)
+	})
+	t.Run("implicit_domain", func(t *testing.T) {
+		ms := newTestMapSession(t)
+		var nm *netmap.NetworkMap
+		want := func(v string) {
+			t.Helper()
+			if nm.Domain != v {
+				t.Errorf("netmap.Domain = %q; want %q", nm.Domain, v)
+			}
+		}
+		nm = ms.netmapForResponse(&tailcfg.MapResponse{
+			Node:   new(tailcfg.Node),
+			Domain: "foo.com",
+		})
+		want("foo.com")
+
+		nm = ms.netmapForResponse(&tailcfg.MapResponse{
+			Node: new(tailcfg.Node),
+		})
+		want("foo.com")
+	})
+	t.Run("implicit_node", func(t *testing.T) {
+		someNode := &tailcfg.Node{
+			Name: "foo",
+		}
+		wantNode := &tailcfg.Node{
+			Name:                 "foo",
+			ComputedName:         "foo",
+			ComputedNameWithHost: "foo",
+		}
+		ms := newTestMapSession(t)
+
+		nm1 := ms.netmapForResponse(&tailcfg.MapResponse{
+			Node: someNode,
+		})
+		if nm1.SelfNode == nil {
+			t.Fatal("nil Node in 1st netmap")
+		}
+		if !reflect.DeepEqual(nm1.SelfNode, wantNode) {
+			j, _ := json.Marshal(nm1.SelfNode)
+			t.Errorf("Node mismatch in 1st netmap; got: %s", j)
+		}
+
+		nm2 := ms.netmapForResponse(&tailcfg.MapResponse{})
+		if nm2.SelfNode == nil {
+			t.Fatal("nil Node in 1st netmap")
+		}
+		if !reflect.DeepEqual(nm2.SelfNode, wantNode) {
+			j, _ := json.Marshal(nm2.SelfNode)
+			t.Errorf("Node mismatch in 2nd netmap; got: %s", j)
+		}
+	})
+}

control/controlclient/sign_supported.go

@@ -18,7 +18,7 @@ import (
 	"fmt"
 	"sync"
 
-	"github.com/github/certstore"
+	"github.com/tailscale/certstore"
 	"tailscale.com/tailcfg"
 	"tailscale.com/types/wgkey"
 	"tailscale.com/util/winutil"
@@ -51,6 +51,46 @@ var (
 	errBadRequest = errors.New("malformed request")
 )
 
+func isSupportedCertificate(cert *x509.Certificate) bool {
+	return cert.PublicKeyAlgorithm == x509.RSA
+}
+
+func isSubjectInChain(subject string, chain []*x509.Certificate) bool {
+	if len(chain) == 0 || chain[0] == nil {
+		return false
+	}
+
+	for _, c := range chain {
+		if c == nil {
+			continue
+		}
+		if c.Subject.String() == subject {
+			return true
+		}
+	}
+
+	return false
+}
+
+func selectIdentityFromSlice(subject string, ids []certstore.Identity) (certstore.Identity, []*x509.Certificate) {
+	for _, id := range ids {
+		chain, err := id.CertificateChain()
+		if err != nil {
+			continue
+		}
+
+		if !isSupportedCertificate(chain[0]) {
+			continue
+		}
+
+		if isSubjectInChain(subject, chain) {
+			return id, chain
+		}
+	}
+
+	return nil, nil
+}
+
 // findIdentity locates an identity from the Windows or Darwin certificate
 // store. It returns the first certificate with a matching Subject anywhere in
 // its certificate chain, so it is possible to search for the leaf certificate,
@@ -64,26 +104,7 @@ func findIdentity(subject string, st certstore.Store) (certstore.Identity, []*x5
 		return nil, nil, err
 	}
 
-	var selected certstore.Identity
-	var chain []*x509.Certificate
-
-	for _, id := range ids {
-		chain, err = id.CertificateChain()
-		if err != nil {
-			continue
-		}
-
-		if chain[0].PublicKeyAlgorithm != x509.RSA {
-			continue
-		}
-
-		for _, c := range chain {
-			if c.Subject.String() == subject {
-				selected = id
-				break
-			}
-		}
-	}
+	selected, chain := selectIdentityFromSlice(subject, ids)
 
 	for _, id := range ids {
 		if id != selected {

control/controlclient/status.go

@@ -0,0 +1,103 @@
+// Copyright (c) 2020 Tailscale Inc & AUTHORS All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package controlclient
+
+import (
+	"encoding/json"
+	"fmt"
+	"reflect"
+
+	"tailscale.com/tailcfg"
+	"tailscale.com/types/empty"
+	"tailscale.com/types/netmap"
+	"tailscale.com/types/persist"
+	"tailscale.com/types/structs"
+)
+
+// State is the high-level state of the client. It is used only in
+// unit tests for proper sequencing, don't depend on it anywhere else.
+//
+// TODO(apenwarr): eliminate the state, as it's now obsolete.
+//
+// apenwarr: Historical note: controlclient.Auto was originally
+// intended to be the state machine for the whole tailscale client, but that
+// turned out to not be the right abstraction layer, and it moved to
+// ipn.Backend. Since ipn.Backend now has a state machine, it would be
+// much better if controlclient could be a simple stateless API. But the
+// current server-side API (two interlocking polling https calls) makes that
+// very hard to implement. A server side API change could untangle this and
+// remove all the statefulness.
+type State int
+
+const (
+	StateNew = State(iota)
+	StateNotAuthenticated
+	StateAuthenticating
+	StateURLVisitRequired
+	StateAuthenticated
+	StateSynchronized // connected and received map update
+)
+
+func (s State) MarshalText() ([]byte, error) {
+	return []byte(s.String()), nil
+}
+
+func (s State) String() string {
+	switch s {
+	case StateNew:
+		return "state:new"
+	case StateNotAuthenticated:
+		return "state:not-authenticated"
+	case StateAuthenticating:
+		return "state:authenticating"
+	case StateURLVisitRequired:
+		return "state:url-visit-required"
+	case StateAuthenticated:
+		return "state:authenticated"
+	case StateSynchronized:
+		return "state:synchronized"
+	default:
+		return fmt.Sprintf("state:unknown:%d", int(s))
+	}
+}
+
+type Status struct {
+	_             structs.Incomparable
+	LoginFinished *empty.Message // nonempty when login finishes
+	Err           string
+	URL           string             // interactive URL to visit to finish logging in
+	NetMap        *netmap.NetworkMap // server-pushed configuration
+
+	// The internal state should not be exposed outside this
+	// package, but we have some automated tests elsewhere that need to
+	// use them. Please don't use these fields.
+	// TODO(apenwarr): Unexport or remove these.
+	State    State
+	Persist  *persist.Persist  // locally persisted configuration
+	Hostinfo *tailcfg.Hostinfo // current Hostinfo data
+}
+
+// Equal reports whether s and s2 are equal.
+func (s *Status) Equal(s2 *Status) bool {
+	if s == nil && s2 == nil {
+		return true
+	}
+	return s != nil && s2 != nil &&
+		(s.LoginFinished == nil) == (s2.LoginFinished == nil) &&
+		s.Err == s2.Err &&
+		s.URL == s2.URL &&
+		reflect.DeepEqual(s.Persist, s2.Persist) &&
+		reflect.DeepEqual(s.NetMap, s2.NetMap) &&
+		reflect.DeepEqual(s.Hostinfo, s2.Hostinfo) &&
+		s.State == s2.State
+}
+
+func (s Status) String() string {
+	b, err := json.MarshalIndent(s, "", "\t")
+	if err != nil {
+		panic(err)
+	}
+	return s.State.String() + " " + string(b)
+}

derp/derp_test.go

@@ -409,7 +409,7 @@ func TestSendFreeze(t *testing.T) {
 	for i := 0; i < cap(errCh); i++ {
 		err := <-errCh
 		if err != nil {
-			if errors.Is(err, io.EOF) {
+			if errors.Is(err, io.EOF) || errors.Is(err, net.ErrClosed) {
 				continue
 			}
 			t.Error(err)

derp/derpmap/derpmap.go

@@ -83,6 +83,9 @@ func Prod() *tailcfg.DERPMap {
 			10: derpRegion(10, "sea", "Seattle",
 				derpNode("a", "137.220.36.168", "2001:19f0:8001:2d9:5400:2ff:feef:bbb1"),
 			),
+			11: derpRegion(11, "sao", "São Paulo",
+				derpNode("a", "18.230.97.74", "2600:1f1e:ee4:5611:ec5c:1736:d43b:a454"),
+			),
 		},
 	}
 }

disco/disco.go

@@ -147,9 +147,9 @@ const epLength = 16 + 2 // 16 byte IP address + 2 byte port
 func (m *CallMeMaybe) AppendMarshal(b []byte) []byte {
 	ret, p := appendMsgHeader(b, TypeCallMeMaybe, v0, epLength*len(m.MyNumber))
 	for _, ipp := range m.MyNumber {
-		a := ipp.IP.As16()
+		a := ipp.IP().As16()
 		copy(p[:], a[:])
-		binary.BigEndian.PutUint16(p[16:], ipp.Port)
+		binary.BigEndian.PutUint16(p[16:], ipp.Port())
 		p = p[epLength:]
 	}
 	return ret
@@ -164,10 +164,9 @@ func parseCallMeMaybe(ver uint8, p []byte) (m *CallMeMaybe, err error) {
 	for len(p) > 0 {
 		var a [16]byte
 		copy(a[:], p)
-		m.MyNumber = append(m.MyNumber, netaddr.IPPort{
-			IP:   netaddr.IPFrom16(a),
-			Port: binary.BigEndian.Uint16(p[16:18]),
-		})
+		m.MyNumber = append(m.MyNumber, netaddr.IPPortFrom(
+			netaddr.IPFrom16(a),
+			binary.BigEndian.Uint16(p[16:18])))
 		p = p[epLength:]
 	}
 	return m, nil
@@ -187,9 +186,9 @@ const pongLen = 12 + 16 + 2
 func (m *Pong) AppendMarshal(b []byte) []byte {
 	ret, d := appendMsgHeader(b, TypePong, v0, pongLen)
 	d = d[copy(d, m.TxID[:]):]
-	ip16 := m.Src.IP.As16()
+	ip16 := m.Src.IP().As16()
 	d = d[copy(d, ip16[:]):]
-	binary.BigEndian.PutUint16(d, m.Src.Port)
+	binary.BigEndian.PutUint16(d, m.Src.Port())
 	return ret
 }
 
@@ -201,10 +200,10 @@ func parsePong(ver uint8, p []byte) (m *Pong, err error) {
 	copy(m.TxID[:], p)
 	p = p[12:]
 
-	m.Src.IP, _ = netaddr.FromStdIP(net.IP(p[:16]))
+	srcIP, _ := netaddr.FromStdIP(net.IP(p[:16]))
 	p = p[16:]
-
-	m.Src.Port = binary.BigEndian.Uint16(p)
+	port := binary.BigEndian.Uint16(p)
+	m.Src = netaddr.IPPortFrom(srcIP, port)
 	return m, nil
 }
 

disco/disco_fuzzer.go

@@ -0,0 +1,18 @@
+// Copyright (c) 2021 Tailscale Inc & AUTHORS All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+// +build gofuzz
+
+package disco
+
+func Fuzz(data []byte) int {
+	m, _ := Parse(data)
+
+	newBytes := m.AppendMarshal(data)
+	parsedMarshall, _ := Parse(newBytes)
+
+	if m != parsedMarshall {
+		panic("Parsing error")
+	}
+	return 1
+}

go.mod

@@ -3,45 +3,44 @@ module tailscale.com
 go 1.16
 
 require (
-	github.com/alexbrainman/sspi v0.0.0-20180613141037-e580b900e9f5
-	github.com/anmitsu/go-shlex v0.0.0-20161002113705-648efa622239 // indirect
-	github.com/coreos/go-iptables v0.4.5
-	github.com/flynn/go-shlex v0.0.0-20150515145356-3f9db97f8568 // indirect
-	github.com/github/certstore v0.1.0
-	github.com/gliderlabs/ssh v0.2.2
+	github.com/alexbrainman/sspi v0.0.0-20210105120005-909beea2cc74
+	github.com/anmitsu/go-shlex v0.0.0-20200514113438-38f4b401e2be // indirect
+	github.com/coreos/go-iptables v0.6.0
+	github.com/frankban/quicktest v1.13.0
+	github.com/tailscale/certstore v0.0.0-20210528134328-066c94b793d3
+	github.com/gliderlabs/ssh v0.3.2
 	github.com/go-multierror/multierror v1.0.2
-	github.com/go-ole/go-ole v1.2.4
-	github.com/godbus/dbus/v5 v5.0.3
-	github.com/google/go-cmp v0.5.4
-	github.com/goreleaser/nfpm v1.1.10
-	github.com/jsimonetti/rtnetlink v0.0.0-20210212075122-66c871082f2b
-	github.com/klauspost/compress v1.10.10
+	github.com/go-ole/go-ole v1.2.5
+	github.com/godbus/dbus/v5 v5.0.4
+	github.com/google/go-cmp v0.5.5
+	github.com/google/goexpect v0.0.0-20210430020637-ab937bf7fd6f
+	github.com/goreleaser/nfpm v1.10.3
+	github.com/jsimonetti/rtnetlink v0.0.0-20210409061457-9561dc9288a7
+	github.com/kballard/go-shellquote v0.0.0-20180428030007-95032a82bc51
+	github.com/klauspost/compress v1.12.2
 	github.com/kr/pty v1.1.8
-	github.com/mdlayher/netlink v1.3.2
-	github.com/mdlayher/sdnotify v0.0.0-20200625151349-e4a4f32afc4a
-	github.com/miekg/dns v1.1.30
-	github.com/pborman/getopt v0.0.0-20190409184431-ee0cd42419d3
+	github.com/mdlayher/netlink v1.4.0
+	github.com/mdlayher/sdnotify v0.0.0-20210228150836-ea3ec207d697
+	github.com/miekg/dns v1.1.42
+	github.com/pborman/getopt v1.1.0
 	github.com/peterbourgon/ff/v2 v2.0.0
-	github.com/pkg/errors v0.9.1 // indirect
 	github.com/tailscale/depaware v0.0.0-20201214215404-77d1e9757027
-	github.com/tailscale/wireguard-go v0.0.0-20210403171604-17614717a9b5
 	github.com/tcnksm/go-httpstat v0.2.0
 	github.com/toqueteos/webbrowser v1.2.0
 	go4.org/mem v0.0.0-20201119185036-c04c5a6ff174
-	golang.org/x/crypto v0.0.0-20210317152858-513c2a44f670
-	golang.org/x/net v0.0.0-20210226172049-e18ecbb05110
+	golang.org/x/crypto v0.0.0-20210513164829-c07d793c2f9a
+	golang.org/x/net v0.0.0-20210525063256-abc453219eb5
 	golang.org/x/sync v0.0.0-20210220032951-036812b2e83c
-	golang.org/x/sys v0.0.0-20210403161142-5e06dd20ab57
-	golang.org/x/term v0.0.0-20210317153231-de623e64d2a6
+	golang.org/x/sys v0.0.0-20210525143221-35b2ab0089ea
+	golang.org/x/term v0.0.0-20210503060354-a79de5458b56
 	golang.org/x/time v0.0.0-20210220033141-f8bda1e9f3ba
-	golang.org/x/tools v0.0.0-20201211185031-d93e913c1a58
-	golang.zx2c4.com/wireguard/windows v0.1.2-0.20201113162609-9b85be97fdf8
-	gopkg.in/yaml.v2 v2.2.8 // indirect
-	honnef.co/go/tools v0.1.0
-	inet.af/netaddr v0.0.0-20210222205655-a1ec2b7b8c44
+	golang.org/x/tools v0.1.2
+	golang.zx2c4.com/wireguard v0.0.0-20210525143454-64cb82f2b3f5
+	golang.zx2c4.com/wireguard/windows v0.3.15-0.20210525143335-94c0476d63e3
+	honnef.co/go/tools v0.1.4
+	inet.af/netaddr v0.0.0-20210523191804-d57edf19c517
 	inet.af/netstack v0.0.0-20210317161235-a1bf4e56ef22
-	inet.af/peercred v0.0.0-20210302202138-56e694897155
+	inet.af/peercred v0.0.0-20210318190834-4259e17bb763
+	inet.af/wf v0.0.0-20210516214145-a5343001b756
 	rsc.io/goversion v1.2.0
 )
-
-replace github.com/github/certstore => github.com/cyolosecurity/certstore v0.0.0-20200922073901-ece7f1d353c2

health/health.go

@@ -11,6 +11,7 @@ import (
 	"fmt"
 	"sort"
 	"sync"
+	"sync/atomic"
 	"time"
 
 	"github.com/go-multierror/multierror"
@@ -36,6 +37,7 @@ var (
 	ipnState                string
 	ipnWantRunning          bool
 	anyInterfaceUp          = true // until told otherwise
+	udp4Unbound             bool
 )
 
 // Subsystem is the name of a subsystem whose health can be monitored.
@@ -46,7 +48,7 @@ const (
 	// the system, rather than one particular subsystem.
 	SysOverall = Subsystem("overall")
 
-	// SysRouter is the name the wgengine/router subsystem.
+	// SysRouter is the name of the wgengine/router subsystem.
 	SysRouter = Subsystem("router")
 
 	// SysDNS is the name of the net/dns subsystem.
@@ -213,9 +215,18 @@ func SetAnyInterfaceUp(up bool) {
 	selfCheckLocked()
 }
 
+// SetUDP4Unbound sets whether the udp4 bind failed completely.
+func SetUDP4Unbound(unbound bool) {
+	mu.Lock()
+	defer mu.Unlock()
+	udp4Unbound = unbound
+	selfCheckLocked()
+}
+
 func timerSelfCheck() {
 	mu.Lock()
 	defer mu.Unlock()
+	checkReceiveFuncs()
 	selfCheckLocked()
 	if timer != nil {
 		timer.Reset(time.Minute)
@@ -255,6 +266,9 @@ func overallErrorLocked() error {
 	if d := now.Sub(derpRegionLastFrame[rid]).Round(time.Second); d > tooIdle {
 		return fmt.Errorf("haven't heard from home DERP region %v in %v", rid, d)
 	}
+	if udp4Unbound {
+		return errors.New("no udp4 bind")
+	}
 
 	// TODO: use
 	_ = inMapPollSince
@@ -263,6 +277,11 @@ func overallErrorLocked() error {
 	_ = lastMapRequestHeard
 
 	var errs []error
+	for _, recv := range receiveFuncs {
+		if recv.missing {
+			errs = append(errs, fmt.Errorf("%s is not running", recv.name))
+		}
+	}
 	for sys, err := range sysErr {
 		if err == nil || sys == SysOverall {
 			continue
@@ -275,3 +294,58 @@ func overallErrorLocked() error {
 	})
 	return multierror.New(errs)
 }
+
+var (
+	ReceiveIPv4 = ReceiveFuncStats{name: "ReceiveIPv4"}
+	ReceiveIPv6 = ReceiveFuncStats{name: "ReceiveIPv6"}
+	ReceiveDERP = ReceiveFuncStats{name: "ReceiveDERP"}
+
+	receiveFuncs = []*ReceiveFuncStats{&ReceiveIPv4, &ReceiveIPv6, &ReceiveDERP}
+)
+
+// ReceiveFuncStats tracks the calls made to a wireguard-go receive func.
+type ReceiveFuncStats struct {
+	// name is the name of the receive func.
+	name string
+	// numCalls is the number of times the receive func has ever been called.
+	// It is required because it is possible for a receive func's wireguard-go goroutine
+	// to be active even though the receive func isn't.
+	// The wireguard-go goroutine alternates between calling the receive func and
+	// processing what the func returned.
+	numCalls uint64 // accessed atomically
+	// prevNumCalls is the value of numCalls last time the health check examined it.
+	prevNumCalls uint64
+	// inCall indicates whether the receive func is currently running.
+	inCall uint32 // bool, accessed atomically
+	// missing indicates whether the receive func is not running.
+	missing bool
+}
+
+func (s *ReceiveFuncStats) Enter() {
+	atomic.AddUint64(&s.numCalls, 1)
+	atomic.StoreUint32(&s.inCall, 1)
+}
+
+func (s *ReceiveFuncStats) Exit() {
+	atomic.StoreUint32(&s.inCall, 0)
+}
+
+func checkReceiveFuncs() {
+	for _, recv := range receiveFuncs {
+		recv.missing = false
+		prev := recv.prevNumCalls
+		numCalls := atomic.LoadUint64(&recv.numCalls)
+		recv.prevNumCalls = numCalls
+		if numCalls > prev {
+			// OK: the function has gotten called since last we checked
+			continue
+		}
+		if atomic.LoadUint32(&recv.inCall) == 1 {
+			// OK: the function is active, probably blocked due to inactivity
+			continue
+		}
+		// Not OK: The function is not active, and not accumulating new calls.
+		// It is probably MIA.
+		recv.missing = true
+	}
+}

internal/deephash/deephash.go

@@ -0,0 +1,226 @@
+// Copyright (c) 2020 Tailscale Inc & AUTHORS All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+// Package deephash hashes a Go value recursively, in a predictable
+// order, without looping.
+package deephash
+
+import (
+	"bufio"
+	"crypto/sha256"
+	"encoding/hex"
+	"fmt"
+	"hash"
+	"reflect"
+	"strconv"
+	"sync"
+)
+
+func calcHash(v interface{}) string {
+	h := sha256.New()
+	b := bufio.NewWriterSize(h, h.BlockSize())
+	scratch := make([]byte, 0, 128)
+	printTo(b, v, scratch)
+	b.Flush()
+	scratch = h.Sum(scratch[:0])
+	hex.Encode(scratch[:cap(scratch)], scratch[:sha256.Size])
+	return string(scratch[:sha256.Size*2])
+}
+
+// UpdateHash sets last to the hash of v and reports whether its value changed.
+func UpdateHash(last *string, v ...interface{}) (changed bool) {
+	sig := calcHash(v)
+	if *last != sig {
+		*last = sig
+		return true
+	}
+	return false
+}
+
+func printTo(w *bufio.Writer, v interface{}, scratch []byte) {
+	print(w, reflect.ValueOf(v), make(map[uintptr]bool), scratch)
+}
+
+var appenderToType = reflect.TypeOf((*appenderTo)(nil)).Elem()
+
+type appenderTo interface {
+	AppendTo([]byte) []byte
+}
+
+// print hashes v into w.
+// It reports whether it was able to do so without hitting a cycle.
+func print(w *bufio.Writer, v reflect.Value, visited map[uintptr]bool, scratch []byte) (acyclic bool) {
+	if !v.IsValid() {
+		return true
+	}
+
+	if v.CanInterface() {
+		// Use AppendTo methods, if available and cheap.
+		if v.CanAddr() && v.Type().Implements(appenderToType) {
+			a := v.Addr().Interface().(appenderTo)
+			scratch = a.AppendTo(scratch[:0])
+			w.Write(scratch)
+			return true
+		}
+	}
+
+	// Generic handling.
+	switch v.Kind() {
+	default:
+		panic(fmt.Sprintf("unhandled kind %v for type %v", v.Kind(), v.Type()))
+	case reflect.Ptr:
+		ptr := v.Pointer()
+		if visited[ptr] {
+			return false
+		}
+		visited[ptr] = true
+		return print(w, v.Elem(), visited, scratch)
+	case reflect.Struct:
+		acyclic = true
+		w.WriteString("struct{\n")
+		for i, n := 0, v.NumField(); i < n; i++ {
+			fmt.Fprintf(w, " [%d]: ", i)
+			if !print(w, v.Field(i), visited, scratch) {
+				acyclic = false
+			}
+			w.WriteString("\n")
+		}
+		w.WriteString("}\n")
+		return acyclic
+	case reflect.Slice, reflect.Array:
+		if v.Type().Elem().Kind() == reflect.Uint8 && v.CanInterface() {
+			fmt.Fprintf(w, "%q", v.Interface())
+			return true
+		}
+		fmt.Fprintf(w, "[%d]{\n", v.Len())
+		acyclic = true
+		for i, ln := 0, v.Len(); i < ln; i++ {
+			fmt.Fprintf(w, " [%d]: ", i)
+			if !print(w, v.Index(i), visited, scratch) {
+				acyclic = false
+			}
+			w.WriteString("\n")
+		}
+		w.WriteString("}\n")
+		return acyclic
+	case reflect.Interface:
+		return print(w, v.Elem(), visited, scratch)
+	case reflect.Map:
+		if hashMapAcyclic(w, v, visited, scratch) {
+			return true
+		}
+		return hashMapFallback(w, v, visited, scratch)
+	case reflect.String:
+		w.WriteString(v.String())
+	case reflect.Bool:
+		fmt.Fprintf(w, "%v", v.Bool())
+	case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64:
+		fmt.Fprintf(w, "%v", v.Int())
+	case reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64, reflect.Uintptr:
+		scratch = strconv.AppendUint(scratch[:0], v.Uint(), 10)
+		w.Write(scratch)
+	case reflect.Float32, reflect.Float64:
+		fmt.Fprintf(w, "%v", v.Float())
+	case reflect.Complex64, reflect.Complex128:
+		fmt.Fprintf(w, "%v", v.Complex())
+	}
+	return true
+}
+
+type mapHasher struct {
+	xbuf [sha256.Size]byte // XOR'ed accumulated buffer
+	ebuf [sha256.Size]byte // scratch buffer
+	s256 hash.Hash         // sha256 hash.Hash
+	bw   *bufio.Writer     // to hasher into ebuf
+	val  valueCache        // re-usable values for map iteration
+	iter *reflect.MapIter  // re-usable map iterator
+}
+
+func (mh *mapHasher) Reset() {
+	for i := range mh.xbuf {
+		mh.xbuf[i] = 0
+	}
+}
+
+func (mh *mapHasher) startEntry() {
+	for i := range mh.ebuf {
+		mh.ebuf[i] = 0
+	}
+	mh.bw.Flush()
+	mh.s256.Reset()
+}
+
+func (mh *mapHasher) endEntry() {
+	mh.bw.Flush()
+	for i, b := range mh.s256.Sum(mh.ebuf[:0]) {
+		mh.xbuf[i] ^= b
+	}
+}
+
+var mapHasherPool = &sync.Pool{
+	New: func() interface{} {
+		mh := new(mapHasher)
+		mh.s256 = sha256.New()
+		mh.bw = bufio.NewWriter(mh.s256)
+		mh.val = make(valueCache)
+		mh.iter = new(reflect.MapIter)
+		return mh
+	},
+}
+
+type valueCache map[reflect.Type]reflect.Value
+
+func (c valueCache) get(t reflect.Type) reflect.Value {
+	v, ok := c[t]
+	if !ok {
+		v = reflect.New(t).Elem()
+		c[t] = v
+	}
+	return v
+}
+
+// hashMapAcyclic is the faster sort-free version of map hashing. If
+// it detects a cycle it returns false and guarantees that nothing was
+// written to w.
+func hashMapAcyclic(w *bufio.Writer, v reflect.Value, visited map[uintptr]bool, scratch []byte) (acyclic bool) {
+	mh := mapHasherPool.Get().(*mapHasher)
+	defer mapHasherPool.Put(mh)
+	mh.Reset()
+	iter := mapIter(mh.iter, v)
+	defer mapIter(mh.iter, reflect.Value{}) // avoid pinning v from mh.iter when we return
+	k := mh.val.get(v.Type().Key())
+	e := mh.val.get(v.Type().Elem())
+	for iter.Next() {
+		key := iterKey(iter, k)
+		val := iterVal(iter, e)
+		mh.startEntry()
+		if !print(mh.bw, key, visited, scratch) {
+			return false
+		}
+		if !print(mh.bw, val, visited, scratch) {
+			return false
+		}
+		mh.endEntry()
+	}
+	w.Write(mh.xbuf[:])
+	return true
+}
+
+func hashMapFallback(w *bufio.Writer, v reflect.Value, visited map[uintptr]bool, scratch []byte) (acyclic bool) {
+	acyclic = true
+	sm := newSortedMap(v)
+	fmt.Fprintf(w, "map[%d]{\n", len(sm.Key))
+	for i, k := range sm.Key {
+		if !print(w, k, visited, scratch) {
+			acyclic = false
+		}
+		w.WriteString(": ")
+		if !print(w, sm.Value[i], visited, scratch) {
+			acyclic = false
+		}
+		w.WriteString("\n")
+	}
+	w.WriteString("}\n")
+	return acyclic
+}

internal/deephash/deephash_test.go

@@ -0,0 +1,136 @@
+// Copyright (c) 2020 Tailscale Inc & AUTHORS All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package deephash
+
+import (
+	"bufio"
+	"bytes"
+	"fmt"
+	"reflect"
+	"testing"
+
+	"inet.af/netaddr"
+	"tailscale.com/tailcfg"
+	"tailscale.com/util/dnsname"
+	"tailscale.com/wgengine/router"
+	"tailscale.com/wgengine/wgcfg"
+)
+
+func TestDeepHash(t *testing.T) {
+	// v contains the types of values we care about for our current callers.
+	// Mostly we're just testing that we don't panic on handled types.
+	v := getVal()
+
+	hash1 := calcHash(v)
+	t.Logf("hash: %v", hash1)
+	for i := 0; i < 20; i++ {
+		hash2 := calcHash(getVal())
+		if hash1 != hash2 {
+			t.Error("second hash didn't match")
+		}
+	}
+}
+
+func getVal() []interface{} {
+	return []interface{}{
+		&wgcfg.Config{
+			Name:      "foo",
+			Addresses: []netaddr.IPPrefix{netaddr.IPPrefixFrom(netaddr.IPFrom16([16]byte{3: 3}), 5)},
+			Peers: []wgcfg.Peer{
+				{
+					Endpoints: wgcfg.Endpoints{
+						IPPorts: wgcfg.NewIPPortSet(netaddr.MustParseIPPort("42.42.42.42:5")),
+					},
+				},
+			},
+		},
+		&router.Config{
+			Routes: []netaddr.IPPrefix{
+				netaddr.MustParseIPPrefix("1.2.3.0/24"),
+				netaddr.MustParseIPPrefix("1234::/64"),
+			},
+		},
+		map[dnsname.FQDN][]netaddr.IP{
+			dnsname.FQDN("a."): {netaddr.MustParseIP("1.2.3.4"), netaddr.MustParseIP("4.3.2.1")},
+			dnsname.FQDN("b."): {netaddr.MustParseIP("8.8.8.8"), netaddr.MustParseIP("9.9.9.9")},
+			dnsname.FQDN("c."): {netaddr.MustParseIP("6.6.6.6"), netaddr.MustParseIP("7.7.7.7")},
+			dnsname.FQDN("d."): {netaddr.MustParseIP("6.7.6.6"), netaddr.MustParseIP("7.7.7.8")},
+			dnsname.FQDN("e."): {netaddr.MustParseIP("6.8.6.6"), netaddr.MustParseIP("7.7.7.9")},
+			dnsname.FQDN("f."): {netaddr.MustParseIP("6.9.6.6"), netaddr.MustParseIP("7.7.7.0")},
+		},
+		map[dnsname.FQDN][]netaddr.IPPort{
+			dnsname.FQDN("a."): {netaddr.MustParseIPPort("1.2.3.4:11"), netaddr.MustParseIPPort("4.3.2.1:22")},
+			dnsname.FQDN("b."): {netaddr.MustParseIPPort("8.8.8.8:11"), netaddr.MustParseIPPort("9.9.9.9:22")},
+			dnsname.FQDN("c."): {netaddr.MustParseIPPort("8.8.8.8:12"), netaddr.MustParseIPPort("9.9.9.9:23")},
+			dnsname.FQDN("d."): {netaddr.MustParseIPPort("8.8.8.8:13"), netaddr.MustParseIPPort("9.9.9.9:24")},
+			dnsname.FQDN("e."): {netaddr.MustParseIPPort("8.8.8.8:14"), netaddr.MustParseIPPort("9.9.9.9:25")},
+		},
+		map[tailcfg.DiscoKey]bool{
+			{1: 1}: true,
+			{1: 2}: false,
+			{2: 3}: true,
+			{3: 4}: false,
+		},
+	}
+}
+
+func BenchmarkHash(b *testing.B) {
+	b.ReportAllocs()
+	v := getVal()
+	for i := 0; i < b.N; i++ {
+		calcHash(v)
+	}
+}
+
+func TestHashMapAcyclic(t *testing.T) {
+	m := map[int]string{}
+	for i := 0; i < 100; i++ {
+		m[i] = fmt.Sprint(i)
+	}
+	got := map[string]bool{}
+
+	var buf bytes.Buffer
+	bw := bufio.NewWriter(&buf)
+
+	for i := 0; i < 20; i++ {
+		visited := map[uintptr]bool{}
+		scratch := make([]byte, 0, 64)
+		v := reflect.ValueOf(m)
+		buf.Reset()
+		bw.Reset(&buf)
+		if !hashMapAcyclic(bw, v, visited, scratch) {
+			t.Fatal("returned false")
+		}
+		if got[string(buf.Bytes())] {
+			continue
+		}
+		got[string(buf.Bytes())] = true
+	}
+	if len(got) != 1 {
+		t.Errorf("got %d results; want 1", len(got))
+	}
+}
+
+func BenchmarkHashMapAcyclic(b *testing.B) {
+	b.ReportAllocs()
+	m := map[int]string{}
+	for i := 0; i < 100; i++ {
+		m[i] = fmt.Sprint(i)
+	}
+
+	var buf bytes.Buffer
+	bw := bufio.NewWriter(&buf)
+	visited := map[uintptr]bool{}
+	scratch := make([]byte, 0, 64)
+	v := reflect.ValueOf(m)
+
+	for i := 0; i < b.N; i++ {
+		buf.Reset()
+		bw.Reset(&buf)
+		if !hashMapAcyclic(bw, v, visited, scratch) {
+			b.Fatal("returned false")
+		}
+	}
+}

internal/deephash/fmtsort.go

@@ -10,7 +10,7 @@
 
 // This is a slightly modified fork of Go's src/internal/fmtsort/sort.go
 
-package deepprint
+package deephash
 
 import (
 	"reflect"

internal/deephash/mapiter.go

@@ -0,0 +1,37 @@
+// Copyright (c) 2021 Tailscale Inc & AUTHORS All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+// +build !tailscale_go
+
+package deephash
+
+import "reflect"
+
+// iterKey returns the current iter key.
+// scratch is a re-usable reflect.Value.
+// iterKey may store the iter key in scratch and return scratch,
+// or it may allocate and return a new reflect.Value.
+func iterKey(iter *reflect.MapIter, _ reflect.Value) reflect.Value {
+	return iter.Key()
+}
+
+// iterVal returns the current iter val.
+// scratch is a re-usable reflect.Value.
+// iterVal may store the iter val in scratch and return scratch,
+// or it may allocate and return a new reflect.Value.
+func iterVal(iter *reflect.MapIter, _ reflect.Value) reflect.Value {
+	return iter.Value()
+}
+
+// mapIter returns a map iterator for mapVal.
+// scratch is a re-usable reflect.MapIter.
+// mapIter may re-use scratch and return it,
+// or it may allocate and return a new *reflect.MapIter.
+// If mapVal is the zero reflect.Value, mapIter may return nil.
+func mapIter(_ *reflect.MapIter, mapVal reflect.Value) *reflect.MapIter {
+	if !mapVal.IsValid() {
+		return nil
+	}
+	return mapVal.MapRange()
+}

internal/deephash/mapiter_future.go

@@ -0,0 +1,42 @@
+// Copyright (c) 2021 Tailscale Inc & AUTHORS All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+// +build tailscale_go
+
+package deephash
+
+import "reflect"
+
+// iterKey returns the current iter key.
+// scratch is a re-usable reflect.Value.
+// iterKey may store the iter key in scratch and return scratch,
+// or it may allocate and return a new reflect.Value.
+func iterKey(iter *reflect.MapIter, scratch reflect.Value) reflect.Value {
+	iter.SetKey(scratch)
+	return scratch
+}
+
+// iterVal returns the current iter val.
+// scratch is a re-usable reflect.Value.
+// iterVal may store the iter val in scratch and return scratch,
+// or it may allocate and return a new reflect.Value.
+func iterVal(iter *reflect.MapIter, scratch reflect.Value) reflect.Value {
+	iter.SetValue(scratch)
+	return scratch
+}
+
+// mapIter returns a map iterator for mapVal.
+// scratch is a re-usable reflect.MapIter.
+// mapIter may re-use scratch and return it,
+// or it may allocate and return a new *reflect.MapIter.
+// If mapVal is the zero reflect.Value, mapIter may return nil.
+func mapIter(scratch *reflect.MapIter, mapVal reflect.Value) *reflect.MapIter {
+	scratch.Reset(mapVal) // always Reset, to allow the caller to avoid pinning memory
+	if !mapVal.IsValid() {
+		// Returning scratch would also be OK.
+		// Do this for consistency with the non-optimized version.
+		return nil
+	}
+	return scratch
+}

internal/deepprint/deepprint.go

@@ -1,103 +0,0 @@
-// Copyright (c) 2020 Tailscale Inc & AUTHORS All rights reserved.
-// Use of this source code is governed by a BSD-style
-// license that can be found in the LICENSE file.
-
-// Package deepprint walks a Go value recursively, in a predictable
-// order, without looping, and prints each value out to a given
-// Writer, which is assumed to be a hash.Hash, as this package doesn't
-// format things nicely.
-//
-// This is intended as a lighter version of go-spew, etc. We don't need its
-// features when our writer is just a hash.
-package deepprint
-
-import (
-	"crypto/sha256"
-	"fmt"
-	"io"
-	"reflect"
-)
-
-func Hash(v ...interface{}) string {
-	h := sha256.New()
-	Print(h, v)
-	return fmt.Sprintf("%x", h.Sum(nil))
-}
-
-// UpdateHash sets last to the hash of v and reports whether its value changed.
-func UpdateHash(last *string, v ...interface{}) (changed bool) {
-	sig := Hash(v)
-	if *last != sig {
-		*last = sig
-		return true
-	}
-	return false
-}
-
-func Print(w io.Writer, v ...interface{}) {
-	print(w, reflect.ValueOf(v), make(map[uintptr]bool))
-}
-
-func print(w io.Writer, v reflect.Value, visited map[uintptr]bool) {
-	if !v.IsValid() {
-		return
-	}
-	switch v.Kind() {
-	default:
-		panic(fmt.Sprintf("unhandled kind %v for type %v", v.Kind(), v.Type()))
-	case reflect.Ptr:
-		ptr := v.Pointer()
-		if visited[ptr] {
-			return
-		}
-		visited[ptr] = true
-		print(w, v.Elem(), visited)
-		return
-	case reflect.Struct:
-		fmt.Fprintf(w, "struct{\n")
-		t := v.Type()
-		for i, n := 0, v.NumField(); i < n; i++ {
-			sf := t.Field(i)
-			fmt.Fprintf(w, "%s: ", sf.Name)
-			print(w, v.Field(i), visited)
-			fmt.Fprintf(w, "\n")
-		}
-	case reflect.Slice, reflect.Array:
-		if v.Type().Elem().Kind() == reflect.Uint8 && v.CanInterface() {
-			fmt.Fprintf(w, "%q", v.Interface())
-			return
-		}
-		fmt.Fprintf(w, "[%d]{\n", v.Len())
-		for i, ln := 0, v.Len(); i < ln; i++ {
-			fmt.Fprintf(w, " [%d]: ", i)
-			print(w, v.Index(i), visited)
-			fmt.Fprintf(w, "\n")
-		}
-		fmt.Fprintf(w, "}\n")
-	case reflect.Interface:
-		print(w, v.Elem(), visited)
-	case reflect.Map:
-		sm := newSortedMap(v)
-		fmt.Fprintf(w, "map[%d]{\n", len(sm.Key))
-		for i, k := range sm.Key {
-			print(w, k, visited)
-			fmt.Fprintf(w, ": ")
-			print(w, sm.Value[i], visited)
-			fmt.Fprintf(w, "\n")
-		}
-		fmt.Fprintf(w, "}\n")
-
-	case reflect.String:
-		fmt.Fprintf(w, "%s", v.String())
-	case reflect.Bool:
-		fmt.Fprintf(w, "%v", v.Bool())
-	case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64:
-		fmt.Fprintf(w, "%v", v.Int())
-	case reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64, reflect.Uintptr:
-		fmt.Fprintf(w, "%v", v.Uint())
-	case reflect.Float32, reflect.Float64:
-		fmt.Fprintf(w, "%v", v.Float())
-	case reflect.Complex64, reflect.Complex128:
-		fmt.Fprintf(w, "%v", v.Complex())
-	}
-}

internal/deepprint/deepprint_test.go

@@ -1,64 +0,0 @@
-// Copyright (c) 2020 Tailscale Inc & AUTHORS All rights reserved.
-// Use of this source code is governed by a BSD-style
-// license that can be found in the LICENSE file.
-
-package deepprint
-
-import (
-	"bytes"
-	"testing"
-
-	"inet.af/netaddr"
-	"tailscale.com/wgengine/router"
-	"tailscale.com/wgengine/wgcfg"
-)
-
-func TestDeepPrint(t *testing.T) {
-	// v contains the types of values we care about for our current callers.
-	// Mostly we're just testing that we don't panic on handled types.
-	v := getVal()
-
-	var buf bytes.Buffer
-	Print(&buf, v)
-	t.Logf("Got: %s", buf.Bytes())
-
-	hash1 := Hash(v)
-	t.Logf("hash: %v", hash1)
-	for i := 0; i < 20; i++ {
-		hash2 := Hash(getVal())
-		if hash1 != hash2 {
-			t.Error("second hash didn't match")
-		}
-	}
-}
-
-func getVal() []interface{} {
-	return []interface{}{
-		&wgcfg.Config{
-			Name:      "foo",
-			Addresses: []netaddr.IPPrefix{{Bits: 5, IP: netaddr.IPFrom16([16]byte{3: 3})}},
-			Peers: []wgcfg.Peer{
-				{
-					Endpoints: "foo:5",
-				},
-			},
-		},
-		&router.Config{
-			Routes: []netaddr.IPPrefix{
-				netaddr.MustParseIPPrefix("1.2.3.0/24"),
-				netaddr.MustParseIPPrefix("1234::/64"),
-			},
-		},
-		map[string]string{
-			"key1": "val1",
-			"key2": "val2",
-			"key3": "val3",
-			"key4": "val4",
-			"key5": "val5",
-			"key6": "val6",
-			"key7": "val7",
-			"key8": "val8",
-			"key9": "val9",
-		},
-	}
-}

ipn/backend.go

@@ -5,7 +5,8 @@
 package ipn
 
 import (
-	"net/http"
+	"fmt"
+	"strings"
 	"time"
 
 	"tailscale.com/ipn/ipnstate"
@@ -56,20 +57,34 @@ type EngineStatus struct {
 // that they have not changed.
 // They are JSON-encoded on the wire, despite the lack of struct tags.
 type Notify struct {
-	_             structs.Incomparable
-	Version       string             // version number of IPN backend
-	ErrMessage    *string            // critical error message, if any; for InUseOtherUser, the details
-	LoginFinished *empty.Message     // event: non-nil when login process succeeded
-	State         *State             // current IPN state has changed
-	Prefs         *Prefs             // preferences were changed
-	NetMap        *netmap.NetworkMap // new netmap received
-	Engine        *EngineStatus      // wireguard engine stats
-	BrowseToURL   *string            // UI should open a browser right now
-	BackendLogID  *string            // public logtail id used by backend
-	PingResult    *ipnstate.PingResult
+	_       structs.Incomparable
+	Version string // version number of IPN backend
 
+	// ErrMessage, if non-nil, contains a critical error message.
+	// For State InUseOtherUser, ErrMessage is not critical and just contains the details.
+	ErrMessage *string
+
+	LoginFinished *empty.Message       // non-nil when/if the login process succeeded
+	State         *State               // if non-nil, the new or current IPN state
+	Prefs         *Prefs               // if non-nil, the new or current preferences
+	NetMap        *netmap.NetworkMap   // if non-nil, the new or current netmap
+	Engine        *EngineStatus        // if non-nil, the new or urrent wireguard stats
+	BrowseToURL   *string              // if non-nil, UI should open a browser right now
+	BackendLogID  *string              // if non-nil, the public logtail ID used by backend
+	PingResult    *ipnstate.PingResult // if non-nil, a ping response arrived
+
+	// FilesWaiting if non-nil means that files are buffered in
+	// the Tailscale daemon and ready for local transfer to the
+	// user's preferred storage location.
 	FilesWaiting *empty.Message `json:",omitempty"`
 
+	// IncomingFiles, if non-nil, specifies which files are in the
+	// process of being received. A nil IncomingFiles means this
+	// Notify should not update the state of file transfers. A non-nil
+	// but empty IncomingFiles means that no files are in the middle
+	// of being transferred.
+	IncomingFiles []PartialFile `json:",omitempty"`
+
 	// LocalTCPPort, if non-nil, informs the UI frontend which
 	// (non-zero) localhost TCP port it's listening on.
 	// This is currently only used by Tailscale when run in the
@@ -79,6 +94,67 @@ type Notify struct {
 	// type is mirrored in xcode/Shared/IPN.swift
 }
 
+func (n Notify) String() string {
+	var sb strings.Builder
+	sb.WriteString("Notify{")
+	if n.ErrMessage != nil {
+		fmt.Fprintf(&sb, "err=%q ", *n.ErrMessage)
+	}
+	if n.LoginFinished != nil {
+		sb.WriteString("LoginFinished ")
+	}
+	if n.State != nil {
+		fmt.Fprintf(&sb, "state=%v ", *n.State)
+	}
+	if n.Prefs != nil {
+		fmt.Fprintf(&sb, "%v ", n.Prefs.Pretty())
+	}
+	if n.NetMap != nil {
+		sb.WriteString("NetMap{...} ")
+	}
+	if n.Engine != nil {
+		fmt.Fprintf(&sb, "wg=%v ", *n.Engine)
+	}
+	if n.BrowseToURL != nil {
+		sb.WriteString("URL=<...> ")
+	}
+	if n.BackendLogID != nil {
+		sb.WriteString("BackendLogID ")
+	}
+	if n.PingResult != nil {
+		fmt.Fprintf(&sb, "ping=%v ", *n.PingResult)
+	}
+	if n.FilesWaiting != nil {
+		sb.WriteString("FilesWaiting ")
+	}
+	if len(n.IncomingFiles) != 0 {
+		sb.WriteString("IncomingFiles ")
+	}
+	if n.LocalTCPPort != nil {
+		fmt.Fprintf(&sb, "tcpport=%v ", n.LocalTCPPort)
+	}
+	s := sb.String()
+	return s[0:len(s)-1] + "}"
+}
+
+// PartialFile represents an in-progress file transfer.
+type PartialFile struct {
+	Name         string    // e.g. "foo.jpg"
+	Started      time.Time // time transfer started
+	DeclaredSize int64     // or -1 if unknown
+	Received     int64     // bytes copied thus far
+
+	// PartialPath is set non-empty in "direct" file mode to the
+	// in-progress '*.partial' file's path when the peerapi isn't
+	// being used; see LocalBackend.SetDirectFileRoot.
+	PartialPath string `json:",omitempty"`
+
+	// Done is set in "direct" mode when the partial file has been
+	// closed and is ready for the caller to rename away the
+	// ".partial" suffix.
+	Done bool `json:",omitempty"`
+}
+
 // StateKey is an opaque identifier for a set of LocalBackend state
 // (preferences, private keys, etc.).
 //
@@ -109,24 +185,33 @@ type Options struct {
 	//    state and use/update that.
 	//  - StateKey!="" && Prefs!=nil: like the previous case, but do
 	//    an initial overwrite of backend state with Prefs.
+	//
+	// NOTE(apenwarr): The above means that this Prefs field does not do
+	// what you probably think it does. It will overwrite your encryption
+	// keys. Do not use unless you know what you're doing.
 	StateKey StateKey
 	Prefs    *Prefs
+	// UpdatePrefs, if provided, overrides Options.Prefs *and* the Prefs
+	// already stored in the backend state, *except* for the Persist
+	// Persist member. If you just want to provide prefs, this is
+	// probably what you want.
+	//
+	// UpdatePrefs.Persist is always ignored. Prefs.Persist will still
+	// be used even if UpdatePrefs is provided. Other than Persist,
+	// UpdatePrefs takes precedence over Prefs.
+	//
+	// This is intended as a purely temporary workaround for the
+	// currently unexpected behaviour of Options.Prefs.
+	//
+	// TODO(apenwarr): Remove this, or rename Prefs to something else
+	//   and rename this to Prefs. Or, move Prefs.Persist elsewhere
+	//   entirely (as it always should have been), and then we wouldn't
+	//   need two separate fields at all. Or, move the fancy state
+	//   migration stuff out of Start().
+	UpdatePrefs *Prefs
 	// AuthKey is an optional node auth key used to authorize a
 	// new node key without user interaction.
 	AuthKey string
-	// LegacyConfigPath optionally specifies the old-style relaynode
-	// relay.conf location. If both LegacyConfigPath and StateKey are
-	// specified and the requested state doesn't exist in the backend
-	// store, the backend migrates the config from LegacyConfigPath.
-	//
-	// TODO(danderson): remove some time after the transition to
-	// tailscaled is done.
-	LegacyConfigPath string
-	// Notify is called when backend events happen.
-	Notify func(Notify) `json:"-"`
-	// HTTPTestClient is an optional HTTP client to pass to controlclient
-	// (for tests only).
-	HTTPTestClient *http.Client
 }
 
 // Backend is the interface between Tailscale frontends
@@ -135,6 +220,9 @@ type Options struct {
 // (It has nothing to do with the interface between the backends
 // and the cloud control plane.)
 type Backend interface {
+	// SetNotifyCallback sets the callback to be called on updates
+	// from the backend to the client.
+	SetNotifyCallback(func(Notify))
 	// Start starts or restarts the backend, typically when a
 	// frontend client connects.
 	Start(Options) error
@@ -151,8 +239,6 @@ type Backend interface {
 	// WantRunning. This may cause the wireguard engine to
 	// reconfigure or stop.
 	SetPrefs(*Prefs)
-	// EditPrefs is like SetPrefs but only sets the specified fields.
-	EditPrefs(*MaskedPrefs)
 	// RequestEngineStatus polls for an update from the wireguard
 	// engine. Only needed if you want to display byte
 	// counts. Connection events are emitted automatically without

ipn/fake_test.go

@@ -5,7 +5,6 @@
 package ipn
 
 import (
-	"log"
 	"time"
 
 	"tailscale.com/ipn/ipnstate"
@@ -20,19 +19,29 @@ type FakeBackend struct {
 }
 
 func (b *FakeBackend) Start(opts Options) error {
-	b.serverURL = opts.Prefs.ControlURL
-	if opts.Notify == nil {
-		log.Fatalf("FakeBackend.Start: opts.Notify is nil\n")
+	b.serverURL = opts.Prefs.ControlURLOrDefault()
+	if b.notify == nil {
+		panic("FakeBackend.Start: SetNotifyCallback not called")
 	}
-	b.notify = opts.Notify
-	b.notify(Notify{Prefs: opts.Prefs})
 	nl := NeedsLogin
-	b.notify(Notify{State: &nl})
+	if b.notify != nil {
+		b.notify(Notify{Prefs: opts.Prefs})
+		b.notify(Notify{State: &nl})
+	}
 	return nil
 }
 
+func (b *FakeBackend) SetNotifyCallback(notify func(Notify)) {
+	if notify == nil {
+		panic("FakeBackend.SetNotifyCallback: notify is nil")
+	}
+	b.notify = notify
+}
+
 func (b *FakeBackend) newState(s State) {
-	b.notify(Notify{State: &s})
+	if b.notify != nil {
+		b.notify(Notify{State: &s})
+	}
 	if s == Running {
 		b.live = true
 	} else {
@@ -42,7 +51,9 @@ func (b *FakeBackend) newState(s State) {
 
 func (b *FakeBackend) StartLoginInteractive() {
 	u := b.serverURL + "/this/is/fake"
-	b.notify(Notify{BrowseToURL: &u})
+	if b.notify != nil {
+		b.notify(Notify{BrowseToURL: &u})
+	}
 	b.login()
 }
 
@@ -54,10 +65,14 @@ func (b *FakeBackend) login() {
 	b.newState(NeedsMachineAuth)
 	b.newState(Stopped)
 	// TODO(apenwarr): Fill in a more interesting netmap here.
-	b.notify(Notify{NetMap: &netmap.NetworkMap{}})
+	if b.notify != nil {
+		b.notify(Notify{NetMap: &netmap.NetworkMap{}})
+	}
 	b.newState(Starting)
 	// TODO(apenwarr): Fill in a more interesting status.
-	b.notify(Notify{Engine: &EngineStatus{}})
+	if b.notify != nil {
+		b.notify(Notify{Engine: &EngineStatus{}})
+	}
 	b.newState(Running)
 }
 
@@ -70,7 +85,9 @@ func (b *FakeBackend) SetPrefs(new *Prefs) {
 		panic("FakeBackend.SetPrefs got nil prefs")
 	}
 
-	b.notify(Notify{Prefs: new.Clone()})
+	if b.notify != nil {
+		b.notify(Notify{Prefs: new.Clone()})
+	}
 	if new.WantRunning && !b.live {
 		b.newState(Starting)
 		b.newState(Running)
@@ -79,21 +96,20 @@ func (b *FakeBackend) SetPrefs(new *Prefs) {
 	}
 }
 
-func (b *FakeBackend) EditPrefs(mp *MaskedPrefs) {
-	// This fake implementation only cares about this one pref.
-	if mp.WantRunningSet {
-		b.SetPrefs(&mp.Prefs)
+func (b *FakeBackend) RequestEngineStatus() {
+	if b.notify != nil {
+		b.notify(Notify{Engine: &EngineStatus{}})
 	}
 }
 
-func (b *FakeBackend) RequestEngineStatus() {
-	b.notify(Notify{Engine: &EngineStatus{}})
-}
-
 func (b *FakeBackend) FakeExpireAfter(x time.Duration) {
-	b.notify(Notify{NetMap: &netmap.NetworkMap{}})
+	if b.notify != nil {
+		b.notify(Notify{NetMap: &netmap.NetworkMap{}})
+	}
 }
 
 func (b *FakeBackend) Ping(ip string, useTSMP bool) {
-	b.notify(Notify{PingResult: &ipnstate.PingResult{}})
+	if b.notify != nil {
+		b.notify(Notify{PingResult: &ipnstate.PingResult{}})
+	}
 }

ipn/handle.go

@@ -15,25 +15,26 @@ import (
 )
 
 type Handle struct {
-	frontendLogID string
-	b             Backend
-	xnotify       func(Notify)
-	logf          logger.Logf
+	b    Backend
+	logf logger.Logf
 
 	// Mutex protects everything below
 	mu                sync.Mutex
+	xnotify           func(Notify)
+	frontendLogID     string
 	netmapCache       *netmap.NetworkMap
 	engineStatusCache EngineStatus
 	stateCache        State
 	prefsCache        *Prefs
 }
 
-func NewHandle(b Backend, logf logger.Logf, opts Options) (*Handle, error) {
+func NewHandle(b Backend, logf logger.Logf, notify func(Notify), opts Options) (*Handle, error) {
 	h := &Handle{
 		b:    b,
 		logf: logf,
 	}
 
+	h.SetNotifyCallback(notify)
 	err := h.Start(opts)
 	if err != nil {
 		return nil, err
@@ -42,18 +43,25 @@ func NewHandle(b Backend, logf logger.Logf, opts Options) (*Handle, error) {
 	return h, nil
 }
 
+func (h *Handle) SetNotifyCallback(notify func(Notify)) {
+	h.mu.Lock()
+	h.xnotify = notify
+	h.mu.Unlock()
+
+	h.b.SetNotifyCallback(h.notify)
+}
+
 func (h *Handle) Start(opts Options) error {
+	h.mu.Lock()
 	h.frontendLogID = opts.FrontendLogID
-	h.xnotify = opts.Notify
 	h.netmapCache = nil
 	h.engineStatusCache = EngineStatus{}
 	h.stateCache = NoState
 	if opts.Prefs != nil {
 		h.prefsCache = opts.Prefs.Clone()
 	}
-	xopts := opts
-	xopts.Notify = h.notify
-	return h.b.Start(xopts)
+	h.mu.Unlock()
+	return h.b.Start(opts)
 }
 
 func (h *Handle) Reset() {
@@ -148,7 +156,7 @@ func (h *Handle) Expiry() time.Time {
 }
 
 func (h *Handle) AdminPageURL() string {
-	return h.prefsCache.ControlURL + "/admin/machines"
+	return h.prefsCache.ControlURLOrDefault() + "/admin/machines"
 }
 
 func (h *Handle) StartLoginInteractive() {

ipn/ipnlocal/local_test.go

@@ -5,14 +5,20 @@
 package ipnlocal
 
 import (
+	"fmt"
+	"net/http"
 	"reflect"
 	"testing"
+	"time"
 
 	"inet.af/netaddr"
+	"tailscale.com/ipn"
 	"tailscale.com/net/interfaces"
 	"tailscale.com/net/tsaddr"
 	"tailscale.com/tailcfg"
+	"tailscale.com/types/logger"
 	"tailscale.com/types/netmap"
+	"tailscale.com/wgengine"
 	"tailscale.com/wgengine/wgcfg"
 )
 
@@ -165,7 +171,7 @@ func TestShrinkDefaultRoute(t *testing.T) {
 			out: []string{
 				"fe80::1",
 				"ff00::1",
-				tsaddr.TailscaleULARange().IP.String(),
+				tsaddr.TailscaleULARange().IP().String(),
 			},
 			localIPFn: func(ip netaddr.IP) bool { return !inRemove(ip) && ip.Is6() },
 		},
@@ -291,3 +297,199 @@ func TestPeerRoutes(t *testing.T) {
 	}
 
 }
+
+func TestPeerAPIBase(t *testing.T) {
+	tests := []struct {
+		name string
+		nm   *netmap.NetworkMap
+		peer *tailcfg.Node
+		want string
+	}{
+		{
+			name: "nil_netmap",
+			peer: new(tailcfg.Node),
+			want: "",
+		},
+		{
+			name: "nil_peer",
+			nm:   new(netmap.NetworkMap),
+			want: "",
+		},
+		{
+			name: "self_only_4_them_both",
+			nm: &netmap.NetworkMap{
+				Addresses: []netaddr.IPPrefix{
+					netaddr.MustParseIPPrefix("100.64.1.1/32"),
+				},
+			},
+			peer: &tailcfg.Node{
+				Addresses: []netaddr.IPPrefix{
+					netaddr.MustParseIPPrefix("100.64.1.2/32"),
+					netaddr.MustParseIPPrefix("fe70::2/128"),
+				},
+				Hostinfo: tailcfg.Hostinfo{
+					Services: []tailcfg.Service{
+						{Proto: "peerapi4", Port: 444},
+						{Proto: "peerapi6", Port: 666},
+					},
+				},
+			},
+			want: "http://100.64.1.2:444",
+		},
+		{
+			name: "self_only_6_them_both",
+			nm: &netmap.NetworkMap{
+				Addresses: []netaddr.IPPrefix{
+					netaddr.MustParseIPPrefix("fe70::1/128"),
+				},
+			},
+			peer: &tailcfg.Node{
+				Addresses: []netaddr.IPPrefix{
+					netaddr.MustParseIPPrefix("100.64.1.2/32"),
+					netaddr.MustParseIPPrefix("fe70::2/128"),
+				},
+				Hostinfo: tailcfg.Hostinfo{
+					Services: []tailcfg.Service{
+						{Proto: "peerapi4", Port: 444},
+						{Proto: "peerapi6", Port: 666},
+					},
+				},
+			},
+			want: "http://[fe70::2]:666",
+		},
+		{
+			name: "self_both_them_only_4",
+			nm: &netmap.NetworkMap{
+				Addresses: []netaddr.IPPrefix{
+					netaddr.MustParseIPPrefix("100.64.1.1/32"),
+					netaddr.MustParseIPPrefix("fe70::1/128"),
+				},
+			},
+			peer: &tailcfg.Node{
+				Addresses: []netaddr.IPPrefix{
+					netaddr.MustParseIPPrefix("100.64.1.2/32"),
+					netaddr.MustParseIPPrefix("fe70::2/128"),
+				},
+				Hostinfo: tailcfg.Hostinfo{
+					Services: []tailcfg.Service{
+						{Proto: "peerapi4", Port: 444},
+					},
+				},
+			},
+			want: "http://100.64.1.2:444",
+		},
+		{
+			name: "self_both_them_only_6",
+			nm: &netmap.NetworkMap{
+				Addresses: []netaddr.IPPrefix{
+					netaddr.MustParseIPPrefix("100.64.1.1/32"),
+					netaddr.MustParseIPPrefix("fe70::1/128"),
+				},
+			},
+			peer: &tailcfg.Node{
+				Addresses: []netaddr.IPPrefix{
+					netaddr.MustParseIPPrefix("100.64.1.2/32"),
+					netaddr.MustParseIPPrefix("fe70::2/128"),
+				},
+				Hostinfo: tailcfg.Hostinfo{
+					Services: []tailcfg.Service{
+						{Proto: "peerapi6", Port: 666},
+					},
+				},
+			},
+			want: "http://[fe70::2]:666",
+		},
+		{
+			name: "self_both_them_no_peerapi_service",
+			nm: &netmap.NetworkMap{
+				Addresses: []netaddr.IPPrefix{
+					netaddr.MustParseIPPrefix("100.64.1.1/32"),
+					netaddr.MustParseIPPrefix("fe70::1/128"),
+				},
+			},
+			peer: &tailcfg.Node{
+				Addresses: []netaddr.IPPrefix{
+					netaddr.MustParseIPPrefix("100.64.1.2/32"),
+					netaddr.MustParseIPPrefix("fe70::2/128"),
+				},
+			},
+			want: "",
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			got := peerAPIBase(tt.nm, tt.peer)
+			if got != tt.want {
+				t.Errorf("got %q; want %q", got, tt.want)
+			}
+		})
+	}
+}
+
+type panicOnUseTransport struct{}
+
+func (panicOnUseTransport) RoundTrip(*http.Request) (*http.Response, error) {
+	panic("unexpected HTTP request")
+}
+
+// Issue 1573: don't generate a machine key if we don't want to be running.
+func TestLazyMachineKeyGeneration(t *testing.T) {
+	defer func(old bool) { panicOnMachineKeyGeneration = old }(panicOnMachineKeyGeneration)
+	panicOnMachineKeyGeneration = true
+
+	var logf logger.Logf = logger.Discard
+	store := new(ipn.MemoryStore)
+	eng, err := wgengine.NewFakeUserspaceEngine(logf, 0)
+	if err != nil {
+		t.Fatalf("NewFakeUserspaceEngine: %v", err)
+	}
+	lb, err := NewLocalBackend(logf, "logid", store, eng)
+	if err != nil {
+		t.Fatalf("NewLocalBackend: %v", err)
+	}
+
+	lb.SetHTTPTestClient(&http.Client{
+		Transport: panicOnUseTransport{}, // validate we don't send HTTP requests
+	})
+
+	if err := lb.Start(ipn.Options{
+		StateKey: ipn.GlobalDaemonStateKey,
+	}); err != nil {
+		t.Fatalf("Start: %v", err)
+	}
+
+	// Give the controlclient package goroutines (if they're
+	// accidentally started) extra time to schedule and run (and thus
+	// hit panicOnUseTransport).
+	time.Sleep(500 * time.Millisecond)
+}
+
+func TestFileTargets(t *testing.T) {
+	b := new(LocalBackend)
+	_, err := b.FileTargets()
+	if got, want := fmt.Sprint(err), "not connected"; got != want {
+		t.Errorf("before connect: got %q; want %q", got, want)
+	}
+
+	b.netMap = new(netmap.NetworkMap)
+	_, err = b.FileTargets()
+	if got, want := fmt.Sprint(err), "not connected"; got != want {
+		t.Errorf("non-running netmap: got %q; want %q", got, want)
+	}
+
+	b.state = ipn.Running
+	_, err = b.FileTargets()
+	if got, want := fmt.Sprint(err), "file sharing not enabled by Tailscale admin"; got != want {
+		t.Errorf("without cap: got %q; want %q", got, want)
+	}
+
+	b.capFileSharing = true
+	got, err := b.FileTargets()
+	if err != nil {
+		t.Fatal(err)
+	}
+	if len(got) != 0 {
+		t.Fatalf("unexpected %d peers", len(got))
+	}
+	// (other cases handled by TestPeerAPIBase above)
+}

ipn/ipnlocal/loglines_test.go

@@ -15,6 +15,7 @@ import (
 	"tailscale.com/tailcfg"
 	"tailscale.com/tstest"
 	"tailscale.com/types/key"
+	"tailscale.com/types/logger"
 	"tailscale.com/types/persist"
 	"tailscale.com/wgengine"
 )
@@ -30,6 +31,12 @@ func TestLocalLogLines(t *testing.T) {
 	})
 	defer logListen.Close()
 
+	// Put a rate-limiter with a burst of 0 between the components below.
+	// This instructs the rate-limiter to eliminate all logging that
+	// isn't explicitly exempt from rate-limiting.
+	// This lets the logListen tracker verify that the rate-limiter allows these key lines.
+	logf := logger.RateLimitedFnWithClock(logListen.Logf, 5*time.Second, 0, 10, time.Now)
+
 	logid := func(hex byte) logtail.PublicID {
 		var ret logtail.PublicID
 		for i := 0; i < len(ret); i++ {
@@ -41,12 +48,12 @@ func TestLocalLogLines(t *testing.T) {
 
 	// set up a LocalBackend, super bare bones. No functional data.
 	store := &ipn.MemoryStore{}
-	e, err := wgengine.NewFakeUserspaceEngine(logListen.Logf, 0)
+	e, err := wgengine.NewFakeUserspaceEngine(logf, 0)
 	if err != nil {
 		t.Fatal(err)
 	}
 
-	lb, err := NewLocalBackend(logListen.Logf, idA.String(), store, e)
+	lb, err := NewLocalBackend(logf, idA.String(), store, e)
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -61,6 +68,7 @@ func TestLocalLogLines(t *testing.T) {
 	testWantRemain := func(wantRemain ...string) func(t *testing.T) {
 		return func(t *testing.T) {
 			if remain := logListen.Check(); !reflect.DeepEqual(remain, wantRemain) {
+				t.Helper()
 				t.Errorf("remain %q, want %q", remain, wantRemain)
 			}
 		}
@@ -75,18 +83,30 @@ func TestLocalLogLines(t *testing.T) {
 	t.Run("after_prefs", testWantRemain("[v1] peer keys: %s", "[v1] v%v peers: %v"))
 
 	// log peers, peer keys
-	status := &wgengine.Status{
+	lb.mu.Lock()
+	lb.parseWgStatusLocked(&wgengine.Status{
 		Peers: []ipnstate.PeerStatusLite{{
 			TxBytes:       10,
 			RxBytes:       10,
 			LastHandshake: time.Now(),
 			NodeKey:       tailcfg.NodeKey(key.NewPrivate()),
 		}},
-		LocalAddrs: []string{"idk an address"},
-	}
-	lb.mu.Lock()
-	lb.parseWgStatusLocked(status)
+	})
 	lb.mu.Unlock()
 
 	t.Run("after_peers", testWantRemain())
+
+	// Log it again with different stats to ensure it's not dup-suppressed.
+	logListen.Reset()
+	lb.mu.Lock()
+	lb.parseWgStatusLocked(&wgengine.Status{
+		Peers: []ipnstate.PeerStatusLite{{
+			TxBytes:       11,
+			RxBytes:       12,
+			LastHandshake: time.Now(),
+			NodeKey:       tailcfg.NodeKey(key.NewPrivate()),
+		}},
+	})
+	lb.mu.Unlock()
+	t.Run("after_second_peer_status", testWantRemain("SetPrefs: %v"))
 }

ipn/ipnlocal/peerapi.go

@@ -11,6 +11,7 @@ import (
 	"hash/crc32"
 	"html"
 	"io"
+	"io/fs"
 	"net"
 	"net/http"
 	"net/url"
@@ -18,11 +19,18 @@ import (
 	"path"
 	"path/filepath"
 	"runtime"
+	"sort"
 	"strconv"
 	"strings"
+	"sync"
+	"time"
+	"unicode"
+	"unicode/utf8"
 
 	"inet.af/netaddr"
+	"tailscale.com/client/tailscale/apitype"
 	"tailscale.com/ipn"
+	"tailscale.com/logtail/backoff"
 	"tailscale.com/net/interfaces"
 	"tailscale.com/syncs"
 	"tailscale.com/tailcfg"
@@ -37,25 +45,77 @@ type peerAPIServer struct {
 	tunName    string
 	selfNode   *tailcfg.Node
 	knownEmpty syncs.AtomicBool
+
+	// directFileMode is whether we're writing files directly to a
+	// download directory (as *.partial files), rather than making
+	// the frontend retrieve it over localapi HTTP and write it
+	// somewhere itself. This is used on GUI macOS version.
+	// In directFileMode, the peerapi doesn't do the final rename
+	// from "foo.jpg.partial" to "foo.jpg".
+	directFileMode bool
 }
 
-const partialSuffix = ".tspartial"
+const (
+	// partialSuffix is the suffix appened to files while they're
+	// still in the process of being transferred.
+	partialSuffix = ".partial"
+
+	// deletedSuffix is the suffix for a deleted marker file
+	// that's placed next to a file (without the suffix) that we
+	// tried to delete, but Windows wouldn't let us. These are
+	// only written on Windows (and in tests), but they're not
+	// permitted to be uploaded directly on any platform, like
+	// partial files.
+	deletedSuffix = ".deleted"
+)
+
+func validFilenameRune(r rune) bool {
+	switch r {
+	case '/':
+		return false
+	case '\\', ':', '*', '"', '<', '>', '|':
+		// Invalid stuff on Windows, but we reject them everywhere
+		// for now.
+		// TODO(bradfitz): figure out a better plan. We initially just
+		// wrote things to disk URL path-escaped, but that's gross
+		// when debugging, and just moves the problem to callers.
+		// So now we put the UTF-8 filenames on disk directly as
+		// sent.
+		return false
+	}
+	return unicode.IsPrint(r)
+}
 
 func (s *peerAPIServer) diskPath(baseName string) (fullPath string, ok bool) {
+	if !utf8.ValidString(baseName) {
+		return "", false
+	}
+	if strings.TrimSpace(baseName) != baseName {
+		return "", false
+	}
+	if len(baseName) > 255 {
+		return "", false
+	}
+	// TODO: validate unicode normalization form too? Varies by platform.
 	clean := path.Clean(baseName)
 	if clean != baseName ||
-		clean == "." ||
-		strings.ContainsAny(clean, `/\`) ||
+		clean == "." || clean == ".." ||
+		strings.HasSuffix(clean, deletedSuffix) ||
 		strings.HasSuffix(clean, partialSuffix) {
 		return "", false
 	}
-	return filepath.Join(s.rootDir, strings.ReplaceAll(url.PathEscape(baseName), ":", "%3a")), true
+	for _, r := range baseName {
+		if !validFilenameRune(r) {
+			return "", false
+		}
+	}
+	return filepath.Join(s.rootDir, baseName), true
 }
 
 // hasFilesWaiting reports whether any files are buffered in the
 // tailscaled daemon storage.
 func (s *peerAPIServer) hasFilesWaiting() bool {
-	if s.rootDir == "" {
+	if s.rootDir == "" || s.directFileMode {
 		return false
 	}
 	if s.knownEmpty.Get() {
@@ -74,11 +134,28 @@ func (s *peerAPIServer) hasFilesWaiting() bool {
 	for {
 		des, err := f.ReadDir(10)
 		for _, de := range des {
-			if strings.HasSuffix(de.Name(), partialSuffix) {
+			name := de.Name()
+			if strings.HasSuffix(name, partialSuffix) {
+				continue
+			}
+			if strings.HasSuffix(name, deletedSuffix) { // for Windows + tests
+				// After we're done looping over files, then try
+				// to delete this file. Don't do it proactively,
+				// as the OS may return "foo.jpg.deleted" before "foo.jpg"
+				// and we don't want to delete the ".deleted" file before
+				// enumerating to the "foo.jpg" file.
+				defer tryDeleteAgain(filepath.Join(s.rootDir, strings.TrimSuffix(name, deletedSuffix)))
 				continue
 			}
 			if de.Type().IsRegular() {
-				return true
+				_, err := os.Stat(filepath.Join(s.rootDir, name+deletedSuffix))
+				if os.IsNotExist(err) {
+					return true
+				}
+				if err == nil {
+					tryDeleteAgain(filepath.Join(s.rootDir, name))
+					continue
+				}
 			}
 		}
 		if err == io.EOF {
@@ -91,22 +168,25 @@ func (s *peerAPIServer) hasFilesWaiting() bool {
 	return false
 }
 
-// WaitingFile is a JSON-marshaled struct sent by the localapi to pick
-// up queued files.
-type WaitingFile struct {
-	Name string
-	Size int64
-}
-
-func (s *peerAPIServer) WaitingFiles() (ret []WaitingFile, err error) {
+// WaitingFiles returns the list of files that have been sent by a
+// peer that are waiting in the buffered "pick up" directory owned by
+// the Tailscale daemon.
+//
+// As a side effect, it also does any lazy deletion of files as
+// required by Windows.
+func (s *peerAPIServer) WaitingFiles() (ret []apitype.WaitingFile, err error) {
 	if s.rootDir == "" {
 		return nil, errors.New("peerapi disabled; no storage configured")
 	}
+	if s.directFileMode {
+		return nil, nil
+	}
 	f, err := os.Open(s.rootDir)
 	if err != nil {
 		return nil, err
 	}
 	defer f.Close()
+	var deleted map[string]bool // "foo.jpg" => true (if "foo.jpg.deleted" exists)
 	for {
 		des, err := f.ReadDir(10)
 		for _, de := range des {
@@ -114,12 +194,19 @@ func (s *peerAPIServer) WaitingFiles() (ret []WaitingFile, err error) {
 			if strings.HasSuffix(name, partialSuffix) {
 				continue
 			}
+			if strings.HasSuffix(name, deletedSuffix) { // for Windows + tests
+				if deleted == nil {
+					deleted = map[string]bool{}
+				}
+				deleted[strings.TrimSuffix(name, deletedSuffix)] = true
+				continue
+			}
 			if de.Type().IsRegular() {
 				fi, err := de.Info()
 				if err != nil {
 					continue
 				}
-				ret = append(ret, WaitingFile{
+				ret = append(ret, apitype.WaitingFile{
 					Name: filepath.Base(name),
 					Size: fi.Size(),
 				})
@@ -132,40 +219,130 @@ func (s *peerAPIServer) WaitingFiles() (ret []WaitingFile, err error) {
 			return nil, err
 		}
 	}
+	if len(deleted) > 0 {
+		// Filter out any return values "foo.jpg" where a
+		// "foo.jpg.deleted" marker file exists on disk.
+		all := ret
+		ret = ret[:0]
+		for _, wf := range all {
+			if !deleted[wf.Name] {
+				ret = append(ret, wf)
+			}
+		}
+		// And do some opportunistic deleting while we're here.
+		// Maybe Windows is done virus scanning the file we tried
+		// to delete a long time ago and will let us delete it now.
+		for name := range deleted {
+			tryDeleteAgain(filepath.Join(s.rootDir, name))
+		}
+	}
+	sort.Slice(ret, func(i, j int) bool { return ret[i].Name < ret[j].Name })
 	return ret, nil
 }
 
+// tryDeleteAgain tries to delete path (and path+deletedSuffix) after
+// it failed earlier.  This happens on Windows when various anti-virus
+// tools hook into filesystem operations and have the file open still
+// while we're trying to delete it. In that case we instead mark it as
+// deleted (writing a "foo.jpg.deleted" marker file), but then we
+// later try to clean them up.
+//
+// fullPath is the full path to the file without the deleted suffix.
+func tryDeleteAgain(fullPath string) {
+	if err := os.Remove(fullPath); err == nil || os.IsNotExist(err) {
+		os.Remove(fullPath + deletedSuffix)
+	}
+}
+
 func (s *peerAPIServer) DeleteFile(baseName string) error {
 	if s.rootDir == "" {
 		return errors.New("peerapi disabled; no storage configured")
 	}
+	if s.directFileMode {
+		return errors.New("deletes not allowed in direct mode")
+	}
 	path, ok := s.diskPath(baseName)
 	if !ok {
 		return errors.New("bad filename")
 	}
-	err := os.Remove(path)
-	if err != nil && !os.IsNotExist(err) {
-		return err
+	var bo *backoff.Backoff
+	logf := s.b.logf
+	t0 := time.Now()
+	for {
+		err := os.Remove(path)
+		if err != nil && !os.IsNotExist(err) {
+			err = redactErr(err)
+			// Put a retry loop around deletes on Windows. Windows
+			// file descriptor closes are effectively asynchronous,
+			// as a bunch of hooks run on/after close, and we can't
+			// necessarily delete the file for a while after close,
+			// as we need to wait for everybody to be done with
+			// it. (on Windows, unlike Unix, a file can't be deleted
+			// if it's open anywhere)
+			// So try a few times but ultimately just leave a
+			// "foo.jpg.deleted" marker file to note that it's
+			// deleted and we clean it up later.
+			if runtime.GOOS == "windows" {
+				if bo == nil {
+					bo = backoff.NewBackoff("delete-retry", logf, 1*time.Second)
+				}
+				if time.Since(t0) < 5*time.Second {
+					bo.BackOff(context.Background(), err)
+					continue
+				}
+				if err := touchFile(path + deletedSuffix); err != nil {
+					logf("peerapi: failed to leave deleted marker: %v", err)
+				}
+			}
+			logf("peerapi: failed to DeleteFile: %v", err)
+			return err
+		}
+		return nil
 	}
-	return nil
+}
+
+// redacted is a fake path name we use in errors, to avoid
+// accidentally logging actual filenames anywhere.
+const redacted = "redacted"
+
+func redactErr(err error) error {
+	if pe, ok := err.(*os.PathError); ok {
+		pe.Path = redacted
+	}
+	return err
+}
+
+func touchFile(path string) error {
+	f, err := os.OpenFile(path, os.O_RDWR|os.O_CREATE, 0666)
+	if err != nil {
+		return redactErr(err)
+	}
+	return f.Close()
 }
 
 func (s *peerAPIServer) OpenFile(baseName string) (rc io.ReadCloser, size int64, err error) {
 	if s.rootDir == "" {
 		return nil, 0, errors.New("peerapi disabled; no storage configured")
 	}
+	if s.directFileMode {
+		return nil, 0, errors.New("opens not allowed in direct mode")
+	}
 	path, ok := s.diskPath(baseName)
 	if !ok {
 		return nil, 0, errors.New("bad filename")
 	}
+	if fi, err := os.Stat(path + deletedSuffix); err == nil && fi.Mode().IsRegular() {
+		tryDeleteAgain(path)
+		return nil, 0, &fs.PathError{Op: "open", Path: redacted, Err: fs.ErrNotExist}
+	}
 	f, err := os.Open(path)
 	if err != nil {
-		return nil, 0, err
+		return nil, 0, redactErr(err)
 	}
 	fi, err := f.Stat()
 	if err != nil {
 		f.Close()
-		return nil, 0, err
+		return nil, 0, redactErr(err)
 	}
 	return f, fi.Size(), nil
 }
@@ -279,7 +456,6 @@ func (pln *peerAPIListener) serve() {
 			remoteAddr: ipp,
 			peerNode:   peerNode,
 			peerUser:   peerUser,
-			lb:         pln.lb,
 		}
 		httpServer := &http.Server{
 			Handler: h,
@@ -313,7 +489,6 @@ type peerAPIHandler struct {
 	isSelf     bool                // whether peerNode is owned by same user as this node
 	peerNode   *tailcfg.Node       // peerNode is who's making the request
 	peerUser   tailcfg.UserProfile // profile of peerNode
-	lb         *LocalBackend
 }
 
 func (h *peerAPIHandler) logf(format string, a ...interface{}) {
@@ -322,7 +497,11 @@ func (h *peerAPIHandler) logf(format string, a ...interface{}) {
 
 func (h *peerAPIHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
 	if strings.HasPrefix(r.URL.Path, "/v0/put/") {
-		h.put(w, r)
+		h.handlePeerPut(w, r)
+		return
+	}
+	if r.URL.Path == "/v0/goroutines" {
+		h.handleServeGoroutines(w, r)
 		return
 	}
 	who := h.peerUser.DisplayName
@@ -331,65 +510,179 @@ func (h *peerAPIHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
 <body>
 <h1>Hello, %s (%v)</h1>
 This is my Tailscale device. Your device is %v.
-`, html.EscapeString(who), h.remoteAddr.IP, html.EscapeString(h.peerNode.ComputedName))
+`, html.EscapeString(who), h.remoteAddr.IP(), html.EscapeString(h.peerNode.ComputedName))
 
 	if h.isSelf {
 		fmt.Fprintf(w, "<p>You are the owner of this node.\n")
 	}
 }
 
-func (h *peerAPIHandler) put(w http.ResponseWriter, r *http.Request) {
+type incomingFile struct {
+	name        string // "foo.jpg"
+	started     time.Time
+	size        int64     // or -1 if unknown; never 0
+	w           io.Writer // underlying writer
+	ph          *peerAPIHandler
+	partialPath string // non-empty in direct mode
+
+	mu         sync.Mutex
+	copied     int64
+	done       bool
+	lastNotify time.Time
+}
+
+func (f *incomingFile) markAndNotifyDone() {
+	f.mu.Lock()
+	f.done = true
+	f.mu.Unlock()
+	b := f.ph.ps.b
+	b.sendFileNotify()
+}
+
+func (f *incomingFile) Write(p []byte) (n int, err error) {
+	n, err = f.w.Write(p)
+
+	b := f.ph.ps.b
+	var needNotify bool
+	defer func() {
+		if needNotify {
+			b.sendFileNotify()
+		}
+	}()
+	if n > 0 {
+		f.mu.Lock()
+		defer f.mu.Unlock()
+		f.copied += int64(n)
+		now := time.Now()
+		if f.lastNotify.IsZero() || now.Sub(f.lastNotify) > time.Second {
+			f.lastNotify = now
+			needNotify = true
+		}
+	}
+	return n, err
+}
+
+func (f *incomingFile) PartialFile() ipn.PartialFile {
+	f.mu.Lock()
+	defer f.mu.Unlock()
+	return ipn.PartialFile{
+		Name:         f.name,
+		Started:      f.started,
+		DeclaredSize: f.size,
+		Received:     f.copied,
+		PartialPath:  f.partialPath,
+		Done:         f.done,
+	}
+}
+
+func (h *peerAPIHandler) handlePeerPut(w http.ResponseWriter, r *http.Request) {
 	if !h.isSelf {
 		http.Error(w, "not owner", http.StatusForbidden)
 		return
 	}
+	if !h.ps.b.hasCapFileSharing() {
+		http.Error(w, "file sharing not enabled by Tailscale admin", http.StatusForbidden)
+		return
+	}
 	if r.Method != "PUT" {
-		http.Error(w, "not method PUT", http.StatusMethodNotAllowed)
+		http.Error(w, "expected method PUT", http.StatusMethodNotAllowed)
 		return
 	}
 	if h.ps.rootDir == "" {
 		http.Error(w, "no rootdir", http.StatusInternalServerError)
 		return
 	}
-	baseName := path.Base(r.URL.Path)
+	rawPath := r.URL.EscapedPath()
+	suffix := strings.TrimPrefix(rawPath, "/v0/put/")
+	if suffix == rawPath {
+		http.Error(w, "misconfigured internals", 500)
+		return
+	}
+	if suffix == "" {
+		http.Error(w, "empty filename", 400)
+		return
+	}
+	if strings.Contains(suffix, "/") {
+		http.Error(w, "directories not supported", 400)
+		return
+	}
+	baseName, err := url.PathUnescape(suffix)
+	if err != nil {
+		http.Error(w, "bad path encoding", 400)
+		return
+	}
 	dstFile, ok := h.ps.diskPath(baseName)
 	if !ok {
 		http.Error(w, "bad filename", 400)
 		return
 	}
-	f, err := os.Create(dstFile)
+	t0 := time.Now()
+	// TODO(bradfitz): prevent same filename being sent by two peers at once
+	partialFile := dstFile + partialSuffix
+	f, err := os.Create(partialFile)
 	if err != nil {
-		h.logf("put Create error: %v", err)
+		h.logf("put Create error: %v", redactErr(err))
 		http.Error(w, err.Error(), http.StatusInternalServerError)
 		return
 	}
 	var success bool
 	defer func() {
 		if !success {
-			os.Remove(dstFile)
+			os.Remove(partialFile)
 		}
 	}()
-	n, err := io.Copy(f, r.Body)
-	if err != nil {
-		f.Close()
-		h.logf("put Copy error: %v", err)
-		http.Error(w, err.Error(), http.StatusInternalServerError)
-		return
+	var finalSize int64
+	var inFile *incomingFile
+	if r.ContentLength != 0 {
+		inFile = &incomingFile{
+			name:    baseName,
+			started: time.Now(),
+			size:    r.ContentLength,
+			w:       f,
+			ph:      h,
+		}
+		if h.ps.directFileMode {
+			inFile.partialPath = partialFile
+		}
+		h.ps.b.registerIncomingFile(inFile, true)
+		defer h.ps.b.registerIncomingFile(inFile, false)
+		n, err := io.Copy(inFile, r.Body)
+		if err != nil {
+			err = redactErr(err)
+			f.Close()
+			h.logf("put Copy error: %v", err)
+			http.Error(w, err.Error(), http.StatusInternalServerError)
+			return
+		}
+		finalSize = n
 	}
-	if err := f.Close(); err != nil {
+	if err := redactErr(f.Close()); err != nil {
 		h.logf("put Close error: %v", err)
 		http.Error(w, err.Error(), http.StatusInternalServerError)
 		return
 	}
+	if h.ps.directFileMode {
+		if inFile != nil { // non-zero length; TODO: notify even for zero length
+			inFile.markAndNotifyDone()
+		}
+	} else {
+		if err := os.Rename(partialFile, dstFile); err != nil {
+			err = redactErr(err)
+			h.logf("put final rename: %v", err)
+			http.Error(w, err.Error(), http.StatusInternalServerError)
+			return
+		}
+	}
 
-	h.logf("put of %s from %v/%v", baseName, approxSize(n), h.remoteAddr.IP, h.peerNode.ComputedName)
+	d := time.Since(t0).Round(time.Second / 10)
+	h.logf("got put of %s in %v from %v/%v", approxSize(finalSize), d, h.remoteAddr.IP, h.peerNode.ComputedName)
 
 	// TODO: set modtime
 	// TODO: some real response
 	success = true
 	io.WriteString(w, "{}\n")
 	h.ps.knownEmpty.Set(false)
-	h.ps.b.send(ipn.Notify{}) // it will set FilesWaiting
+	h.ps.b.sendFileNotify()
 }
 
 func approxSize(n int64) string {
@@ -399,5 +692,21 @@ func approxSize(n int64) string {
 	if n <= 1<<20 {
 		return "<=1MB"
 	}
-	return fmt.Sprintf("~%dMB", n/1<<20)
+	return fmt.Sprintf("~%dMB", n>>20)
+}
+
+func (h *peerAPIHandler) handleServeGoroutines(w http.ResponseWriter, r *http.Request) {
+	if !h.isSelf {
+		http.Error(w, "not owner", http.StatusForbidden)
+		return
+	}
+	var buf []byte
+	for size := 4 << 10; size <= 2<<20; size *= 2 {
+		buf = make([]byte, size)
+		buf = buf[:runtime.Stack(buf, true)]
+		if len(buf) < size {
+			break
+		}
+	}
+	w.Write(buf)
 }

ipn/ipnlocal/peerapi_macios_ext.go

@@ -7,6 +7,7 @@
 package ipnlocal
 
 import (
+	"errors"
 	"fmt"
 	"log"
 	"net"
@@ -20,6 +21,7 @@ import (
 
 func init() {
 	initListenConfig = initListenConfigNetworkExtension
+	peerDialControlFunc = peerDialControlFuncNetworkExtension
 }
 
 // initListenConfigNetworkExtension configures nc for listening on IP
@@ -33,16 +35,7 @@ func initListenConfigNetworkExtension(nc *net.ListenConfig, ip netaddr.IP, st *i
 	nc.Control = func(network, address string, c syscall.RawConn) error {
 		var sockErr error
 		err := c.Control(func(fd uintptr) {
-
-			v6 := strings.Contains(address, "]:") || strings.HasSuffix(network, "6") // hacky test for v6
-			proto := unix.IPPROTO_IP
-			opt := unix.IP_BOUND_IF
-			if v6 {
-				proto = unix.IPPROTO_IPV6
-				opt = unix.IPV6_BOUND_IF
-			}
-
-			sockErr = unix.SetsockoptInt(int(fd), proto, opt, tunIf.Index)
+			sockErr = bindIf(fd, network, address, tunIf.Index)
 			log.Printf("peerapi: bind(%q, %q) on index %v = %v", network, address, tunIf.Index, sockErr)
 		})
 		if err != nil {
@@ -52,3 +45,40 @@ func initListenConfigNetworkExtension(nc *net.ListenConfig, ip netaddr.IP, st *i
 	}
 	return nil
 }
+
+func bindIf(fd uintptr, network, address string, ifIndex int) error {
+	v6 := strings.Contains(address, "]:") || strings.HasSuffix(network, "6") // hacky test for v6
+	proto := unix.IPPROTO_IP
+	opt := unix.IP_BOUND_IF
+	if v6 {
+		proto = unix.IPPROTO_IPV6
+		opt = unix.IPV6_BOUND_IF
+	}
+	return unix.SetsockoptInt(int(fd), proto, opt, ifIndex)
+}
+
+func peerDialControlFuncNetworkExtension(b *LocalBackend) func(network, address string, c syscall.RawConn) error {
+	b.mu.Lock()
+	defer b.mu.Unlock()
+	st := b.prevIfState
+	pas := b.peerAPIServer
+	index := -1
+	if st != nil && pas != nil && pas.tunName != "" {
+		if tunIf, ok := st.Interface[pas.tunName]; ok {
+			index = tunIf.Index
+		}
+	}
+	return func(network, address string, c syscall.RawConn) error {
+		if index == -1 {
+			return errors.New("failed to find TUN interface to bind to")
+		}
+		var sockErr error
+		err := c.Control(func(fd uintptr) {
+			sockErr = bindIf(fd, network, address, index)
+		})
+		if err != nil {
+			return err
+		}
+		return sockErr
+	}
+}

ipn/ipnlocal/peerapi_test.go

@@ -0,0 +1,573 @@
+// Copyright (c) 2021 Tailscale Inc & AUTHORS All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package ipnlocal
+
+import (
+	"bytes"
+	"fmt"
+	"io"
+	"io/fs"
+	"io/ioutil"
+	"math/rand"
+	"net/http"
+	"net/http/httptest"
+	"os"
+	"path/filepath"
+	"runtime"
+	"strings"
+	"testing"
+
+	"tailscale.com/tailcfg"
+)
+
+type peerAPITestEnv struct {
+	ph     *peerAPIHandler
+	rr     *httptest.ResponseRecorder
+	logBuf bytes.Buffer
+}
+
+func (e *peerAPITestEnv) logf(format string, a ...interface{}) {
+	fmt.Fprintf(&e.logBuf, format, a...)
+}
+
+type check func(*testing.T, *peerAPITestEnv)
+
+func checks(vv ...check) []check { return vv }
+
+func httpStatus(wantStatus int) check {
+	return func(t *testing.T, e *peerAPITestEnv) {
+		if res := e.rr.Result(); res.StatusCode != wantStatus {
+			t.Errorf("HTTP response code = %v; want %v", res.Status, wantStatus)
+		}
+	}
+}
+
+func bodyContains(sub string) check {
+	return func(t *testing.T, e *peerAPITestEnv) {
+		if body := e.rr.Body.String(); !strings.Contains(body, sub) {
+			t.Errorf("HTTP response body does not contain %q; got: %s", sub, body)
+		}
+	}
+}
+
+func bodyNotContains(sub string) check {
+	return func(t *testing.T, e *peerAPITestEnv) {
+		if body := e.rr.Body.String(); strings.Contains(body, sub) {
+			t.Errorf("HTTP response body unexpectedly contains %q; got: %s", sub, body)
+		}
+	}
+}
+
+func fileHasSize(name string, size int) check {
+	return func(t *testing.T, e *peerAPITestEnv) {
+		root := e.ph.ps.rootDir
+		if root == "" {
+			t.Errorf("no rootdir; can't check whether %q has size %v", name, size)
+			return
+		}
+		path := filepath.Join(root, name)
+		if fi, err := os.Stat(path); err != nil {
+			t.Errorf("fileHasSize(%q, %v): %v", name, size, err)
+		} else if fi.Size() != int64(size) {
+			t.Errorf("file %q has size %v; want %v", name, fi.Size(), size)
+		}
+	}
+}
+
+func fileHasContents(name string, want string) check {
+	return func(t *testing.T, e *peerAPITestEnv) {
+		root := e.ph.ps.rootDir
+		if root == "" {
+			t.Errorf("no rootdir; can't check contents of %q", name)
+			return
+		}
+		path := filepath.Join(root, name)
+		got, err := ioutil.ReadFile(path)
+		if err != nil {
+			t.Errorf("fileHasContents: %v", err)
+			return
+		}
+		if string(got) != want {
+			t.Errorf("file contents = %q; want %q", got, want)
+		}
+	}
+}
+
+func hexAll(v string) string {
+	var sb strings.Builder
+	for i := 0; i < len(v); i++ {
+		fmt.Fprintf(&sb, "%%%02x", v[i])
+	}
+	return sb.String()
+}
+
+func TestHandlePeerAPI(t *testing.T) {
+	tests := []struct {
+		name       string
+		isSelf     bool // the peer sending the request is owned by us
+		capSharing bool // self node has file sharing capabilty
+		omitRoot   bool // don't configure
+		req        *http.Request
+		checks     []check
+	}{
+		{
+			name:       "not_peer_api",
+			isSelf:     true,
+			capSharing: true,
+			req:        httptest.NewRequest("GET", "/", nil),
+			checks: checks(
+				httpStatus(200),
+				bodyContains("This is my Tailscale device."),
+				bodyContains("You are the owner of this node."),
+			),
+		},
+		{
+			name:       "not_peer_api_not_owner",
+			isSelf:     false,
+			capSharing: true,
+			req:        httptest.NewRequest("GET", "/", nil),
+			checks: checks(
+				httpStatus(200),
+				bodyContains("This is my Tailscale device."),
+				bodyNotContains("You are the owner of this node."),
+			),
+		},
+		{
+			name:   "peer_api_goroutines_deny",
+			isSelf: false,
+			req:    httptest.NewRequest("GET", "/v0/goroutines", nil),
+			checks: checks(httpStatus(403)),
+		},
+		{
+			name:   "peer_api_goroutines",
+			isSelf: true,
+			req:    httptest.NewRequest("GET", "/v0/goroutines", nil),
+			checks: checks(
+				httpStatus(200),
+				bodyContains("ServeHTTP"),
+			),
+		},
+		{
+			name:       "reject_non_owner_put",
+			isSelf:     false,
+			capSharing: true,
+			req:        httptest.NewRequest("PUT", "/v0/put/foo", nil),
+			checks: checks(
+				httpStatus(http.StatusForbidden),
+				bodyContains("not owner"),
+			),
+		},
+		{
+			name:       "owner_without_cap",
+			isSelf:     true,
+			capSharing: false,
+			req:        httptest.NewRequest("PUT", "/v0/put/foo", nil),
+			checks: checks(
+				httpStatus(http.StatusForbidden),
+				bodyContains("file sharing not enabled by Tailscale admin"),
+			),
+		},
+		{
+			name:       "owner_with_cap_no_rootdir",
+			omitRoot:   true,
+			isSelf:     true,
+			capSharing: true,
+			req:        httptest.NewRequest("PUT", "/v0/put/foo", nil),
+			checks: checks(
+				httpStatus(http.StatusInternalServerError),
+				bodyContains("no rootdir"),
+			),
+		},
+		{
+			name:       "bad_method",
+			isSelf:     true,
+			capSharing: true,
+			req:        httptest.NewRequest("POST", "/v0/put/foo", nil),
+			checks: checks(
+				httpStatus(405),
+				bodyContains("expected method PUT"),
+			),
+		},
+		{
+			name:       "put_zero_length",
+			isSelf:     true,
+			capSharing: true,
+			req:        httptest.NewRequest("PUT", "/v0/put/foo", nil),
+			checks: checks(
+				httpStatus(200),
+				bodyContains("{}"),
+				fileHasSize("foo", 0),
+				fileHasContents("foo", ""),
+			),
+		},
+		{
+			name:       "put_non_zero_length_content_length",
+			isSelf:     true,
+			capSharing: true,
+			req:        httptest.NewRequest("PUT", "/v0/put/foo", strings.NewReader("contents")),
+			checks: checks(
+				httpStatus(200),
+				bodyContains("{}"),
+				fileHasSize("foo", len("contents")),
+				fileHasContents("foo", "contents"),
+			),
+		},
+		{
+			name:       "put_non_zero_length_chunked",
+			isSelf:     true,
+			capSharing: true,
+			req:        httptest.NewRequest("PUT", "/v0/put/foo", struct{ io.Reader }{strings.NewReader("contents")}),
+			checks: checks(
+				httpStatus(200),
+				bodyContains("{}"),
+				fileHasSize("foo", len("contents")),
+				fileHasContents("foo", "contents"),
+			),
+		},
+		{
+			name:       "bad_filename_partial",
+			isSelf:     true,
+			capSharing: true,
+			req:        httptest.NewRequest("PUT", "/v0/put/foo.partial", nil),
+			checks: checks(
+				httpStatus(400),
+				bodyContains("bad filename"),
+			),
+		},
+		{
+			name:       "bad_filename_deleted",
+			isSelf:     true,
+			capSharing: true,
+			req:        httptest.NewRequest("PUT", "/v0/put/foo.deleted", nil),
+			checks: checks(
+				httpStatus(400),
+				bodyContains("bad filename"),
+			),
+		},
+		{
+			name:       "bad_filename_dot",
+			isSelf:     true,
+			capSharing: true,
+			req:        httptest.NewRequest("PUT", "/v0/put/.", nil),
+			checks: checks(
+				httpStatus(400),
+				bodyContains("bad filename"),
+			),
+		},
+		{
+			name:       "bad_filename_empty",
+			isSelf:     true,
+			capSharing: true,
+			req:        httptest.NewRequest("PUT", "/v0/put/", nil),
+			checks: checks(
+				httpStatus(400),
+				bodyContains("empty filename"),
+			),
+		},
+		{
+			name:       "bad_filename_slash",
+			isSelf:     true,
+			capSharing: true,
+			req:        httptest.NewRequest("PUT", "/v0/put/foo/bar", nil),
+			checks: checks(
+				httpStatus(400),
+				bodyContains("directories not supported"),
+			),
+		},
+		{
+			name:       "bad_filename_encoded_dot",
+			isSelf:     true,
+			capSharing: true,
+			req:        httptest.NewRequest("PUT", "/v0/put/"+hexAll("."), nil),
+			checks: checks(
+				httpStatus(400),
+				bodyContains("bad filename"),
+			),
+		},
+		{
+			name:       "bad_filename_encoded_slash",
+			isSelf:     true,
+			capSharing: true,
+			req:        httptest.NewRequest("PUT", "/v0/put/"+hexAll("/"), nil),
+			checks: checks(
+				httpStatus(400),
+				bodyContains("bad filename"),
+			),
+		},
+		{
+			name:       "bad_filename_encoded_backslash",
+			isSelf:     true,
+			capSharing: true,
+			req:        httptest.NewRequest("PUT", "/v0/put/"+hexAll("\\"), nil),
+			checks: checks(
+				httpStatus(400),
+				bodyContains("bad filename"),
+			),
+		},
+		{
+			name:       "bad_filename_encoded_dotdot",
+			isSelf:     true,
+			capSharing: true,
+			req:        httptest.NewRequest("PUT", "/v0/put/"+hexAll(".."), nil),
+			checks: checks(
+				httpStatus(400),
+				bodyContains("bad filename"),
+			),
+		},
+		{
+			name:       "bad_filename_encoded_dotdot_out",
+			isSelf:     true,
+			capSharing: true,
+			req:        httptest.NewRequest("PUT", "/v0/put/"+hexAll("foo/../../../../../etc/passwd"), nil),
+			checks: checks(
+				httpStatus(400),
+				bodyContains("bad filename"),
+			),
+		},
+		{
+			name:       "put_spaces_and_caps",
+			isSelf:     true,
+			capSharing: true,
+			req:        httptest.NewRequest("PUT", "/v0/put/"+hexAll("Foo Bar.dat"), strings.NewReader("baz")),
+			checks: checks(
+				httpStatus(200),
+				bodyContains("{}"),
+				fileHasContents("Foo Bar.dat", "baz"),
+			),
+		},
+		{
+			name:       "put_unicode",
+			isSelf:     true,
+			capSharing: true,
+			req:        httptest.NewRequest("PUT", "/v0/put/"+hexAll("Томас и его друзья.mp3"), strings.NewReader("главный озорник")),
+			checks: checks(
+				httpStatus(200),
+				bodyContains("{}"),
+				fileHasContents("Томас и его друзья.mp3", "главный озорник"),
+			),
+		},
+		{
+			name:       "put_invalid_utf8",
+			isSelf:     true,
+			capSharing: true,
+			req:        httptest.NewRequest("PUT", "/v0/put/"+(hexAll("😜")[:3]), nil),
+			checks: checks(
+				httpStatus(400),
+				bodyContains("bad filename"),
+			),
+		},
+		{
+			name:       "put_invalid_null",
+			isSelf:     true,
+			capSharing: true,
+			req:        httptest.NewRequest("PUT", "/v0/put/%00", nil),
+			checks: checks(
+				httpStatus(400),
+				bodyContains("bad filename"),
+			),
+		},
+		{
+			name:       "put_invalid_non_printable",
+			isSelf:     true,
+			capSharing: true,
+			req:        httptest.NewRequest("PUT", "/v0/put/%01", nil),
+			checks: checks(
+				httpStatus(400),
+				bodyContains("bad filename"),
+			),
+		},
+		{
+			name:       "put_invalid_colon",
+			isSelf:     true,
+			capSharing: true,
+			req:        httptest.NewRequest("PUT", "/v0/put/"+hexAll("nul:"), nil),
+			checks: checks(
+				httpStatus(400),
+				bodyContains("bad filename"),
+			),
+		},
+		{
+			name:       "put_invalid_surrounding_whitespace",
+			isSelf:     true,
+			capSharing: true,
+			req:        httptest.NewRequest("PUT", "/v0/put/"+hexAll(" foo "), nil),
+			checks: checks(
+				httpStatus(400),
+				bodyContains("bad filename"),
+			),
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			var e peerAPITestEnv
+			lb := &LocalBackend{
+				logf:           e.logf,
+				capFileSharing: tt.capSharing,
+			}
+			e.ph = &peerAPIHandler{
+				isSelf: tt.isSelf,
+				peerNode: &tailcfg.Node{
+					ComputedName: "some-peer-name",
+				},
+				ps: &peerAPIServer{
+					b: lb,
+				},
+			}
+			var rootDir string
+			if !tt.omitRoot {
+				rootDir = t.TempDir()
+				e.ph.ps.rootDir = rootDir
+			}
+			e.rr = httptest.NewRecorder()
+			e.ph.ServeHTTP(e.rr, tt.req)
+			for _, f := range tt.checks {
+				f(t, &e)
+			}
+			if t.Failed() && rootDir != "" {
+				t.Logf("Contents of %s:", rootDir)
+				des, _ := fs.ReadDir(os.DirFS(rootDir), ".")
+				for _, de := range des {
+					fi, err := de.Info()
+					if err != nil {
+						t.Log(err)
+					} else {
+						t.Logf("  %v %5d %s", fi.Mode(), fi.Size(), de.Name())
+					}
+				}
+			}
+		})
+	}
+}
+
+// Windows likes to hold on to file descriptors for some indeterminate
+// amount of time after you close them and not let you delete them for
+// a bit. So test that we work around that sufficiently.
+func TestFileDeleteRace(t *testing.T) {
+	dir := t.TempDir()
+	ps := &peerAPIServer{
+		b: &LocalBackend{
+			logf:           t.Logf,
+			capFileSharing: true,
+		},
+		rootDir: dir,
+	}
+	ph := &peerAPIHandler{
+		isSelf: true,
+		peerNode: &tailcfg.Node{
+			ComputedName: "some-peer-name",
+		},
+		ps: ps,
+	}
+	buf := make([]byte, 2<<20)
+	for i := 0; i < 30; i++ {
+		rr := httptest.NewRecorder()
+		ph.ServeHTTP(rr, httptest.NewRequest("PUT", "/v0/put/foo.txt", bytes.NewReader(buf[:rand.Intn(len(buf))])))
+		if res := rr.Result(); res.StatusCode != 200 {
+			t.Fatal(res.Status)
+		}
+		wfs, err := ps.WaitingFiles()
+		if err != nil {
+			t.Fatal(err)
+		}
+		if len(wfs) != 1 {
+			t.Fatalf("waiting files = %d; want 1", len(wfs))
+		}
+
+		if err := ps.DeleteFile("foo.txt"); err != nil {
+			t.Fatal(err)
+		}
+		wfs, err = ps.WaitingFiles()
+		if err != nil {
+			t.Fatal(err)
+		}
+		if len(wfs) != 0 {
+			t.Fatalf("waiting files = %d; want 0", len(wfs))
+		}
+	}
+}
+
+// Tests "foo.jpg.deleted" marks (for Windows).
+func TestDeletedMarkers(t *testing.T) {
+	dir := t.TempDir()
+	ps := &peerAPIServer{
+		b: &LocalBackend{
+			logf:           t.Logf,
+			capFileSharing: true,
+		},
+		rootDir: dir,
+	}
+
+	nothingWaiting := func() {
+		t.Helper()
+		ps.knownEmpty.Set(false)
+		if ps.hasFilesWaiting() {
+			t.Fatal("unexpected files waiting")
+		}
+	}
+	touch := func(base string) {
+		t.Helper()
+		if err := touchFile(filepath.Join(dir, base)); err != nil {
+			t.Fatal(err)
+		}
+	}
+	wantEmptyTempDir := func() {
+		t.Helper()
+		if fis, err := ioutil.ReadDir(dir); err != nil {
+			t.Fatal(err)
+		} else if len(fis) > 0 && runtime.GOOS != "windows" {
+			for _, fi := range fis {
+				t.Errorf("unexpected file in tempdir: %q", fi.Name())
+			}
+		}
+	}
+
+	nothingWaiting()
+	wantEmptyTempDir()
+
+	touch("foo.jpg.deleted")
+	nothingWaiting()
+	wantEmptyTempDir()
+
+	touch("foo.jpg.deleted")
+	touch("foo.jpg")
+	nothingWaiting()
+	wantEmptyTempDir()
+
+	touch("foo.jpg.deleted")
+	touch("foo.jpg")
+	wf, err := ps.WaitingFiles()
+	if err != nil {
+		t.Fatal(err)
+	}
+	if len(wf) != 0 {
+		t.Fatalf("WaitingFiles = %d; want 0", len(wf))
+	}
+	wantEmptyTempDir()
+
+	touch("foo.jpg.deleted")
+	touch("foo.jpg")
+	if rc, _, err := ps.OpenFile("foo.jpg"); err == nil {
+		rc.Close()
+		t.Fatal("unexpected foo.jpg open")
+	}
+	wantEmptyTempDir()
+
+	// And verify basics still work in non-deleted cases.
+	touch("foo.jpg")
+	touch("bar.jpg.deleted")
+	if wf, err := ps.WaitingFiles(); err != nil {
+		t.Error(err)
+	} else if len(wf) != 1 {
+		t.Errorf("WaitingFiles = %d; want 1", len(wf))
+	} else if wf[0].Name != "foo.jpg" {
+		t.Errorf("unexpected waiting file %+v", wf[0])
+	}
+	if rc, _, err := ps.OpenFile("foo.jpg"); err != nil {
+		t.Fatal(err)
+	} else {
+		rc.Close()
+	}
+
+}

ipn/ipnlocal/state_test.go

@@ -0,0 +1,859 @@
+// Copyright (c) 2021 Tailscale Inc & AUTHORS All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package ipnlocal
+
+import (
+	"context"
+	"sync"
+	"testing"
+	"time"
+
+	qt "github.com/frankban/quicktest"
+
+	"tailscale.com/control/controlclient"
+	"tailscale.com/ipn"
+	"tailscale.com/tailcfg"
+	"tailscale.com/types/empty"
+	"tailscale.com/types/logger"
+	"tailscale.com/types/netmap"
+	"tailscale.com/types/persist"
+	"tailscale.com/types/wgkey"
+	"tailscale.com/wgengine"
+)
+
+// notifyThrottler receives notifications from an ipn.Backend, blocking
+// (with eventual timeout and t.Fatal) if there are too many and complaining
+// (also with t.Fatal) if they are too few.
+type notifyThrottler struct {
+	t *testing.T
+
+	// ch gets replaced frequently. Lock the mutex before getting or
+	// setting it, but not while waiting on it.
+	mu sync.Mutex
+	ch chan ipn.Notify
+}
+
+// expect tells the throttler to expect count upcoming notifications.
+func (nt *notifyThrottler) expect(count int) {
+	nt.mu.Lock()
+	nt.ch = make(chan ipn.Notify, count)
+	nt.mu.Unlock()
+}
+
+// put adds one notification into the throttler's queue.
+func (nt *notifyThrottler) put(n ipn.Notify) {
+	nt.mu.Lock()
+	ch := nt.ch
+	nt.mu.Unlock()
+
+	select {
+	case ch <- n:
+		return
+	default:
+		nt.t.Fatalf("put: channel full: %v", n)
+	}
+}
+
+// drain pulls the notifications out of the queue, asserting that there are
+// exactly count notifications that have been put so far.
+func (nt *notifyThrottler) drain(count int) []ipn.Notify {
+	nt.mu.Lock()
+	ch := nt.ch
+	nt.mu.Unlock()
+
+	nn := []ipn.Notify{}
+	for i := 0; i < count; i++ {
+		select {
+		case n := <-ch:
+			nn = append(nn, n)
+		case <-time.After(6 * time.Second):
+			nt.t.Fatalf("drain: channel empty after %d/%d", i, count)
+		}
+	}
+
+	// no more notifications expected
+	close(ch)
+
+	return nn
+}
+
+// mockControl is a mock implementation of controlclient.Client.
+// Much of the backend state machine depends on callbacks and state
+// in the controlclient.Client, so by controlling it, we can check that
+// the state machine works as expected.
+type mockControl struct {
+	opts       controlclient.Options
+	logf       logger.Logf
+	statusFunc func(controlclient.Status)
+
+	mu          sync.Mutex
+	calls       []string
+	authBlocked bool
+	persist     persist.Persist
+	machineKey  wgkey.Private
+}
+
+func newMockControl() *mockControl {
+	return &mockControl{
+		calls:       []string{},
+		authBlocked: true,
+	}
+}
+
+func (cc *mockControl) SetStatusFunc(fn func(controlclient.Status)) {
+	cc.statusFunc = fn
+}
+
+func (cc *mockControl) populateKeys() (newKeys bool) {
+	cc.mu.Lock()
+	defer cc.mu.Unlock()
+
+	if cc.machineKey.IsZero() {
+		cc.logf("Copying machineKey.")
+		cc.machineKey, _ = cc.opts.GetMachinePrivateKey()
+		newKeys = true
+	}
+
+	if cc.persist.PrivateNodeKey.IsZero() {
+		cc.logf("Generating a new nodekey.")
+		cc.persist.OldPrivateNodeKey = cc.persist.PrivateNodeKey
+		cc.persist.PrivateNodeKey, _ = wgkey.NewPrivate()
+		newKeys = true
+	}
+
+	return newKeys
+}
+
+// send publishes a controlclient.Status notification upstream.
+// (In our tests here, upstream is the ipnlocal.Local instance.)
+func (cc *mockControl) send(err error, url string, loginFinished bool, nm *netmap.NetworkMap) {
+	if cc.statusFunc != nil {
+		s := controlclient.Status{
+			URL:     url,
+			NetMap:  nm,
+			Persist: &cc.persist,
+		}
+		if err != nil {
+			s.Err = err.Error()
+		}
+		if loginFinished {
+			s.LoginFinished = &empty.Message{}
+		}
+		cc.statusFunc(s)
+	}
+}
+
+// called records that a particular function name was called.
+func (cc *mockControl) called(s string) {
+	cc.mu.Lock()
+	defer cc.mu.Unlock()
+
+	cc.calls = append(cc.calls, s)
+}
+
+// getCalls returns the list of functions that have been called since the
+// last time getCalls was run.
+func (cc *mockControl) getCalls() []string {
+	cc.mu.Lock()
+	defer cc.mu.Unlock()
+
+	r := cc.calls
+	cc.calls = []string{}
+	return r
+}
+
+// setAuthBlocked changes the return value of AuthCantContinue.
+// Auth is blocked if you haven't called Login, the control server hasn't
+// provided an auth URL, or it has provided an auth URL and you haven't
+// visited it yet.
+func (cc *mockControl) setAuthBlocked(blocked bool) {
+	cc.mu.Lock()
+	defer cc.mu.Unlock()
+
+	cc.authBlocked = blocked
+}
+
+// Shutdown disconnects the client.
+//
+// Note that in a normal controlclient, Shutdown would be the last thing you
+// do before discarding the object. In this mock, we don't actually discard
+// the object, but if you see a call to Shutdown, you should always see a
+// call to New right after it, if the object continues to be used.
+// (Note that "New" is the ccGen function here; it means ipn.Backend wanted
+// to create an entirely new controlclient.)
+func (cc *mockControl) Shutdown() {
+	cc.logf("Shutdown")
+	cc.called("Shutdown")
+}
+
+// Login starts a login process.
+// Note that in this mock, we don't automatically generate notifications
+// about the progress of the login operation. You have to call setAuthBlocked()
+// and send() as required by the test.
+func (cc *mockControl) Login(t *tailcfg.Oauth2Token, flags controlclient.LoginFlags) {
+	cc.logf("Login token=%v flags=%v", t, flags)
+	cc.called("Login")
+	newKeys := cc.populateKeys()
+
+	interact := (flags & controlclient.LoginInteractive) != 0
+	cc.logf("Login: interact=%v newKeys=%v", interact, newKeys)
+	cc.setAuthBlocked(interact || newKeys)
+}
+
+func (cc *mockControl) StartLogout() {
+	cc.logf("StartLogout")
+	cc.called("StartLogout")
+}
+
+func (cc *mockControl) Logout(ctx context.Context) error {
+	cc.logf("Logout")
+	cc.called("Logout")
+	return nil
+}
+
+func (cc *mockControl) SetPaused(paused bool) {
+	cc.logf("SetPaused=%v", paused)
+	if paused {
+		cc.called("pause")
+	} else {
+		cc.called("unpause")
+	}
+}
+
+func (cc *mockControl) AuthCantContinue() bool {
+	cc.mu.Lock()
+	defer cc.mu.Unlock()
+
+	return cc.authBlocked
+}
+
+func (cc *mockControl) SetHostinfo(hi *tailcfg.Hostinfo) {
+	cc.logf("SetHostinfo: %v", *hi)
+	cc.called("SetHostinfo")
+}
+
+func (cc *mockControl) SetNetInfo(ni *tailcfg.NetInfo) {
+	cc.called("SetNetinfo")
+	cc.logf("SetNetInfo: %v", *ni)
+	cc.called("SetNetInfo")
+}
+
+func (cc *mockControl) UpdateEndpoints(localPort uint16, endpoints []tailcfg.Endpoint) {
+	// validate endpoint information here?
+	cc.logf("UpdateEndpoints: lp=%v ep=%v", localPort, endpoints)
+	cc.called("UpdateEndpoints")
+}
+
+// A very precise test of the sequence of function calls generated by
+// ipnlocal.Local into its controlclient instance, and the events it
+// produces upstream into the UI.
+//
+// [apenwarr] Normally I'm not a fan of "mock" style tests, but the precise
+// sequence of this state machine is so important for writing our multiple
+// frontends, that it's worth validating it all in one place.
+//
+// Any changes that affect this test will most likely require carefully
+// re-testing all our GUIs (and the CLI) to make sure we didn't break
+// anything.
+//
+// Note also that this test doesn't have any timers, goroutines, or duplicate
+// detection. It expects messages to be produced in exactly the right order,
+// with no duplicates, without doing network activity (other than through
+// controlclient, which we fake, so there's no network activity there either).
+//
+// TODO: A few messages that depend on magicsock (which actually might have
+// network delays) are just ignored for now, which makes the test
+// predictable, but maybe a bit less thorough. This is more of an overall
+// state machine test than a test of the wgengine+magicsock integration.
+func TestStateMachine(t *testing.T) {
+	c := qt.New(t)
+
+	logf := t.Logf
+	store := new(ipn.MemoryStore)
+	e, err := wgengine.NewFakeUserspaceEngine(logf, 0)
+	if err != nil {
+		t.Fatalf("NewFakeUserspaceEngine: %v", err)
+	}
+
+	cc := newMockControl()
+	b, err := NewLocalBackend(logf, "logid", store, e)
+	if err != nil {
+		t.Fatalf("NewLocalBackend: %v", err)
+	}
+	b.SetControlClientGetterForTesting(func(opts controlclient.Options) (controlclient.Client, error) {
+		cc.mu.Lock()
+		cc.opts = opts
+		cc.logf = opts.Logf
+		cc.authBlocked = true
+		cc.persist = cc.opts.Persist
+		cc.mu.Unlock()
+
+		cc.logf("ccGen: new mockControl.")
+		cc.called("New")
+		return cc, nil
+	})
+
+	notifies := &notifyThrottler{t: t}
+	notifies.expect(0)
+
+	b.SetNotifyCallback(func(n ipn.Notify) {
+		if n.State != nil ||
+			n.Prefs != nil ||
+			n.BrowseToURL != nil ||
+			n.LoginFinished != nil {
+			logf("\n%v\n\n", n)
+			notifies.put(n)
+		} else {
+			logf("\n(ignored) %v\n\n", n)
+		}
+	})
+
+	// Check that it hasn't called us right away.
+	// The state machine should be idle until we call Start().
+	c.Assert(cc.getCalls(), qt.HasLen, 0)
+
+	// Start the state machine.
+	// Since !WantRunning by default, it'll create a controlclient,
+	// but not ask it to do anything yet.
+	t.Logf("\n\nStart")
+	notifies.expect(2)
+	c.Assert(b.Start(ipn.Options{StateKey: ipn.GlobalDaemonStateKey}), qt.IsNil)
+	{
+		// BUG: strictly, it should pause, not unpause, here, since !WantRunning.
+		c.Assert([]string{"New", "unpause"}, qt.DeepEquals, cc.getCalls())
+
+		nn := notifies.drain(2)
+		c.Assert(cc.getCalls(), qt.HasLen, 0)
+		c.Assert(nn[0].Prefs, qt.Not(qt.IsNil))
+		c.Assert(nn[1].State, qt.Not(qt.IsNil))
+		prefs := *nn[0].Prefs
+		// Note: a totally fresh system has Prefs.LoggedOut=false by
+		// default. We are logged out, but not because the user asked
+		// for it, so it doesn't count as Prefs.LoggedOut==true.
+		c.Assert(nn[0].Prefs.LoggedOut, qt.IsFalse)
+		c.Assert(prefs.WantRunning, qt.IsFalse)
+		c.Assert(ipn.NeedsLogin, qt.Equals, *nn[1].State)
+		c.Assert(ipn.NeedsLogin, qt.Equals, b.State())
+	}
+
+	// Restart the state machine.
+	// It's designed to handle frontends coming and going sporadically.
+	// Make the sure the restart not only works, but generates the same
+	// events as the first time, so UIs always know what to expect.
+	t.Logf("\n\nStart2")
+	notifies.expect(2)
+	c.Assert(b.Start(ipn.Options{StateKey: ipn.GlobalDaemonStateKey}), qt.IsNil)
+	{
+		// BUG: strictly, it should pause, not unpause, here, since !WantRunning.
+		c.Assert([]string{"Shutdown", "New", "unpause"}, qt.DeepEquals, cc.getCalls())
+
+		nn := notifies.drain(2)
+		c.Assert(cc.getCalls(), qt.HasLen, 0)
+		c.Assert(nn[0].Prefs, qt.Not(qt.IsNil))
+		c.Assert(nn[1].State, qt.Not(qt.IsNil))
+		c.Assert(nn[0].Prefs.LoggedOut, qt.IsFalse)
+		c.Assert(nn[0].Prefs.WantRunning, qt.IsFalse)
+		c.Assert(ipn.NeedsLogin, qt.Equals, *nn[1].State)
+		c.Assert(ipn.NeedsLogin, qt.Equals, b.State())
+	}
+
+	// Start non-interactive login with no token.
+	// This will ask controlclient to start its own Login() process,
+	// then wait for us to respond.
+	t.Logf("\n\nLogin (noninteractive)")
+	notifies.expect(0)
+	b.Login(nil)
+	{
+		c.Assert(cc.getCalls(), qt.DeepEquals, []string{"Login"})
+		notifies.drain(0)
+		// Note: WantRunning isn't true yet. It'll switch to true
+		// after a successful login finishes.
+		// (This behaviour is needed so that b.Login() won't
+		// start connecting to an old account right away, if one
+		// exists when you launch another login.)
+	}
+
+	// Attempted non-interactive login with no key; indicate that
+	// the user needs to visit a login URL.
+	t.Logf("\n\nLogin (url response)")
+	notifies.expect(1)
+	url1 := "http://localhost:1/1"
+	cc.send(nil, url1, false, nil)
+	{
+		c.Assert(cc.getCalls(), qt.DeepEquals, []string{})
+
+		// ...but backend eats that notification, because the user
+		// didn't explicitly request interactive login yet, and
+		// we're already in NeedsLogin state.
+		nn := notifies.drain(1)
+
+		c.Assert(nn[0].Prefs, qt.Not(qt.IsNil))
+		c.Assert(nn[0].Prefs.LoggedOut, qt.IsFalse)
+		c.Assert(nn[0].Prefs.WantRunning, qt.IsFalse)
+	}
+
+	// Now we'll try an interactive login.
+	// Since we provided an interactive URL earlier, this shouldn't
+	// ask control to do anything. Instead backend will emit an event
+	// indicating that the UI should browse to the given URL.
+	t.Logf("\n\nLogin (interactive)")
+	notifies.expect(1)
+	b.StartLoginInteractive()
+	{
+		nn := notifies.drain(1)
+		// BUG: UpdateEndpoints shouldn't be called yet.
+		// We're still not logged in so there's nothing we can do
+		// with it. (And empirically, it's providing an empty list
+		// of endpoints.)
+		c.Assert([]string{"UpdateEndpoints"}, qt.DeepEquals, cc.getCalls())
+		c.Assert(nn[0].BrowseToURL, qt.Not(qt.IsNil))
+		c.Assert(url1, qt.Equals, *nn[0].BrowseToURL)
+	}
+
+	// Sometimes users press the Login button again, in the middle of
+	// a login sequence. For example, they might have closed their
+	// browser window without logging in, or they waited too long and
+	// the login URL expired. If they start another interactive login,
+	// we must always get a *new* login URL first.
+	t.Logf("\n\nLogin2 (interactive)")
+	notifies.expect(0)
+	b.StartLoginInteractive()
+	{
+		notifies.drain(0)
+		// backend asks control for another login sequence
+		c.Assert([]string{"Login"}, qt.DeepEquals, cc.getCalls())
+	}
+
+	// Provide a new interactive login URL.
+	t.Logf("\n\nLogin2 (url response)")
+	notifies.expect(1)
+	url2 := "http://localhost:1/2"
+	cc.send(nil, url2, false, nil)
+	{
+		// BUG: UpdateEndpoints again, this is getting silly.
+		c.Assert([]string{"UpdateEndpoints"}, qt.DeepEquals, cc.getCalls())
+
+		// This time, backend should emit it to the UI right away,
+		// because the UI is anxiously awaiting a new URL to visit.
+		nn := notifies.drain(1)
+		c.Assert(nn[0].BrowseToURL, qt.Not(qt.IsNil))
+		c.Assert(url2, qt.Equals, *nn[0].BrowseToURL)
+	}
+
+	// Pretend that the interactive login actually happened.
+	// Controlclient always sends the netmap and LoginFinished at the
+	// same time.
+	// The backend should propagate this upward for the UI.
+	t.Logf("\n\nLoginFinished")
+	notifies.expect(3)
+	cc.setAuthBlocked(false)
+	cc.persist.LoginName = "user1"
+	cc.send(nil, "", true, &netmap.NetworkMap{})
+	{
+		nn := notifies.drain(3)
+		// BUG: still too soon for UpdateEndpoints.
+		//
+		// Arguably it makes sense to unpause now, since the machine
+		// authorization status is part of the netmap.
+		//
+		// BUG: backend unblocks wgengine at this point, even though
+		// our machine key is not authorized. It probably should
+		// wait until it gets into Starting.
+		// TODO: (Currently this test doesn't detect that bug, but
+		// it's visible in the logs)
+		c.Assert([]string{"unpause", "UpdateEndpoints"}, qt.DeepEquals, cc.getCalls())
+		c.Assert(nn[0].LoginFinished, qt.Not(qt.IsNil))
+		c.Assert(nn[1].Prefs, qt.Not(qt.IsNil))
+		c.Assert(nn[2].State, qt.Not(qt.IsNil))
+		c.Assert(nn[1].Prefs.Persist.LoginName, qt.Equals, "user1")
+		c.Assert(ipn.NeedsMachineAuth, qt.Equals, *nn[2].State)
+	}
+
+	// Pretend that the administrator has authorized our machine.
+	t.Logf("\n\nMachineAuthorized")
+	notifies.expect(1)
+	// BUG: the real controlclient sends LoginFinished with every
+	// notification while it's in StateAuthenticated, but not StateSynced.
+	// We should send it exactly once, or every time we're authenticated,
+	// but the current code is brittle.
+	// (ie. I suspect it would be better to change false->true in send()
+	// below, and do the same in the real controlclient.)
+	cc.send(nil, "", false, &netmap.NetworkMap{
+		MachineStatus: tailcfg.MachineAuthorized,
+	})
+	{
+		nn := notifies.drain(1)
+		c.Assert([]string{"unpause", "UpdateEndpoints"}, qt.DeepEquals, cc.getCalls())
+		c.Assert(nn[0].State, qt.Not(qt.IsNil))
+		c.Assert(ipn.Starting, qt.Equals, *nn[0].State)
+	}
+
+	// TODO: add a fake DERP server to our fake netmap, so we can
+	// transition to the Running state here.
+
+	// TODO: test what happens when the admin forcibly deletes our key.
+	// (ie. unsolicited logout)
+
+	// TODO: test what happens when our key expires, client side.
+	// (and when it gets close to expiring)
+
+	// The user changes their preference to !WantRunning.
+	t.Logf("\n\nWantRunning -> false")
+	notifies.expect(2)
+	b.EditPrefs(&ipn.MaskedPrefs{
+		WantRunningSet: true,
+		Prefs:          ipn.Prefs{WantRunning: false},
+	})
+	{
+		nn := notifies.drain(2)
+		c.Assert([]string{"pause"}, qt.DeepEquals, cc.getCalls())
+		// BUG: I would expect Prefs to change first, and state after.
+		c.Assert(nn[0].State, qt.Not(qt.IsNil))
+		c.Assert(nn[1].Prefs, qt.Not(qt.IsNil))
+		c.Assert(ipn.Stopped, qt.Equals, *nn[0].State)
+	}
+
+	// The user changes their preference to WantRunning after all.
+	t.Logf("\n\nWantRunning -> true")
+	notifies.expect(2)
+	b.EditPrefs(&ipn.MaskedPrefs{
+		WantRunningSet: true,
+		Prefs:          ipn.Prefs{WantRunning: true},
+	})
+	{
+		nn := notifies.drain(2)
+		// BUG: UpdateEndpoints isn't needed here.
+		// BUG: Login isn't needed here. We never logged out.
+		c.Assert([]string{"Login", "unpause", "UpdateEndpoints"}, qt.DeepEquals, cc.getCalls())
+		// BUG: I would expect Prefs to change first, and state after.
+		c.Assert(nn[0].State, qt.Not(qt.IsNil))
+		c.Assert(nn[1].Prefs, qt.Not(qt.IsNil))
+		c.Assert(ipn.Starting, qt.Equals, *nn[0].State)
+	}
+
+	// Test the fast-path frontend reconnection.
+	// This one is very finicky, so we have to force State==Running.
+	// TODO: actually get to State==Running, rather than cheating.
+	//  That'll require spinning up a fake DERP server and putting it in
+	//  the netmap.
+	t.Logf("\n\nFastpath Start()")
+	notifies.expect(1)
+	b.state = ipn.Running
+	c.Assert(b.Start(ipn.Options{StateKey: ipn.GlobalDaemonStateKey}), qt.IsNil)
+	{
+		nn := notifies.drain(1)
+		c.Assert(cc.getCalls(), qt.HasLen, 0)
+		c.Assert(nn[0].State, qt.Not(qt.IsNil))
+		c.Assert(nn[0].LoginFinished, qt.Not(qt.IsNil))
+		c.Assert(nn[0].NetMap, qt.Not(qt.IsNil))
+		// BUG: Prefs should be sent too, or the UI could end up in
+		// a bad state. (iOS, the only current user of this feature,
+		// probably wouldn't notice because it happens to not display
+		// any prefs. Maybe exit nodes will look weird?)
+	}
+
+	// undo the state hack above.
+	b.state = ipn.Starting
+
+	// User wants to logout.
+	t.Logf("\n\nLogout (async)")
+	notifies.expect(2)
+	b.Logout()
+	{
+		nn := notifies.drain(2)
+		// BUG: now is not the time to unpause.
+		c.Assert([]string{"unpause", "StartLogout"}, qt.DeepEquals, cc.getCalls())
+		c.Assert(nn[0].State, qt.Not(qt.IsNil))
+		c.Assert(nn[1].Prefs, qt.Not(qt.IsNil))
+		c.Assert(ipn.NeedsLogin, qt.Equals, *nn[0].State)
+		c.Assert(nn[1].Prefs.LoggedOut, qt.IsTrue)
+		c.Assert(nn[1].Prefs.WantRunning, qt.IsFalse)
+		c.Assert(ipn.NeedsLogin, qt.Equals, b.State())
+	}
+
+	// Let's make the logout succeed.
+	t.Logf("\n\nLogout (async) - succeed")
+	notifies.expect(0)
+	cc.setAuthBlocked(true)
+	cc.send(nil, "", false, nil)
+	{
+		notifies.drain(0)
+		c.Assert(cc.getCalls(), qt.HasLen, 0)
+		c.Assert(b.Prefs().LoggedOut, qt.IsTrue)
+		c.Assert(b.Prefs().WantRunning, qt.IsFalse)
+		c.Assert(ipn.NeedsLogin, qt.Equals, b.State())
+	}
+
+	// A second logout should do nothing, since the prefs haven't changed.
+	t.Logf("\n\nLogout2 (async)")
+	notifies.expect(0)
+	b.Logout()
+	{
+		notifies.drain(0)
+		// BUG: the backend has already called StartLogout, and we're
+		// still logged out. So it shouldn't call it again.
+		c.Assert([]string{"StartLogout"}, qt.DeepEquals, cc.getCalls())
+		c.Assert(cc.getCalls(), qt.HasLen, 0)
+		c.Assert(b.Prefs().LoggedOut, qt.IsTrue)
+		c.Assert(b.Prefs().WantRunning, qt.IsFalse)
+		c.Assert(ipn.NeedsLogin, qt.Equals, b.State())
+	}
+
+	// Let's acknowledge the second logout too.
+	t.Logf("\n\nLogout2 (async) - succeed")
+	notifies.expect(0)
+	cc.setAuthBlocked(true)
+	cc.send(nil, "", false, nil)
+	{
+		notifies.drain(0)
+		c.Assert(cc.getCalls(), qt.HasLen, 0)
+		c.Assert(cc.getCalls(), qt.HasLen, 0)
+		c.Assert(b.Prefs().LoggedOut, qt.IsTrue)
+		c.Assert(b.Prefs().WantRunning, qt.IsFalse)
+		c.Assert(ipn.NeedsLogin, qt.Equals, b.State())
+	}
+
+	// Try the synchronous logout feature.
+	t.Logf("\n\nLogout3 (sync)")
+	notifies.expect(0)
+	b.LogoutSync(context.Background())
+	// NOTE: This returns as soon as cc.Logout() returns, which is okay
+	// I guess, since that's supposed to be synchronous.
+	{
+		notifies.drain(0)
+		c.Assert([]string{"Logout"}, qt.DeepEquals, cc.getCalls())
+		c.Assert(cc.getCalls(), qt.HasLen, 0)
+		c.Assert(b.Prefs().LoggedOut, qt.IsTrue)
+		c.Assert(b.Prefs().WantRunning, qt.IsFalse)
+		c.Assert(ipn.NeedsLogin, qt.Equals, b.State())
+	}
+
+	// Generate the third logout event.
+	t.Logf("\n\nLogout3 (sync) - succeed")
+	notifies.expect(0)
+	cc.setAuthBlocked(true)
+	cc.send(nil, "", false, nil)
+	{
+		notifies.drain(0)
+		c.Assert(cc.getCalls(), qt.HasLen, 0)
+		c.Assert(cc.getCalls(), qt.HasLen, 0)
+		c.Assert(b.Prefs().LoggedOut, qt.IsTrue)
+		c.Assert(b.Prefs().WantRunning, qt.IsFalse)
+		c.Assert(ipn.NeedsLogin, qt.Equals, b.State())
+	}
+
+	// Shut down the backend.
+	t.Logf("\n\nShutdown")
+	notifies.expect(0)
+	b.Shutdown()
+	{
+		notifies.drain(0)
+		// BUG: I expect a transition to ipn.NoState here.
+		c.Assert(cc.getCalls(), qt.DeepEquals, []string{"Shutdown"})
+	}
+
+	// Oh, you thought we were done? Ha! Now we have to test what
+	// happens if the user exits and restarts while logged out.
+	// Note that it's explicitly okay to call b.Start() over and over
+	// again, every time the frontend reconnects.
+
+	// TODO: test user switching between statekeys.
+
+	// The frontend restarts!
+	t.Logf("\n\nStart3")
+	notifies.expect(2)
+	c.Assert(b.Start(ipn.Options{StateKey: ipn.GlobalDaemonStateKey}), qt.IsNil)
+	{
+		// BUG: We already called Shutdown(), no need to do it again.
+		// BUG: Way too soon for UpdateEndpoints.
+		// BUG: don't unpause because we're not logged in.
+		c.Assert([]string{"Shutdown", "New", "UpdateEndpoints", "unpause"}, qt.DeepEquals, cc.getCalls())
+
+		nn := notifies.drain(2)
+		c.Assert(cc.getCalls(), qt.HasLen, 0)
+		c.Assert(nn[0].Prefs, qt.Not(qt.IsNil))
+		c.Assert(nn[1].State, qt.Not(qt.IsNil))
+		c.Assert(nn[0].Prefs.LoggedOut, qt.IsTrue)
+		c.Assert(nn[0].Prefs.WantRunning, qt.IsFalse)
+		c.Assert(ipn.NeedsLogin, qt.Equals, *nn[1].State)
+		c.Assert(ipn.NeedsLogin, qt.Equals, b.State())
+	}
+
+	// Let's break the rules a little. Our control server accepts
+	// your invalid login attempt, with no need for an interactive login.
+	// (This simulates an admin reviving a key that you previously
+	// disabled.)
+	t.Logf("\n\nLoginFinished3")
+	notifies.expect(3)
+	cc.setAuthBlocked(false)
+	cc.persist.LoginName = "user2"
+	cc.send(nil, "", true, &netmap.NetworkMap{
+		MachineStatus: tailcfg.MachineAuthorized,
+	})
+	{
+		nn := notifies.drain(3)
+		c.Assert([]string{"unpause"}, qt.DeepEquals, cc.getCalls())
+		c.Assert(nn[0].LoginFinished, qt.Not(qt.IsNil))
+		c.Assert(nn[1].Prefs, qt.Not(qt.IsNil))
+		c.Assert(nn[2].State, qt.Not(qt.IsNil))
+		// Prefs after finishing the login, so LoginName updated.
+		c.Assert(nn[1].Prefs.Persist.LoginName, qt.Equals, "user2")
+		c.Assert(nn[1].Prefs.LoggedOut, qt.IsFalse)
+		c.Assert(nn[1].Prefs.WantRunning, qt.IsTrue)
+		c.Assert(ipn.Starting, qt.Equals, *nn[2].State)
+	}
+
+	// Now we've logged in successfully. Let's disconnect.
+	t.Logf("\n\nWantRunning -> false")
+	notifies.expect(2)
+	b.EditPrefs(&ipn.MaskedPrefs{
+		WantRunningSet: true,
+		Prefs:          ipn.Prefs{WantRunning: false},
+	})
+	{
+		nn := notifies.drain(2)
+		c.Assert([]string{"pause"}, qt.DeepEquals, cc.getCalls())
+		// BUG: I would expect Prefs to change first, and state after.
+		c.Assert(nn[0].State, qt.Not(qt.IsNil))
+		c.Assert(nn[1].Prefs, qt.Not(qt.IsNil))
+		c.Assert(ipn.Stopped, qt.Equals, *nn[0].State)
+		c.Assert(nn[1].Prefs.LoggedOut, qt.IsFalse)
+	}
+
+	// One more restart, this time with a valid key, but WantRunning=false.
+	t.Logf("\n\nStart4")
+	notifies.expect(2)
+	c.Assert(b.Start(ipn.Options{StateKey: ipn.GlobalDaemonStateKey}), qt.IsNil)
+	{
+		// NOTE: cc.Shutdown() is correct here, since we didn't call
+		// b.Shutdown() explicitly ourselves.
+		// BUG: UpdateEndpoints should be called here since we're not WantRunning.
+		// Note: unpause happens because ipn needs to get at least one netmap
+		//  on startup, otherwise UIs can't show the node list, login
+		//  name, etc when in state ipn.Stopped.
+		//  Arguably they shouldn't try. But they currently do.
+		c.Assert([]string{"Shutdown", "New", "UpdateEndpoints", "Login", "unpause"}, qt.DeepEquals, cc.getCalls())
+
+		nn := notifies.drain(2)
+		c.Assert(cc.getCalls(), qt.HasLen, 0)
+		c.Assert(nn[0].Prefs, qt.Not(qt.IsNil))
+		c.Assert(nn[1].State, qt.Not(qt.IsNil))
+		c.Assert(nn[0].Prefs.WantRunning, qt.IsFalse)
+		c.Assert(nn[0].Prefs.LoggedOut, qt.IsFalse)
+		c.Assert(ipn.Stopped, qt.Equals, *nn[1].State)
+	}
+
+	// Request connection.
+	// The state machine didn't call Login() earlier, so now it needs to.
+	t.Logf("\n\nWantRunning4 -> true")
+	notifies.expect(2)
+	b.EditPrefs(&ipn.MaskedPrefs{
+		WantRunningSet: true,
+		Prefs:          ipn.Prefs{WantRunning: true},
+	})
+	{
+		nn := notifies.drain(2)
+		c.Assert([]string{"Login", "unpause"}, qt.DeepEquals, cc.getCalls())
+		// BUG: I would expect Prefs to change first, and state after.
+		c.Assert(nn[0].State, qt.Not(qt.IsNil))
+		c.Assert(nn[1].Prefs, qt.Not(qt.IsNil))
+		c.Assert(ipn.Starting, qt.Equals, *nn[0].State)
+	}
+
+	// Disconnect.
+	t.Logf("\n\nStop")
+	notifies.expect(2)
+	b.EditPrefs(&ipn.MaskedPrefs{
+		WantRunningSet: true,
+		Prefs:          ipn.Prefs{WantRunning: false},
+	})
+	{
+		nn := notifies.drain(2)
+		c.Assert([]string{"unpause"}, qt.DeepEquals, cc.getCalls())
+		// BUG: I would expect Prefs to change first, and state after.
+		c.Assert(nn[0].State, qt.Not(qt.IsNil))
+		c.Assert(nn[1].Prefs, qt.Not(qt.IsNil))
+		c.Assert(ipn.Stopped, qt.Equals, *nn[0].State)
+	}
+
+	// We want to try logging in as a different user, while Stopped.
+	// First, start the login process (without logging out first).
+	t.Logf("\n\nLoginDifferent")
+	notifies.expect(2)
+	b.StartLoginInteractive()
+	url3 := "http://localhost:1/3"
+	cc.send(nil, url3, false, nil)
+	{
+		nn := notifies.drain(2)
+		// It might seem like WantRunning should switch to true here,
+		// but that would be risky since we already have a valid
+		// user account. It might try to reconnect to the old account
+		// before the new one is ready. So no change yet.
+		c.Assert([]string{"Login", "unpause"}, qt.DeepEquals, cc.getCalls())
+		c.Assert(nn[0].BrowseToURL, qt.Not(qt.IsNil))
+		c.Assert(nn[1].State, qt.Not(qt.IsNil))
+		c.Assert(*nn[0].BrowseToURL, qt.Equals, url3)
+		c.Assert(ipn.NeedsLogin, qt.Equals, *nn[1].State)
+	}
+
+	// Now, let's say the interactive login completed, using a different
+	// user account than before.
+	t.Logf("\n\nLoginDifferent URL visited")
+	notifies.expect(3)
+	cc.persist.LoginName = "user3"
+	cc.send(nil, "", true, &netmap.NetworkMap{
+		MachineStatus: tailcfg.MachineAuthorized,
+	})
+	{
+		nn := notifies.drain(3)
+		c.Assert([]string{"unpause"}, qt.DeepEquals, cc.getCalls())
+		c.Assert(nn[0].LoginFinished, qt.Not(qt.IsNil))
+		c.Assert(nn[1].Prefs, qt.Not(qt.IsNil))
+		c.Assert(nn[2].State, qt.Not(qt.IsNil))
+		// Prefs after finishing the login, so LoginName updated.
+		c.Assert(nn[1].Prefs.Persist.LoginName, qt.Equals, "user3")
+		c.Assert(nn[1].Prefs.LoggedOut, qt.IsFalse)
+		c.Assert(nn[1].Prefs.WantRunning, qt.IsTrue)
+		c.Assert(ipn.Starting, qt.Equals, *nn[2].State)
+	}
+
+	// The last test case is the most common one: restarting when both
+	// logged in and WantRunning.
+	t.Logf("\n\nStart5")
+	notifies.expect(1)
+	c.Assert(b.Start(ipn.Options{StateKey: ipn.GlobalDaemonStateKey}), qt.IsNil)
+	{
+		// NOTE: cc.Shutdown() is correct here, since we didn't call
+		// b.Shutdown() ourselves.
+		c.Assert([]string{"Shutdown", "New", "UpdateEndpoints", "Login"}, qt.DeepEquals, cc.getCalls())
+
+		nn := notifies.drain(1)
+		c.Assert(cc.getCalls(), qt.HasLen, 0)
+		c.Assert(nn[0].Prefs, qt.Not(qt.IsNil))
+		c.Assert(nn[0].Prefs.LoggedOut, qt.IsFalse)
+		c.Assert(nn[0].Prefs.WantRunning, qt.IsTrue)
+		c.Assert(ipn.NoState, qt.Equals, b.State())
+	}
+
+	// Control server accepts our valid key from before.
+	t.Logf("\n\nLoginFinished5")
+	notifies.expect(1)
+	cc.setAuthBlocked(false)
+	cc.send(nil, "", true, &netmap.NetworkMap{
+		MachineStatus: tailcfg.MachineAuthorized,
+	})
+	{
+		nn := notifies.drain(1)
+		c.Assert([]string{"unpause"}, qt.DeepEquals, cc.getCalls())
+		// NOTE: No LoginFinished message since no interactive
+		// login was needed.
+		c.Assert(nn[0].State, qt.Not(qt.IsNil))
+		c.Assert(ipn.Starting, qt.Equals, *nn[0].State)
+		// NOTE: No prefs change this time. WantRunning stays true.
+		// We were in Starting in the first place, so that doesn't
+		// change either.
+		c.Assert(ipn.Starting, qt.Equals, b.State())
+	}
+}

ipn/ipnserver/server.go

@@ -6,7 +6,9 @@ package ipnserver
 
 import (
 	"bufio"
+	"bytes"
 	"context"
+	"encoding/json"
 	"errors"
 	"fmt"
 	"io"
@@ -63,16 +65,6 @@ type Options struct {
 	// waits for a frontend to start it.
 	AutostartStateKey ipn.StateKey
 
-	// LegacyConfigPath optionally specifies the old-style relaynode
-	// relay.conf location. If both LegacyConfigPath and
-	// AutostartStateKey are specified and the requested state doesn't
-	// exist in the backend store, the backend migrates the config
-	// from LegacyConfigPath.
-	//
-	// TODO(danderson): remove some time after the transition to
-	// tailscaled is done.
-	LegacyConfigPath string
-
 	// SurviveDisconnects specifies how the server reacts to its
 	// frontend disconnecting. If true, the server keeps running on
 	// its existing state, and accepts new frontend connections. If
@@ -155,7 +147,7 @@ func (s *server) getConnIdentity(c net.Conn) (ci connIdentity, err error) {
 	if err != nil {
 		return ci, fmt.Errorf("parsing local remote: %w", err)
 	}
-	if !la.IP.IsLoopback() || !ra.IP.IsLoopback() {
+	if !la.IP().IsLoopback() || !ra.IP().IsLoopback() {
 		return ci, errors.New("non-loopback connection")
 	}
 	tab, err := netstat.Get()
@@ -261,8 +253,7 @@ func (s *server) serveConn(ctx context.Context, c net.Conn, logf logger.Logf) {
 			return
 		}
 		defer c.Close()
-		serverToClient := func(b []byte) { ipn.WriteMsg(c, b) }
-		bs := ipn.NewBackendServer(logf, nil, serverToClient)
+		bs := ipn.NewBackendServer(logf, nil, jsonNotifier(c, s.logf))
 		_, occupied := err.(inUseOtherUserError)
 		if occupied {
 			bs.SendInUseOtherUserErrorMessage(err.Error())
@@ -297,7 +288,7 @@ func (s *server) serveConn(ctx context.Context, c net.Conn, logf logger.Logf) {
 	defer s.removeAndCloseConn(c)
 	logf("[v1] incoming control connection")
 
-	if isReadonlyConn(ci, logf) {
+	if isReadonlyConn(ci, s.b.OperatorUserID(), logf) {
 		ctx = ipn.ReadonlyContextOf(ctx)
 	}
 
@@ -323,7 +314,7 @@ func (s *server) serveConn(ctx context.Context, c net.Conn, logf logger.Logf) {
 	}
 }
 
-func isReadonlyConn(ci connIdentity, logf logger.Logf) bool {
+func isReadonlyConn(ci connIdentity, operatorUID string, logf logger.Logf) bool {
 	if runtime.GOOS == "windows" {
 		// Windows doesn't need/use this mechanism, at least yet. It
 		// has a different last-user-wins auth model.
@@ -352,6 +343,10 @@ func isReadonlyConn(ci connIdentity, logf logger.Logf) bool {
 		logf("connection from userid %v; connection from non-root user matching daemon has access", uid)
 		return rw
 	}
+	if operatorUID != "" && uid == operatorUID {
+		logf("connection from userid %v; is configured operator", uid)
+		return rw
+	}
 	var adminGroupID string
 	switch runtime.GOOS {
 	case "darwin":
@@ -445,7 +440,7 @@ func (s *server) localAPIPermissions(ci connIdentity) (read, write bool) {
 		return false, false
 	}
 	if ci.IsUnixSock {
-		return true, !isReadonlyConn(ci, logger.Discard)
+		return true, !isReadonlyConn(ci, s.b.OperatorUserID(), logger.Discard)
 	}
 	return false, false
 }
@@ -482,9 +477,7 @@ func (s *server) addConn(c net.Conn, isHTTP bool) (ci connIdentity, err error) {
 	defer func() {
 		if doReset {
 			s.logf("identity changed; resetting server")
-			s.bsMu.Lock()
-			s.bs.Reset(context.TODO())
-			s.bsMu.Unlock()
+			s.b.ResetForClientDisconnect()
 		}
 	}()
 
@@ -534,9 +527,7 @@ func (s *server) removeAndCloseConn(c net.Conn) {
 			s.logf("client disconnected; staying alive in server mode")
 		} else {
 			s.logf("client disconnected; stopping server")
-			s.bsMu.Lock()
-			s.bs.Reset(context.TODO())
-			s.bsMu.Unlock()
+			s.b.ResetForClientDisconnect()
 		}
 	}
 	c.Close()
@@ -577,7 +568,9 @@ func (s *server) setServerModeUserLocked() {
 	}
 }
 
-func (s *server) writeToClients(b []byte) {
+var jsonEscapedZero = []byte(`\u0000`)
+
+func (s *server) writeToClients(n ipn.Notify) {
 	inServerMode := s.b.InServerMode()
 
 	s.mu.Lock()
@@ -594,14 +587,24 @@ func (s *server) writeToClients(b []byte) {
 		}
 	}
 
-	for c := range s.clients {
-		ipn.WriteMsg(c, b)
+	if len(s.clients) == 0 {
+		// Common case (at least on busy servers): nobody
+		// connected (no GUI, etc), so return before
+		// serializing JSON.
+		return
+	}
+
+	if b, ok := marshalNotify(n, s.logf); ok {
+		for c := range s.clients {
+			ipn.WriteMsg(c, b)
+		}
 	}
 }
 
 // Run runs a Tailscale backend service.
 // The getEngine func is called repeatedly, once per connection, until it returns an engine successfully.
 func Run(ctx context.Context, logf logger.Logf, logid string, getEngine func() (wgengine.Engine, error), opts Options) error {
+	getEngine = getEngineUntilItWorksWrapper(getEngine)
 	runDone := make(chan struct{})
 	defer close(runDone)
 
@@ -662,46 +665,6 @@ func Run(ctx context.Context, logf logger.Logf, logid string, getEngine func() (
 	eng, err := getEngine()
 	if err != nil {
 		logf("ipnserver: initial getEngine call: %v", err)
-
-		// Issue 1187: on Windows, in unattended mode,
-		// sometimes we try 5 times and fail to create the
-		// engine before the system's ready. Hack until the
-		// bug if fixed properly: if we're running in
-		// unattended mode on Windows, keep trying forever,
-		// waiting for the machine to be ready (networking to
-		// come up?) and then dial our own safesocket TCP
-		// listener to wake up the usual mechanism that lets
-		// us surface getEngine errors to UI clients. (We
-		// don't want to just call getEngine in a loop without
-		// the listener.Accept, as we do want to handle client
-		// connections so we can tell them about errors)
-
-		bootRaceWaitForEngine, bootRaceWaitForEngineCancel := context.WithTimeout(context.Background(), time.Minute)
-		if runtime.GOOS == "windows" && opts.AutostartStateKey != "" {
-			logf("ipnserver: in unattended mode, waiting for engine availability")
-			getEngine = getEngineUntilItWorksWrapper(getEngine)
-			// Wait for it to be ready.
-			go func() {
-				defer bootRaceWaitForEngineCancel()
-				t0 := time.Now()
-				for {
-					time.Sleep(10 * time.Second)
-					if _, err := getEngine(); err != nil {
-						logf("ipnserver: unattended mode engine load: %v", err)
-						continue
-					}
-					c, err := net.Dial("tcp", listen.Addr().String())
-					logf("ipnserver: engine created after %v; waking up Accept: Dial error: %v", time.Since(t0).Round(time.Second), err)
-					if err == nil {
-						c.Close()
-					}
-					break
-				}
-			}()
-		} else {
-			bootRaceWaitForEngineCancel()
-		}
-
 		for i := 1; ctx.Err() == nil; i++ {
 			c, err := listen.Accept()
 			if err != nil {
@@ -709,7 +672,6 @@ func Run(ctx context.Context, logf logger.Logf, logid string, getEngine func() (
 				bo.BackOff(ctx, err)
 				continue
 			}
-			<-bootRaceWaitForEngine.Done()
 			logf("ipnserver: try%d: trying getEngine again...", i)
 			eng, err = getEngine()
 			if err == nil {
@@ -721,8 +683,7 @@ func Run(ctx context.Context, logf logger.Logf, logid string, getEngine func() (
 			errMsg := err.Error()
 			go func() {
 				defer c.Close()
-				serverToClient := func(b []byte) { ipn.WriteMsg(c, b) }
-				bs := ipn.NewBackendServer(logf, nil, serverToClient)
+				bs := ipn.NewBackendServer(logf, nil, jsonNotifier(c, logf))
 				bs.SendErrorMessage(errMsg)
 				time.Sleep(time.Second)
 			}()
@@ -754,10 +715,7 @@ func Run(ctx context.Context, logf logger.Logf, logid string, getEngine func() (
 		server.bs.GotCommand(context.TODO(), &ipn.Command{
 			Version: version.Long,
 			Start: &ipn.StartArgs{
-				Opts: ipn.Options{
-					StateKey:         opts.AutostartStateKey,
-					LegacyConfigPath: opts.LegacyConfigPath,
-				},
+				Opts: ipn.Options{StateKey: opts.AutostartStateKey},
 			},
 		})
 	}
@@ -1015,3 +973,25 @@ func peerPid(entries []netstat.Entry, la, ra netaddr.IPPort) int {
 	}
 	return 0
 }
+
+// jsonNotifier returns a notify-writer func that writes ipn.Notify
+// messages to w.
+func jsonNotifier(w io.Writer, logf logger.Logf) func(ipn.Notify) {
+	return func(n ipn.Notify) {
+		if b, ok := marshalNotify(n, logf); ok {
+			ipn.WriteMsg(w, b)
+		}
+	}
+}
+
+func marshalNotify(n ipn.Notify, logf logger.Logf) (b []byte, ok bool) {
+	b, err := json.Marshal(n)
+	if err != nil {
+		logf("ipnserver: [unexpected] error serializing JSON: %v", err)
+		return nil, false
+	}
+	if bytes.Contains(b, jsonEscapedZero) {
+		logf("[unexpected] zero byte in BackendServer.send notify message: %q", b)
+	}
+	return b, true
+}

ipn/ipnstate/ipnstate.go

@@ -65,13 +65,15 @@ type PeerStatusLite struct {
 }
 
 type PeerStatus struct {
+	ID        tailcfg.StableNodeID
 	PublicKey key.Public
 	HostName  string // HostInfo's Hostname (not a DNS name or necessarily unique)
 	DNSName   string
 	OS        string // HostInfo.OS
 	UserID    tailcfg.UserID
 
-	TailAddr string // Tailscale IP
+	TailAddrDeprecated string       `json:"TailAddr"` // Tailscale IP
+	TailscaleIPs       []netaddr.IP // Tailscale IP(s) assigned to this node
 
 	// Endpoints:
 	Addrs   []string
@@ -87,7 +89,8 @@ type PeerStatus struct {
 	KeepAlive     bool
 	ExitNode      bool // true if this is the currently selected exit node.
 
-	PeerAPIURL []string
+	PeerAPIURL   []string
+	Capabilities []string `json:",omitempty"`
 
 	// ShareeNode indicates this node exists in the netmap because
 	// it's owned by a shared-to user and that node might connect
@@ -201,6 +204,9 @@ func (sb *StatusBuilder) AddPeer(peer key.Public, st *PeerStatus) {
 		return
 	}
 
+	if v := st.ID; v != "" {
+		e.ID = v
+	}
 	if v := st.HostName; v != "" {
 		e.HostName = v
 	}
@@ -213,8 +219,11 @@ func (sb *StatusBuilder) AddPeer(peer key.Public, st *PeerStatus) {
 	if v := st.UserID; v != 0 {
 		e.UserID = v
 	}
-	if v := st.TailAddr; v != "" {
-		e.TailAddr = v
+	if v := st.TailAddrDeprecated; v != "" {
+		e.TailAddrDeprecated = v
+	}
+	if v := st.TailscaleIPs; v != nil {
+		e.TailscaleIPs = v
 	}
 	if v := st.OS; v != "" {
 		e.OS = st.OS
@@ -343,13 +352,17 @@ table tbody tr:nth-child(even) td { background-color: #f5f5f5; }
 			hostNameHTML = "<br>" + html.EscapeString(hostName)
 		}
 
+		var tailAddr string
+		if len(ps.TailscaleIPs) > 0 {
+			tailAddr = ps.TailscaleIPs[0].String()
+		}
 		f("<tr><td>%s</td><td class=acenter>%s</td>"+
 			"<td><b>%s</b>%s<div class=\"tailaddr\">%s</div></td><td class=\"acenter owner\">%s</td><td class=\"aright\">%v</td><td class=\"aright\">%v</td><td class=\"aright\">%v</td>",
 			ps.PublicKey.ShortString(),
 			osEmoji(ps.OS),
 			html.EscapeString(dnsName),
 			hostNameHTML,
-			ps.TailAddr,
+			tailAddr,
 			html.EscapeString(owner),
 			ps.RxBytes,
 			ps.TxBytes,
@@ -437,5 +450,9 @@ func sortKey(ps *PeerStatus) string {
 	if ps.HostName != "" {
 		return ps.HostName
 	}
-	return ps.TailAddr
+	// TODO(bradfitz): add PeerStatus.Less and avoid these allocs in a Less func.
+	if len(ps.TailscaleIPs) > 0 {
+		return ps.TailscaleIPs[0].String()
+	}
+	return string(ps.PublicKey[:])
 }

ipn/localapi/localapi.go

@@ -9,16 +9,23 @@ import (
 	"crypto/rand"
 	"encoding/hex"
 	"encoding/json"
+	"errors"
 	"fmt"
 	"io"
+	"net"
 	"net/http"
+	"net/http/httputil"
 	"net/url"
+	"reflect"
 	"runtime"
 	"strconv"
 	"strings"
+	"sync"
 	"time"
 
 	"inet.af/netaddr"
+	"tailscale.com/client/tailscale/apitype"
+	"tailscale.com/ipn"
 	"tailscale.com/ipn/ipnlocal"
 	"tailscale.com/ipn/ipnstate"
 	"tailscale.com/tailcfg"
@@ -72,6 +79,10 @@ func (h *Handler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
 		h.serveFiles(w, r)
 		return
 	}
+	if strings.HasPrefix(r.URL.Path, "/localapi/v0/file-put/") {
+		h.serveFilePut(w, r)
+		return
+	}
 	switch r.URL.Path {
 	case "/localapi/v0/whois":
 		h.serveWhoIs(w, r)
@@ -79,12 +90,20 @@ func (h *Handler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
 		h.serveGoroutines(w, r)
 	case "/localapi/v0/status":
 		h.serveStatus(w, r)
+	case "/localapi/v0/logout":
+		h.serveLogout(w, r)
+	case "/localapi/v0/prefs":
+		h.servePrefs(w, r)
 	case "/localapi/v0/check-ip-forwarding":
 		h.serveCheckIPForwarding(w, r)
 	case "/localapi/v0/bugreport":
 		h.serveBugReport(w, r)
-	default:
+	case "/localapi/v0/file-targets":
+		h.serveFileTargets(w, r)
+	case "/":
 		io.WriteString(w, "tailscaled\n")
+	default:
+		http.Error(w, "404 not found", 404)
 	}
 }
 
@@ -126,7 +145,7 @@ func (h *Handler) serveWhoIs(w http.ResponseWriter, r *http.Request) {
 		http.Error(w, "no match for IP:port", 404)
 		return
 	}
-	res := &tailcfg.WhoIsResponse{
+	res := &apitype.WhoIsResponse{
 		Node:        n,
 		UserProfile: &u,
 	}
@@ -186,6 +205,58 @@ func (h *Handler) serveStatus(w http.ResponseWriter, r *http.Request) {
 	e.Encode(st)
 }
 
+func (h *Handler) serveLogout(w http.ResponseWriter, r *http.Request) {
+	if !h.PermitWrite {
+		http.Error(w, "logout access denied", http.StatusForbidden)
+		return
+	}
+	if r.Method != "POST" {
+		http.Error(w, "want POST", 400)
+		return
+	}
+	err := h.b.LogoutSync(r.Context())
+	if err == nil {
+		w.WriteHeader(http.StatusNoContent)
+		return
+	}
+	http.Error(w, err.Error(), 500)
+}
+
+func (h *Handler) servePrefs(w http.ResponseWriter, r *http.Request) {
+	if !h.PermitRead {
+		http.Error(w, "prefs access denied", http.StatusForbidden)
+		return
+	}
+	var prefs *ipn.Prefs
+	switch r.Method {
+	case "PATCH":
+		if !h.PermitWrite {
+			http.Error(w, "prefs write access denied", http.StatusForbidden)
+			return
+		}
+		mp := new(ipn.MaskedPrefs)
+		if err := json.NewDecoder(r.Body).Decode(mp); err != nil {
+			http.Error(w, err.Error(), 400)
+			return
+		}
+		var err error
+		prefs, err = h.b.EditPrefs(mp)
+		if err != nil {
+			http.Error(w, err.Error(), 400)
+			return
+		}
+	case "GET", "HEAD":
+		prefs = h.b.Prefs()
+	default:
+		http.Error(w, "unsupported method", http.StatusMethodNotAllowed)
+		return
+	}
+	w.Header().Set("Content-Type", "application/json")
+	e := json.NewEncoder(w)
+	e.SetIndent("", "\t")
+	e.Encode(prefs)
+}
+
 func (h *Handler) serveFiles(w http.ResponseWriter, r *http.Request) {
 	if !h.PermitWrite {
 		http.Error(w, "file access denied", http.StatusForbidden)
@@ -229,6 +300,108 @@ func (h *Handler) serveFiles(w http.ResponseWriter, r *http.Request) {
 	io.Copy(w, rc)
 }
 
+func writeErrorJSON(w http.ResponseWriter, err error) {
+	if err == nil {
+		err = errors.New("unexpected nil error")
+	}
+	w.Header().Set("Content-Type", "application/json")
+	w.WriteHeader(500)
+	type E struct {
+		Error string `json:"error"`
+	}
+	json.NewEncoder(w).Encode(E{err.Error()})
+}
+
+func (h *Handler) serveFileTargets(w http.ResponseWriter, r *http.Request) {
+	if !h.PermitRead {
+		http.Error(w, "access denied", http.StatusForbidden)
+		return
+	}
+	if r.Method != "GET" {
+		http.Error(w, "want GET to list targets", 400)
+		return
+	}
+	fts, err := h.b.FileTargets()
+	if err != nil {
+		writeErrorJSON(w, err)
+		return
+	}
+	makeNonNil(&fts)
+	w.Header().Set("Content-Type", "application/json")
+	json.NewEncoder(w).Encode(fts)
+}
+
+func (h *Handler) serveFilePut(w http.ResponseWriter, r *http.Request) {
+	if !h.PermitWrite {
+		http.Error(w, "file access denied", http.StatusForbidden)
+		return
+	}
+	if r.Method != "PUT" {
+		http.Error(w, "want PUT to put file", 400)
+		return
+	}
+	fts, err := h.b.FileTargets()
+	if err != nil {
+		http.Error(w, err.Error(), 500)
+		return
+	}
+
+	upath := strings.TrimPrefix(r.URL.EscapedPath(), "/localapi/v0/file-put/")
+	slash := strings.Index(upath, "/")
+	if slash == -1 {
+		http.Error(w, "bogus URL", 400)
+		return
+	}
+	stableID, filenameEscaped := tailcfg.StableNodeID(upath[:slash]), upath[slash+1:]
+
+	var ft *apitype.FileTarget
+	for _, x := range fts {
+		if x.Node.StableID == stableID {
+			ft = x
+			break
+		}
+	}
+	if ft == nil {
+		http.Error(w, "node not found", 404)
+		return
+	}
+	dstURL, err := url.Parse(ft.PeerAPIURL)
+	if err != nil {
+		http.Error(w, "bogus peer URL", 500)
+		return
+	}
+	outReq, err := http.NewRequestWithContext(r.Context(), "PUT", "http://peer/v0/put/"+filenameEscaped, r.Body)
+	if err != nil {
+		http.Error(w, "bogus outreq", 500)
+		return
+	}
+	outReq.ContentLength = r.ContentLength
+
+	rp := httputil.NewSingleHostReverseProxy(dstURL)
+	rp.Transport = getDialPeerTransport(h.b)
+	rp.ServeHTTP(w, outReq)
+}
+
+var dialPeerTransportOnce struct {
+	sync.Once
+	v *http.Transport
+}
+
+func getDialPeerTransport(b *ipnlocal.LocalBackend) *http.Transport {
+	dialPeerTransportOnce.Do(func() {
+		t := http.DefaultTransport.(*http.Transport).Clone()
+		t.Dial = nil //lint:ignore SA1019 yes I know I'm setting it to nil defensively
+		dialer := net.Dialer{
+			Timeout:   30 * time.Second,
+			KeepAlive: 30 * time.Second,
+			Control:   b.PeerDialControlFunc(),
+		}
+		t.DialContext = dialer.DialContext
+		dialPeerTransportOnce.v = t
+	})
+	return dialPeerTransportOnce.v
+}
+
 func defBool(a string, def bool) bool {
 	if a == "" {
 		return def
@@ -239,3 +412,30 @@ func defBool(a string, def bool) bool {
 	}
 	return v
 }
+
+// makeNonNil takes a pointer to a Go data structure
+// (currently only a slice or a map) and makes sure it's non-nil for
+// JSON serialization. (In particular, JavaScript clients usually want
+// the field to be defined after they decode the JSON.)
+func makeNonNil(ptr interface{}) {
+	if ptr == nil {
+		panic("nil interface")
+	}
+	rv := reflect.ValueOf(ptr)
+	if rv.Kind() != reflect.Ptr {
+		panic(fmt.Sprintf("kind %v, not Ptr", rv.Kind()))
+	}
+	if rv.Pointer() == 0 {
+		panic("nil pointer")
+	}
+	rv = rv.Elem()
+	if rv.Pointer() != 0 {
+		return
+	}
+	switch rv.Type().Kind() {
+	case reflect.Slice:
+		rv.Set(reflect.MakeSlice(rv.Type(), 0, 0))
+	case reflect.Map:
+		rv.Set(reflect.MakeMap(rv.Type()))
+	}
+}

ipn/message.go

@@ -80,7 +80,6 @@ type Command struct {
 	Login                 *tailcfg.Oauth2Token
 	Logout                *NoArgs
 	SetPrefs              *SetPrefsArgs
-	EditPrefs             *MaskedPrefs
 	RequestEngineStatus   *NoArgs
 	RequestStatus         *NoArgs
 	FakeExpireAfter       *FakeExpireAfterArgs
@@ -89,29 +88,36 @@ type Command struct {
 
 type BackendServer struct {
 	logf          logger.Logf
-	b             Backend              // the Backend we are serving up
-	sendNotifyMsg func(jsonMsg []byte) // send a notification message
-	GotQuit       bool                 // a Quit command was received
+	b             Backend      // the Backend we are serving up
+	sendNotifyMsg func(Notify) // send a notification message
+	GotQuit       bool         // a Quit command was received
 }
 
-func NewBackendServer(logf logger.Logf, b Backend, sendNotifyMsg func(b []byte)) *BackendServer {
-	return &BackendServer{
+// NewBackendServer creates a new BackendServer using b.
+//
+// If sendNotifyMsg is non-nil, it additionally sets the Backend's
+// notification callback to call the func with ipn.Notify messages in
+// JSON form. If nil, it does not change the notification callback.
+func NewBackendServer(logf logger.Logf, b Backend, sendNotifyMsg func(Notify)) *BackendServer {
+	bs := &BackendServer{
 		logf:          logf,
 		b:             b,
 		sendNotifyMsg: sendNotifyMsg,
 	}
+	// b may be nil if the BackendServer is being created just to
+	// encapsulate and send an error message.
+	if sendNotifyMsg != nil && b != nil {
+		b.SetNotifyCallback(bs.send)
+	}
+	return bs
 }
 
 func (bs *BackendServer) send(n Notify) {
+	if bs.sendNotifyMsg == nil {
+		return
+	}
 	n.Version = version.Long
-	b, err := json.Marshal(n)
-	if err != nil {
-		log.Fatalf("Failed json.Marshal(notify): %v\n%#v", err, n)
-	}
-	if bytes.Contains(b, jsonEscapedZero) {
-		log.Printf("[unexpected] zero byte in BackendServer.send notify message: %q", b)
-	}
-	bs.sendNotifyMsg(b)
+	bs.sendNotifyMsg(n)
 }
 
 func (bs *BackendServer) SendErrorMessage(msg string) {
@@ -142,11 +148,6 @@ func (bs *BackendServer) GotCommandMsg(ctx context.Context, b []byte) error {
 	return bs.GotCommand(ctx, cmd)
 }
 
-func (bs *BackendServer) GotFakeCommand(ctx context.Context, cmd *Command) error {
-	cmd.Version = version.Long
-	return bs.GotCommand(ctx, cmd)
-}
-
 // ErrMsgPermissionDenied is the Notify.ErrMessage value used an
 // operation was done from a user/context that didn't have permission.
 const ErrMsgPermissionDenied = "permission denied"
@@ -190,7 +191,6 @@ func (bs *BackendServer) GotCommand(ctx context.Context, cmd *Command) error {
 		return errors.New("Quit command received")
 	} else if c := cmd.Start; c != nil {
 		opts := c.Opts
-		opts.Notify = bs.send
 		return bs.b.Start(opts)
 	} else if c := cmd.StartLoginInteractive; c != nil {
 		bs.b.StartLoginInteractive()
@@ -204,9 +204,6 @@ func (bs *BackendServer) GotCommand(ctx context.Context, cmd *Command) error {
 	} else if c := cmd.SetPrefs; c != nil {
 		bs.b.SetPrefs(c.New)
 		return nil
-	} else if c := cmd.EditPrefs; c != nil {
-		bs.b.EditPrefs(c)
-		return nil
 	} else if c := cmd.FakeExpireAfter; c != nil {
 		bs.b.FakeExpireAfter(c.Duration)
 		return nil
@@ -214,12 +211,6 @@ func (bs *BackendServer) GotCommand(ctx context.Context, cmd *Command) error {
 	return fmt.Errorf("BackendServer.Do: no command specified")
 }
 
-func (bs *BackendServer) Reset(ctx context.Context) error {
-	// Tell the backend we got a Logout command, which will cause it
-	// to forget all its authentication information.
-	return bs.GotFakeCommand(ctx, &Command{Logout: &NoArgs{}})
-}
-
 type BackendClient struct {
 	logf           logger.Logf
 	sendCommandMsg func(jsonb []byte)
@@ -247,7 +238,7 @@ func (bc *BackendClient) GotNotifyMsg(b []byte) {
 	}
 	n := Notify{}
 	if err := json.Unmarshal(b, &n); err != nil {
-		log.Fatalf("BackendClient.Notify: cannot decode message (length=%d)\n%#v", len(b), string(b))
+		log.Fatalf("BackendClient.Notify: cannot decode message (length=%d, %#q): %v", len(b), b, err)
 	}
 	if n.Version != version.Long && !bc.AllowVersionSkew {
 		vs := fmt.Sprintf("GotNotify: Version mismatch! frontend=%#v backend=%#v",
@@ -287,8 +278,6 @@ func (bc *BackendClient) Quit() error {
 }
 
 func (bc *BackendClient) Start(opts Options) error {
-	bc.notify = opts.Notify
-	opts.Notify = nil // server can't call our function pointer
 	bc.send(Command{Start: &StartArgs{Opts: opts}})
 	return nil // remote Start() errors must be handled remotely
 }
@@ -309,10 +298,6 @@ func (bc *BackendClient) SetPrefs(new *Prefs) {
 	bc.send(Command{SetPrefs: &SetPrefsArgs{New: new}})
 }
 
-func (bc *BackendClient) EditPrefs(mp *MaskedPrefs) {
-	bc.send(Command{EditPrefs: mp})
-}
-
 func (bc *BackendClient) RequestEngineStatus() {
 	bc.send(Command{RequestEngineStatus: &NoArgs{}})
 }

ipn/message_test.go

@@ -7,6 +7,7 @@ package ipn
 import (
 	"bytes"
 	"context"
+	"encoding/json"
 	"testing"
 	"time"
 
@@ -74,7 +75,11 @@ func TestClientServer(t *testing.T) {
 			bc.GotNotifyMsg(b)
 		}
 	}()
-	serverToClient := func(b []byte) {
+	serverToClient := func(n Notify) {
+		b, err := json.Marshal(n)
+		if err != nil {
+			panic(err.Error())
+		}
 		serverToClientCh <- append([]byte{}, b...)
 	}
 	clientToServer := func(b []byte) {
@@ -87,16 +92,16 @@ func TestClientServer(t *testing.T) {
 		t.Logf("c: "+fmt, args...)
 	}
 	bs = NewBackendServer(slogf, b, serverToClient)
+	// Verify that this doesn't break bs's callback:
+	NewBackendServer(slogf, b, nil)
 	bc = NewBackendClient(clogf, clientToServer)
 
 	ch := make(chan Notify, 256)
-	h, err := NewHandle(bc, clogf, Options{
+	notify := func(n Notify) { ch <- n }
+	h, err := NewHandle(bc, clogf, notify, Options{
 		Prefs: &Prefs{
 			ControlURL: "http://example.com/fake",
 		},
-		Notify: func(n Notify) {
-			ch <- n
-		},
 	})
 	if err != nil {
 		t.Fatalf("NewHandle error: %v\n", err)
@@ -182,3 +187,17 @@ func TestClientServer(t *testing.T) {
 	})
 	flushUntil(Running)
 }
+
+func TestNilBackend(t *testing.T) {
+	var called *Notify
+	bs := NewBackendServer(t.Logf, nil, func(n Notify) {
+		called = &n
+	})
+	bs.SendErrorMessage("Danger, Will Robinson!")
+	if called == nil {
+		t.Errorf("expect callback to be called, wasn't")
+	}
+	if called.ErrMessage == nil || *called.ErrMessage != "Danger, Will Robinson!" {
+		t.Errorf("callback got wrong error: %v", called.ErrMessage)
+	}
+}

ipn/prefs.go

@@ -25,9 +25,27 @@ import (
 
 //go:generate go run tailscale.com/cmd/cloner -type=Prefs -output=prefs_clone.go
 
+// DefaultControlURL returns the URL base of the control plane
+// ("coordination server") for use when no explicit one is configured.
+// The default control plane is the hosted version run by Tailscale.com.
+const DefaultControlURL = "https://login.tailscale.com"
+
 // Prefs are the user modifiable settings of the Tailscale node agent.
 type Prefs struct {
 	// ControlURL is the URL of the control server to use.
+	//
+	// If empty, the default for new installs, DefaultControlURL
+	// is used. It's set non-empty once the daemon has been started
+	// for the first time.
+	//
+	// TODO(apenwarr): Make it safe to update this with SetPrefs().
+	// Right now, you have to pass it in the initial prefs in Start(),
+	// which is the only code that actually uses the ControlURL value.
+	// It would be more consistent to restart controlclient
+	// automatically whenever this variable changes.
+	//
+	// Meanwhile, you have to provide this as part of Options.Prefs or
+	// Options.UpdatePrefs when calling Backend.Start().
 	ControlURL string
 
 	// RouteAll specifies whether to accept subnets advertised by
@@ -66,6 +84,10 @@ type Prefs struct {
 	ExitNodeID tailcfg.StableNodeID
 	ExitNodeIP netaddr.IP
 
+	// ExitNodeAllowLANAccess indicates whether locally accessible subnets should be
+	// routed directly or via the exit node.
+	ExitNodeAllowLANAccess bool
+
 	// CorpDNS specifies whether to install the Tailscale network's
 	// DNS configuration, if it exists.
 	CorpDNS bool
@@ -74,6 +96,14 @@ type Prefs struct {
 	// this node.
 	WantRunning bool
 
+	// LoggedOut indicates whether the user intends to be logged out.
+	// There are other reasons we may be logged out, including no valid
+	// keys.
+	// We need to remember this state so that, on next startup, we can
+	// generate the "Login" vs "Connect" buttons correctly, without having
+	// to contact the server to confirm our nodekey status first.
+	LoggedOut bool
+
 	// ShieldsUp indicates whether to block all incoming connections,
 	// regardless of the control-provided packet filter. If false, we
 	// use the packet filter as provided. If true, we block incoming
@@ -140,6 +170,10 @@ type Prefs struct {
 	// Tailscale, if at all.
 	NetfilterMode preftype.NetfilterMode
 
+	// OperatorUser is the local machine user name who is allowed to
+	// operate tailscaled without being root or using sudo.
+	OperatorUser string `json:",omitempty"`
+
 	// The Persist field is named 'Config' in the file for backward
 	// compatibility with earlier versions.
 	// TODO(apenwarr): We should move this out of here, it's not a pref.
@@ -152,23 +186,26 @@ type Prefs struct {
 type MaskedPrefs struct {
 	Prefs
 
-	ControlURLSet       bool `json:",omitempty"`
-	RouteAllSet         bool `json:",omitempty"`
-	AllowSingleHostsSet bool `json:",omitempty"`
-	ExitNodeIDSet       bool `json:",omitempty"`
-	ExitNodeIPSet       bool `json:",omitempty"`
-	CorpDNSSet          bool `json:",omitempty"`
-	WantRunningSet      bool `json:",omitempty"`
-	ShieldsUpSet        bool `json:",omitempty"`
-	AdvertiseTagsSet    bool `json:",omitempty"`
-	HostnameSet         bool `json:",omitempty"`
-	OSVersionSet        bool `json:",omitempty"`
-	DeviceModelSet      bool `json:",omitempty"`
-	NotepadURLsSet      bool `json:",omitempty"`
-	ForceDaemonSet      bool `json:",omitempty"`
-	AdvertiseRoutesSet  bool `json:",omitempty"`
-	NoSNATSet           bool `json:",omitempty"`
-	NetfilterModeSet    bool `json:",omitempty"`
+	ControlURLSet             bool `json:",omitempty"`
+	RouteAllSet               bool `json:",omitempty"`
+	AllowSingleHostsSet       bool `json:",omitempty"`
+	ExitNodeIDSet             bool `json:",omitempty"`
+	ExitNodeIPSet             bool `json:",omitempty"`
+	ExitNodeAllowLANAccessSet bool `json:",omitempty"`
+	CorpDNSSet                bool `json:",omitempty"`
+	WantRunningSet            bool `json:",omitempty"`
+	LoggedOutSet              bool `json:",omitempty"`
+	ShieldsUpSet              bool `json:",omitempty"`
+	AdvertiseTagsSet          bool `json:",omitempty"`
+	HostnameSet               bool `json:",omitempty"`
+	OSVersionSet              bool `json:",omitempty"`
+	DeviceModelSet            bool `json:",omitempty"`
+	NotepadURLsSet            bool `json:",omitempty"`
+	ForceDaemonSet            bool `json:",omitempty"`
+	AdvertiseRoutesSet        bool `json:",omitempty"`
+	NoSNATSet                 bool `json:",omitempty"`
+	NetfilterModeSet          bool `json:",omitempty"`
+	OperatorUserSet           bool `json:",omitempty"`
 }
 
 // ApplyEdits mutates p, assigning fields from m.Prefs for each MaskedPrefs
@@ -227,6 +264,9 @@ func (p *Prefs) pretty(goos string) string {
 		sb.WriteString("mesh=false ")
 	}
 	fmt.Fprintf(&sb, "dns=%v want=%v ", p.CorpDNS, p.WantRunning)
+	if p.LoggedOut {
+		sb.WriteString("loggedout=true ")
+	}
 	if p.ForceDaemon {
 		sb.WriteString("server=true ")
 	}
@@ -237,9 +277,9 @@ func (p *Prefs) pretty(goos string) string {
 		sb.WriteString("shields=true ")
 	}
 	if !p.ExitNodeIP.IsZero() {
-		fmt.Fprintf(&sb, "exit=%v ", p.ExitNodeIP)
+		fmt.Fprintf(&sb, "exit=%v lan=%t ", p.ExitNodeIP, p.ExitNodeAllowLANAccess)
 	} else if !p.ExitNodeID.IsZero() {
-		fmt.Fprintf(&sb, "exit=%v ", p.ExitNodeID)
+		fmt.Fprintf(&sb, "exit=%v lan=%t ", p.ExitNodeID, p.ExitNodeAllowLANAccess)
 	}
 	if len(p.AdvertiseRoutes) > 0 || goos == "linux" {
 		fmt.Fprintf(&sb, "routes=%v ", p.AdvertiseRoutes)
@@ -253,9 +293,15 @@ func (p *Prefs) pretty(goos string) string {
 	if goos == "linux" {
 		fmt.Fprintf(&sb, "nf=%v ", p.NetfilterMode)
 	}
-	if p.ControlURL != "" && p.ControlURL != "https://login.tailscale.com" {
+	if p.ControlURL != "" && p.ControlURL != DefaultControlURL {
 		fmt.Fprintf(&sb, "url=%q ", p.ControlURL)
 	}
+	if p.Hostname != "" {
+		fmt.Fprintf(&sb, "host=%q ", p.Hostname)
+	}
+	if p.OperatorUser != "" {
+		fmt.Fprintf(&sb, "op=%q ", p.OperatorUser)
+	}
 	if p.Persist != nil {
 		sb.WriteString(p.Persist.Pretty())
 	} else {
@@ -287,12 +333,15 @@ func (p *Prefs) Equals(p2 *Prefs) bool {
 		p.AllowSingleHosts == p2.AllowSingleHosts &&
 		p.ExitNodeID == p2.ExitNodeID &&
 		p.ExitNodeIP == p2.ExitNodeIP &&
+		p.ExitNodeAllowLANAccess == p2.ExitNodeAllowLANAccess &&
 		p.CorpDNS == p2.CorpDNS &&
 		p.WantRunning == p2.WantRunning &&
+		p.LoggedOut == p2.LoggedOut &&
 		p.NotepadURLs == p2.NotepadURLs &&
 		p.ShieldsUp == p2.ShieldsUp &&
 		p.NoSNAT == p2.NoSNAT &&
 		p.NetfilterMode == p2.NetfilterMode &&
+		p.OperatorUser == p2.OperatorUser &&
 		p.Hostname == p2.Hostname &&
 		p.OSVersion == p2.OSVersion &&
 		p.DeviceModel == p2.DeviceModel &&
@@ -326,12 +375,19 @@ func compareStrings(a, b []string) bool {
 	return true
 }
 
+// NewPrefs returns the default preferences to use.
 func NewPrefs() *Prefs {
+	// Provide default values for options which might be missing
+	// from the json data for any reason. The json can still
+	// override them to false.
 	return &Prefs{
-		// Provide default values for options which might be missing
-		// from the json data for any reason. The json can still
-		// override them to false.
-		ControlURL:       "https://login.tailscale.com",
+		// ControlURL is explicitly not set to signal that
+		// it's not yet configured, which relaxes the CLI "up"
+		// safety net features. It will get set to DefaultControlURL
+		// on first up. Or, if not, DefaultControlURL will be used
+		// later anyway.
+		ControlURL: "",
+
 		RouteAll:         true,
 		AllowSingleHosts: true,
 		CorpDNS:          true,
@@ -340,6 +396,15 @@ func NewPrefs() *Prefs {
 	}
 }
 
+// ControlURLOrDefault returns the coordination server's URL base.
+// If not configured, DefaultControlURL is returned instead.
+func (p *Prefs) ControlURLOrDefault() string {
+	if p.ControlURL != "" {
+		return p.ControlURL
+	}
+	return DefaultControlURL
+}
+
 // PrefsFromBytes deserializes Prefs from a JSON blob. If
 // enforceDefaults is true, Prefs.RouteAll and Prefs.AllowSingleHosts
 // are forced on.

ipn/prefs_clone.go

@@ -33,22 +33,25 @@ func (src *Prefs) Clone() *Prefs {
 // A compilation failure here means this code must be regenerated, with command:
 //   tailscale.com/cmd/cloner -type Prefs
 var _PrefsNeedsRegeneration = Prefs(struct {
-	ControlURL       string
-	RouteAll         bool
-	AllowSingleHosts bool
-	ExitNodeID       tailcfg.StableNodeID
-	ExitNodeIP       netaddr.IP
-	CorpDNS          bool
-	WantRunning      bool
-	ShieldsUp        bool
-	AdvertiseTags    []string
-	Hostname         string
-	OSVersion        string
-	DeviceModel      string
-	NotepadURLs      bool
-	ForceDaemon      bool
-	AdvertiseRoutes  []netaddr.IPPrefix
-	NoSNAT           bool
-	NetfilterMode    preftype.NetfilterMode
-	Persist          *persist.Persist
+	ControlURL             string
+	RouteAll               bool
+	AllowSingleHosts       bool
+	ExitNodeID             tailcfg.StableNodeID
+	ExitNodeIP             netaddr.IP
+	ExitNodeAllowLANAccess bool
+	CorpDNS                bool
+	WantRunning            bool
+	LoggedOut              bool
+	ShieldsUp              bool
+	AdvertiseTags          []string
+	Hostname               string
+	OSVersion              string
+	DeviceModel            string
+	NotepadURLs            bool
+	ForceDaemon            bool
+	AdvertiseRoutes        []netaddr.IPPrefix
+	NoSNAT                 bool
+	NetfilterMode          preftype.NetfilterMode
+	OperatorUser           string
+	Persist                *persist.Persist
 }{})

ipn/prefs_test.go

@@ -33,7 +33,29 @@ func fieldsOf(t reflect.Type) (fields []string) {
 func TestPrefsEqual(t *testing.T) {
 	tstest.PanicOnLog()
 
-	prefsHandles := []string{"ControlURL", "RouteAll", "AllowSingleHosts", "ExitNodeID", "ExitNodeIP", "CorpDNS", "WantRunning", "ShieldsUp", "AdvertiseTags", "Hostname", "OSVersion", "DeviceModel", "NotepadURLs", "ForceDaemon", "AdvertiseRoutes", "NoSNAT", "NetfilterMode", "Persist"}
+	prefsHandles := []string{
+		"ControlURL",
+		"RouteAll",
+		"AllowSingleHosts",
+		"ExitNodeID",
+		"ExitNodeIP",
+		"ExitNodeAllowLANAccess",
+		"CorpDNS",
+		"WantRunning",
+		"LoggedOut",
+		"ShieldsUp",
+		"AdvertiseTags",
+		"Hostname",
+		"OSVersion",
+		"DeviceModel",
+		"NotepadURLs",
+		"ForceDaemon",
+		"AdvertiseRoutes",
+		"NoSNAT",
+		"NetfilterMode",
+		"OperatorUser",
+		"Persist",
+	}
 	if have := fieldsOf(reflect.TypeOf(Prefs{})); !reflect.DeepEqual(have, prefsHandles) {
 		t.Errorf("Prefs.Equal check might be out of sync\nfields: %q\nhandled: %q\n",
 			have, prefsHandles)
@@ -124,6 +146,17 @@ func TestPrefsEqual(t *testing.T) {
 			true,
 		},
 
+		{
+			&Prefs{},
+			&Prefs{ExitNodeAllowLANAccess: true},
+			false,
+		},
+		{
+			&Prefs{ExitNodeAllowLANAccess: true},
+			&Prefs{ExitNodeAllowLANAccess: true},
+			true,
+		},
+
 		{
 			&Prefs{CorpDNS: true},
 			&Prefs{CorpDNS: false},
@@ -384,14 +417,36 @@ func TestPrefsPretty(t *testing.T) {
 				ExitNodeIP: netaddr.MustParseIP("1.2.3.4"),
 			},
 			"linux",
-			`Prefs{ra=false mesh=false dns=false want=false exit=1.2.3.4 routes=[] nf=off Persist=nil}`,
+			`Prefs{ra=false mesh=false dns=false want=false exit=1.2.3.4 lan=false routes=[] nf=off Persist=nil}`,
 		},
 		{
 			Prefs{
 				ExitNodeID: tailcfg.StableNodeID("myNodeABC"),
 			},
 			"linux",
-			`Prefs{ra=false mesh=false dns=false want=false exit=myNodeABC routes=[] nf=off Persist=nil}`,
+			`Prefs{ra=false mesh=false dns=false want=false exit=myNodeABC lan=false routes=[] nf=off Persist=nil}`,
+		},
+		{
+			Prefs{
+				ExitNodeID:             tailcfg.StableNodeID("myNodeABC"),
+				ExitNodeAllowLANAccess: true,
+			},
+			"linux",
+			`Prefs{ra=false mesh=false dns=false want=false exit=myNodeABC lan=true routes=[] nf=off Persist=nil}`,
+		},
+		{
+			Prefs{
+				ExitNodeAllowLANAccess: true,
+			},
+			"linux",
+			`Prefs{ra=false mesh=false dns=false want=false routes=[] nf=off Persist=nil}`,
+		},
+		{
+			Prefs{
+				Hostname: "foo",
+			},
+			"linux",
+			`Prefs{ra=false mesh=false dns=false want=false routes=[] nf=off host="foo" Persist=nil}`,
 		},
 	}
 	for i, tt := range tests {

logtail/filch/filch.go

@@ -15,6 +15,7 @@ import (
 	"sync"
 )
 
+//lint:ignore U1000 work around false positive: https://github.com/dominikh/go-tools/issues/983
 var stderrFD = 2 // a variable for testing
 
 type Options struct {

logtail/id.go

@@ -121,14 +121,17 @@ func (id PublicID) MarshalText() ([]byte, error) {
 }
 
 func (id *PublicID) UnmarshalText(s []byte) error {
-	b, err := hex.DecodeString(string(s))
-	if err != nil {
-		return fmt.Errorf("logtail.PublicID.UnmarshalText: %v", err)
+	if len(s) != len(id)*2 {
+		return fmt.Errorf("logtail.PublicID.UnmarshalText: invalid hex length: %d", len(s))
 	}
-	if len(b) != len(id) {
-		return fmt.Errorf("logtail.PublicID.UnmarshalText: invalid hex length: %d", len(b))
+	for i := range id {
+		a, ok1 := fromHexChar(s[i*2+0])
+		b, ok2 := fromHexChar(s[i*2+1])
+		if !ok1 || !ok2 {
+			return errors.New("invalid hex character")
+		}
+		id[i] = (a << 4) | b
 	}
-	copy(id[:], b)
 	return nil
 }
 

logtail/logtail.go

@@ -15,6 +15,7 @@ import (
 	"net/http"
 	"os"
 	"strconv"
+	"sync/atomic"
 	"time"
 
 	"tailscale.com/logtail/backoff"
@@ -72,7 +73,7 @@ func NewLogger(cfg Config, logf tslogger.Logf) *Logger {
 	}
 	l := &Logger{
 		stderr:         cfg.Stderr,
-		stderrLevel:    cfg.StderrLevel,
+		stderrLevel:    int64(cfg.StderrLevel),
 		httpc:          cfg.HTTPC,
 		url:            cfg.BaseURL + "/c/" + cfg.Collection + "/" + cfg.PrivateID.String(),
 		lowMem:         cfg.LowMemory,
@@ -103,7 +104,7 @@ func NewLogger(cfg Config, logf tslogger.Logf) *Logger {
 // logging facilities and uploading to a log server.
 type Logger struct {
 	stderr         io.Writer
-	stderrLevel    int
+	stderrLevel    int64 // accessed atomically
 	httpc          *http.Client
 	url            string
 	lowMem         bool
@@ -125,10 +126,8 @@ type Logger struct {
 // SetVerbosityLevel controls the verbosity level that should be
 // written to stderr. 0 is the default (not verbose). Levels 1 or higher
 // are increasingly verbose.
-//
-// It should not be changed concurrently with log writes.
 func (l *Logger) SetVerbosityLevel(level int) {
-	l.stderrLevel = level
+	atomic.StoreInt64(&l.stderrLevel, int64(level))
 }
 
 // SetLinkMonitor sets the optional the link monitor.
@@ -514,7 +513,7 @@ func (l *Logger) Write(buf []byte) (int, error) {
 		return 0, nil
 	}
 	level, buf := parseAndRemoveLogLevel(buf)
-	if l.stderr != nil && l.stderr != ioutil.Discard && level <= l.stderrLevel {
+	if l.stderr != nil && l.stderr != ioutil.Discard && int64(level) <= atomic.LoadInt64(&l.stderrLevel) {
 		if buf[len(buf)-1] == '\n' {
 			l.stderr.Write(buf)
 		} else {

logtail/logtail_test.go

@@ -303,3 +303,26 @@ func TestParseAndRemoveLogLevel(t *testing.T) {
 		}
 	}
 }
+
+func TestPublicIDUnmarshalText(t *testing.T) {
+	const hexStr = "6c60a9e0e7af57170bb1347b2d477e4cbc27d4571a4923b21651456f931e3d55"
+	x := []byte(hexStr)
+
+	var id PublicID
+	if err := id.UnmarshalText(x); err != nil {
+		t.Fatal(err)
+	}
+	if id.String() != hexStr {
+		t.Errorf("String = %q; want %q", id.String(), hexStr)
+	}
+
+	n := int(testing.AllocsPerRun(1000, func() {
+		var id PublicID
+		if err := id.UnmarshalText(x); err != nil {
+			t.Fatal(err)
+		}
+	}))
+	if n != 0 {
+		t.Errorf("allocs = %v; want 0", n)
+	}
+}

net/dns/config.go

@@ -5,7 +5,10 @@
 package dns
 
 import (
+	"sort"
+
 	"inet.af/netaddr"
+	"tailscale.com/util/dnsname"
 )
 
 // Config is a DNS configuration.
@@ -19,27 +22,85 @@ type Config struct {
 	// for queries that fall within that suffix.
 	// If a query doesn't match any entry in Routes, the
 	// DefaultResolvers are used.
-	Routes map[string][]netaddr.IPPort
+	// A Routes entry with no resolvers means the route should be
+	// authoritatively answered using the contents of Hosts.
+	Routes map[dnsname.FQDN][]netaddr.IPPort
 	// SearchDomains are DNS suffixes to try when expanding
 	// single-label queries.
-	SearchDomains []string
+	SearchDomains []dnsname.FQDN
 	// Hosts maps DNS FQDNs to their IPs, which can be a mix of IPv4
 	// and IPv6.
-	// Queries matching entries in Hosts are resolved locally without
-	// recursing off-machine.
-	Hosts map[string][]netaddr.IP
-	// AuthoritativeSuffixes is a list of fully-qualified DNS suffixes
-	// for which the in-process Tailscale resolver is authoritative.
-	// Queries for names within AuthoritativeSuffixes can only be
-	// fulfilled by entries in Hosts. Queries with no match in Hosts
-	// return NXDOMAIN.
-	AuthoritativeSuffixes []string
+	// Queries matching entries in Hosts are resolved locally by
+	// 100.100.100.100 without leaving the machine.
+	// Adding an entry to Hosts merely creates the record. If you want
+	// it to resolve, you also need to add appropriate routes to
+	// Routes.
+	Hosts map[dnsname.FQDN][]netaddr.IP
 }
 
-// OSConfig is an OS DNS configuration.
-type OSConfig struct {
-	// Nameservers are the IP addresses of the nameservers to use.
-	Nameservers []netaddr.IP
-	// Domains are the search domains to use.
-	Domains []string
+// needsAnyResolvers reports whether c requires a resolver to be set
+// at the OS level.
+func (c Config) needsOSResolver() bool {
+	return c.hasDefaultResolvers() || c.hasRoutes()
+}
+
+func (c Config) hasRoutes() bool {
+	return len(c.Routes) > 0
+}
+
+// hasDefaultResolversOnly reports whether the only resolvers in c are
+// DefaultResolvers.
+func (c Config) hasDefaultResolversOnly() bool {
+	return c.hasDefaultResolvers() && !c.hasRoutes()
+}
+
+func (c Config) hasDefaultResolvers() bool {
+	return len(c.DefaultResolvers) > 0
+}
+
+// singleResolverSet returns the resolvers used by c.Routes if all
+// routes use the same resolvers, or nil if multiple sets of resolvers
+// are specified.
+func (c Config) singleResolverSet() []netaddr.IPPort {
+	var (
+		prev            []netaddr.IPPort
+		prevInitialized bool
+	)
+	for _, resolvers := range c.Routes {
+		if !prevInitialized {
+			prev = resolvers
+			prevInitialized = true
+			continue
+		}
+		if !sameIPPorts(prev, resolvers) {
+			return nil
+		}
+	}
+	return prev
+}
+
+// matchDomains returns the list of match suffixes needed by Routes.
+func (c Config) matchDomains() []dnsname.FQDN {
+	ret := make([]dnsname.FQDN, 0, len(c.Routes))
+	for suffix := range c.Routes {
+		ret = append(ret, suffix)
+	}
+	sort.Slice(ret, func(i, j int) bool {
+		return ret[i].WithTrailingDot() < ret[j].WithTrailingDot()
+	})
+	return ret
+}
+
+func sameIPPorts(a, b []netaddr.IPPort) bool {
+	if len(a) != len(b) {
+		return false
+	}
+
+	for i := range a {
+		if a[i] != b[i] {
+			return false
+		}
+	}
+
+	return true
 }

net/dns/debian_resolvconf.go

@@ -0,0 +1,186 @@
+// Copyright (c) 2020 Tailscale Inc & AUTHORS All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+// +build linux freebsd openbsd
+
+package dns
+
+import (
+	"bufio"
+	"bytes"
+	_ "embed"
+	"fmt"
+	"io/ioutil"
+	"os"
+	"os/exec"
+	"path/filepath"
+
+	"tailscale.com/atomicfile"
+	"tailscale.com/types/logger"
+)
+
+//go:embed resolvconf-workaround.sh
+var workaroundScript []byte
+
+// resolvconfConfigName is the name of the config submitted to
+// resolvconf.
+// The name starts with 'tun' in order to match the hardcoded
+// interface order in debian resolvconf, which will place this
+// configuration ahead of regular network links. In theory, this
+// doesn't matter because we then fix things up to ensure our config
+// is the only one in use, but in case that fails, this will make our
+// configuration slightly preferred.
+// The 'inet' suffix has no specific meaning, but conventionally
+// resolvconf implementations encourage adding a suffix roughly
+// indicating where the config came from, and "inet" is the "none of
+// the above" value (rather than, say, "ppp" or "dhcp").
+const resolvconfConfigName = "tun-tailscale.inet"
+
+// resolvconfLibcHookPath is the directory containing libc update
+// scripts, which are run by Debian resolvconf when /etc/resolv.conf
+// has been updated.
+const resolvconfLibcHookPath = "/etc/resolvconf/update-libc.d"
+
+// resolvconfHookPath is the name of the libc hook script we install
+// to force Tailscale's DNS config to take effect.
+var resolvconfHookPath = filepath.Join(resolvconfLibcHookPath, "tailscale")
+
+// resolvconfManager manages DNS configuration using the Debian
+// implementation of the `resolvconf` program, written by Thomas Hood.
+type resolvconfManager struct {
+	logf            logger.Logf
+	listRecordsPath string
+	interfacesDir   string
+	scriptInstalled bool // libc update script has been installed
+}
+
+func newDebianResolvconfManager(logf logger.Logf) (*resolvconfManager, error) {
+	ret := &resolvconfManager{
+		logf:            logf,
+		listRecordsPath: "/lib/resolvconf/list-records",
+		interfacesDir:   "/etc/resolvconf/run/interface", // panic fallback if nothing seems to work
+	}
+
+	if _, err := os.Stat(ret.listRecordsPath); os.IsNotExist(err) {
+		// This might be a Debian system from before the big /usr
+		// merge, try /usr instead.
+		ret.listRecordsPath = "/usr" + ret.listRecordsPath
+	}
+	// The runtime directory is currently (2020-04) canonically
+	// /etc/resolvconf/run, but the manpage is making noise about
+	// switching to /run/resolvconf and dropping the /etc path. So,
+	// let's probe the possible directories and use the first one
+	// that works.
+	for _, path := range []string{
+		"/etc/resolvconf/run/interface",
+		"/run/resolvconf/interface",
+		"/var/run/resolvconf/interface",
+	} {
+		if _, err := os.Stat(path); err == nil {
+			ret.interfacesDir = path
+			break
+		}
+	}
+	if ret.interfacesDir == "" {
+		// None of the paths seem to work, use the canonical location
+		// that the current manpage says to use.
+		ret.interfacesDir = "/etc/resolvconf/run/interfaces"
+	}
+
+	return ret, nil
+}
+
+func (m *resolvconfManager) deleteTailscaleConfig() error {
+	cmd := exec.Command("resolvconf", "-d", resolvconfConfigName)
+	out, err := cmd.CombinedOutput()
+	if err != nil {
+		return fmt.Errorf("running %s: %s", cmd, out)
+	}
+	return nil
+}
+
+func (m *resolvconfManager) SetDNS(config OSConfig) error {
+	if !m.scriptInstalled {
+		m.logf("injecting resolvconf workaround script")
+		if err := os.MkdirAll(resolvconfLibcHookPath, 0755); err != nil {
+			return err
+		}
+		if err := atomicfile.WriteFile(resolvconfHookPath, workaroundScript, 0755); err != nil {
+			return err
+		}
+		m.scriptInstalled = true
+	}
+
+	if config.IsZero() {
+		if err := m.deleteTailscaleConfig(); err != nil {
+			return err
+		}
+	} else {
+		stdin := new(bytes.Buffer)
+		writeResolvConf(stdin, config.Nameservers, config.SearchDomains) // dns_direct.go
+
+		// This resolvconf implementation doesn't support exclusive
+		// mode or interface priorities, so it will end up blending
+		// our configuration with other sources. However, this will
+		// get fixed up by the script we injected above.
+		cmd := exec.Command("resolvconf", "-a", resolvconfConfigName)
+		cmd.Stdin = stdin
+		out, err := cmd.CombinedOutput()
+		if err != nil {
+			return fmt.Errorf("running %s: %s", cmd, out)
+		}
+	}
+
+	return nil
+}
+
+func (m *resolvconfManager) SupportsSplitDNS() bool {
+	return false
+}
+
+func (m *resolvconfManager) GetBaseConfig() (OSConfig, error) {
+	var bs bytes.Buffer
+
+	cmd := exec.Command(m.listRecordsPath)
+	// list-records assumes it's being run with CWD set to the
+	// interfaces runtime dir, and returns nonsense otherwise.
+	cmd.Dir = m.interfacesDir
+	cmd.Stdout = &bs
+	if err := cmd.Run(); err != nil {
+		return OSConfig{}, err
+	}
+
+	var conf bytes.Buffer
+	sc := bufio.NewScanner(&bs)
+	for sc.Scan() {
+		if sc.Text() == resolvconfConfigName {
+			continue
+		}
+		bs, err := ioutil.ReadFile(filepath.Join(m.interfacesDir, sc.Text()))
+		if err != nil {
+			if os.IsNotExist(err) {
+				// Probably raced with a deletion, that's okay.
+				continue
+			}
+			return OSConfig{}, err
+		}
+		conf.Write(bs)
+		conf.WriteByte('\n')
+	}
+
+	return readResolv(&conf)
+}
+
+func (m *resolvconfManager) Close() error {
+	if err := m.deleteTailscaleConfig(); err != nil {
+		return err
+	}
+
+	if m.scriptInstalled {
+		m.logf("removing resolvconf workaround script")
+		os.Remove(resolvconfHookPath) // Best-effort
+	}
+
+	return nil
+}

net/dns/direct.go

@@ -9,7 +9,6 @@ package dns
 import (
 	"bufio"
 	"bytes"
-	"errors"
 	"fmt"
 	"io"
 	"io/ioutil"
@@ -20,16 +19,16 @@ import (
 
 	"inet.af/netaddr"
 	"tailscale.com/atomicfile"
+	"tailscale.com/util/dnsname"
 )
 
 const (
-	tsConf     = "/etc/resolv.tailscale.conf"
 	backupConf = "/etc/resolv.pre-tailscale-backup.conf"
 	resolvConf = "/etc/resolv.conf"
 )
 
 // writeResolvConf writes DNS configuration in resolv.conf format to the given writer.
-func writeResolvConf(w io.Writer, servers []netaddr.IP, domains []string) {
+func writeResolvConf(w io.Writer, servers []netaddr.IP, domains []dnsname.FQDN) {
 	io.WriteString(w, "# resolv.conf(5) file generated by tailscale\n")
 	io.WriteString(w, "# DO NOT EDIT THIS FILE BY HAND -- CHANGES WILL BE OVERWRITTEN\n\n")
 	for _, ns := range servers {
@@ -41,22 +40,14 @@ func writeResolvConf(w io.Writer, servers []netaddr.IP, domains []string) {
 		io.WriteString(w, "search")
 		for _, domain := range domains {
 			io.WriteString(w, " ")
-			io.WriteString(w, domain)
+			io.WriteString(w, domain.WithoutTrailingDot())
 		}
 		io.WriteString(w, "\n")
 	}
 }
 
-// readResolvConf reads DNS configuration from /etc/resolv.conf.
-func readResolvConf() (OSConfig, error) {
-	var config OSConfig
-
-	f, err := os.Open("/etc/resolv.conf")
-	if err != nil {
-		return config, err
-	}
-
-	scanner := bufio.NewScanner(f)
+func readResolv(r io.Reader) (config OSConfig, err error) {
+	scanner := bufio.NewScanner(r)
 	for scanner.Scan() {
 		line := strings.TrimSpace(scanner.Text())
 
@@ -65,7 +56,7 @@ func readResolvConf() (OSConfig, error) {
 			nameserver = strings.TrimSpace(nameserver)
 			ip, err := netaddr.ParseIP(nameserver)
 			if err != nil {
-				return config, err
+				return OSConfig{}, err
 			}
 			config.Nameservers = append(config.Nameservers, ip)
 			continue
@@ -74,7 +65,11 @@ func readResolvConf() (OSConfig, error) {
 		if strings.HasPrefix(line, "search") {
 			domain := strings.TrimPrefix(line, "search")
 			domain = strings.TrimSpace(domain)
-			config.Domains = append(config.Domains, domain)
+			fqdn, err := dnsname.ToFQDN(domain)
+			if err != nil {
+				return OSConfig{}, fmt.Errorf("parsing search domains %q: %w", line, err)
+			}
+			config.SearchDomains = append(config.SearchDomains, fqdn)
 			continue
 		}
 	}
@@ -82,6 +77,53 @@ func readResolvConf() (OSConfig, error) {
 	return config, nil
 }
 
+func readResolvFile(path string) (OSConfig, error) {
+	var config OSConfig
+
+	f, err := os.Open(path)
+	if err != nil {
+		return config, err
+	}
+	defer f.Close()
+
+	return readResolv(f)
+}
+
+// readResolvConf reads DNS configuration from /etc/resolv.conf.
+func readResolvConf() (OSConfig, error) {
+	return readResolvFile(resolvConf)
+}
+
+// resolvOwner returns the apparent owner of the resolv.conf
+// configuration in bs - one of "resolvconf", "systemd-resolved" or
+// "NetworkManager", or "" if no known owner was found.
+func resolvOwner(bs []byte) string {
+	b := bytes.NewBuffer(bs)
+	for {
+		line, err := b.ReadString('\n')
+		if err != nil {
+			return ""
+		}
+		line = strings.TrimSpace(line)
+		if line == "" {
+			continue
+		}
+		if line[0] != '#' {
+			// First non-empty, non-comment line. Assume the owner
+			// isn't hiding further down.
+			return ""
+		}
+
+		if strings.Contains(line, "systemd-resolved") {
+			return "systemd-resolved"
+		} else if strings.Contains(line, "NetworkManager") {
+			return "NetworkManager"
+		} else if strings.Contains(line, "resolvconf") {
+			return "resolvconf"
+		}
+	}
+}
+
 // isResolvedRunning reports whether systemd-resolved is running on the system,
 // even if it is not managing the system DNS settings.
 func isResolvedRunning() bool {
@@ -110,77 +152,171 @@ func isResolvedRunning() bool {
 // or as cleanup if the program terminates unexpectedly.
 type directManager struct{}
 
-func newDirectManager() directManager {
-	return directManager{}
+func newDirectManager() (directManager, error) {
+	return directManager{}, nil
 }
 
-func (m directManager) SetDNS(config OSConfig) error {
-	// Write the tsConf file.
-	buf := new(bytes.Buffer)
-	writeResolvConf(buf, config.Nameservers, config.Domains)
-	if err := atomicfile.WriteFile(tsConf, buf.Bytes(), 0644); err != nil {
-		return err
-	}
-
-	if linkPath, err := os.Readlink(resolvConf); err != nil {
-		// Remove any old backup that may exist.
-		os.Remove(backupConf)
-
-		// Backup the existing /etc/resolv.conf file.
-		contents, err := ioutil.ReadFile(resolvConf)
-		// If the original did not exist, still back up an empty file.
-		// The presence of a backup file is the way we know that Up ran.
-		if err != nil && !errors.Is(err, os.ErrNotExist) {
-			return err
-		}
-		if err := atomicfile.WriteFile(backupConf, contents, 0644); err != nil {
-			return err
-		}
-	} else if linkPath != tsConf {
-		// Backup the existing symlink.
-		os.Remove(backupConf)
-		if err := os.Symlink(linkPath, backupConf); err != nil {
-			return err
-		}
-	} else {
-		// Nothing to do, resolvConf already points to tsConf.
-		return nil
-	}
-
-	os.Remove(resolvConf)
-	if err := os.Symlink(tsConf, resolvConf); err != nil {
-		return err
-	}
-
-	if isResolvedRunning() {
-		exec.Command("systemctl", "restart", "systemd-resolved.service").Run() // Best-effort.
-	}
-
-	return nil
-}
-
-func (m directManager) RoutingMode() RoutingMode {
-	return RoutingModeNone
-}
-
-func (m directManager) Close() error {
-	if _, err := os.Stat(backupConf); err != nil {
-		// If the backup file does not exist, then Up never ran successfully.
+// ownedByTailscale reports whether /etc/resolv.conf seems to be a
+// tailscale-managed file.
+func (m directManager) ownedByTailscale() (bool, error) {
+	st, err := os.Stat(resolvConf)
+	if err != nil {
 		if os.IsNotExist(err) {
+			return false, nil
+		}
+		return false, err
+	}
+	if !st.Mode().IsRegular() {
+		return false, nil
+	}
+	bs, err := ioutil.ReadFile(resolvConf)
+	if err != nil {
+		return false, err
+	}
+	if bytes.Contains(bs, []byte("generated by tailscale")) {
+		return true, nil
+	}
+	return false, nil
+}
+
+// backupConfig creates or updates a backup of /etc/resolv.conf, if
+// resolv.conf does not currently contain a Tailscale-managed config.
+func (m directManager) backupConfig() error {
+	if _, err := os.Stat(resolvConf); err != nil {
+		if os.IsNotExist(err) {
+			// No resolv.conf, nothing to back up. Also get rid of any
+			// existing backup file, to avoid restoring something old.
+			os.Remove(backupConf)
 			return nil
 		}
 		return err
 	}
 
-	if ln, err := os.Readlink(resolvConf); err != nil {
+	owned, err := m.ownedByTailscale()
+	if err != nil {
 		return err
-	} else if ln != tsConf {
-		return fmt.Errorf("resolv.conf is not a symlink to %s", tsConf)
 	}
+	if owned {
+		return nil
+	}
+
+	return os.Rename(resolvConf, backupConf)
+}
+
+func (m directManager) restoreBackup() error {
+	if _, err := os.Stat(backupConf); err != nil {
+		if os.IsNotExist(err) {
+			// No backup, nothing we can do.
+			return nil
+		}
+		return err
+	}
+	owned, err := m.ownedByTailscale()
+	if err != nil {
+		return err
+	}
+	if _, err := os.Stat(resolvConf); err != nil && !os.IsNotExist(err) {
+		return err
+	}
+	resolvConfExists := !os.IsNotExist(err)
+
+	if resolvConfExists && !owned {
+		// There's already a non-tailscale config in place, get rid of
+		// our backup.
+		os.Remove(backupConf)
+		return nil
+	}
+
+	// We own resolv.conf, and a backup exists.
+	if err := os.Rename(backupConf, resolvConf); err != nil {
+		return err
+	}
+
+	return nil
+}
+
+func (m directManager) SetDNS(config OSConfig) error {
+	if config.IsZero() {
+		if err := m.restoreBackup(); err != nil {
+			return err
+		}
+	} else {
+		if err := m.backupConfig(); err != nil {
+			return err
+		}
+
+		buf := new(bytes.Buffer)
+		writeResolvConf(buf, config.Nameservers, config.SearchDomains)
+		if err := atomicfile.WriteFile(resolvConf, buf.Bytes(), 0644); err != nil {
+			return err
+		}
+	}
+
+	// We might have taken over a configuration managed by resolved,
+	// in which case it will notice this on restart and gracefully
+	// start using our configuration. This shouldn't happen because we
+	// try to manage DNS through resolved when it's around, but as a
+	// best-effort fallback if we messed up the detection, try to
+	// restart resolved to make the system configuration consistent.
+	if isResolvedRunning() {
+		exec.Command("systemctl", "restart", "systemd-resolved.service").Run()
+	}
+
+	return nil
+}
+
+func (m directManager) SupportsSplitDNS() bool {
+	return false
+}
+
+func (m directManager) GetBaseConfig() (OSConfig, error) {
+	owned, err := m.ownedByTailscale()
+	if err != nil {
+		return OSConfig{}, err
+	}
+	fileToRead := resolvConf
+	if owned {
+		fileToRead = backupConf
+	}
+
+	return readResolvFile(fileToRead)
+}
+
+func (m directManager) Close() error {
+	// We used to keep a file for the tailscale config and symlinked
+	// to it, but then we stopped because /etc/resolv.conf being a
+	// symlink to surprising places breaks snaps and other sandboxing
+	// things. Clean it up if it's still there.
+	os.Remove("/etc/resolv.tailscale.conf")
+
+	if _, err := os.Stat(backupConf); err != nil {
+		if os.IsNotExist(err) {
+			// No backup, nothing we can do.
+			return nil
+		}
+		return err
+	}
+	owned, err := m.ownedByTailscale()
+	if err != nil {
+		return err
+	}
+	_, err = os.Stat(resolvConf)
+	if err != nil && !os.IsNotExist(err) {
+		return err
+	}
+	resolvConfExists := !os.IsNotExist(err)
+
+	if resolvConfExists && !owned {
+		// There's already a non-tailscale config in place, get rid of
+		// our backup.
+		os.Remove(backupConf)
+		return nil
+	}
+
+	// We own resolv.conf, and a backup exists.
 	if err := os.Rename(backupConf, resolvConf); err != nil {
 		return err
 	}
-	os.Remove(tsConf)
 
 	if isResolvedRunning() {
 		exec.Command("systemctl", "restart", "systemd-resolved.service").Run() // Best-effort.

net/dns/manager.go

@@ -5,12 +5,14 @@
 package dns
 
 import (
+	"runtime"
 	"time"
 
 	"inet.af/netaddr"
 	"tailscale.com/net/dns/resolver"
 	"tailscale.com/net/tsaddr"
 	"tailscale.com/types/logger"
+	"tailscale.com/util/dnsname"
 	"tailscale.com/wgengine/monitor"
 )
 
@@ -45,7 +47,6 @@ func NewManager(logf logger.Logf, oscfg OSConfigurator, linkMon *monitor.Mon) *M
 		resolver: resolver.New(logf, linkMon),
 		os:       oscfg,
 	}
-
 	m.logf("using %T", m.os)
 	return m
 }
@@ -53,46 +54,136 @@ func NewManager(logf logger.Logf, oscfg OSConfigurator, linkMon *monitor.Mon) *M
 func (m *Manager) Set(cfg Config) error {
 	m.logf("Set: %+v", cfg)
 
-	if len(cfg.DefaultResolvers) == 0 {
-		// TODO: make other settings work even if you didn't set a
-		// default resolver. For now, no default resolvers == no
-		// managed DNS config.
-		cfg = Config{}
-	}
-
-	resolverCfg := resolver.Config{
-		Hosts:        cfg.Hosts,
-		LocalDomains: cfg.AuthoritativeSuffixes,
-		Routes:       map[string][]netaddr.IPPort{},
-	}
-	osCfg := OSConfig{
-		Domains: cfg.SearchDomains,
-	}
-	// We must proxy through quad-100 if MagicDNS hosts are in
-	// use, or there are any per-domain routes.
-	mustProxy := len(cfg.Hosts) > 0 || len(cfg.Routes) > 0
-	if mustProxy {
-		osCfg.Nameservers = []netaddr.IP{tsaddr.TailscaleServiceIP()}
-		resolverCfg.Routes["."] = cfg.DefaultResolvers
-		for suffix, resolvers := range cfg.Routes {
-			resolverCfg.Routes[suffix] = resolvers
-		}
-	} else {
-		for _, resolver := range cfg.DefaultResolvers {
-			osCfg.Nameservers = append(osCfg.Nameservers, resolver.IP)
-		}
-	}
-
-	if err := m.resolver.SetConfig(resolverCfg); err != nil {
+	rcfg, ocfg, err := m.compileConfig(cfg)
+	if err != nil {
 		return err
 	}
-	if err := m.os.SetDNS(osCfg); err != nil {
+
+	m.logf("Resolvercfg: %+v", rcfg)
+	m.logf("OScfg: %+v", ocfg)
+
+	if err := m.resolver.SetConfig(rcfg); err != nil {
+		return err
+	}
+	if err := m.os.SetDNS(ocfg); err != nil {
 		return err
 	}
 
 	return nil
 }
 
+// compileConfig converts cfg into a quad-100 resolver configuration
+// and an OS-level configuration.
+func (m *Manager) compileConfig(cfg Config) (rcfg resolver.Config, ocfg OSConfig, err error) {
+	// The internal resolver always gets MagicDNS hosts and
+	// authoritative suffixes, even if we don't propagate MagicDNS to
+	// the OS.
+	rcfg.Hosts = cfg.Hosts
+	routes := map[dnsname.FQDN][]netaddr.IPPort{} // assigned conditionally to rcfg.Routes below.
+	for suffix, resolvers := range cfg.Routes {
+		if len(resolvers) == 0 {
+			rcfg.LocalDomains = append(rcfg.LocalDomains, suffix)
+		} else {
+			routes[suffix] = resolvers
+		}
+	}
+	// Similarly, the OS always gets search paths.
+	ocfg.SearchDomains = cfg.SearchDomains
+
+	// Deal with trivial configs first.
+	switch {
+	case !cfg.needsOSResolver():
+		// Set search domains, but nothing else. This also covers the
+		// case where cfg is entirely zero, in which case these
+		// configs clear all Tailscale DNS settings.
+		return rcfg, ocfg, nil
+	case cfg.hasDefaultResolversOnly():
+		// Trivial CorpDNS configuration, just override the OS
+		// resolver.
+		ocfg.Nameservers = toIPsOnly(cfg.DefaultResolvers)
+		return rcfg, ocfg, nil
+	case cfg.hasDefaultResolvers():
+		// Default resolvers plus other stuff always ends up proxying
+		// through quad-100.
+		rcfg.Routes = routes
+		rcfg.Routes["."] = cfg.DefaultResolvers
+		ocfg.Nameservers = []netaddr.IP{tsaddr.TailscaleServiceIP()}
+		return rcfg, ocfg, nil
+	}
+
+	// From this point on, we're figuring out split DNS
+	// configurations. The possible cases don't return directly any
+	// more, because as a final step we have to handle the case where
+	// the OS can't do split DNS.
+
+	// Workaround for
+	// https://github.com/tailscale/corp/issues/1662. Even though
+	// Windows natively supports split DNS, it only configures linux
+	// containers using whatever the primary is, and doesn't apply
+	// NRPT rules to DNS traffic coming from WSL.
+	//
+	// In order to make WSL work okay when the host Windows is using
+	// Tailscale, we need to set up quad-100 as a "full proxy"
+	// resolver, regardless of whether Windows itself can do split
+	// DNS. We still make Windows do split DNS itself when it can, but
+	// quad-100 will still have the full split configuration as well,
+	// and so can service WSL requests correctly.
+	//
+	// This bool is used in a couple of places below to implement this
+	// workaround.
+	isWindows := runtime.GOOS == "windows"
+	if cfg.singleResolverSet() != nil && m.os.SupportsSplitDNS() && !isWindows {
+		// Split DNS configuration requested, where all split domains
+		// go to the same resolvers. We can let the OS do it.
+		ocfg.Nameservers = toIPsOnly(cfg.singleResolverSet())
+		ocfg.MatchDomains = cfg.matchDomains()
+		return rcfg, ocfg, nil
+	}
+
+	// Split DNS configuration with either multiple upstream routes,
+	// or routes + MagicDNS, or just MagicDNS, or on an OS that cannot
+	// split-DNS. Install a split config pointing at quad-100.
+	rcfg.Routes = routes
+	ocfg.Nameservers = []netaddr.IP{tsaddr.TailscaleServiceIP()}
+
+	// If the OS can't do native split-dns, read out the underlying
+	// resolver config and blend it into our config.
+	if m.os.SupportsSplitDNS() {
+		ocfg.MatchDomains = cfg.matchDomains()
+	}
+	if !m.os.SupportsSplitDNS() || isWindows {
+		bcfg, err := m.os.GetBaseConfig()
+		if err != nil {
+			return resolver.Config{}, OSConfig{}, err
+		}
+		rcfg.Routes["."] = toIPPorts(bcfg.Nameservers)
+		ocfg.SearchDomains = append(ocfg.SearchDomains, bcfg.SearchDomains...)
+	}
+
+	return rcfg, ocfg, nil
+}
+
+// toIPsOnly returns only the IP portion of ipps.
+// TODO: this discards port information on the assumption that we're
+// always pointing at port 53.
+// https://github.com/tailscale/tailscale/issues/1666 tracks making
+// that not true, if we ever want to.
+func toIPsOnly(ipps []netaddr.IPPort) (ret []netaddr.IP) {
+	ret = make([]netaddr.IP, 0, len(ipps))
+	for _, ipp := range ipps {
+		ret = append(ret, ipp.IP())
+	}
+	return ret
+}
+
+func toIPPorts(ips []netaddr.IP) (ret []netaddr.IPPort) {
+	ret = make([]netaddr.IPPort, 0, len(ips))
+	for _, ip := range ips {
+		ret = append(ret, netaddr.IPPortFrom(ip, 53))
+	}
+	return ret
+}
+
 func (m *Manager) EnqueueRequest(bs []byte, from netaddr.IPPort) error {
 	return m.resolver.EnqueueRequest(bs, from)
 }
@@ -113,7 +204,11 @@ func (m *Manager) Down() error {
 // in case the Tailscale daemon terminated without closing the router.
 // No other state needs to be instantiated before this runs.
 func Cleanup(logf logger.Logf, interfaceName string) {
-	oscfg := NewOSConfigurator(logf, interfaceName)
+	oscfg, err := NewOSConfigurator(logf, interfaceName)
+	if err != nil {
+		logf("creating dns cleanup: %v", err)
+		return
+	}
 	dns := NewManager(logf, oscfg, nil)
 	if err := dns.Down(); err != nil {
 		logf("dns down: %v", err)

net/dns/manager_default.go

@@ -8,7 +8,7 @@ package dns
 
 import "tailscale.com/types/logger"
 
-func NewOSConfigurator(logger.Logf, string) OSConfigurator {
+func NewOSConfigurator(logger.Logf, string) (OSConfigurator, error) {
 	// TODO(dmytro): on darwin, we should use a macOS-specific method such as scutil.
 	// This is currently not implemented. Editing /etc/resolv.conf does not work,
 	// as most applications use the system resolver, which disregards it.

net/dns/manager_freebsd.go

@@ -4,11 +4,25 @@
 
 package dns
 
-import "tailscale.com/types/logger"
+import (
+	"fmt"
+	"io/ioutil"
+	"os"
 
-func NewOSConfigurator(logf logger.Logf, _ string) OSConfigurator {
-	switch {
-	case isResolvconfActive():
+	"tailscale.com/types/logger"
+)
+
+func NewOSConfigurator(logf logger.Logf, _ string) (OSConfigurator, error) {
+	bs, err := ioutil.ReadFile("/etc/resolv.conf")
+	if os.IsNotExist(err) {
+		return newDirectManager()
+	}
+	if err != nil {
+		return nil, fmt.Errorf("reading /etc/resolv.conf: %w", err)
+	}
+
+	switch resolvOwner(bs) {
+	case "resolvconf":
 		return newResolvconfManager(logf)
 	default:
 		return newDirectManager()

net/dns/manager_linux.go

@@ -4,18 +4,237 @@
 
 package dns
 
-import "tailscale.com/types/logger"
+import (
+	"bytes"
+	"context"
+	"errors"
+	"fmt"
+	"io/ioutil"
+	"os"
+	"os/exec"
+	"time"
 
-func NewOSConfigurator(logf logger.Logf, interfaceName string) OSConfigurator {
-	switch {
-	// TODO: rework NetworkManager and resolved support.
-	// case isResolvedActive():
-	// 	return newResolvedManager()
-	// case isNMActive():
-	// 	return newNMManager(interfaceName)
-	case isResolvconfActive():
+	"github.com/godbus/dbus/v5"
+	"tailscale.com/types/logger"
+	"tailscale.com/util/cmpver"
+)
+
+type kv struct {
+	k, v string
+}
+
+func (kv kv) String() string {
+	return fmt.Sprintf("%s=%s", kv.k, kv.v)
+}
+
+func NewOSConfigurator(logf logger.Logf, interfaceName string) (ret OSConfigurator, err error) {
+	var debug []kv
+	dbg := func(k, v string) {
+		debug = append(debug, kv{k, v})
+	}
+	defer func() {
+		if ret != nil {
+			dbg("ret", fmt.Sprintf("%T", ret))
+		}
+		logf("dns: %v", debug)
+	}()
+
+	bs, err := ioutil.ReadFile("/etc/resolv.conf")
+	if os.IsNotExist(err) {
+		dbg("rc", "missing")
+		return newDirectManager()
+	}
+	if err != nil {
+		return nil, fmt.Errorf("reading /etc/resolv.conf: %w", err)
+	}
+
+	switch resolvOwner(bs) {
+	case "systemd-resolved":
+		dbg("rc", "resolved")
+		if err := dbusPing("org.freedesktop.resolve1", "/org/freedesktop/resolve1"); err != nil {
+			dbg("resolved", "no")
+			return newDirectManager()
+		}
+		if err := dbusPing("org.freedesktop.NetworkManager", "/org/freedesktop/NetworkManager/DnsManager"); err != nil {
+			dbg("nm", "no")
+			return newResolvedManager(logf, interfaceName)
+		}
+		dbg("nm", "yes")
+		if err := nmIsUsingResolved(); err != nil {
+			dbg("nm-resolved", "no")
+			return newResolvedManager(logf, interfaceName)
+		}
+		dbg("nm-resolved", "yes")
+
+		// Version of NetworkManager before 1.26.6 programmed resolved
+		// incorrectly, such that NM's settings would always take
+		// precedence over other settings set by other resolved
+		// clients.
+		//
+		// If we're dealing with such a version, we have to set our
+		// DNS settings through NM to have them take.
+		//
+		// However, versions 1.26.6 later both fixed the resolved
+		// programming issue _and_ started ignoring DNS settings for
+		// "unmanaged" interfaces - meaning NM 1.26.6 and later
+		// actively ignore DNS configuration we give it. So, for those
+		// NM versions, we can and must use resolved directly.
+		old, err := nmVersionOlderThan("1.26.6")
+		if err != nil {
+			// Failed to figure out NM's version, can't make a correct
+			// decision.
+			return nil, fmt.Errorf("checking NetworkManager version: %v", err)
+		}
+		if old {
+			dbg("nm-old", "yes")
+			return newNMManager(interfaceName)
+		}
+		dbg("nm-old", "no")
+		return newResolvedManager(logf, interfaceName)
+	case "resolvconf":
+		dbg("rc", "resolvconf")
+		if err := resolvconfSourceIsNM(bs); err == nil {
+			dbg("src-is-nm", "yes")
+			if err := dbusPing("org.freedesktop.NetworkManager", "/org/freedesktop/NetworkManager/DnsManager"); err == nil {
+				dbg("nm", "yes")
+				old, err := nmVersionOlderThan("1.26.6")
+				if err != nil {
+					return nil, fmt.Errorf("checking NetworkManager version: %v", err)
+				}
+				if old {
+					dbg("nm-old", "yes")
+					return newNMManager(interfaceName)
+				} else {
+					dbg("nm-old", "no")
+				}
+			} else {
+				dbg("nm", "no")
+			}
+		} else {
+			dbg("src-is-nm", "no")
+		}
+		if _, err := exec.LookPath("resolvconf"); err != nil {
+			dbg("resolvconf", "no")
+			return newDirectManager()
+		}
+		dbg("resolvconf", "yes")
 		return newResolvconfManager(logf)
+	case "NetworkManager":
+		dbg("rc", "nm")
+		if err := dbusPing("org.freedesktop.NetworkManager", "/org/freedesktop/NetworkManager/DnsManager"); err != nil {
+			dbg("nm", "no")
+			return newDirectManager()
+		}
+		dbg("nm", "yes")
+		old, err := nmVersionOlderThan("1.26.6")
+		if err != nil {
+			return nil, fmt.Errorf("checking NetworkManager version: %v", err)
+		}
+		if old {
+			dbg("nm-old", "yes")
+			return newNMManager(interfaceName)
+		}
+		dbg("nm-old", "no")
+		return newDirectManager()
 	default:
+		dbg("rc", "unknown")
 		return newDirectManager()
 	}
 }
+
+func resolvconfSourceIsNM(resolvDotConf []byte) error {
+	b := bytes.NewBuffer(resolvDotConf)
+	cfg, err := readResolv(b)
+	if err != nil {
+		return fmt.Errorf("parsing /etc/resolv.conf: %w", err)
+	}
+
+	var (
+		paths = []string{
+			"/etc/resolvconf/run/interface/NetworkManager",
+			"/run/resolvconf/interface/NetworkManager",
+			"/var/run/resolvconf/interface/NetworkManager",
+			"/run/resolvconf/interfaces/NetworkManager",
+			"/var/run/resolvconf/interfaces/NetworkManager",
+		}
+		nmCfg OSConfig
+		found bool
+	)
+	for _, path := range paths {
+		nmCfg, err = readResolvFile(path)
+		if os.IsNotExist(err) {
+			continue
+		} else if err != nil {
+			return err
+		}
+		found = true
+		break
+	}
+	if !found {
+		return errors.New("NetworkManager resolvconf snippet not found")
+	}
+
+	if !nmCfg.Equal(cfg) {
+		return errors.New("NetworkManager config not applied by resolvconf")
+	}
+
+	return nil
+}
+
+func nmVersionOlderThan(want string) (bool, error) {
+	conn, err := dbus.SystemBus()
+	if err != nil {
+		// DBus probably not running.
+		return false, err
+	}
+
+	nm := conn.Object("org.freedesktop.NetworkManager", dbus.ObjectPath("/org/freedesktop/NetworkManager"))
+	v, err := nm.GetProperty("org.freedesktop.NetworkManager.Version")
+	if err != nil {
+		return false, err
+	}
+
+	version, ok := v.Value().(string)
+	if !ok {
+		return false, fmt.Errorf("unexpected type %T for NM version", v.Value())
+	}
+
+	return cmpver.Compare(version, want) < 0, nil
+}
+
+func nmIsUsingResolved() error {
+	conn, err := dbus.SystemBus()
+	if err != nil {
+		// DBus probably not running.
+		return err
+	}
+
+	nm := conn.Object("org.freedesktop.NetworkManager", dbus.ObjectPath("/org/freedesktop/NetworkManager/DnsManager"))
+	v, err := nm.GetProperty("org.freedesktop.NetworkManager.DnsManager.Mode")
+	if err != nil {
+		return fmt.Errorf("getting NM mode: %w", err)
+	}
+	mode, ok := v.Value().(string)
+	if !ok {
+		return fmt.Errorf("unexpected type %T for NM DNS mode", v.Value())
+	}
+	if mode != "systemd-resolved" {
+		return errors.New("NetworkManager is not using systemd-resolved for DNS")
+	}
+	return nil
+}
+
+func dbusPing(name, objectPath string) error {
+	ctx, cancel := context.WithTimeout(context.Background(), time.Second)
+	defer cancel()
+
+	conn, err := dbus.SystemBus()
+	if err != nil {
+		// DBus probably not running.
+		return err
+	}
+
+	obj := conn.Object(name, dbus.ObjectPath(objectPath))
+	call := obj.CallWithContext(ctx, "org.freedesktop.DBus.Peer.Ping", 0)
+	return call.Err
+}

net/dns/manager_openbsd.go

@@ -6,6 +6,6 @@ package dns
 
 import "tailscale.com/types/logger"
 
-func NewOSConfigurator(logger.Logf, string) OSConfigurator {
+func NewOSConfigurator(logger.Logf, string) (OSConfigurator, error) {
 	return newDirectManager()
 }

net/dns/manager_test.go

@@ -0,0 +1,465 @@
+// Copyright (c) 2021 Tailscale Inc & AUTHORS All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package dns
+
+import (
+	"runtime"
+	"testing"
+
+	"github.com/google/go-cmp/cmp"
+	"github.com/google/go-cmp/cmp/cmpopts"
+	"inet.af/netaddr"
+	"tailscale.com/net/dns/resolver"
+	"tailscale.com/util/dnsname"
+)
+
+type fakeOSConfigurator struct {
+	SplitDNS   bool
+	BaseConfig OSConfig
+
+	OSConfig       OSConfig
+	ResolverConfig resolver.Config
+}
+
+func (c *fakeOSConfigurator) SetDNS(cfg OSConfig) error {
+	if !c.SplitDNS && len(cfg.MatchDomains) > 0 {
+		panic("split DNS config passed to non-split OSConfigurator")
+	}
+	c.OSConfig = cfg
+	return nil
+}
+
+func (c *fakeOSConfigurator) SetResolver(cfg resolver.Config) {
+	c.ResolverConfig = cfg
+}
+
+func (c *fakeOSConfigurator) SupportsSplitDNS() bool {
+	return c.SplitDNS
+}
+
+func (c *fakeOSConfigurator) GetBaseConfig() (OSConfig, error) {
+	return c.BaseConfig, nil
+}
+
+func (c *fakeOSConfigurator) Close() error { return nil }
+
+func TestManager(t *testing.T) {
+	if runtime.GOOS == "windows" {
+		t.Skipf("test's assumptions break because of https://github.com/tailscale/corp/issues/1662")
+	}
+
+	// Note: these tests assume that it's safe to switch the
+	// OSConfigurator's split-dns support on and off between Set
+	// calls. Empirically this is currently true, because we reprobe
+	// the support every time we generate configs. It would be
+	// reasonable to make this unsupported as well, in which case
+	// these tests will need tweaking.
+	tests := []struct {
+		name  string
+		in    Config
+		split bool
+		bs    OSConfig
+		os    OSConfig
+		rs    resolver.Config
+	}{
+		{
+			name: "empty",
+		},
+		{
+			name: "search-only",
+			in: Config{
+				SearchDomains: fqdns("tailscale.com", "universe.tf"),
+			},
+			os: OSConfig{
+				SearchDomains: fqdns("tailscale.com", "universe.tf"),
+			},
+		},
+		{
+			// Regression test for https://github.com/tailscale/tailscale/issues/1886
+			name: "hosts-only",
+			in: Config{
+				Hosts: hosts(
+					"dave.ts.com.", "1.2.3.4",
+					"bradfitz.ts.com.", "2.3.4.5"),
+			},
+			rs: resolver.Config{
+				Hosts: hosts(
+					"dave.ts.com.", "1.2.3.4",
+					"bradfitz.ts.com.", "2.3.4.5"),
+			},
+		},
+		{
+			name: "corp",
+			in: Config{
+				DefaultResolvers: mustIPPs("1.1.1.1:53", "9.9.9.9:53"),
+				SearchDomains:    fqdns("tailscale.com", "universe.tf"),
+			},
+			os: OSConfig{
+				Nameservers:   mustIPs("1.1.1.1", "9.9.9.9"),
+				SearchDomains: fqdns("tailscale.com", "universe.tf"),
+			},
+		},
+		{
+			name: "corp-split",
+			in: Config{
+				DefaultResolvers: mustIPPs("1.1.1.1:53", "9.9.9.9:53"),
+				SearchDomains:    fqdns("tailscale.com", "universe.tf"),
+			},
+			split: true,
+			os: OSConfig{
+				Nameservers:   mustIPs("1.1.1.1", "9.9.9.9"),
+				SearchDomains: fqdns("tailscale.com", "universe.tf"),
+			},
+		},
+		{
+			name: "corp-magic",
+			in: Config{
+				DefaultResolvers: mustIPPs("1.1.1.1:53", "9.9.9.9:53"),
+				SearchDomains:    fqdns("tailscale.com", "universe.tf"),
+				Routes:           upstreams("ts.com", ""),
+				Hosts: hosts(
+					"dave.ts.com.", "1.2.3.4",
+					"bradfitz.ts.com.", "2.3.4.5"),
+			},
+			os: OSConfig{
+				Nameservers:   mustIPs("100.100.100.100"),
+				SearchDomains: fqdns("tailscale.com", "universe.tf"),
+			},
+			rs: resolver.Config{
+				Routes: upstreams(".", "1.1.1.1:53", "9.9.9.9:53"),
+				Hosts: hosts(
+					"dave.ts.com.", "1.2.3.4",
+					"bradfitz.ts.com.", "2.3.4.5"),
+				LocalDomains: fqdns("ts.com."),
+			},
+		},
+		{
+			name: "corp-magic-split",
+			in: Config{
+				DefaultResolvers: mustIPPs("1.1.1.1:53", "9.9.9.9:53"),
+				SearchDomains:    fqdns("tailscale.com", "universe.tf"),
+				Routes:           upstreams("ts.com", ""),
+				Hosts: hosts(
+					"dave.ts.com.", "1.2.3.4",
+					"bradfitz.ts.com.", "2.3.4.5"),
+			},
+			split: true,
+			os: OSConfig{
+				Nameservers:   mustIPs("100.100.100.100"),
+				SearchDomains: fqdns("tailscale.com", "universe.tf"),
+			},
+			rs: resolver.Config{
+				Routes: upstreams(".", "1.1.1.1:53", "9.9.9.9:53"),
+				Hosts: hosts(
+					"dave.ts.com.", "1.2.3.4",
+					"bradfitz.ts.com.", "2.3.4.5"),
+				LocalDomains: fqdns("ts.com."),
+			},
+		},
+		{
+			name: "corp-routes",
+			in: Config{
+				DefaultResolvers: mustIPPs("1.1.1.1:53", "9.9.9.9:53"),
+				Routes:           upstreams("corp.com", "2.2.2.2:53"),
+				SearchDomains:    fqdns("tailscale.com", "universe.tf"),
+			},
+			os: OSConfig{
+				Nameservers:   mustIPs("100.100.100.100"),
+				SearchDomains: fqdns("tailscale.com", "universe.tf"),
+			},
+			rs: resolver.Config{
+				Routes: upstreams(
+					".", "1.1.1.1:53", "9.9.9.9:53",
+					"corp.com.", "2.2.2.2:53"),
+			},
+		},
+		{
+			name: "corp-routes-split",
+			in: Config{
+				DefaultResolvers: mustIPPs("1.1.1.1:53", "9.9.9.9:53"),
+				Routes:           upstreams("corp.com", "2.2.2.2:53"),
+				SearchDomains:    fqdns("tailscale.com", "universe.tf"),
+			},
+			split: true,
+			os: OSConfig{
+				Nameservers:   mustIPs("100.100.100.100"),
+				SearchDomains: fqdns("tailscale.com", "universe.tf"),
+			},
+			rs: resolver.Config{
+				Routes: upstreams(
+					".", "1.1.1.1:53", "9.9.9.9:53",
+					"corp.com.", "2.2.2.2:53"),
+			},
+		},
+		{
+			name: "routes",
+			in: Config{
+				Routes:        upstreams("corp.com", "2.2.2.2:53"),
+				SearchDomains: fqdns("tailscale.com", "universe.tf"),
+			},
+			bs: OSConfig{
+				Nameservers:   mustIPs("8.8.8.8"),
+				SearchDomains: fqdns("coffee.shop"),
+			},
+			os: OSConfig{
+				Nameservers:   mustIPs("100.100.100.100"),
+				SearchDomains: fqdns("tailscale.com", "universe.tf", "coffee.shop"),
+			},
+			rs: resolver.Config{
+				Routes: upstreams(
+					".", "8.8.8.8:53",
+					"corp.com.", "2.2.2.2:53"),
+			},
+		},
+		{
+			name: "routes-split",
+			in: Config{
+				Routes:        upstreams("corp.com", "2.2.2.2:53"),
+				SearchDomains: fqdns("tailscale.com", "universe.tf"),
+			},
+			split: true,
+			os: OSConfig{
+				Nameservers:   mustIPs("2.2.2.2"),
+				SearchDomains: fqdns("tailscale.com", "universe.tf"),
+				MatchDomains:  fqdns("corp.com"),
+			},
+		},
+		{
+			name: "routes-multi",
+			in: Config{
+				Routes: upstreams(
+					"corp.com", "2.2.2.2:53",
+					"bigco.net", "3.3.3.3:53"),
+				SearchDomains: fqdns("tailscale.com", "universe.tf"),
+			},
+			bs: OSConfig{
+				Nameservers:   mustIPs("8.8.8.8"),
+				SearchDomains: fqdns("coffee.shop"),
+			},
+			os: OSConfig{
+				Nameservers:   mustIPs("100.100.100.100"),
+				SearchDomains: fqdns("tailscale.com", "universe.tf", "coffee.shop"),
+			},
+			rs: resolver.Config{
+				Routes: upstreams(
+					".", "8.8.8.8:53",
+					"corp.com.", "2.2.2.2:53",
+					"bigco.net.", "3.3.3.3:53"),
+			},
+		},
+		{
+			name: "routes-multi-split",
+			in: Config{
+				Routes: upstreams(
+					"corp.com", "2.2.2.2:53",
+					"bigco.net", "3.3.3.3:53"),
+				SearchDomains: fqdns("tailscale.com", "universe.tf"),
+			},
+			split: true,
+			os: OSConfig{
+				Nameservers:   mustIPs("100.100.100.100"),
+				SearchDomains: fqdns("tailscale.com", "universe.tf"),
+				MatchDomains:  fqdns("bigco.net", "corp.com"),
+			},
+			rs: resolver.Config{
+				Routes: upstreams(
+					"corp.com.", "2.2.2.2:53",
+					"bigco.net.", "3.3.3.3:53"),
+			},
+		},
+		{
+			name: "magic",
+			in: Config{
+				Hosts: hosts(
+					"dave.ts.com.", "1.2.3.4",
+					"bradfitz.ts.com.", "2.3.4.5"),
+				Routes:        upstreams("ts.com", ""),
+				SearchDomains: fqdns("tailscale.com", "universe.tf"),
+			},
+			bs: OSConfig{
+				Nameservers:   mustIPs("8.8.8.8"),
+				SearchDomains: fqdns("coffee.shop"),
+			},
+			os: OSConfig{
+				Nameservers:   mustIPs("100.100.100.100"),
+				SearchDomains: fqdns("tailscale.com", "universe.tf", "coffee.shop"),
+			},
+			rs: resolver.Config{
+				Routes: upstreams(".", "8.8.8.8:53"),
+				Hosts: hosts(
+					"dave.ts.com.", "1.2.3.4",
+					"bradfitz.ts.com.", "2.3.4.5"),
+				LocalDomains: fqdns("ts.com."),
+			},
+		},
+		{
+			name: "magic-split",
+			in: Config{
+				Hosts: hosts(
+					"dave.ts.com.", "1.2.3.4",
+					"bradfitz.ts.com.", "2.3.4.5"),
+				Routes:        upstreams("ts.com", ""),
+				SearchDomains: fqdns("tailscale.com", "universe.tf"),
+			},
+			split: true,
+			os: OSConfig{
+				Nameservers:   mustIPs("100.100.100.100"),
+				SearchDomains: fqdns("tailscale.com", "universe.tf"),
+				MatchDomains:  fqdns("ts.com"),
+			},
+			rs: resolver.Config{
+				Hosts: hosts(
+					"dave.ts.com.", "1.2.3.4",
+					"bradfitz.ts.com.", "2.3.4.5"),
+				LocalDomains: fqdns("ts.com."),
+			},
+		},
+		{
+			name: "routes-magic",
+			in: Config{
+				Routes: upstreams("corp.com", "2.2.2.2:53", "ts.com", ""),
+				Hosts: hosts(
+					"dave.ts.com.", "1.2.3.4",
+					"bradfitz.ts.com.", "2.3.4.5"),
+				SearchDomains: fqdns("tailscale.com", "universe.tf"),
+			},
+			bs: OSConfig{
+				Nameservers:   mustIPs("8.8.8.8"),
+				SearchDomains: fqdns("coffee.shop"),
+			},
+			os: OSConfig{
+				Nameservers:   mustIPs("100.100.100.100"),
+				SearchDomains: fqdns("tailscale.com", "universe.tf", "coffee.shop"),
+			},
+			rs: resolver.Config{
+				Routes: upstreams(
+					"corp.com.", "2.2.2.2:53",
+					".", "8.8.8.8:53"),
+				Hosts: hosts(
+					"dave.ts.com.", "1.2.3.4",
+					"bradfitz.ts.com.", "2.3.4.5"),
+				LocalDomains: fqdns("ts.com."),
+			},
+		},
+		{
+			name: "routes-magic-split",
+			in: Config{
+				Routes: upstreams(
+					"corp.com", "2.2.2.2:53",
+					"ts.com", ""),
+				Hosts: hosts(
+					"dave.ts.com.", "1.2.3.4",
+					"bradfitz.ts.com.", "2.3.4.5"),
+				SearchDomains: fqdns("tailscale.com", "universe.tf"),
+			},
+			split: true,
+			os: OSConfig{
+				Nameservers:   mustIPs("100.100.100.100"),
+				SearchDomains: fqdns("tailscale.com", "universe.tf"),
+				MatchDomains:  fqdns("corp.com", "ts.com"),
+			},
+			rs: resolver.Config{
+				Routes: upstreams("corp.com.", "2.2.2.2:53"),
+				Hosts: hosts(
+					"dave.ts.com.", "1.2.3.4",
+					"bradfitz.ts.com.", "2.3.4.5"),
+				LocalDomains: fqdns("ts.com."),
+			},
+		},
+	}
+
+	for _, test := range tests {
+		t.Run(test.name, func(t *testing.T) {
+			f := fakeOSConfigurator{
+				SplitDNS:   test.split,
+				BaseConfig: test.bs,
+			}
+			m := NewManager(t.Logf, &f, nil)
+			m.resolver.TestOnlySetHook(f.SetResolver)
+
+			if err := m.Set(test.in); err != nil {
+				t.Fatalf("m.Set: %v", err)
+			}
+			trIP := cmp.Transformer("ipStr", func(ip netaddr.IP) string { return ip.String() })
+			trIPPort := cmp.Transformer("ippStr", func(ipp netaddr.IPPort) string { return ipp.String() })
+			if diff := cmp.Diff(f.OSConfig, test.os, trIP, trIPPort, cmpopts.EquateEmpty()); diff != "" {
+				t.Errorf("wrong OSConfig (-got+want)\n%s", diff)
+			}
+			if diff := cmp.Diff(f.ResolverConfig, test.rs, trIP, trIPPort, cmpopts.EquateEmpty()); diff != "" {
+				t.Errorf("wrong resolver.Config (-got+want)\n%s", diff)
+			}
+		})
+	}
+}
+
+func mustIPs(strs ...string) (ret []netaddr.IP) {
+	for _, s := range strs {
+		ret = append(ret, netaddr.MustParseIP(s))
+	}
+	return ret
+}
+
+func mustIPPs(strs ...string) (ret []netaddr.IPPort) {
+	for _, s := range strs {
+		ret = append(ret, netaddr.MustParseIPPort(s))
+	}
+	return ret
+}
+
+func fqdns(strs ...string) (ret []dnsname.FQDN) {
+	for _, s := range strs {
+		fqdn, err := dnsname.ToFQDN(s)
+		if err != nil {
+			panic(err)
+		}
+		ret = append(ret, fqdn)
+	}
+	return ret
+}
+
+func hosts(strs ...string) (ret map[dnsname.FQDN][]netaddr.IP) {
+	var key dnsname.FQDN
+	ret = map[dnsname.FQDN][]netaddr.IP{}
+	for _, s := range strs {
+		if ip, err := netaddr.ParseIP(s); err == nil {
+			if key == "" {
+				panic("IP provided before name")
+			}
+			ret[key] = append(ret[key], ip)
+		} else {
+			fqdn, err := dnsname.ToFQDN(s)
+			if err != nil {
+				panic(err)
+			}
+			key = fqdn
+		}
+	}
+	return ret
+}
+
+func upstreams(strs ...string) (ret map[dnsname.FQDN][]netaddr.IPPort) {
+	var key dnsname.FQDN
+	ret = map[dnsname.FQDN][]netaddr.IPPort{}
+	for _, s := range strs {
+		if s == "" {
+			if key == "" {
+				panic("IPPort provided before suffix")
+			}
+			ret[key] = nil
+		} else if ipp, err := netaddr.ParseIPPort(s); err == nil {
+			if key == "" {
+				panic("IPPort provided before suffix")
+			}
+			ret[key] = append(ret[key], ipp)
+		} else {
+			fqdn, err := dnsname.ToFQDN(s)
+			if err != nil {
+				panic(err)
+			}
+			key = fqdn
+		}
+	}
+	return ret
+}

net/dns/manager_windows.go

@@ -5,31 +5,59 @@
 package dns
 
 import (
+	"errors"
 	"fmt"
 	"os/exec"
+	"sort"
 	"strings"
 	"syscall"
 	"time"
 
+	"golang.org/x/sys/windows"
 	"golang.org/x/sys/windows/registry"
+	"golang.zx2c4.com/wireguard/windows/tunnel/winipcfg"
+	"inet.af/netaddr"
 	"tailscale.com/types/logger"
+	"tailscale.com/util/dnsname"
 )
 
 const (
 	ipv4RegBase = `SYSTEM\CurrentControlSet\Services\Tcpip\Parameters`
 	ipv6RegBase = `SYSTEM\CurrentControlSet\Services\Tcpip6\Parameters`
+
+	// the GUID is randomly generated. At present, Tailscale installs
+	// zero or one NRPT rules, so hardcoding a single GUID everywhere
+	// is fine.
+	nrptBase        = `SYSTEM\CurrentControlSet\services\Dnscache\Parameters\DnsPolicyConfig\{5abe529b-675b-4486-8459-25a634dacc23}`
+	nrptOverrideDNS = 0x8 // bitmask value for "use the provided override DNS resolvers"
+
+	versionKey = `SOFTWARE\Microsoft\Windows NT\CurrentVersion`
 )
 
 type windowsManager struct {
-	logf logger.Logf
-	guid string
+	logf      logger.Logf
+	guid      string
+	nrptWorks bool
 }
 
-func NewOSConfigurator(logf logger.Logf, interfaceName string) OSConfigurator {
-	return windowsManager{
-		logf: logf,
-		guid: interfaceName,
+func NewOSConfigurator(logf logger.Logf, interfaceName string) (OSConfigurator, error) {
+	ret := windowsManager{
+		logf:      logf,
+		guid:      interfaceName,
+		nrptWorks: !isWindows7(),
 	}
+
+	// Best-effort: if our NRPT rule exists, try to delete it. Unlike
+	// per-interface configuration, NRPT rules survive the unclean
+	// termination of the Tailscale process, and depending on the
+	// rule, it may prevent us from reaching login.tailscale.com to
+	// boot up. The bootstrap resolver logic will save us, but it
+	// slows down start-up a bunch.
+	if ret.nrptWorks {
+		ret.delKey(nrptBase)
+	}
+
+	return ret, nil
 }
 
 // keyOpenTimeout is how long we wait for a registry key to
@@ -38,37 +66,86 @@ func NewOSConfigurator(logf logger.Logf, interfaceName string) OSConfigurator {
 // can end up racing with that.
 const keyOpenTimeout = 20 * time.Second
 
-func setRegistryString(path, name, value string) error {
+func (m windowsManager) openKey(path string) (registry.Key, error) {
 	key, err := openKeyWait(registry.LOCAL_MACHINE, path, registry.SET_VALUE, keyOpenTimeout)
 	if err != nil {
-		return fmt.Errorf("opening %s: %w", path, err)
+		return 0, fmt.Errorf("opening %s: %w", path, err)
 	}
-	defer key.Close()
+	return key, nil
+}
 
-	err = key.SetStringValue(name, value)
-	if err != nil {
-		return fmt.Errorf("setting %s[%s]: %w", path, name, err)
+func (m windowsManager) ifPath(basePath string) string {
+	return fmt.Sprintf(`%s\Interfaces\%s`, basePath, m.guid)
+}
+
+func (m windowsManager) delKey(path string) error {
+	if err := registry.DeleteKey(registry.LOCAL_MACHINE, path); err != nil && err != registry.ErrNotExist {
+		return err
 	}
 	return nil
 }
 
-func (m windowsManager) setNameservers(basePath string, nameservers []string) error {
-	path := fmt.Sprintf(`%s\Interfaces\%s`, basePath, m.guid)
-	value := strings.Join(nameservers, ",")
-	return setRegistryString(path, "NameServer", value)
+func delValue(key registry.Key, name string) error {
+	if err := key.DeleteValue(name); err != nil && err != registry.ErrNotExist {
+		return err
+	}
+	return nil
 }
 
-func (m windowsManager) setDomains(basePath string, domains []string) error {
-	path := fmt.Sprintf(`%s\Interfaces\%s`, basePath, m.guid)
-	value := strings.Join(domains, ",")
-	return setRegistryString(path, "SearchList", value)
+// setSplitDNS configures an NRPT (Name Resolution Policy Table) rule
+// to resolve queries for domains using resolvers, rather than the
+// system's "primary" resolver.
+//
+// If no resolvers are provided, the Tailscale NRPT rule is deleted.
+func (m windowsManager) setSplitDNS(resolvers []netaddr.IP, domains []dnsname.FQDN) error {
+	if len(resolvers) == 0 {
+		return m.delKey(nrptBase)
+	}
+
+	servers := make([]string, 0, len(resolvers))
+	for _, resolver := range resolvers {
+		servers = append(servers, resolver.String())
+	}
+	doms := make([]string, 0, len(domains))
+	for _, domain := range domains {
+		// NRPT rules must have a leading dot, which is not usual for
+		// DNS search paths.
+		doms = append(doms, "."+domain.WithoutTrailingDot())
+	}
+
+	// CreateKey is actually open-or-create, which suits us fine.
+	key, _, err := registry.CreateKey(registry.LOCAL_MACHINE, nrptBase, registry.SET_VALUE)
+	if err != nil {
+		return fmt.Errorf("opening %s: %w", nrptBase, err)
+	}
+	defer key.Close()
+	if err := key.SetDWordValue("Version", 1); err != nil {
+		return err
+	}
+	if err := key.SetStringsValue("Name", doms); err != nil {
+		return err
+	}
+	if err := key.SetStringValue("GenericDNSServers", strings.Join(servers, "; ")); err != nil {
+		return err
+	}
+	if err := key.SetDWordValue("ConfigOptions", nrptOverrideDNS); err != nil {
+		return err
+	}
+
+	return nil
 }
 
-func (m windowsManager) SetDNS(config OSConfig) error {
+// setPrimaryDNS sets the given resolvers and domains as the Tailscale
+// interface's DNS configuration.
+// If resolvers is non-empty, those resolvers become the system's
+// "primary" resolvers.
+// domains can be set without resolvers, which just contributes new
+// paths to the global DNS search list.
+func (m windowsManager) setPrimaryDNS(resolvers []netaddr.IP, domains []dnsname.FQDN) error {
 	var ipsv4 []string
 	var ipsv6 []string
 
-	for _, ip := range config.Nameservers {
+	for _, ip := range resolvers {
 		if ip.Is4() {
 			ipsv4 = append(ipsv4, ip.String())
 		} else {
@@ -76,19 +153,111 @@ func (m windowsManager) SetDNS(config OSConfig) error {
 		}
 	}
 
-	if err := m.setNameservers(ipv4RegBase, ipsv4); err != nil {
+	domStrs := make([]string, 0, len(domains))
+	for _, dom := range domains {
+		domStrs = append(domStrs, dom.WithoutTrailingDot())
+	}
+
+	key4, err := m.openKey(m.ifPath(ipv4RegBase))
+	if err != nil {
 		return err
 	}
-	if err := m.setDomains(ipv4RegBase, config.Domains); err != nil {
+	defer key4.Close()
+
+	if len(ipsv4) == 0 {
+		if err := delValue(key4, "NameServer"); err != nil {
+			return err
+		}
+	} else if err := key4.SetStringValue("NameServer", strings.Join(ipsv4, ",")); err != nil {
 		return err
 	}
 
-	if err := m.setNameservers(ipv6RegBase, ipsv6); err != nil {
+	if len(domains) == 0 {
+		if err := delValue(key4, "SearchList"); err != nil {
+			return err
+		}
+	} else if err := key4.SetStringValue("SearchList", strings.Join(domStrs, ",")); err != nil {
 		return err
 	}
-	if err := m.setDomains(ipv6RegBase, config.Domains); err != nil {
+
+	key6, err := m.openKey(m.ifPath(ipv6RegBase))
+	if err != nil {
 		return err
 	}
+	defer key6.Close()
+
+	if len(ipsv6) == 0 {
+		if err := delValue(key6, "NameServer"); err != nil {
+			return err
+		}
+	} else if err := key6.SetStringValue("NameServer", strings.Join(ipsv6, ",")); err != nil {
+		return err
+	}
+
+	if len(domains) == 0 {
+		if err := delValue(key6, "SearchList"); err != nil {
+			return err
+		}
+	} else if err := key6.SetStringValue("SearchList", strings.Join(domStrs, ",")); err != nil {
+		return err
+	}
+
+	// Disable LLMNR on the Tailscale interface. We don't do
+	// multicast, and we certainly don't do LLMNR, so it's pointless
+	// to make Windows try it.
+	if err := key4.SetDWordValue("EnableMulticast", 0); err != nil {
+		return err
+	}
+	if err := key6.SetDWordValue("EnableMulticast", 0); err != nil {
+		return err
+	}
+
+	return nil
+}
+
+func (m windowsManager) SetDNS(cfg OSConfig) error {
+	// We can configure Windows DNS in one of two ways:
+	//
+	//  - In primary DNS mode, we set the NameServer and SearchList
+	//    registry keys on our interface. Because our interface metric
+	//    is very low, this turns us into the one and only "primary"
+	//    resolver for the OS, i.e. all queries flow to the
+	//    resolver(s) we specify.
+	//  - In split DNS mode, we set the Domain registry key on our
+	//    interface (which adds that domain to the global search list,
+	//    but does not contribute other DNS configuration from the
+	//    interface), and configure an NRPT (Name Resolution Policy
+	//    Table) rule to route queries for our suffixes to the
+	//    provided resolver.
+	//
+	// When switching modes, we delete all the configuration related
+	// to the other mode, so these two are an XOR.
+	//
+	// Windows actually supports much more advanced configurations as
+	// well, with arbitrary routing of hosts and suffixes to arbitrary
+	// resolvers. However, we use it in a "simple" split domain
+	// configuration only, routing one set of things to the "split"
+	// resolver and the rest to the primary.
+
+	if len(cfg.MatchDomains) == 0 {
+		if err := m.setSplitDNS(nil, nil); err != nil {
+			return err
+		}
+		if err := m.setPrimaryDNS(cfg.Nameservers, cfg.SearchDomains); err != nil {
+			return err
+		}
+	} else if !m.nrptWorks {
+		return errors.New("cannot set per-domain resolvers on Windows 7")
+	} else {
+		if err := m.setSplitDNS(cfg.Nameservers, cfg.MatchDomains); err != nil {
+			return err
+		}
+		// Still set search domains on the interface, since NRPT only
+		// handles query routing and not search domain expansion.
+		if err := m.setPrimaryDNS(nil, cfg.SearchDomains); err != nil {
+			return err
+		}
+	}
 
 	// Force DNS re-registration in Active Directory. What we actually
 	// care about is that this command invokes the undocumented hidden
@@ -97,26 +266,163 @@ func (m windowsManager) SetDNS(config OSConfig) error {
 	// effect.
 	//
 	// This command can take a few seconds to run, so run it async, best effort.
+	//
+	// After re-registering DNS, also flush the DNS cache to clear out
+	// any cached split-horizon queries that are no longer the correct
+	// answer.
 	go func() {
 		t0 := time.Now()
 		m.logf("running ipconfig /registerdns ...")
 		cmd := exec.Command("ipconfig", "/registerdns")
 		cmd.SysProcAttr = &syscall.SysProcAttr{HideWindow: true}
+		err := cmd.Run()
 		d := time.Since(t0).Round(time.Millisecond)
-		if err := cmd.Run(); err != nil {
+		if err != nil {
 			m.logf("error running ipconfig /registerdns after %v: %v", d, err)
 		} else {
 			m.logf("ran ipconfig /registerdns in %v", d)
 		}
+
+		t0 = time.Now()
+		m.logf("running ipconfig /registerdns ...")
+		cmd = exec.Command("ipconfig", "/flushdns")
+		cmd.SysProcAttr = &syscall.SysProcAttr{HideWindow: true}
+		err = cmd.Run()
+		d = time.Since(t0).Round(time.Millisecond)
+		if err != nil {
+			m.logf("error running ipconfig /flushdns after %v: %v", d, err)
+		} else {
+			m.logf("ran ipconfig /flushdns in %v", d)
+		}
 	}()
 
 	return nil
 }
 
-func (m windowsManager) RoutingMode() RoutingMode {
-	return RoutingModeNone
+func (m windowsManager) SupportsSplitDNS() bool {
+	return m.nrptWorks
 }
 
 func (m windowsManager) Close() error {
 	return m.SetDNS(OSConfig{})
 }
+
+func (m windowsManager) GetBaseConfig() (OSConfig, error) {
+	resolvers, err := m.getBasePrimaryResolver()
+	if err != nil {
+		return OSConfig{}, err
+	}
+	return OSConfig{
+		Nameservers: resolvers,
+		// Don't return any search domains here, because even Windows
+		// 7 correctly handles blending search domains from multiple
+		// sources, and any search domains we add here will get tacked
+		// onto the Tailscale config unnecessarily.
+	}, nil
+}
+
+// getBasePrimaryResolver returns a guess of the non-Tailscale primary
+// resolver on the system.
+// It's used on Windows 7 to emulate split DNS by trying to figure out
+// what the "previous" primary resolver was. It might be wrong, or
+// incomplete.
+func (m windowsManager) getBasePrimaryResolver() (resolvers []netaddr.IP, err error) {
+	tsGUID, err := windows.GUIDFromString(m.guid)
+	if err != nil {
+		return nil, err
+	}
+	tsLUID, err := winipcfg.LUIDFromGUID(&tsGUID)
+	if err != nil {
+		return nil, err
+	}
+	ifrows, err := winipcfg.GetIPInterfaceTable(windows.AF_INET)
+	if err == windows.ERROR_NOT_FOUND {
+		// IPv4 seems disabled, try to get interface metrics from IPv6 instead.
+		ifrows, err = winipcfg.GetIPInterfaceTable(windows.AF_INET6)
+	}
+	if err != nil {
+		return nil, err
+	}
+
+	type candidate struct {
+		id     winipcfg.LUID
+		metric uint32
+	}
+	var candidates []candidate
+	for _, row := range ifrows {
+		if !row.Connected {
+			continue
+		}
+		if row.InterfaceLUID == tsLUID {
+			continue
+		}
+		candidates = append(candidates, candidate{row.InterfaceLUID, row.Metric})
+	}
+	if len(candidates) == 0 {
+		// No resolvers set outside of Tailscale.
+		return nil, nil
+	}
+
+	sort.Slice(candidates, func(i, j int) bool { return candidates[i].metric < candidates[j].metric })
+
+	for _, candidate := range candidates {
+		ips, err := candidate.id.DNS()
+		if err != nil {
+			return nil, err
+		}
+
+	ipLoop:
+		for _, stdip := range ips {
+			ip, ok := netaddr.FromStdIP(stdip)
+			if !ok {
+				continue
+			}
+			// Skip IPv6 site-local resolvers. These are an ancient
+			// and obsolete IPv6 RFC, which Windows still faithfully
+			// implements. The net result is that some low-metric
+			// interfaces can "have" DNS resolvers, but they're just
+			// site-local resolver IPs that don't go anywhere. So, we
+			// skip the site-local resolvers in order to find the
+			// first interface that has real DNS servers configured.
+			for _, sl := range siteLocalResolvers {
+				if ip.WithZone("") == sl {
+					continue ipLoop
+				}
+			}
+			resolvers = append(resolvers, ip)
+		}
+
+		if len(resolvers) > 0 {
+			// Found some resolvers, we're done.
+			break
+		}
+	}
+
+	return resolvers, nil
+}
+
+var siteLocalResolvers = []netaddr.IP{
+	netaddr.MustParseIP("fec0:0:0:ffff::1"),
+	netaddr.MustParseIP("fec0:0:0:ffff::2"),
+	netaddr.MustParseIP("fec0:0:0:ffff::3"),
+}
+
+func isWindows7() bool {
+	key, err := registry.OpenKey(registry.LOCAL_MACHINE, versionKey, registry.READ)
+	if err != nil {
+		// Fail safe, assume Windows 7.
+		return true
+	}
+	ver, _, err := key.GetStringValue("CurrentVersion")
+	if err != nil {
+		return true
+	}
+	// Careful to not assume anything about version numbers beyond
+	// 6.3, Microsoft deprecated this registry key and locked its
+	// value to what it was in Windows 8.1. We can only use this to
+	// probe for versions before that. Good thing we only need Windows
+	// 7 (so far).
+	//
+	// And yes, Windows 7 is version 6.1. Don't ask.
+	return ver == "6.1"
+}

net/dns/nm.go

@@ -9,66 +9,68 @@
 package dns
 
 import (
-	"bufio"
-	"bytes"
 	"context"
 	"fmt"
-	"os"
-	"os/exec"
+	"sort"
+	"time"
 
 	"github.com/godbus/dbus/v5"
+	"inet.af/netaddr"
+	"tailscale.com/net/interfaces"
+	"tailscale.com/util/dnsname"
 	"tailscale.com/util/endian"
 )
 
-// isNMActive determines if NetworkManager is currently managing system DNS settings.
-func isNMActive() bool {
-	// This is somewhat tricky because NetworkManager supports a number
-	// of DNS configuration modes. In all cases, we expect it to be installed
-	// and /etc/resolv.conf to contain a mention of NetworkManager in the comments.
-	_, err := exec.LookPath("NetworkManager")
-	if err != nil {
-		return false
-	}
-
-	f, err := os.Open("/etc/resolv.conf")
-	if err != nil {
-		return false
-	}
-	defer f.Close()
-
-	scanner := bufio.NewScanner(f)
-	for scanner.Scan() {
-		line := scanner.Bytes()
-		// Look for the word "NetworkManager" until comments end.
-		if len(line) > 0 && line[0] != '#' {
-			return false
-		}
-		if bytes.Contains(line, []byte("NetworkManager")) {
-			return true
-		}
-	}
-	return false
-}
+const (
+	highestPriority = int32(-1 << 31)
+	mediumPriority  = int32(1)   // Highest priority that doesn't hard-override
+	lowerPriority   = int32(200) // lower than all builtin auto priorities
+)
 
 // nmManager uses the NetworkManager DBus API.
 type nmManager struct {
 	interfaceName string
+	manager       dbus.BusObject
+	dnsManager    dbus.BusObject
 }
 
-func newNMManager(interfaceName string) nmManager {
-	return nmManager{
-		interfaceName: interfaceName,
+func newNMManager(interfaceName string) (*nmManager, error) {
+	conn, err := dbus.SystemBus()
+	if err != nil {
+		return nil, err
 	}
+
+	return &nmManager{
+		interfaceName: interfaceName,
+		manager:       conn.Object("org.freedesktop.NetworkManager", dbus.ObjectPath("/org/freedesktop/NetworkManager")),
+		dnsManager:    conn.Object("org.freedesktop.NetworkManager", dbus.ObjectPath("/org/freedesktop/NetworkManager/DnsManager")),
+	}, nil
 }
 
 type nmConnectionSettings map[string]map[string]dbus.Variant
 
-func (m nmManager) SetDNS(config OSConfig) error {
+func (m *nmManager) SetDNS(config OSConfig) error {
 	ctx, cancel := context.WithTimeout(context.Background(), reconfigTimeout)
 	defer cancel()
 
-	// conn is a shared connection whose lifecycle is managed by the dbus package.
-	// We should not interfere with that by closing it.
+	// NetworkManager only lets you set DNS settings on "active"
+	// connections, which requires an assigned IP address. This got
+	// configured before the DNS manager was invoked, but it might
+	// take a little time for the netlink notifications to propagate
+	// up. So, keep retrying for the duration of the reconfigTimeout.
+	var err error
+	for ctx.Err() == nil {
+		err = m.trySet(ctx, config)
+		if err == nil {
+			break
+		}
+		time.Sleep(10 * time.Millisecond)
+	}
+
+	return err
+}
+
+func (m *nmManager) trySet(ctx context.Context, config OSConfig) error {
 	conn, err := dbus.SystemBus()
 	if err != nil {
 		return fmt.Errorf("connecting to system bus: %w", err)
@@ -136,43 +138,101 @@ func (m nmManager) SetDNS(config OSConfig) error {
 		}
 	}
 
+	// NetworkManager wipes out IPv6 address configuration unless we
+	// tell it explicitly to keep it. Read out the current interface
+	// settings and mirror them out to NetworkManager.
+	var addrs6 []map[string]interface{}
+	addrs, _, err := interfaces.Tailscale()
+	if err == nil {
+		for _, a := range addrs {
+			if a.Is6() {
+				addrs6 = append(addrs6, map[string]interface{}{
+					"address": a.String(),
+					"prefix":  uint32(128),
+				})
+			}
+		}
+	}
+
+	seen := map[dnsname.FQDN]bool{}
+	var search []string
+	for _, dom := range config.SearchDomains {
+		if seen[dom] {
+			continue
+		}
+		seen[dom] = true
+		search = append(search, dom.WithTrailingDot())
+	}
+	for _, dom := range config.MatchDomains {
+		if seen[dom] {
+			continue
+		}
+		seen[dom] = true
+		search = append(search, "~"+dom.WithTrailingDot())
+	}
+	if len(config.MatchDomains) == 0 {
+		// Non-split routing requested, add an all-domains match.
+		search = append(search, "~.")
+	}
+
+	// Ideally we would like to disable LLMNR and mdns on the
+	// interface here, but older NetworkManagers don't understand
+	// those settings and choke on them, so we don't. Both LLMNR and
+	// mdns will fail since tailscale0 doesn't do multicast, so it's
+	// effectively fine. We used to try and enforce LLMNR and mdns
+	// settings here, but that led to #1870.
+
 	ipv4Map := settings["ipv4"]
 	ipv4Map["dns"] = dbus.MakeVariant(dnsv4)
-	ipv4Map["dns-search"] = dbus.MakeVariant(config.Domains)
+	ipv4Map["dns-search"] = dbus.MakeVariant(search)
 	// We should only request priority if we have nameservers to set.
 	if len(dnsv4) == 0 {
-		ipv4Map["dns-priority"] = dbus.MakeVariant(100)
+		ipv4Map["dns-priority"] = dbus.MakeVariant(lowerPriority)
+	} else if len(config.MatchDomains) > 0 {
+		// Set a fairly high priority, but don't override all other
+		// configs when in split-DNS mode.
+		ipv4Map["dns-priority"] = dbus.MakeVariant(mediumPriority)
 	} else {
-		// dns-priority = -1 ensures that we have priority
-		// over other interfaces, except those exploiting this same trick.
-		// Ref: https://bugs.launchpad.net/ubuntu/+source/network-manager/+bug/1211110/comments/92.
-		ipv4Map["dns-priority"] = dbus.MakeVariant(-1)
+		// Negative priority means only the settings from the most
+		// negative connection get used. The way this mixes with
+		// per-domain routing is unclear, but it _seems_ that the
+		// priority applies after routing has found possible
+		// candidates for a resolution.
+		ipv4Map["dns-priority"] = dbus.MakeVariant(highestPriority)
 	}
-	// In principle, we should not need set this to true,
-	// as our interface does not configure any automatic DNS settings (presumably via DHCP).
-	// All the same, better to be safe.
-	ipv4Map["ignore-auto-dns"] = dbus.MakeVariant(true)
 
 	ipv6Map := settings["ipv6"]
-	// This is a hack.
-	// Methods "disabled", "ignore", "link-local" (IPv6 default) prevent us from setting DNS.
-	// It seems that our only recourse is "manual" or "auto".
-	// "manual" requires addresses, so we use "auto", which will assign us a random IPv6 /64.
+	// In IPv6 settings, you're only allowed to provide additional
+	// static DNS settings in "auto" (SLAAC) or "manual" mode. In
+	// "manual" mode you also have to specify IP addresses, so we use
+	// "auto".
+	//
+	// NM actually documents that to set just DNS servers, you should
+	// use "auto" mode and then set ignore auto routes and DNS, which
+	// basically means "autoconfigure but ignore any autoconfiguration
+	// results you might get". As a safety, we also say that
+	// NetworkManager should never try to make us the default route
+	// (none of its business anyway, we handle our own default
+	// routing).
 	ipv6Map["method"] = dbus.MakeVariant("auto")
-	// Our IPv6 config is a fake, so it should never become the default route.
-	ipv6Map["never-default"] = dbus.MakeVariant(true)
-	// Moreover, we should ignore all autoconfigured routes (hopefully none), as they are bogus.
-	ipv6Map["ignore-auto-routes"] = dbus.MakeVariant(true)
-
-	// Finally, set the actual DNS config.
-	ipv6Map["dns"] = dbus.MakeVariant(dnsv6)
-	ipv6Map["dns-search"] = dbus.MakeVariant(config.Domains)
-	if len(dnsv6) == 0 {
-		ipv6Map["dns-priority"] = dbus.MakeVariant(100)
-	} else {
-		ipv6Map["dns-priority"] = dbus.MakeVariant(-1)
+	if len(addrs6) > 0 {
+		ipv6Map["address-data"] = dbus.MakeVariant(addrs6)
 	}
+	ipv6Map["ignore-auto-routes"] = dbus.MakeVariant(true)
 	ipv6Map["ignore-auto-dns"] = dbus.MakeVariant(true)
+	ipv6Map["never-default"] = dbus.MakeVariant(true)
+
+	ipv6Map["dns"] = dbus.MakeVariant(dnsv6)
+	ipv6Map["dns-search"] = dbus.MakeVariant(search)
+	if len(dnsv6) == 0 {
+		ipv6Map["dns-priority"] = dbus.MakeVariant(lowerPriority)
+	} else if len(config.MatchDomains) > 0 {
+		// Set a fairly high priority, but don't override all other
+		// configs when in split-DNS mode.
+		ipv6Map["dns-priority"] = dbus.MakeVariant(mediumPriority)
+	} else {
+		ipv6Map["dns-priority"] = dbus.MakeVariant(highestPriority)
+	}
 
 	// deprecatedProperties are the properties in interface settings
 	// that are deprecated by NetworkManager.
@@ -189,19 +249,134 @@ func (m nmManager) SetDNS(config OSConfig) error {
 		delete(ipv6Map, property)
 	}
 
-	err = device.CallWithContext(
-		ctx, "org.freedesktop.NetworkManager.Device.Reapply", 0,
-		settings, version, uint32(0),
-	).Store()
-	if err != nil {
-		return fmt.Errorf("reapply: %w", err)
+	if call := device.CallWithContext(ctx, "org.freedesktop.NetworkManager.Device.Reapply", 0, settings, version, uint32(0)); call.Err != nil {
+		return fmt.Errorf("reapply: %w", call.Err)
 	}
 
 	return nil
 }
 
-func (m nmManager) RoutingMode() RoutingMode { return RoutingModeNone }
+func (m *nmManager) SupportsSplitDNS() bool {
+	var mode string
+	v, err := m.dnsManager.GetProperty("org.freedesktop.NetworkManager.DnsManager.Mode")
+	if err != nil {
+		return false
+	}
+	mode, ok := v.Value().(string)
+	if !ok {
+		return false
+	}
 
-func (m nmManager) Close() error {
-	return m.SetDNS(OSConfig{})
+	// Per NM's documentation, it only does split-DNS when it's
+	// programming dnsmasq or systemd-resolved. All other modes are
+	// primary-only.
+	return mode == "dnsmasq" || mode == "systemd-resolved"
+}
+
+func (m *nmManager) GetBaseConfig() (OSConfig, error) {
+	conn, err := dbus.SystemBus()
+	if err != nil {
+		return OSConfig{}, err
+	}
+
+	nm := conn.Object("org.freedesktop.NetworkManager", dbus.ObjectPath("/org/freedesktop/NetworkManager/DnsManager"))
+	v, err := nm.GetProperty("org.freedesktop.NetworkManager.DnsManager.Configuration")
+	if err != nil {
+		return OSConfig{}, err
+	}
+	cfgs, ok := v.Value().([]map[string]dbus.Variant)
+	if !ok {
+		return OSConfig{}, fmt.Errorf("unexpected NM config type %T", v.Value())
+	}
+
+	if len(cfgs) == 0 {
+		return OSConfig{}, nil
+	}
+
+	type dnsPrio struct {
+		resolvers []netaddr.IP
+		domains   []string
+		priority  int32
+	}
+	order := make([]dnsPrio, 0, len(cfgs)-1)
+
+	for _, cfg := range cfgs {
+		if name, ok := cfg["interface"]; ok {
+			if s, ok := name.Value().(string); ok && s == m.interfaceName {
+				// Config for the taislcale interface, skip.
+				continue
+			}
+		}
+
+		var p dnsPrio
+
+		if v, ok := cfg["nameservers"]; ok {
+			if ips, ok := v.Value().([]string); ok {
+				for _, s := range ips {
+					ip, err := netaddr.ParseIP(s)
+					if err != nil {
+						// hmm, what do? Shouldn't really happen.
+						continue
+					}
+					p.resolvers = append(p.resolvers, ip)
+				}
+			}
+		}
+		if v, ok := cfg["domains"]; ok {
+			if domains, ok := v.Value().([]string); ok {
+				p.domains = domains
+			}
+		}
+		if v, ok := cfg["priority"]; ok {
+			if prio, ok := v.Value().(int32); ok {
+				p.priority = prio
+			}
+		}
+
+		order = append(order, p)
+	}
+
+	sort.Slice(order, func(i, j int) bool {
+		return order[i].priority < order[j].priority
+	})
+
+	var (
+		ret           OSConfig
+		seenResolvers = map[netaddr.IP]bool{}
+		seenSearch    = map[string]bool{}
+	)
+
+	for _, cfg := range order {
+		for _, resolver := range cfg.resolvers {
+			if seenResolvers[resolver] {
+				continue
+			}
+			ret.Nameservers = append(ret.Nameservers, resolver)
+			seenResolvers[resolver] = true
+		}
+		for _, dom := range cfg.domains {
+			if seenSearch[dom] {
+				continue
+			}
+			fqdn, err := dnsname.ToFQDN(dom)
+			if err != nil {
+				continue
+			}
+			ret.SearchDomains = append(ret.SearchDomains, fqdn)
+			seenSearch[dom] = true
+		}
+		if cfg.priority < 0 {
+			// exclusive configurations preempt all other
+			// configurations, so we're done.
+			break
+		}
+	}
+
+	return ret, nil
+}
+
+func (m *nmManager) Close() error {
+	// No need to do anything on close, NetworkManager will delete our
+	// settings when the tailscale interface goes away.
+	return nil
 }

net/dns/noop.go

@@ -6,10 +6,13 @@ package dns
 
 type noopManager struct{}
 
-func (m noopManager) SetDNS(OSConfig) error    { return nil }
-func (m noopManager) RoutingMode() RoutingMode { return RoutingModeNone }
-func (m noopManager) Close() error             { return nil }
-
-func NewNoopManager() noopManager {
-	return noopManager{}
+func (m noopManager) SetDNS(OSConfig) error  { return nil }
+func (m noopManager) SupportsSplitDNS() bool { return false }
+func (m noopManager) Close() error           { return nil }
+func (m noopManager) GetBaseConfig() (OSConfig, error) {
+	return OSConfig{}, ErrGetBaseConfigNotSupported
+}
+
+func NewNoopManager() (noopManager, error) {
+	return noopManager{}, nil
 }

net/dns/openresolv.go

@@ -0,0 +1,94 @@
+// Copyright (c) 2021 Tailscale Inc & AUTHORS All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+// +build linux freebsd openbsd
+
+package dns
+
+import (
+	"bytes"
+	"fmt"
+	"os/exec"
+	"strings"
+)
+
+// openresolvManager manages DNS configuration using the openresolv
+// implementation of the `resolvconf` program.
+type openresolvManager struct{}
+
+func newOpenresolvManager() (openresolvManager, error) {
+	return openresolvManager{}, nil
+}
+
+func (m openresolvManager) deleteTailscaleConfig() error {
+	cmd := exec.Command("resolvconf", "-f", "-d", "tailscale")
+	out, err := cmd.CombinedOutput()
+	if err != nil {
+		return fmt.Errorf("running %s: %s", cmd, out)
+	}
+	return nil
+}
+
+func (m openresolvManager) SetDNS(config OSConfig) error {
+	if config.IsZero() {
+		return m.deleteTailscaleConfig()
+	}
+
+	var stdin bytes.Buffer
+	writeResolvConf(&stdin, config.Nameservers, config.SearchDomains)
+
+	cmd := exec.Command("resolvconf", "-m", "0", "-x", "-a", "tailscale")
+	cmd.Stdin = &stdin
+	out, err := cmd.CombinedOutput()
+	if err != nil {
+		return fmt.Errorf("running %s: %s", cmd, out)
+	}
+	return nil
+}
+
+func (m openresolvManager) SupportsSplitDNS() bool {
+	return false
+}
+
+func (m openresolvManager) GetBaseConfig() (OSConfig, error) {
+	// List the names of all config snippets openresolv is aware
+	// of. Snippets get listed in priority order (most to least),
+	// which we'll exploit later.
+	bs, err := exec.Command("resolvconf", "-i").CombinedOutput()
+	if err != nil {
+		return OSConfig{}, err
+	}
+
+	// Remove the "tailscale" snippet from the list.
+	args := []string{"-l"}
+	for _, f := range strings.Split(strings.TrimSpace(string(bs)), " ") {
+		if f == "tailscale" {
+			continue
+		}
+		args = append(args, f)
+	}
+
+	// List all resolvconf snippets except our own, and parse that as
+	// a resolv.conf. This effectively generates a blended config of
+	// "everyone except tailscale", which is what would be in use if
+	// tailscale hadn't set exclusive mode.
+	//
+	// Note that this is not _entirely_ true. To be perfectly correct,
+	// we should be looking for other interfaces marked exclusive that
+	// predated tailscale, and stick to only those. However, in
+	// practice, openresolv uses are generally quite limited, and boil
+	// down to 1-2 DHCP leases, for which the correct outcome is a
+	// blended config like the one we produce here.
+	var buf bytes.Buffer
+	cmd := exec.Command("resolvconf", args...)
+	cmd.Stdout = &buf
+	if err := cmd.Run(); err != nil {
+		return OSConfig{}, err
+	}
+	return readResolv(&buf)
+}
+
+func (m openresolvManager) Close() error {
+	return m.deleteTailscaleConfig()
+}

net/dns/osconfig.go

@@ -4,21 +4,11 @@
 
 package dns
 
-// DNSRoutingMode describes the type of per-domain DNS routing that
-// the OS is capable of.
-type RoutingMode int
+import (
+	"errors"
 
-const (
-	// RoutingModeNone means the OS only supports setting a single
-	// primary set of DNS resolvers.
-	RoutingModeNone RoutingMode = iota
-	// RoutingModeSingle means the OS supports a set of
-	// primary resolvers, as well as one set of additional per-suffix
-	// resolvers per network interface.
-	RoutingModeSingle
-	// RoutingModeMulti means the OS supports a set of primary
-	// resolvers, as well as an arbitrary overlay of DNS routes.
-	RoutingModeMulti
+	"inet.af/netaddr"
+	"tailscale.com/util/dnsname"
 )
 
 // An OSConfigurator applies DNS settings to the operating system.
@@ -28,9 +18,75 @@ type OSConfigurator interface {
 	// configuration is removed.
 	// SetDNS must not be called after Close.
 	SetDNS(cfg OSConfig) error
-	// DNSRoutingMode reports the DNS routing capabilities of this OS
-	// configurator.
-	RoutingMode() RoutingMode
+	// SupportsSplitDNS reports whether the configurator is capable of
+	// installing a resolver only for specific DNS suffixes. If false,
+	// the configurator can only set a global resolver.
+	SupportsSplitDNS() bool
+	// GetBaseConfig returns the OS's "base" configuration, i.e. the
+	// resolver settings the OS would use without Tailscale
+	// contributing any configuration.
+	// GetBaseConfig must return the tailscale-free base config even
+	// after SetDNS has been called to set a Tailscale configuration.
+	// Only works when SupportsSplitDNS=false.
+
+	// Implementations that don't support getting the base config must
+	// return ErrGetBaseConfigNotSupported.
+	GetBaseConfig() (OSConfig, error)
 	// Close removes Tailscale-related DNS configuration from the OS.
 	Close() error
 }
+
+// OSConfig is an OS DNS configuration.
+type OSConfig struct {
+	// Nameservers are the IP addresses of the nameservers to use.
+	Nameservers []netaddr.IP
+	// SearchDomains are the domain suffixes to use when expanding
+	// single-label name queries. SearchDomains is additive to
+	// whatever non-Tailscale search domains the OS has.
+	SearchDomains []dnsname.FQDN
+	// MatchDomains are the DNS suffixes for which Nameservers should
+	// be used. If empty, Nameservers is installed as the "primary" resolver.
+	// A non-empty MatchDomains requests a "split DNS" configuration
+	// from the OS, which will only work with OSConfigurators that
+	// report SupportsSplitDNS()=true.
+	MatchDomains []dnsname.FQDN
+}
+
+func (o OSConfig) IsZero() bool {
+	return len(o.Nameservers) == 0 && len(o.SearchDomains) == 0 && len(o.MatchDomains) == 0
+}
+
+func (a OSConfig) Equal(b OSConfig) bool {
+	if len(a.Nameservers) != len(b.Nameservers) {
+		return false
+	}
+	if len(a.SearchDomains) != len(b.SearchDomains) {
+		return false
+	}
+	if len(a.MatchDomains) != len(b.MatchDomains) {
+		return false
+	}
+
+	for i := range a.Nameservers {
+		if a.Nameservers[i] != b.Nameservers[i] {
+			return false
+		}
+	}
+	for i := range a.SearchDomains {
+		if a.SearchDomains[i] != b.SearchDomains[i] {
+			return false
+		}
+	}
+	for i := range a.MatchDomains {
+		if a.MatchDomains[i] != b.MatchDomains[i] {
+			return false
+		}
+	}
+
+	return true
+}
+
+// ErrGetBaseConfigNotSupported is the error
+// OSConfigurator.GetBaseConfig returns when the OSConfigurator
+// doesn't support reading the underlying configuration out of the OS.
+var ErrGetBaseConfigNotSupported = errors.New("getting OS base config is not supported")

net/dns/resolvconf-workaround.sh

@@ -0,0 +1,63 @@
+#!/bin/sh
+# Copyright (c) 2021 Tailscale Inc & AUTHORS All rights reserved.
+# Use of this source code is governed by a BSD-style
+# license that can be found in the LICENSE file.
+#
+# This script is a workaround for a vpn-unfriendly behavior of the
+# original resolvconf by Thomas Hood. Unlike the `openresolv`
+# implementation (whose binary is also called resolvconf,
+# confusingly), the original resolvconf lacks a way to specify
+# "exclusive mode" for a provider configuration. In practice, this
+# means that if Tailscale wants to install a DNS configuration, that
+# config will get "blended" with the configs from other sources,
+# rather than override those other sources.
+#
+# This script gets installed at /etc/resolvconf/update-libc.d, which
+# is a directory of hook scripts that get run after resolvconf's libc
+# helper has finished rewriting /etc/resolv.conf. It's meant to notify
+# consumers of resolv.conf of a new configuration.
+#
+# Instead, we use that hook mechanism to reach into resolvconf's
+# stuff, and rewrite the libc-generated resolv.conf to exclusively
+# contain Tailscale's configuration - effectively implementing
+# exclusive mode ourselves in post-production.
+
+set -e
+
+if [ -n "$TAILSCALE_RESOLVCONF_HOOK_LOOP" ]; then
+	# Hook script being invoked by itself, skip.
+	exit 0
+fi
+
+if [ ! -f tun-tailscale.inet ]; then
+	# Tailscale isn't trying to manage DNS, do nothing.
+	exit 0
+fi
+
+if ! grep resolvconf /etc/resolv.conf >/dev/null; then
+	# resolvconf isn't managing /etc/resolv.conf, do nothing.
+	exit 0
+fi
+
+# Write out a modified /etc/resolv.conf containing just our config.
+(
+	if [ -f /etc/resolvconf/resolv.conf.d/head ]; then
+		cat /etc/resolvconf/resolv.conf.d/head
+	fi
+	echo "# Tailscale workaround applied to set exclusive DNS configuration."
+	cat tun-tailscale.inet
+	if [ -f /etc/resolvconf/resolv.conf.d/base ]; then
+		# Keep options and sortlist, discard other base things since
+		# they're the things we're trying to override.
+		grep -e 'sortlist ' -e 'options ' /etc/resolvconf/resolv.conf.d/base || true
+	fi
+	if [ -f /etc/resolvconf/resolv.conf.d/tail ]; then
+		cat /etc/resolvconf/resolv.conf.d/tail
+	fi
+) >/etc/resolv.conf
+
+if [ -d /etc/resolvconf/update-libc.d ] ; then
+	# Re-notify libc watchers that we've changed resolv.conf again.
+	export TAILSCALE_RESOLVCONF_HOOK_LOOP=1
+	exec run-parts /etc/resolvconf/update-libc.d
+fi

net/dns/resolvconf.go

@@ -1,161 +1,29 @@
-// Copyright (c) 2020 Tailscale Inc & AUTHORS All rights reserved.
+// Copyright (c) 2021 Tailscale Inc & AUTHORS All rights reserved.
 // Use of this source code is governed by a BSD-style
 // license that can be found in the LICENSE file.
 
-// +build linux freebsd
+// +build linux freebsd openbsd
 
 package dns
 
 import (
-	"bufio"
-	"bytes"
-	"fmt"
-	"os"
 	"os/exec"
 
 	"tailscale.com/types/logger"
 )
 
-// isResolvconfActive indicates whether the system appears to be using resolvconf.
-// If this is true, then directManager should be avoided:
-// resolvconf has exclusive ownership of /etc/resolv.conf.
-func isResolvconfActive() bool {
-	// Sanity-check first: if there is no resolvconf binary, then this is fruitless.
-	//
-	// However, this binary may be a shim like the one systemd-resolved provides.
-	// Such a shim may not behave as expected: in particular, systemd-resolved
-	// does not seem to respect the exclusive mode -x, saying:
-	//   -x            Send DNS traffic preferably over this interface
-	// whereas e.g. openresolv sends DNS traffix _exclusively_ over that interface,
-	// or not at all (in case of another exclusive-mode request later in time).
-	//
-	// Moreover, resolvconf may be installed but unused, in which case we should
-	// not use it either, lest we clobber existing configuration.
-	//
-	// To handle all the above correctly, we scan the comments in /etc/resolv.conf
-	// to ensure that it was generated by a resolvconf implementation.
-	_, err := exec.LookPath("resolvconf")
+func newResolvconfManager(logf logger.Logf) (OSConfigurator, error) {
+	_, err := exec.Command("resolvconf", "--version").CombinedOutput()
 	if err != nil {
-		return false
-	}
-
-	f, err := os.Open("/etc/resolv.conf")
-	if err != nil {
-		return false
-	}
-	defer f.Close()
-
-	scanner := bufio.NewScanner(f)
-	for scanner.Scan() {
-		line := scanner.Bytes()
-		// Look for the word "resolvconf" until comments end.
-		if len(line) > 0 && line[0] != '#' {
-			return false
-		}
-		if bytes.Contains(line, []byte("resolvconf")) {
-			return true
+		if exitErr, ok := err.(*exec.ExitError); ok && exitErr.ExitCode() == 99 {
+			// Debian resolvconf doesn't understand --version, and
+			// exits with a specific error code.
+			return newDebianResolvconfManager(logf)
 		}
 	}
-	return false
-}
-
-// resolvconfImpl enumerates supported implementations of the resolvconf CLI.
-type resolvconfImpl uint8
-
-const (
-	// resolvconfOpenresolv is the implementation packaged as "openresolv" on Ubuntu.
-	// It supports exclusive mode and interface metrics.
-	resolvconfOpenresolv resolvconfImpl = iota
-	// resolvconfLegacy is the implementation by Thomas Hood packaged as "resolvconf" on Ubuntu.
-	// It does not support exclusive mode or interface metrics.
-	resolvconfLegacy
-)
-
-func (impl resolvconfImpl) String() string {
-	switch impl {
-	case resolvconfOpenresolv:
-		return "openresolv"
-	case resolvconfLegacy:
-		return "legacy"
-	default:
-		return "unknown"
-	}
-}
-
-// getResolvconfImpl returns the implementation of resolvconf that appears to be in use.
-func getResolvconfImpl() resolvconfImpl {
-	err := exec.Command("resolvconf", "-v").Run()
-	if err != nil {
-		if exitErr, ok := err.(*exec.ExitError); ok {
-			// Thomas Hood's resolvconf has a minimal flag set
-			// and exits with code 99 when passed an unknown flag.
-			if exitErr.ExitCode() == 99 {
-				return resolvconfLegacy
-			}
-		}
-	}
-	return resolvconfOpenresolv
-}
-
-type resolvconfManager struct {
-	impl resolvconfImpl
-}
-
-func newResolvconfManager(logf logger.Logf) resolvconfManager {
-	impl := getResolvconfImpl()
-	logf("resolvconf implementation is %s", impl)
-
-	return resolvconfManager{
-		impl: impl,
-	}
-}
-
-// resolvconfConfigName is the name of the config submitted to resolvconf.
-// It has this form to match the "tun*" rule in interface-order
-// when running resolvconfLegacy, hopefully placing our config first.
-const resolvconfConfigName = "tun-tailscale.inet"
-
-func (m resolvconfManager) SetDNS(config OSConfig) error {
-	stdin := new(bytes.Buffer)
-	writeResolvConf(stdin, config.Nameservers, config.Domains) // dns_direct.go
-
-	var cmd *exec.Cmd
-	switch m.impl {
-	case resolvconfOpenresolv:
-		// Request maximal priority (metric 0) and exclusive mode.
-		cmd = exec.Command("resolvconf", "-m", "0", "-x", "-a", resolvconfConfigName)
-	case resolvconfLegacy:
-		// This does not quite give us the desired behavior (queries leak),
-		// but there is nothing else we can do without messing with other interfaces' settings.
-		cmd = exec.Command("resolvconf", "-a", resolvconfConfigName)
-	}
-	cmd.Stdin = stdin
-	out, err := cmd.CombinedOutput()
-	if err != nil {
-		return fmt.Errorf("running %s: %s", cmd, out)
-	}
-
-	return nil
-}
-
-func (m resolvconfManager) RoutingMode() RoutingMode {
-	return RoutingModeNone
-}
-
-func (m resolvconfManager) Close() error {
-	var cmd *exec.Cmd
-	switch m.impl {
-	case resolvconfOpenresolv:
-		cmd = exec.Command("resolvconf", "-f", "-d", resolvconfConfigName)
-	case resolvconfLegacy:
-		// resolvconfLegacy lacks the -f flag.
-		// Instead, it succeeds even when the config does not exist.
-		cmd = exec.Command("resolvconf", "-d", resolvconfConfigName)
-	}
-	out, err := cmd.CombinedOutput()
-	if err != nil {
-		return fmt.Errorf("running %s: %s", cmd, out)
-	}
-
-	return nil
+	// If --version works, or we got some surprising error while
+	// probing, use openresolv. It's the more common implementation,
+	// so in cases where we can't figure things out, it's the least
+	// likely to misbehave.
+	return newOpenresolvManager()
 }

net/dns/resolved.go

@@ -12,12 +12,13 @@ import (
 	"context"
 	"errors"
 	"fmt"
-	"os/exec"
+	"net"
 
 	"github.com/godbus/dbus/v5"
 	"golang.org/x/sys/unix"
 	"inet.af/netaddr"
-	"tailscale.com/net/interfaces"
+	"tailscale.com/types/logger"
+	"tailscale.com/util/dnsname"
 )
 
 // resolvedListenAddr is the listen address of the resolved stub resolver.
@@ -51,15 +52,20 @@ type resolvedLinkDomain struct {
 
 // isResolvedActive determines if resolved is currently managing system DNS settings.
 func isResolvedActive() bool {
-	// systemd-resolved is never installed without systemd.
-	_, err := exec.LookPath("systemctl")
+	ctx, cancel := context.WithTimeout(context.Background(), reconfigTimeout)
+	defer cancel()
+
+	conn, err := dbus.SystemBus()
 	if err != nil {
+		// Probably no DBus on the system, or we're not allowed to use
+		// it. Cannot control resolved.
 		return false
 	}
 
-	// is-active exits with code 3 if the service is not active.
-	err = exec.Command("systemctl", "is-active", "systemd-resolved").Run()
-	if err != nil {
+	rd := conn.Object("org.freedesktop.resolve1", dbus.ObjectPath("/org/freedesktop/resolve1"))
+	call := rd.CallWithContext(ctx, "org.freedesktop.DBus.Peer.Ping", 0)
+	if call.Err != nil {
+		// Can't talk to resolved.
 		return false
 	}
 
@@ -77,39 +83,35 @@ func isResolvedActive() bool {
 }
 
 // resolvedManager uses the systemd-resolved DBus API.
-type resolvedManager struct{}
+type resolvedManager struct {
+	logf     logger.Logf
+	ifidx    int
+	resolved dbus.BusObject
+}
 
-func newResolvedManager() resolvedManager {
-	return resolvedManager{}
+func newResolvedManager(logf logger.Logf, interfaceName string) (*resolvedManager, error) {
+	conn, err := dbus.SystemBus()
+	if err != nil {
+		return nil, err
+	}
+
+	iface, err := net.InterfaceByName(interfaceName)
+	if err != nil {
+		return nil, err
+	}
+
+	return &resolvedManager{
+		logf:     logf,
+		ifidx:    iface.Index,
+		resolved: conn.Object("org.freedesktop.resolve1", dbus.ObjectPath("/org/freedesktop/resolve1")),
+	}, nil
 }
 
 // Up implements managerImpl.
-func (m resolvedManager) SetDNS(config OSConfig) error {
+func (m *resolvedManager) SetDNS(config OSConfig) error {
 	ctx, cancel := context.WithTimeout(context.Background(), reconfigTimeout)
 	defer cancel()
 
-	// conn is a shared connection whose lifecycle is managed by the dbus package.
-	// We should not interfere with that by closing it.
-	conn, err := dbus.SystemBus()
-	if err != nil {
-		return fmt.Errorf("connecting to system bus: %w", err)
-	}
-
-	resolved := conn.Object(
-		"org.freedesktop.resolve1",
-		dbus.ObjectPath("/org/freedesktop/resolve1"),
-	)
-
-	// In principle, we could persist this in the manager struct
-	// if we knew that interface indices are persistent. This does not seem to be the case.
-	_, iface, err := interfaces.Tailscale()
-	if err != nil {
-		return fmt.Errorf("getting interface index: %w", err)
-	}
-	if iface == nil {
-		return errNotReady
-	}
-
 	var linkNameservers = make([]resolvedLinkNameserver, len(config.Nameservers))
 	for i, server := range config.Nameservers {
 		ip := server.As16()
@@ -126,67 +128,104 @@ func (m resolvedManager) SetDNS(config OSConfig) error {
 		}
 	}
 
-	err = resolved.CallWithContext(
+	err := m.resolved.CallWithContext(
 		ctx, "org.freedesktop.resolve1.Manager.SetLinkDNS", 0,
-		iface.Index, linkNameservers,
+		m.ifidx, linkNameservers,
 	).Store()
 	if err != nil {
 		return fmt.Errorf("setLinkDNS: %w", err)
 	}
 
-	var linkDomains = make([]resolvedLinkDomain, len(config.Domains))
-	for i, domain := range config.Domains {
-		linkDomains[i] = resolvedLinkDomain{
-			Domain:      domain,
-			RoutingOnly: false,
+	linkDomains := make([]resolvedLinkDomain, 0, len(config.SearchDomains)+len(config.MatchDomains))
+	seenDomains := map[dnsname.FQDN]bool{}
+	for _, domain := range config.SearchDomains {
+		if seenDomains[domain] {
+			continue
 		}
+		seenDomains[domain] = true
+		linkDomains = append(linkDomains, resolvedLinkDomain{
+			Domain:      domain.WithTrailingDot(),
+			RoutingOnly: false,
+		})
+	}
+	for _, domain := range config.MatchDomains {
+		if seenDomains[domain] {
+			// Search domains act as both search and match in
+			// resolved, so it's correct to skip.
+			continue
+		}
+		seenDomains[domain] = true
+		linkDomains = append(linkDomains, resolvedLinkDomain{
+			Domain:      domain.WithTrailingDot(),
+			RoutingOnly: true,
+		})
+	}
+	if len(config.MatchDomains) == 0 && len(config.Nameservers) > 0 {
+		// Caller requested full DNS interception, install a
+		// routing-only root domain.
+		linkDomains = append(linkDomains, resolvedLinkDomain{
+			Domain:      ".",
+			RoutingOnly: true,
+		})
 	}
 
-	err = resolved.CallWithContext(
+	err = m.resolved.CallWithContext(
 		ctx, "org.freedesktop.resolve1.Manager.SetLinkDomains", 0,
-		iface.Index, linkDomains,
+		m.ifidx, linkDomains,
 	).Store()
 	if err != nil {
 		return fmt.Errorf("setLinkDomains: %w", err)
 	}
 
+	if call := m.resolved.CallWithContext(ctx, "org.freedesktop.resolve1.Manager.SetLinkDefaultRoute", 0, m.ifidx, len(config.MatchDomains) == 0); call.Err != nil {
+		return fmt.Errorf("setLinkDefaultRoute: %w", err)
+	}
+
+	// Some best-effort setting of things, but resolved should do the
+	// right thing if these fail (e.g. a really old resolved version
+	// or something).
+
+	// Disable LLMNR, we don't do multicast.
+	if call := m.resolved.CallWithContext(ctx, "org.freedesktop.resolve1.Manager.SetLinkLLMNR", 0, m.ifidx, "no"); call.Err != nil {
+		m.logf("[v1] failed to disable LLMNR: %v", call.Err)
+	}
+
+	// Disable mdns.
+	if call := m.resolved.CallWithContext(ctx, "org.freedesktop.resolve1.Manager.SetLinkMulticastDNS", 0, m.ifidx, "no"); call.Err != nil {
+		m.logf("[v1] failed to disable mdns: %v", call.Err)
+	}
+
+	// We don't support dnssec consistently right now, force it off to
+	// avoid partial failures when we split DNS internally.
+	if call := m.resolved.CallWithContext(ctx, "org.freedesktop.resolve1.Manager.SetLinkDNSSEC", 0, m.ifidx, "no"); call.Err != nil {
+		m.logf("[v1] failed to disable DNSSEC: %v", call.Err)
+	}
+
+	if call := m.resolved.CallWithContext(ctx, "org.freedesktop.resolve1.Manager.SetLinkDNSOverTLS", 0, m.ifidx, "no"); call.Err != nil {
+		m.logf("[v1] failed to disable DoT: %v", call.Err)
+	}
+
+	if call := m.resolved.CallWithContext(ctx, "org.freedesktop.resolve1.Manager.FlushCaches", 0); call.Err != nil {
+		m.logf("failed to flush resolved DNS cache: %v", call.Err)
+	}
+
 	return nil
 }
 
-func (m resolvedManager) RoutingMode() RoutingMode {
-	return RoutingModeNone
+func (m *resolvedManager) SupportsSplitDNS() bool {
+	return true
 }
 
-func (m resolvedManager) Close() error {
+func (m *resolvedManager) GetBaseConfig() (OSConfig, error) {
+	return OSConfig{}, ErrGetBaseConfigNotSupported
+}
+
+func (m *resolvedManager) Close() error {
 	ctx, cancel := context.WithTimeout(context.Background(), reconfigTimeout)
 	defer cancel()
 
-	// conn is a shared connection whose lifecycle is managed by the dbus package.
-	// We should not interfere with that by closing it.
-	conn, err := dbus.SystemBus()
-	if err != nil {
-		return fmt.Errorf("connecting to system bus: %w", err)
-	}
-
-	resolved := conn.Object(
-		"org.freedesktop.resolve1",
-		dbus.ObjectPath("/org/freedesktop/resolve1"),
-	)
-
-	_, iface, err := interfaces.Tailscale()
-	if err != nil {
-		return fmt.Errorf("getting interface index: %w", err)
-	}
-	if iface == nil {
-		return errNotReady
-	}
-
-	err = resolved.CallWithContext(
-		ctx, "org.freedesktop.resolve1.Manager.RevertLink", 0,
-		iface.Index,
-	).Store()
-	if err != nil {
-		return fmt.Errorf("RevertLink: %w", err)
+	if call := m.resolved.CallWithContext(ctx, "org.freedesktop.resolve1.Manager.RevertLink", 0, m.ifidx); call.Err != nil {
+		return fmt.Errorf("RevertLink: %w", call.Err)
 	}
 
 	return nil

net/dns/resolver/forwarder.go

@@ -13,14 +13,12 @@ import (
 	"hash/crc32"
 	"math/rand"
 	"net"
-	"os"
 	"sync"
 	"time"
 
 	dns "golang.org/x/net/dns/dnsmessage"
 	"inet.af/netaddr"
 	"tailscale.com/logtail/backoff"
-	"tailscale.com/net/netns"
 	"tailscale.com/types/logger"
 	"tailscale.com/util/dnsname"
 )
@@ -40,8 +38,6 @@ const (
 
 var errNoUpstreams = errors.New("upstream nameservers not set")
 
-var aLongTimeAgo = time.Unix(0, 1)
-
 type forwardingRecord struct {
 	src       netaddr.IPPort
 	createdAt time.Time
@@ -103,7 +99,7 @@ func getTxID(packet []byte) txid {
 }
 
 type route struct {
-	suffix    string
+	suffix    dnsname.FQDN
 	resolvers []netaddr.IPPort
 }
 
@@ -273,7 +269,7 @@ func (f *forwarder) forward(query packet) error {
 
 	var resolvers []netaddr.IPPort
 	for _, route := range routes {
-		if route.suffix != "." && !dnsname.HasSuffix(domain, route.suffix) {
+		if route.suffix != "." && !route.suffix.Contains(domain) {
 			continue
 		}
 		resolvers = route.resolvers
@@ -304,8 +300,6 @@ type fwdConn struct {
 	// logf allows a fwdConn to log.
 	logf logger.Logf
 
-	// wg tracks the number of outstanding conn.Read and conn.Write calls.
-	wg sync.WaitGroup
 	// change allows calls to read to block until a the network connection has been replaced.
 	change *sync.Cond
 
@@ -353,15 +347,12 @@ func (c *fwdConn) send(packet []byte, dst netaddr.IPPort) {
 		}
 		c.mu.Unlock()
 
-		a := dst.UDPAddr()
-		c.wg.Add(1)
-		_, err := conn.WriteTo(packet, a)
-		c.wg.Done()
+		_, err := conn.WriteTo(packet, dst.UDPAddr())
 		if err == nil {
 			// Success
 			return
 		}
-		if errors.Is(err, os.ErrDeadlineExceeded) {
+		if errors.Is(err, net.ErrClosed) {
 			// We intentionally closed this connection.
 			// It has been replaced by a new connection. Try again.
 			continue
@@ -430,14 +421,12 @@ func (c *fwdConn) read(out []byte) int {
 		}
 		c.mu.Unlock()
 
-		c.wg.Add(1)
 		n, _, err := conn.ReadFrom(out)
-		c.wg.Done()
 		if err == nil {
 			// Success.
 			return n
 		}
-		if errors.Is(err, os.ErrDeadlineExceeded) {
+		if errors.Is(err, net.ErrClosed) {
 			// We intentionally closed this connection.
 			// It has been replaced by a new connection. Try again.
 			continue
@@ -453,7 +442,7 @@ func (c *fwdConn) read(out []byte) int {
 func (c *fwdConn) reconnectLocked() {
 	c.closeConnLocked()
 	// Make a new connection.
-	conn, err := netns.Listener().ListenPacket(context.Background(), "udp", "")
+	conn, err := net.ListenPacket("udp", "")
 	if err != nil {
 		c.logf("ListenPacket failed: %v", err)
 	} else {
@@ -469,10 +458,7 @@ func (c *fwdConn) closeConnLocked() {
 	if c.conn == nil {
 		return
 	}
-	// Unblock all readers/writers, wait for them, close the connection.
-	c.conn.SetDeadline(aLongTimeAgo)
-	c.wg.Wait()
-	c.conn.Close()
+	c.conn.Close() // unblocks all readers/writers, they'll pick up the next connection.
 	c.conn = nil
 }
 
@@ -490,7 +476,7 @@ func (c *fwdConn) close() {
 }
 
 // nameFromQuery extracts the normalized query name from bs.
-func nameFromQuery(bs []byte) (string, error) {
+func nameFromQuery(bs []byte) (dnsname.FQDN, error) {
 	var parser dns.Parser
 
 	hdr, err := parser.Start(bs)
@@ -507,5 +493,5 @@ func nameFromQuery(bs []byte) (string, error) {
 	}
 
 	n := q.Name.Data[:q.Name.Length]
-	return rawNameToLower(n), nil
+	return dnsname.ToFQDN(rawNameToLower(n))
 }

net/dns/resolver/tsdns.go

@@ -9,7 +9,6 @@ package resolver
 import (
 	"encoding/hex"
 	"errors"
-	"fmt"
 	"sort"
 	"strings"
 	"sync"
@@ -59,12 +58,12 @@ type Config struct {
 	// queries within that suffix.
 	// Queries only match the most specific suffix.
 	// To register a "default route", add an entry for ".".
-	Routes map[string][]netaddr.IPPort
+	Routes map[dnsname.FQDN][]netaddr.IPPort
 	// LocalHosts is a map of FQDNs to corresponding IPs.
-	Hosts map[string][]netaddr.IP
+	Hosts map[dnsname.FQDN][]netaddr.IP
 	// LocalDomains is a list of DNS name suffixes that should not be
 	// routed to upstream resolvers.
-	LocalDomains []string
+	LocalDomains []dnsname.FQDN
 }
 
 // Resolver is a DNS resolver for nodes on the Tailscale network,
@@ -72,9 +71,10 @@ type Config struct {
 // If it is asked to resolve a domain that is not of that form,
 // it delegates to upstream nameservers if any are set.
 type Resolver struct {
-	logf         logger.Logf
-	linkMon      *monitor.Mon // or nil
-	unregLinkMon func()       // or nil
+	logf               logger.Logf
+	linkMon            *monitor.Mon     // or nil
+	unregLinkMon       func()           // or nil
+	saveConfigForTests func(cfg Config) // used in tests to capture resolver config
 	// forwarder forwards requests to upstream nameservers.
 	forwarder *forwarder
 
@@ -91,9 +91,9 @@ type Resolver struct {
 
 	// mu guards the following fields from being updated while used.
 	mu           sync.Mutex
-	localDomains []string
-	hostToIP     map[string][]netaddr.IP
-	ipToHost     map[netaddr.IP]string
+	localDomains []dnsname.FQDN
+	hostToIP     map[dnsname.FQDN][]netaddr.IP
+	ipToHost     map[netaddr.IP]dnsname.FQDN
 }
 
 // New returns a new resolver.
@@ -106,8 +106,8 @@ func New(logf logger.Logf, linkMon *monitor.Mon) *Resolver {
 		responses: make(chan packet),
 		errors:    make(chan error),
 		closed:    make(chan struct{}),
-		hostToIP:  map[string][]netaddr.IP{},
-		ipToHost:  map[netaddr.IP]string{},
+		hostToIP:  map[dnsname.FQDN][]netaddr.IP{},
+		ipToHost:  map[netaddr.IP]dnsname.FQDN{},
 	}
 	r.forwarder = newForwarder(r.logf, r.responses)
 	if r.linkMon != nil {
@@ -120,32 +120,23 @@ func New(logf logger.Logf, linkMon *monitor.Mon) *Resolver {
 	return r
 }
 
-func isFQDN(s string) bool {
-	return strings.HasSuffix(s, ".")
-}
+func (r *Resolver) TestOnlySetHook(hook func(Config)) { r.saveConfigForTests = hook }
 
 func (r *Resolver) SetConfig(cfg Config) error {
+	if r.saveConfigForTests != nil {
+		r.saveConfigForTests(cfg)
+	}
+
 	routes := make([]route, 0, len(cfg.Routes))
-	reverse := make(map[netaddr.IP]string, len(cfg.Hosts))
+	reverse := make(map[netaddr.IP]dnsname.FQDN, len(cfg.Hosts))
 
 	for host, ips := range cfg.Hosts {
-		if !isFQDN(host) {
-			return fmt.Errorf("host entry %q is not a FQDN", host)
-		}
 		for _, ip := range ips {
 			reverse[ip] = host
 		}
 	}
-	for _, domain := range cfg.LocalDomains {
-		if !isFQDN(domain) {
-			return fmt.Errorf("local domain %q is not a FQDN", domain)
-		}
-	}
 
 	for suffix, ips := range cfg.Routes {
-		if !strings.HasSuffix(suffix, ".") {
-			return fmt.Errorf("route suffix %q is not a FQDN", suffix)
-		}
 		routes = append(routes, route{
 			suffix:    suffix,
 			resolvers: ips,
@@ -153,7 +144,7 @@ func (r *Resolver) SetConfig(cfg Config) error {
 	}
 	// Sort from longest prefix to shortest.
 	sort.Slice(routes, func(i, j int) bool {
-		return dnsname.NumLabels(routes[i].suffix) > dnsname.NumLabels(routes[j].suffix)
+		return routes[i].suffix.NumLabels() > routes[j].suffix.NumLabels()
 	})
 
 	r.forwarder.setRoutes(routes)
@@ -222,12 +213,11 @@ func (r *Resolver) NextResponse() (packet []byte, to netaddr.IPPort, err error)
 // resolveLocal returns an IP for the given domain, if domain is in
 // the local hosts map and has an IP corresponding to the requested
 // typ (A, AAAA, ALL).
-// The domain name must be in canonical form (with a trailing period).
 // Returns dns.RCodeRefused to indicate that the local map is not
 // authoritative for domain.
-func (r *Resolver) resolveLocal(domain string, typ dns.Type) (netaddr.IP, dns.RCode) {
+func (r *Resolver) resolveLocal(domain dnsname.FQDN, typ dns.Type) (netaddr.IP, dns.RCode) {
 	// Reject .onion domains per RFC 7686.
-	if dnsname.HasSuffix(domain, ".onion") {
+	if dnsname.HasSuffix(domain.WithoutTrailingDot(), ".onion") {
 		return netaddr.IP{}, dns.RCodeNameError
 	}
 
@@ -239,7 +229,7 @@ func (r *Resolver) resolveLocal(domain string, typ dns.Type) (netaddr.IP, dns.RC
 	addrs, found := hosts[domain]
 	if !found {
 		for _, suffix := range localDomains {
-			if dnsname.HasSuffix(domain, suffix) {
+			if suffix.Contains(domain) {
 				// We are authoritative for the queried domain.
 				return netaddr.IP{}, dns.RCodeNameError
 			}
@@ -297,8 +287,7 @@ func (r *Resolver) resolveLocal(domain string, typ dns.Type) (netaddr.IP, dns.RC
 }
 
 // resolveReverse returns the unique domain name that maps to the given address.
-// The returned domain name is in canonical form (with a trailing period).
-func (r *Resolver) resolveLocalReverse(ip netaddr.IP) (string, dns.RCode) {
+func (r *Resolver) resolveLocalReverse(ip netaddr.IP) (dnsname.FQDN, dns.RCode) {
 	r.mu.Lock()
 	ips := r.ipToHost
 	r.mu.Unlock()
@@ -355,7 +344,7 @@ type response struct {
 	Header   dns.Header
 	Question dns.Question
 	// Name is the response to a PTR query.
-	Name string
+	Name dnsname.FQDN
 	// IP is the response to an A, AAAA, or ALL query.
 	IP netaddr.IP
 }
@@ -418,7 +407,7 @@ func marshalAAAARecord(name dns.Name, ip netaddr.IP, builder *dns.Builder) error
 
 // marshalPTRRecord serializes a PTR record into an active builder.
 // The caller may continue using the builder following the call.
-func marshalPTRRecord(queryName dns.Name, name string, builder *dns.Builder) error {
+func marshalPTRRecord(queryName dns.Name, name dnsname.FQDN, builder *dns.Builder) error {
 	var answer dns.PTRResource
 	var err error
 
@@ -428,7 +417,7 @@ func marshalPTRRecord(queryName dns.Name, name string, builder *dns.Builder) err
 		Class: dns.ClassINET,
 		TTL:   uint32(defaultTTL / time.Second),
 	}
-	answer.PTR, err = dns.NewName(name)
+	answer.PTR, err = dns.NewName(name.WithTrailingDot())
 	if err != nil {
 		return err
 	}
@@ -501,12 +490,13 @@ const (
 //   r._dns-sd._udp.<domain>.
 //  dr._dns-sd._udp.<domain>.
 //  lb._dns-sd._udp.<domain>.
-func hasRDNSBonjourPrefix(s string) bool {
+func hasRDNSBonjourPrefix(name dnsname.FQDN) bool {
 	// Even the shortest name containing a Bonjour prefix is long,
 	// so check length (cheap) and bail early if possible.
-	if len(s) < len("*._dns-sd._udp.0.0.0.0.in-addr.arpa.") {
+	if len(name) < len("*._dns-sd._udp.0.0.0.0.in-addr.arpa.") {
 		return false
 	}
+	s := name.WithTrailingDot()
 	dot := strings.IndexByte(s, '.')
 	if dot == -1 {
 		return false // shouldn't happen
@@ -541,9 +531,9 @@ func rawNameToLower(name []byte) string {
 //   4.3.2.1.in-addr.arpa
 // is transformed to
 //   1.2.3.4
-func rdnsNameToIPv4(name string) (ip netaddr.IP, ok bool) {
-	name = strings.TrimSuffix(name, rdnsv4Suffix)
-	ip, err := netaddr.ParseIP(string(name))
+func rdnsNameToIPv4(name dnsname.FQDN) (ip netaddr.IP, ok bool) {
+	s := strings.TrimSuffix(name.WithTrailingDot(), rdnsv4Suffix)
+	ip, err := netaddr.ParseIP(s)
 	if err != nil {
 		return netaddr.IP{}, false
 	}
@@ -560,21 +550,21 @@ func rdnsNameToIPv4(name string) (ip netaddr.IP, ok bool) {
 //   b.a.9.8.7.6.5.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.8.b.d.0.1.0.0.2.ip6.arpa.
 // is transformed to
 //   2001:db8::567:89ab
-func rdnsNameToIPv6(name string) (ip netaddr.IP, ok bool) {
+func rdnsNameToIPv6(name dnsname.FQDN) (ip netaddr.IP, ok bool) {
 	var b [32]byte
 	var ipb [16]byte
 
-	name = strings.TrimSuffix(name, rdnsv6Suffix)
+	s := strings.TrimSuffix(name.WithTrailingDot(), rdnsv6Suffix)
 	// 32 nibbles and 31 dots between them.
-	if len(name) != 63 {
+	if len(s) != 63 {
 		return netaddr.IP{}, false
 	}
 
 	// Dots and hex digits alternate.
 	prevDot := true
 	// i ranges over name backward; j ranges over b forward.
-	for i, j := len(name)-1, 0; i >= 0; i-- {
-		thisDot := (name[i] == '.')
+	for i, j := len(s)-1, 0; i >= 0; i-- {
+		thisDot := (s[i] == '.')
 		if prevDot == thisDot {
 			return netaddr.IP{}, false
 		}
@@ -583,7 +573,7 @@ func rdnsNameToIPv6(name string) (ip netaddr.IP, ok bool) {
 		if !thisDot {
 			// This is safe assuming alternation.
 			// We do not check that non-dots are hex digits: hex.Decode below will do that.
-			b[j] = name[i]
+			b[j] = s[i]
 			j++
 		}
 	}
@@ -598,7 +588,7 @@ func rdnsNameToIPv6(name string) (ip netaddr.IP, ok bool) {
 
 // respondReverse returns a DNS response to a PTR query.
 // It is assumed that resp.Question is populated by respond before this is called.
-func (r *Resolver) respondReverse(query []byte, name string, resp *response) ([]byte, error) {
+func (r *Resolver) respondReverse(query []byte, name dnsname.FQDN, resp *response) ([]byte, error) {
 	if hasRDNSBonjourPrefix(name) {
 		return nil, errNotOurName
 	}
@@ -606,9 +596,9 @@ func (r *Resolver) respondReverse(query []byte, name string, resp *response) ([]
 	var ip netaddr.IP
 	var ok bool
 	switch {
-	case strings.HasSuffix(name, rdnsv4Suffix):
+	case strings.HasSuffix(name.WithTrailingDot(), rdnsv4Suffix):
 		ip, ok = rdnsNameToIPv4(name)
-	case strings.HasSuffix(name, rdnsv6Suffix):
+	case strings.HasSuffix(name.WithTrailingDot(), rdnsv6Suffix):
 		ip, ok = rdnsNameToIPv6(name)
 	default:
 		return nil, errNotOurName
@@ -649,7 +639,12 @@ func (r *Resolver) respond(query []byte) ([]byte, error) {
 		return marshalResponse(resp)
 	}
 	rawName := resp.Question.Name.Data[:resp.Question.Name.Length]
-	name := rawNameToLower(rawName)
+	name, err := dnsname.ToFQDN(rawNameToLower(rawName))
+	if err != nil {
+		// DNS packet unexpectedly contains an invalid FQDN.
+		resp.Header.RCode = dns.RCodeFormatError
+		return marshalResponse(resp)
+	}
 
 	// Always try to handle reverse lookups; delegate inside when not found.
 	// This way, queries for existent nodes do not leak,

net/dns/resolver/tsdns_test.go

@@ -13,23 +13,24 @@ import (
 	dns "golang.org/x/net/dns/dnsmessage"
 	"inet.af/netaddr"
 	"tailscale.com/tstest"
+	"tailscale.com/util/dnsname"
 )
 
 var testipv4 = netaddr.MustParseIP("1.2.3.4")
 var testipv6 = netaddr.MustParseIP("0001:0203:0405:0607:0809:0a0b:0c0d:0e0f")
 
 var dnsCfg = Config{
-	Hosts: map[string][]netaddr.IP{
+	Hosts: map[dnsname.FQDN][]netaddr.IP{
 		"test1.ipn.dev.": []netaddr.IP{testipv4},
 		"test2.ipn.dev.": []netaddr.IP{testipv6},
 	},
-	LocalDomains: []string{"ipn.dev."},
+	LocalDomains: []dnsname.FQDN{"ipn.dev."},
 }
 
-func dnspacket(domain string, tp dns.Type) []byte {
+func dnspacket(domain dnsname.FQDN, tp dns.Type) []byte {
 	var dnsHeader dns.Header
 	question := dns.Question{
-		Name:  dns.MustNewName(domain),
+		Name:  dns.MustNewName(domain.WithTrailingDot()),
 		Type:  tp,
 		Class: dns.ClassINET,
 	}
@@ -44,7 +45,7 @@ func dnspacket(domain string, tp dns.Type) []byte {
 
 type dnsResponse struct {
 	ip    netaddr.IP
-	name  string
+	name  dnsname.FQDN
 	rcode dns.RCode
 }
 
@@ -94,7 +95,10 @@ func unpackResponse(payload []byte) (dnsResponse, error) {
 		if err != nil {
 			return response, err
 		}
-		response.name = res.NS.String()
+		response.name, err = dnsname.ToFQDN(res.NS.String())
+		if err != nil {
+			return response, err
+		}
 	default:
 		return response, errors.New("type not in {A, AAAA, NS}")
 	}
@@ -119,7 +123,7 @@ func mustIP(str string) netaddr.IP {
 func TestRDNSNameToIPv4(t *testing.T) {
 	tests := []struct {
 		name   string
-		input  string
+		input  dnsname.FQDN
 		wantIP netaddr.IP
 		wantOK bool
 	}{
@@ -144,7 +148,7 @@ func TestRDNSNameToIPv4(t *testing.T) {
 func TestRDNSNameToIPv6(t *testing.T) {
 	tests := []struct {
 		name   string
-		input  string
+		input  dnsname.FQDN
 		wantIP netaddr.IP
 		wantOK bool
 	}{
@@ -194,7 +198,7 @@ func TestResolveLocal(t *testing.T) {
 
 	tests := []struct {
 		name  string
-		qname string
+		qname dnsname.FQDN
 		qtype dns.Type
 		ip    netaddr.IP
 		code  dns.RCode
@@ -235,7 +239,7 @@ func TestResolveLocalReverse(t *testing.T) {
 	tests := []struct {
 		name string
 		ip   netaddr.IP
-		want string
+		want dnsname.FQDN
 		code dns.RCode
 	}{
 		{"ipv4", testipv4, "test1.ipn.dev.", dns.RCodeSuccess},
@@ -285,7 +289,7 @@ func TestDelegate(t *testing.T) {
 	defer r.Close()
 
 	cfg := dnsCfg
-	cfg.Routes = map[string][]netaddr.IPPort{
+	cfg.Routes = map[dnsname.FQDN][]netaddr.IPPort{
 		".": {
 			netaddr.MustParseIPPort(v4server.PacketConn.LocalAddr().String()),
 			netaddr.MustParseIPPort(v6server.PacketConn.LocalAddr().String()),
@@ -360,7 +364,7 @@ func TestDelegateSplitRoute(t *testing.T) {
 	defer r.Close()
 
 	cfg := dnsCfg
-	cfg.Routes = map[string][]netaddr.IPPort{
+	cfg.Routes = map[dnsname.FQDN][]netaddr.IPPort{
 		".":      {netaddr.MustParseIPPort(server1.PacketConn.LocalAddr().String())},
 		"other.": {netaddr.MustParseIPPort(server2.PacketConn.LocalAddr().String())},
 	}
@@ -417,7 +421,7 @@ func TestDelegateCollision(t *testing.T) {
 	defer r.Close()
 
 	cfg := dnsCfg
-	cfg.Routes = map[string][]netaddr.IPPort{
+	cfg.Routes = map[dnsname.FQDN][]netaddr.IPPort{
 		".": {
 			netaddr.MustParseIPPort(server.PacketConn.LocalAddr().String()),
 		},
@@ -425,12 +429,12 @@ func TestDelegateCollision(t *testing.T) {
 	r.SetConfig(cfg)
 
 	packets := []struct {
-		qname string
+		qname dnsname.FQDN
 		qtype dns.Type
 		addr  netaddr.IPPort
 	}{
-		{"test.site.", dns.TypeA, netaddr.IPPort{IP: netaddr.IPv4(1, 1, 1, 1), Port: 1001}},
-		{"test.site.", dns.TypeAAAA, netaddr.IPPort{IP: netaddr.IPv4(1, 1, 1, 1), Port: 1002}},
+		{"test.site.", dns.TypeA, netaddr.IPPortFrom(netaddr.IPv4(1, 1, 1, 1), 1001)},
+		{"test.site.", dns.TypeAAAA, netaddr.IPPortFrom(netaddr.IPv4(1, 1, 1, 1), 1002)},
 	}
 
 	// packets will have the same dns txid.
@@ -692,7 +696,7 @@ func TestAllocs(t *testing.T) {
 
 func TestTrimRDNSBonjourPrefix(t *testing.T) {
 	tests := []struct {
-		in   string
+		in   dnsname.FQDN
 		want bool
 	}{
 		{"b._dns-sd._udp.0.10.20.172.in-addr.arpa.", true},
@@ -702,7 +706,6 @@ func TestTrimRDNSBonjourPrefix(t *testing.T) {
 		{"lb._dns-sd._udp.0.10.20.172.in-addr.arpa.", true},
 		{"qq._dns-sd._udp.0.10.20.172.in-addr.arpa.", false},
 		{"0.10.20.172.in-addr.arpa.", false},
-		{"i-have-no-dot", false},
 	}
 
 	for _, test := range tests {
@@ -722,7 +725,7 @@ func BenchmarkFull(b *testing.B) {
 	defer r.Close()
 
 	cfg := dnsCfg
-	cfg.Routes = map[string][]netaddr.IPPort{
+	cfg.Routes = map[dnsname.FQDN][]netaddr.IPPort{
 		".": {
 			netaddr.MustParseIPPort(server.PacketConn.LocalAddr().String()),
 		},