control/controlclient: remove some channels (quit, updateCh)

Updates #cleanup

Change-Id: Ie4820b64f1ad276129e4f81a609fa14e7e04604b
Co-authored-by: Maisem Ali <maisem@tailscale.com>
Signed-off-by: Brad Fitzpatrick <bradfitz@tailscale.com>

.github/workflows/flakehub-publish-tagged.yml

@@ -0,0 +1,27 @@
+name: update-flakehub
+
+on:
+  push:
+    tags:
+      - "v[0-9]+.*[02468].[0-9]+"
+  workflow_dispatch:
+    inputs:
+      tag:
+        description: "The existing tag to publish to FlakeHub"
+        type: "string"
+        required: true
+jobs:
+  flakehub-publish:
+    runs-on: "ubuntu-latest"
+    permissions:
+      id-token: "write"
+      contents: "read"
+    steps:
+      - uses: "actions/checkout@v3"
+        with:
+          ref: "${{ (inputs.tag != null) && format('refs/tags/{0}', inputs.tag) || '' }}"
+      - uses: "DeterminateSystems/nix-installer-action@main"
+      - uses: "DeterminateSystems/flakehub-push@main"
+        with:
+          visibility: "public"
+          tag: "${{ inputs.tag }}"

.github/workflows/test.yml

@@ -74,6 +74,7 @@ jobs:
       env:
         GOARCH: ${{ matrix.goarch }}
     - name: build variant CLIs
+      if: matrix.buildflags == '' # skip on race builder
       run: |
         export TS_USE_TOOLCHAIN=1
         ./build_dist.sh --extra-small ./cmd/tailscaled
@@ -83,7 +84,7 @@ jobs:
       env:
         GOARCH: ${{ matrix.goarch }}
     - name: get qemu # for tstest/archtest
-      if: matrix.goarch == 'amd64' && matrix.variant == ''
+      if: matrix.goarch == 'amd64' && matrix.buildflags == ''
       run: |
         sudo apt-get -y update
         sudo apt-get -y install qemu-user
@@ -94,7 +95,7 @@ jobs:
       env:
         GOARCH: ${{ matrix.goarch }}
     - name: bench all
-      run: PATH=$PWD/tool:$PATH /tmp/testwrapper ./... ${{matrix.buildflags}} -bench=. -benchtime=1x -run=^$ 
+      run: ./tool/go test ${{matrix.buildflags}} -bench=. -benchtime=1x -run=^$ $(for x in $(git grep -l "^func Benchmark" | xargs dirname | sort | uniq); do echo "./$x"; done)
       env:
         GOARCH: ${{ matrix.goarch }}
     - name: check that no tracked files changed
@@ -141,10 +142,12 @@ jobs:
           ${{ github.job }}-${{ runner.os }}-go-2-${{ hashFiles('**/go.sum') }}
           ${{ github.job }}-${{ runner.os }}-go-2-
     - name: test
+      run: go run ./cmd/testwrapper ./...
+    - name: bench all
       # Don't use -bench=. -benchtime=1x.
       # Somewhere in the layers (powershell?)
       # the equals signs cause great confusion.
-      run: go run ./cmd/testwrapper ./... -bench . -benchtime 1x
+      run: go test ./... -bench . -benchtime 1x -run "^$"
 
   vm:
     runs-on: ["self-hosted", "linux", "vm"]
@@ -194,6 +197,9 @@ jobs:
             goarch: amd64
           - goos: openbsd
             goarch: amd64
+          # Plan9
+          - goos: plan9
+            goarch: amd64
 
     runs-on: ubuntu-22.04
     steps:

.gitignore

@@ -38,7 +38,7 @@ cmd/tailscaled/tailscaled
 # Ignore web client node modules
 .vite/
 client/web/node_modules
-client/web/build
+client/web/build/assets
 
 /gocross
 /dist

CODEOWNERS

@@ -0,0 +1 @@
+/tailcfg/ @tailscale/control-protocol-owners

Makefile

@@ -36,6 +36,9 @@ buildlinuxarm: ## Build tailscale CLI for linux/arm
 buildwasm: ## Build tailscale CLI for js/wasm
 	GOOS=js GOARCH=wasm ./tool/go install ./cmd/tsconnect/wasm ./cmd/tailscale/cli
 
+buildplan9:
+	GOOS=plan9 GOARCH=amd64 ./tool/go install ./cmd/tailscale ./cmd/tailscaled
+
 buildlinuxloong64: ## Build tailscale CLI for linux/loong64
 	GOOS=linux GOARCH=loong64 ./tool/go install tailscale.com/cmd/tailscale tailscale.com/cmd/tailscaled
 

README.md

@@ -57,6 +57,17 @@ If your distro has conventions that preclude the use of
 `build_dist.sh`, please do the equivalent of what it does in your
 distro's way, so that bug reports contain useful version information.
 
+## Building the web client
+
+To include the embedded web client (accessed via the `tailscale web` command),
+you'll need to build the client assets using:
+
+```
+./tool/yarn --cwd client/web build
+```
+
+Do this before building the `tailscale.com/cmd/tailscale` binary.
+
 ## Bugs
 
 Please file any issues about this code or the hosted service on

build_dist.sh

@@ -5,6 +5,9 @@
 # information into the binaries, so that we can track down user
 # issues.
 #
+# To include the embedded web client, build the web client assets
+# before running this script. See README.md for details.
+#
 # If you're packaging Tailscale for a distro, please consider using
 # this script, or executing equivalent commands in your
 # distro-specific build system.

client/tailscale/apitype/controltype.go

@@ -4,13 +4,13 @@
 package apitype
 
 type DNSConfig struct {
-	Resolvers         []DNSResolver            `json:"resolvers"`
-	FallbackResolvers []DNSResolver            `json:"fallbackResolvers"`
-	Routes            map[string][]DNSResolver `json:"routes"`
-	Domains           []string                 `json:"domains"`
-	Nameservers       []string                 `json:"nameservers"`
-	Proxied           bool                     `json:"proxied"`
-	DNSFilterURL      string                   `json:"DNSFilterURL"`
+	Resolvers          []DNSResolver            `json:"resolvers"`
+	FallbackResolvers  []DNSResolver            `json:"fallbackResolvers"`
+	Routes             map[string][]DNSResolver `json:"routes"`
+	Domains            []string                 `json:"domains"`
+	Nameservers        []string                 `json:"nameservers"`
+	Proxied            bool                     `json:"proxied"`
+	TempCorpIssue13969 string                   `json:"TempCorpIssue13969,omitempty"`
 }
 
 type DNSResolver struct {

client/tailscale/localclient.go

@@ -37,6 +37,7 @@ import (
 	"tailscale.com/tka"
 	"tailscale.com/types/key"
 	"tailscale.com/types/tkatype"
+	"tailscale.com/util/cmpx"
 )
 
 // defaultLocalClient is the default LocalClient when using the legacy
@@ -391,15 +392,51 @@ func (lc *LocalClient) DebugAction(ctx context.Context, action string) error {
 	return nil
 }
 
+// DebugPortmapOpts contains options for the DebugPortmap command.
+type DebugPortmapOpts struct {
+	// Duration is how long the mapping should be created for. It defaults
+	// to 5 seconds if not set.
+	Duration time.Duration
+
+	// Type is the kind of portmap to debug. The empty string instructs the
+	// portmap client to perform all known types. Other valid options are
+	// "pmp", "pcp", and "upnp".
+	Type string
+
+	// GatewayAddr specifies the gateway address used during portmapping.
+	// If set, SelfAddr must also be set. If unset, it will be
+	// autodetected.
+	GatewayAddr netip.Addr
+
+	// SelfAddr specifies the gateway address used during portmapping. If
+	// set, GatewayAddr must also be set. If unset, it will be
+	// autodetected.
+	SelfAddr netip.Addr
+
+	// LogHTTP instructs the debug-portmap endpoint to print all HTTP
+	// requests and responses made to the logs.
+	LogHTTP bool
+}
+
 // DebugPortmap invokes the debug-portmap endpoint, and returns an
 // io.ReadCloser that can be used to read the logs that are printed during this
 // process.
-func (lc *LocalClient) DebugPortmap(ctx context.Context, duration time.Duration, ty, gwSelf string) (io.ReadCloser, error) {
+//
+// opts can be nil; if so, default values will be used.
+func (lc *LocalClient) DebugPortmap(ctx context.Context, opts *DebugPortmapOpts) (io.ReadCloser, error) {
 	vals := make(url.Values)
-	vals.Set("duration", duration.String())
-	vals.Set("type", ty)
-	if gwSelf != "" {
-		vals.Set("gateway_and_self", gwSelf)
+	if opts == nil {
+		opts = &DebugPortmapOpts{}
+	}
+
+	vals.Set("duration", cmpx.Or(opts.Duration, 5*time.Second).String())
+	vals.Set("type", opts.Type)
+	vals.Set("log_http", strconv.FormatBool(opts.LogHTTP))
+
+	if opts.GatewayAddr.IsValid() != opts.SelfAddr.IsValid() {
+		return nil, fmt.Errorf("both GatewayAddr and SelfAddr must be provided if one is")
+	} else if opts.GatewayAddr.IsValid() {
+		vals.Set("gateway_and_self", fmt.Sprintf("%s/%s", opts.GatewayAddr, opts.SelfAddr))
 	}
 
 	req, err := http.NewRequestWithContext(ctx, "GET", "http://"+apitype.LocalAPIHost+"/localapi/v0/debug-portmap?"+vals.Encode(), nil)
@@ -1057,6 +1094,29 @@ func (lc *LocalClient) NetworkLockDisable(ctx context.Context, secret []byte) er
 	return nil
 }
 
+// StreamServe returns an io.ReadCloser that streams serve/Funnel
+// connections made to the provided HostPort.
+//
+// If Serve and Funnel were not already enabled for the HostPort in the ServeConfig,
+// the backend enables it for the duration of the context's lifespan and
+// then turns it back off once the context is closed. If either are already enabled,
+// then they remain that way but logs are still streamed
+func (lc *LocalClient) StreamServe(ctx context.Context, hp ipn.ServeStreamRequest) (io.ReadCloser, error) {
+	req, err := http.NewRequestWithContext(ctx, "POST", "http://"+apitype.LocalAPIHost+"/localapi/v0/stream-serve", jsonBody(hp))
+	if err != nil {
+		return nil, err
+	}
+	res, err := lc.doLocalRequestNiceError(req)
+	if err != nil {
+		return nil, err
+	}
+	if res.StatusCode != 200 {
+		res.Body.Close()
+		return nil, errors.New(res.Status)
+	}
+	return res.Body, nil
+}
+
 // GetServeConfig return the current serve config.
 //
 // If the serve config is empty, it returns (nil, nil).

client/web/auth-redirect.html

@@ -1,57 +0,0 @@
-<html>
-<head>
-	<title>Redirecting...</title>
-	<style>
-		html,
-		body {
-			height: 100%;
-		}
-
-		html {
-			background-color: rgb(249, 247, 246);
-			font-family: ui-sans-serif, system-ui, -apple-system, BlinkMacSystemFont, "Segoe UI", Roboto, "Helvetica Neue", Arial, "Noto Sans", sans-serif, "Apple Color Emoji", "Segoe UI Emoji", "Segoe UI Symbol", "Noto Color Emoji";
-			line-height: 1.5;
-			-webkit-text-size-adjust: 100%;
-			-webkit-font-smoothing: antialiased;
-			-moz-osx-font-smoothing: grayscale;
-		}
-
-		body {
-			display: flex;
-			flex-direction: column;
-			align-items: center;
-			justify-content: center;
-		}
-
-		.spinner {
-			margin-bottom: 2rem;
-			border: 4px rgba(112, 110, 109, 0.5) solid;
-			border-left-color: transparent;
-			border-radius: 9999px;
-			width: 4rem;
-			height: 4rem;
-			-webkit-animation: spin 700ms linear infinite;
-      animation: spin 800ms linear infinite;
-		}
-
-		.label {
-			color: rgb(112, 110, 109);
-			padding-left: 0.4rem;
-		}
-
-		@-webkit-keyframes spin {
-			to {
-				transform: rotate(360deg);
-			}
-		}
-
-		@keyframes spin {
-			to {
-				transform: rotate(360deg);
-			}
-		}
-	</style>
-</head> <body>
-	<div class="spinner"></div>
-	<div class="label">Redirecting...</div>
-</body>

client/web/build/index.html

@@ -0,0 +1,28 @@
+<!doctype html>
+<html class="bg-gray-50">
+  <head>
+    <title>Tailscale</title>
+    <meta charset="utf-8" />
+    <meta name="viewport" content="width=device-width, initial-scale=1" />
+    <link rel="shortcut icon" href="data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAACAAAAAgCAQAAADZc7J/AAAABGdBTUEAALGPC/xhBQAAACBjSFJNAAB6JgAAgIQAAPoAAACA6AAAdTAAAOpgAAA6mAAAF3CculE8AAAAAmJLR0QA/4ePzL8AAAAHdElNRQflAx4QGA4EvmzDAAAA30lEQVRIx2NgGAWMCKa8JKM4A8Ovt88ekyLCDGOoyDBJMjExMbFy8zF8/EKsCAMDE8yAPyIwFps48SJIBpAL4AZwvoSx/r0lXgQpDN58EWL5x/7/H+vL20+JFxluQKVe5b3Ke5V+0kQQCamfoYKBg4GDwUKI8d0BYkWQkrLKewYBKPPDHUFiRaiZkBgmwhj/F5IgggyUJ6i8V3mv0kCayDAAeEsklXqGAgYGhgV3CnGrwVciYSYk0kokhgS44/JxqqFpiYSZbEgskd4dEBRk1GD4wdB5twKXmlHAwMDAAACdEZau06NQUwAAACV0RVh0ZGF0ZTpjcmVhdGUAMjAyMC0wNy0xNVQxNTo1Mzo0MCswMDowMCVXsDIAAAAldEVYdGRhdGU6bW9kaWZ5ADIwMjAtMDctMTVUMTU6NTM6NDArMDA6MDBUCgiOAAAAAElFTkSuQmCC" />
+    
+    <script type="module" crossorigin src="./assets/index-f8beba53.js"></script>
+    <link rel="stylesheet" href="./assets/index-8612dca6.css">
+  </head>
+  <body>
+    <noscript>
+      <p class="mb-2">You need to enable Javascript to access the Tailscale web client.</p>
+      <p>If you need any help, feel free to <a href="mailto:support+webclient@tailscale.com" class="link">contact us</a>.</p>
+    </noscript>
+    
+    <script>
+      window.addEventListener("load", () => {
+        if (!window.Tailscale) {
+          const rootEl = document.createElement("p")
+          rootEl.innerHTML = 'Tailscale was built without the web client. See <a href="https://github.com/tailscale/tailscale#building-the-web-client">Building the web client</a> for more information.'
+          document.body.append(rootEl)
+        }
+      });
+    </script>
+  </body>
+</html>

client/web/index.html

@@ -8,22 +8,19 @@
     <link rel="stylesheet" type="text/css" href="/src/index.css" />
   </head>
   <body>
-    <div class="min-h-screen py-10 flex justify-center items-center" style="display: none">
-      <div class="max-w-md">
-        <h3 class="font-semibold text-lg mb-4">Your web browser is unsupported.</h3>
-        <p class="mb-2">
-          Update to a modern browser to access the Tailscale web client. You can use
-          <a class="link" href="https://www.mozilla.org/en-US/firefox/new/" target="_blank">Firefox</a>,
-          <a class="link" href="https://www.microsoft.com/en-us/edge" target="_blank">Edge</a>,
-          <a class="link" href="https://www.apple.com/safari/" target="_blank">Safari</a>,
-          or <a class="link" href="https://www.google.com/chrome/" target="_blank">Chrome</a>.</p>
-        <p>If you need any help, feel free to <a href="mailto:support+webclient@tailscale.com" class="link">contact us</a></p>
-      </div>
-    </div>
     <noscript>
       <p class="mb-2">You need to enable Javascript to access the Tailscale web client.</p>
       <p>If you need any help, feel free to <a href="mailto:support+webclient@tailscale.com" class="link">contact us</a>.</p>
     </noscript>
     <script type="module" src="/src/index.tsx"></script>
+    <script>
+      window.addEventListener("load", () => {
+        if (!window.Tailscale) {
+          const rootEl = document.createElement("p")
+          rootEl.innerHTML = 'Tailscale was built without the web client. See <a href="https://github.com/tailscale/tailscale#building-the-web-client">Building the web client</a> for more information.'
+          document.body.append(rootEl)
+        }
+      });
+    </script>
   </body>
 </html>

client/web/package.json

@@ -8,10 +8,12 @@
   },
   "private": true,
   "dependencies": {
+    "classnames": "^2.3.1",
     "react": "^18.2.0",
     "react-dom": "^18.2.0"
   },
   "devDependencies": {
+    "@types/classnames": "^2.2.10",
     "@types/react": "^18.0.20",
     "@types/react-dom": "^18.0.6",
     "@vitejs/plugin-react-swc": "^3.3.2",

client/web/qnap.go

@@ -0,0 +1,130 @@
+// Copyright (c) Tailscale Inc & AUTHORS
+// SPDX-License-Identifier: BSD-3-Clause
+
+// qnap.go contains handlers and logic, such as authentication,
+// that is specific to running the web client on QNAP.
+
+package web
+
+import (
+	"crypto/tls"
+	"encoding/xml"
+	"fmt"
+	"io"
+	"log"
+	"net/http"
+	"net/url"
+)
+
+const qnapPrefix = "/cgi-bin/qpkg/Tailscale/index.cgi/"
+
+// authorizeQNAP authenticates the logged-in QNAP user and verifies
+// that they are authorized to use the web client.  It returns true if the
+// request was handled and no further processing is required.
+func authorizeQNAP(w http.ResponseWriter, r *http.Request) (handled bool) {
+	_, resp, err := qnapAuthn(r)
+	if err != nil {
+		http.Error(w, err.Error(), http.StatusUnauthorized)
+		return true
+	}
+	if resp.IsAdmin == 0 {
+		http.Error(w, "user is not an admin", http.StatusForbidden)
+		return true
+	}
+
+	return false
+}
+
+type qnapAuthResponse struct {
+	AuthPassed int    `xml:"authPassed"`
+	IsAdmin    int    `xml:"isAdmin"`
+	AuthSID    string `xml:"authSid"`
+	ErrorValue int    `xml:"errorValue"`
+}
+
+func qnapAuthn(r *http.Request) (string, *qnapAuthResponse, error) {
+	user, err := r.Cookie("NAS_USER")
+	if err != nil {
+		return "", nil, err
+	}
+	token, err := r.Cookie("qtoken")
+	if err == nil {
+		return qnapAuthnQtoken(r, user.Value, token.Value)
+	}
+	sid, err := r.Cookie("NAS_SID")
+	if err == nil {
+		return qnapAuthnSid(r, user.Value, sid.Value)
+	}
+	return "", nil, fmt.Errorf("not authenticated by any mechanism")
+}
+
+// qnapAuthnURL returns the auth URL to use by inferring where the UI is
+// running based on the request URL. This is necessary because QNAP has so
+// many options, see https://github.com/tailscale/tailscale/issues/7108
+// and https://github.com/tailscale/tailscale/issues/6903
+func qnapAuthnURL(requestUrl string, query url.Values) string {
+	in, err := url.Parse(requestUrl)
+	scheme := ""
+	host := ""
+	if err != nil || in.Scheme == "" {
+		log.Printf("Cannot parse QNAP login URL %v", err)
+
+		// try localhost and hope for the best
+		scheme = "http"
+		host = "localhost"
+	} else {
+		scheme = in.Scheme
+		host = in.Host
+	}
+
+	u := url.URL{
+		Scheme:   scheme,
+		Host:     host,
+		Path:     "/cgi-bin/authLogin.cgi",
+		RawQuery: query.Encode(),
+	}
+
+	return u.String()
+}
+
+func qnapAuthnQtoken(r *http.Request, user, token string) (string, *qnapAuthResponse, error) {
+	query := url.Values{
+		"qtoken": []string{token},
+		"user":   []string{user},
+	}
+	return qnapAuthnFinish(user, qnapAuthnURL(r.URL.String(), query))
+}
+
+func qnapAuthnSid(r *http.Request, user, sid string) (string, *qnapAuthResponse, error) {
+	query := url.Values{
+		"sid": []string{sid},
+	}
+	return qnapAuthnFinish(user, qnapAuthnURL(r.URL.String(), query))
+}
+
+func qnapAuthnFinish(user, url string) (string, *qnapAuthResponse, error) {
+	// QNAP Force HTTPS mode uses a self-signed certificate. Even importing
+	// the QNAP root CA isn't enough, the cert doesn't have a usable CN nor
+	// SAN. See https://github.com/tailscale/tailscale/issues/6903
+	tr := &http.Transport{
+		TLSClientConfig: &tls.Config{InsecureSkipVerify: true},
+	}
+	client := &http.Client{Transport: tr}
+	resp, err := client.Get(url)
+	if err != nil {
+		return "", nil, err
+	}
+	defer resp.Body.Close()
+	out, err := io.ReadAll(resp.Body)
+	if err != nil {
+		return "", nil, err
+	}
+	authResp := &qnapAuthResponse{}
+	if err := xml.Unmarshal(out, authResp); err != nil {
+		return "", nil, err
+	}
+	if authResp.AuthPassed == 0 {
+		return "", nil, fmt.Errorf("not authenticated")
+	}
+	return user, authResp, nil
+}

client/web/src/api.ts

@@ -0,0 +1,67 @@
+let csrfToken: string
+let unraidCsrfToken: string | undefined // required for unraid POST requests (#8062)
+
+// apiFetch wraps the standard JS fetch function with csrf header
+// management and param additions specific to the web client.
+//
+// apiFetch adds the `api` prefix to the request URL,
+// so endpoint should be provided without the `api` prefix
+// (i.e. provide `/data` rather than `api/data`).
+export function apiFetch(
+  endpoint: string,
+  method: "GET" | "POST",
+  body?: any,
+  params?: Record<string, string>
+): Promise<Response> {
+  const urlParams = new URLSearchParams(window.location.search)
+  const nextParams = new URLSearchParams(params)
+  const token = urlParams.get("SynoToken")
+  if (token) {
+    nextParams.set("SynoToken", token)
+  }
+  const search = nextParams.toString()
+  const url = `api${endpoint}${search ? `?${search}` : ""}`
+
+  var contentType: string
+  if (unraidCsrfToken) {
+    const params = new URLSearchParams()
+    params.append("csrf_token", unraidCsrfToken)
+    if (body) {
+      params.append("ts_data", JSON.stringify(body))
+    }
+    body = params.toString()
+    contentType = "application/x-www-form-urlencoded;charset=UTF-8"
+  } else {
+    body = body ? JSON.stringify(body) : undefined
+    contentType = "application/json"
+  }
+
+  return fetch(url, {
+    method: method,
+    headers: {
+      Accept: "application/json",
+      "Content-Type": contentType,
+      "X-CSRF-Token": csrfToken,
+    },
+    body,
+  }).then((r) => {
+    updateCsrfToken(r)
+    if (!r.ok) {
+      return r.text().then((err) => {
+        throw new Error(err)
+      })
+    }
+    return r
+  })
+}
+
+function updateCsrfToken(r: Response) {
+  const tok = r.headers.get("X-CSRF-Token")
+  if (tok) {
+    csrfToken = tok
+  }
+}
+
+export function setUnraidCsrfToken(token?: string) {
+  unraidCsrfToken = token
+}

client/web/src/components/app.tsx

@@ -3,7 +3,9 @@ import { Footer, Header, IP, State } from "src/components/legacy"
 import useNodeData from "src/hooks/node-data"
 
 export default function App() {
-  const data = useNodeData()
+  // TODO(sonia): use isPosting value from useNodeData
+  // to fill loading states.
+  const { data, refreshData, updateNode } = useNodeData()
 
   return (
     <div className="py-14">
@@ -13,9 +15,13 @@ export default function App() {
       ) : (
         <>
           <main className="container max-w-lg mx-auto mb-8 py-6 px-8 bg-white rounded-md shadow-2xl">
-            <Header data={data} />
+            <Header
+              data={data}
+              refreshData={refreshData}
+              updateNode={updateNode}
+            />
             <IP data={data} />
-            <State data={data} />
+            <State data={data} updateNode={updateNode} />
           </main>
           <Footer data={data} />
         </>

client/web/src/components/legacy.tsx

@@ -1,14 +1,22 @@
+import cx from "classnames"
 import React from "react"
-import { NodeData } from "src/hooks/node-data"
+import { apiFetch } from "src/api"
+import { NodeData, NodeUpdate } from "src/hooks/node-data"
 
 // TODO(tailscale/corp#13775): legacy.tsx contains a set of components
 // that (crudely) implement the pre-2023 web client. These are implemented
 // purely to ease migration to the new React-based web client, and will
 // eventually be completely removed.
 
-export function Header(props: { data: NodeData }) {
-  const { data } = props
-
+export function Header({
+  data,
+  refreshData,
+  updateNode,
+}: {
+  data: NodeData
+  refreshData: () => void
+  updateNode: (update: NodeUpdate) => void
+}) {
   return (
     <header className="flex justify-between items-center min-width-0 py-2 mb-8">
       <svg
@@ -60,41 +68,56 @@ export function Header(props: { data: NodeData }) {
         ></circle>
       </svg>
       <div className="flex items-center justify-end space-x-2 w-2/3">
-        {data.Profile && (
-          <>
-            <div className="text-right w-full leading-4">
-              <h4 className="truncate leading-normal">
-                {data.Profile.LoginName}
-              </h4>
-              <div className="text-xs text-gray-500 text-right">
-                <a href="#" className="hover:text-gray-700 js-loginButton">
-                  Switch account
-                </a>{" "}
-                |{" "}
-                <a href="#" className="hover:text-gray-700 js-loginButton">
-                  Reauthenticate
-                </a>{" "}
-                |{" "}
-                <a href="#" className="hover:text-gray-700 js-logoutButton">
-                  Logout
-                </a>
+        {data.Profile &&
+          data.Status !== "NoState" &&
+          data.Status !== "NeedsLogin" && (
+            <>
+              <div className="text-right w-full leading-4">
+                <h4 className="truncate leading-normal">
+                  {data.Profile.LoginName}
+                </h4>
+                <div className="text-xs text-gray-500 text-right">
+                  <button
+                    onClick={() => updateNode({ Reauthenticate: true })}
+                    className="hover:text-gray-700"
+                  >
+                    Switch account
+                  </button>{" "}
+                  |{" "}
+                  <button
+                    onClick={() => updateNode({ Reauthenticate: true })}
+                    className="hover:text-gray-700"
+                  >
+                    Reauthenticate
+                  </button>{" "}
+                  |{" "}
+                  <button
+                    onClick={() =>
+                      apiFetch("/local/v0/logout", "POST")
+                        .then(refreshData)
+                        .catch((err) => alert("Logout failed: " + err.message))
+                    }
+                    className="hover:text-gray-700"
+                  >
+                    Logout
+                  </button>
+                </div>
               </div>
-            </div>
-            <div className="relative flex-shrink-0 w-8 h-8 rounded-full overflow-hidden">
-              {data.Profile.ProfilePicURL ? (
-                <div
-                  className="w-8 h-8 flex pointer-events-none rounded-full bg-gray-200"
-                  style={{
-                    backgroundImage: `url(${data.Profile.ProfilePicURL})`,
-                    backgroundSize: "cover",
-                  }}
-                />
-              ) : (
-                <div className="w-8 h-8 flex pointer-events-none rounded-full border border-gray-400 border-dashed" />
-              )}
-            </div>
-          </>
-        )}
+              <div className="relative flex-shrink-0 w-8 h-8 rounded-full overflow-hidden">
+                {data.Profile.ProfilePicURL ? (
+                  <div
+                    className="w-8 h-8 flex pointer-events-none rounded-full bg-gray-200"
+                    style={{
+                      backgroundImage: `url(${data.Profile.ProfilePicURL})`,
+                      backgroundSize: "cover",
+                    }}
+                  />
+                ) : (
+                  <div className="w-8 h-8 flex pointer-events-none rounded-full border border-gray-400 border-dashed" />
+                )}
+              </div>
+            </>
+          )}
       </div>
     </header>
   )
@@ -128,9 +151,9 @@ export function IP(props: { data: NodeData }) {
             <line x1="6" y1="6" x2="6.01" y2="6"></line>
             <line x1="6" y1="18" x2="6.01" y2="18"></line>
           </svg>
-          <div>
-            <h4 className="font-semibold truncate mr-2">{data.DeviceName}</h4>
-          </div>
+          <h4 className="font-semibold truncate mr-2">
+            {data.DeviceName || "Your device"}
+          </h4>
         </div>
         <h5>{data.IP}</h5>
       </div>
@@ -162,9 +185,13 @@ export function IP(props: { data: NodeData }) {
   )
 }
 
-export function State(props: { data: NodeData }) {
-  const { data } = props
-
+export function State({
+  data,
+  updateNode,
+}: {
+  data: NodeData
+  updateNode: (update: NodeUpdate) => void
+}) {
   switch (data.Status) {
     case "NeedsLogin":
     case "NoState":
@@ -185,11 +212,12 @@ export function State(props: { data: NodeData }) {
                 .
               </p>
             </div>
-            <a href="#" className="mb-4 js-loginButton" target="_blank">
-              <button className="button button-blue w-full">
-                Reauthenticate
-              </button>
-            </a>
+            <button
+              onClick={() => updateNode({ Reauthenticate: true })}
+              className="button button-blue w-full mb-4"
+            >
+              Reauthenticate
+            </button>
           </>
         )
       } else {
@@ -210,9 +238,12 @@ export function State(props: { data: NodeData }) {
                 .
               </p>
             </div>
-            <a href="#" className="mb-4 js-loginButton" target="_blank">
-              <button className="button button-blue w-full">Log In</button>
-            </a>
+            <button
+              onClick={() => updateNode({ Reauthenticate: true })}
+              className="button button-blue w-full mb-4"
+            >
+              Log In
+            </button>
           </>
         )
       }
@@ -232,25 +263,20 @@ export function State(props: { data: NodeData }) {
               device name or IP address above.
             </p>
           </div>
-          <div className="mb-4">
-            <a href="#" className="mb-4 js-advertiseExitNode">
-              {data.AdvertiseExitNode ? (
-                <button
-                  className="button button-red button-medium"
-                  id="enabled"
-                >
-                  Stop advertising Exit Node
-                </button>
-              ) : (
-                <button
-                  className="button button-blue button-medium"
-                  id="enabled"
-                >
-                  Advertise as Exit Node
-                </button>
-              )}
-            </a>
-          </div>
+          <button
+            className={cx("button button-medium mb-4", {
+              "button-red": data.AdvertiseExitNode,
+              "button-blue": !data.AdvertiseExitNode,
+            })}
+            id="enabled"
+            onClick={() =>
+              updateNode({ AdvertiseExitNode: !data.AdvertiseExitNode })
+            }
+          >
+            {data.AdvertiseExitNode
+              ? "Stop advertising Exit Node"
+              : "Advertise as Exit Node"}
+          </button>
         </>
       )
   }

client/web/src/hooks/node-data.ts

@@ -1,4 +1,5 @@
-import { useEffect, useState } from "react"
+import { useCallback, useEffect, useState } from "react"
+import { apiFetch, setUnraidCsrfToken } from "src/api"
 
 export type NodeData = {
   Profile: UserProfile
@@ -22,16 +23,93 @@ export type UserProfile = {
   ProfilePicURL: string
 }
 
+export type NodeUpdate = {
+  AdvertiseRoutes?: string
+  AdvertiseExitNode?: boolean
+  Reauthenticate?: boolean
+  ForceLogout?: boolean
+}
+
 // useNodeData returns basic data about the current node.
 export default function useNodeData() {
   const [data, setData] = useState<NodeData>()
+  const [isPosting, setIsPosting] = useState<boolean>(false)
 
-  useEffect(() => {
-    fetch("/api/data")
-      .then((response) => response.json())
-      .then((json) => setData(json))
-      .catch((error) => console.error(error))
-  }, [])
+  const refreshData = useCallback(
+    () =>
+      apiFetch("/data", "GET")
+        .then((r) => r.json())
+        .then((d: NodeData) => {
+          setData(d)
+          setUnraidCsrfToken(d.IsUnraid ? d.UnraidToken : undefined)
+        })
+        .catch((error) => console.error(error)),
+    [setData]
+  )
 
-  return data
+  const updateNode = useCallback(
+    (update: NodeUpdate) => {
+      // The contents of this function are mostly copied over
+      // from the legacy client's web.html file.
+      // It makes all data updates through one API endpoint.
+      // As we build out the web client in React,
+      // this endpoint will eventually be deprecated.
+
+      if (isPosting || !data) {
+        return
+      }
+      setIsPosting(true)
+
+      update = {
+        ...update,
+        // Default to current data value for any unset fields.
+        AdvertiseRoutes:
+          update.AdvertiseRoutes !== undefined
+            ? update.AdvertiseRoutes
+            : data.AdvertiseRoutes,
+        AdvertiseExitNode:
+          update.AdvertiseExitNode !== undefined
+            ? update.AdvertiseExitNode
+            : data.AdvertiseExitNode,
+      }
+
+      apiFetch("/data", "POST", update, { up: "true" })
+        .then((r) => r.json())
+        .then((r) => {
+          setIsPosting(false)
+          const err = r["error"]
+          if (err) {
+            throw new Error(err)
+          }
+          const url = r["url"]
+          if (url) {
+            window.open(url, "_blank")
+          }
+          refreshData()
+        })
+        .catch((err) => alert("Failed operation: " + err.message))
+    },
+    [data]
+  )
+
+  useEffect(
+    () => {
+      // Initial data load.
+      refreshData()
+
+      // Refresh on browser tab focus.
+      const onVisibilityChange = () => {
+        document.visibilityState === "visible" && refreshData()
+      }
+      window.addEventListener("visibilitychange", onVisibilityChange)
+      return () => {
+        // Cleanup browser tab listener.
+        window.removeEventListener("visibilitychange", onVisibilityChange)
+      }
+    },
+    // Run once.
+    []
+  )
+
+  return { data, refreshData, updateNode, isPosting }
 }

client/web/src/index.tsx

@@ -2,6 +2,10 @@ import React from "react"
 import { createRoot } from "react-dom/client"
 import App from "src/components/app"
 
+declare var window: any
+// This is used to determine if the react client is built.
+window.Tailscale = true
+
 const rootEl = document.createElement("div")
 rootEl.id = "app-root"
 rootEl.classList.add("relative", "z-0")

client/web/synology.go

@@ -0,0 +1,78 @@
+// Copyright (c) Tailscale Inc & AUTHORS
+// SPDX-License-Identifier: BSD-3-Clause
+
+// synology.go contains handlers and logic, such as authentication,
+// that is specific to running the web client on Synology.
+
+package web
+
+import (
+	"fmt"
+	"net/http"
+	"os/exec"
+	"strings"
+
+	"tailscale.com/util/groupmember"
+)
+
+const synologyPrefix = "/webman/3rdparty/Tailscale/index.cgi/"
+
+// authorizeSynology authenticates the logged-in Synology user and verifies
+// that they are authorized to use the web client.  It returns true if the
+// request was handled and no further processing is required.
+func authorizeSynology(w http.ResponseWriter, r *http.Request) (handled bool) {
+	if synoTokenRedirect(w, r) {
+		return true
+	}
+
+	// authenticate the Synology user
+	cmd := exec.Command("/usr/syno/synoman/webman/modules/authenticate.cgi")
+	out, err := cmd.CombinedOutput()
+	if err != nil {
+		http.Error(w, fmt.Sprintf("auth: %v: %s", err, out), http.StatusUnauthorized)
+		return true
+	}
+	user := strings.TrimSpace(string(out))
+
+	// check if the user is in the administrators group
+	isAdmin, err := groupmember.IsMemberOfGroup("administrators", user)
+	if err != nil {
+		http.Error(w, err.Error(), http.StatusForbidden)
+		return true
+	}
+	if !isAdmin {
+		http.Error(w, "not a member of administrators group", http.StatusForbidden)
+		return true
+	}
+
+	return false
+}
+
+func synoTokenRedirect(w http.ResponseWriter, r *http.Request) bool {
+	if r.Header.Get("X-Syno-Token") != "" {
+		return false
+	}
+	if r.URL.Query().Get("SynoToken") != "" {
+		return false
+	}
+	if r.Method == "POST" && r.FormValue("SynoToken") != "" {
+		return false
+	}
+	// We need a SynoToken for authenticate.cgi.
+	// So we tell the client to get one.
+	_, _ = fmt.Fprint(w, synoTokenRedirectHTML)
+	return true
+}
+
+const synoTokenRedirectHTML = `<html>
+Redirecting with session token...
+<script>
+  fetch("/webman/login.cgi")
+    .then(r => r.json())
+    .then(data => {
+	u = new URL(window.location)
+	u.searchParams.set("SynoToken", data.SynoToken)
+	document.location = u
+    })
+</script>
+`

client/web/vite.config.ts

@@ -20,7 +20,7 @@ filteringLogger.info = (...args) => {
 
 // https://vitejs.dev/config/
 export default defineConfig({
-  base: "/",
+  base: "./",
   plugins: [
     paths(),
     svgr(),

client/web/web.go

@@ -5,46 +5,50 @@
 package web
 
 import (
-	"bytes"
 	"context"
-	"crypto/tls"
-	_ "embed"
+	"crypto/rand"
+	"embed"
 	"encoding/json"
-	"encoding/xml"
 	"fmt"
-	"html/template"
 	"io"
+	"io/fs"
 	"log"
 	"net/http"
 	"net/http/httputil"
 	"net/netip"
-	"net/url"
 	"os"
-	"os/exec"
+	"path/filepath"
+	"slices"
 	"strings"
 
+	"github.com/gorilla/csrf"
 	"tailscale.com/client/tailscale"
+	"tailscale.com/client/tailscale/apitype"
 	"tailscale.com/envknob"
 	"tailscale.com/ipn"
 	"tailscale.com/ipn/ipnstate"
 	"tailscale.com/licenses"
 	"tailscale.com/net/netutil"
 	"tailscale.com/tailcfg"
-	"tailscale.com/util/groupmember"
 	"tailscale.com/util/httpm"
+	"tailscale.com/util/must"
 	"tailscale.com/version/distro"
 )
 
-//go:embed web.html
-var webHTML string
+// This contains all files needed to build the frontend assets.
+// Because we assign this to the blank identifier, it does not actually embed the files.
+// However, this does cause `go mod vendor` to include the files when vendoring the package.
+// External packages that use the web client can `go mod vendor`, run `yarn build` to
+// build the assets, then those asset bundles will be embedded.
+//
+//go:embed yarn.lock index.html *.js *.json src/*
+var _ embed.FS
 
-//go:embed web.css
-var webCSS string
+//go:embed build/*
+var embeddedFS embed.FS
 
-//go:embed auth-redirect.html
-var authenticationRedirectHTML string
-
-var tmpl *template.Template
+// staticfiles serves static files from the build directory.
+var staticfiles http.Handler
 
 // Server is the backend server for a Tailscale web client.
 type Server struct {
@@ -52,270 +56,150 @@ type Server struct {
 
 	devMode  bool
 	devProxy *httputil.ReverseProxy // only filled when devMode is on
+
+	cgiMode    bool
+	cgiPath    string
+	apiHandler http.Handler // csrf-protected api handler
+}
+
+// ServerOpts contains options for constructing a new Server.
+type ServerOpts struct {
+	DevMode bool
+
+	// CGIMode indicates if the server is running as a CGI script.
+	CGIMode bool
+
+	// If running in CGIMode, CGIPath is the URL path prefix to the CGI script.
+	CGIPath string
+
+	// LocalClient is the tailscale.LocalClient to use for this web server.
+	// If nil, a new one will be created.
+	LocalClient *tailscale.LocalClient
 }
 
 // NewServer constructs a new Tailscale web client server.
-//
-// lc is an optional parameter. When not filled, NewServer
-// initializes its own tailscale.LocalClient.
-func NewServer(devMode bool, lc *tailscale.LocalClient) (s *Server, cleanup func()) {
-	if lc == nil {
-		lc = &tailscale.LocalClient{}
+// The provided context should live for the duration of the Server's lifetime.
+func NewServer(ctx context.Context, opts ServerOpts) (s *Server, cleanup func()) {
+	if opts.LocalClient == nil {
+		opts.LocalClient = &tailscale.LocalClient{}
 	}
 	s = &Server{
-		devMode: devMode,
-		lc:      lc,
+		devMode: opts.DevMode,
+		lc:      opts.LocalClient,
+		cgiMode: opts.CGIMode,
+		cgiPath: opts.CGIPath,
 	}
 	cleanup = func() {}
 	if s.devMode {
 		cleanup = s.startDevServer()
 		s.addProxyToDevServer()
 	}
+
+	// Create handler for "/api" requests with CSRF protection.
+	// We don't require secure cookies, since the web client is regularly used
+	// on network appliances that are served on local non-https URLs.
+	// The client is secured by limiting the interface it listens on,
+	// or by authenticating requests before they reach the web client.
+	csrfProtect := csrf.Protect(s.csrfKey(), csrf.Secure(false))
+	s.apiHandler = csrfProtect(http.HandlerFunc(s.serveAPI))
+
+	s.lc.IncrementCounter(context.Background(), "web_client_initialization", 1)
 	return s, cleanup
 }
 
 func init() {
-	tmpl = template.Must(template.New("web.html").Parse(webHTML))
-	template.Must(tmpl.New("web.css").Parse(webCSS))
+	buildFiles := must.Get(fs.Sub(embeddedFS, "build"))
+	staticfiles = http.FileServer(http.FS(buildFiles))
 }
 
-// authorize returns the name of the user accessing the web UI after verifying
-// whether the user has access to the web UI. The function will write the
-// error to the provided http.ResponseWriter.
-// Note: This is different from a tailscale user, and is typically the local
-// user on the node.
-func authorize(w http.ResponseWriter, r *http.Request) (string, error) {
-	switch distro.Get() {
-	case distro.Synology:
-		user, err := synoAuthn()
-		if err != nil {
-			http.Error(w, err.Error(), http.StatusUnauthorized)
-			return "", err
-		}
-		if err := authorizeSynology(user); err != nil {
-			http.Error(w, err.Error(), http.StatusForbidden)
-			return "", err
-		}
-		return user, nil
-	case distro.QNAP:
-		user, resp, err := qnapAuthn(r)
-		if err != nil {
-			http.Error(w, err.Error(), http.StatusUnauthorized)
-			return "", err
-		}
-		if resp.IsAdmin == 0 {
-			http.Error(w, err.Error(), http.StatusForbidden)
-			return "", err
-		}
-		return user, nil
-	}
-	return "", nil
-}
-
-// authorizeSynology checks whether the provided user has access to the web UI
-// by consulting the membership of the "administrators" group.
-func authorizeSynology(name string) error {
-	yes, err := groupmember.IsMemberOfGroup("administrators", name)
-	if err != nil {
-		return err
-	}
-	if !yes {
-		return fmt.Errorf("not a member of administrators group")
-	}
-	return nil
-}
-
-type qnapAuthResponse struct {
-	AuthPassed int    `xml:"authPassed"`
-	IsAdmin    int    `xml:"isAdmin"`
-	AuthSID    string `xml:"authSid"`
-	ErrorValue int    `xml:"errorValue"`
-}
-
-func qnapAuthn(r *http.Request) (string, *qnapAuthResponse, error) {
-	user, err := r.Cookie("NAS_USER")
-	if err != nil {
-		return "", nil, err
-	}
-	token, err := r.Cookie("qtoken")
-	if err == nil {
-		return qnapAuthnQtoken(r, user.Value, token.Value)
-	}
-	sid, err := r.Cookie("NAS_SID")
-	if err == nil {
-		return qnapAuthnSid(r, user.Value, sid.Value)
-	}
-	return "", nil, fmt.Errorf("not authenticated by any mechanism")
-}
-
-// qnapAuthnURL returns the auth URL to use by inferring where the UI is
-// running based on the request URL. This is necessary because QNAP has so
-// many options, see https://github.com/tailscale/tailscale/issues/7108
-// and https://github.com/tailscale/tailscale/issues/6903
-func qnapAuthnURL(requestUrl string, query url.Values) string {
-	in, err := url.Parse(requestUrl)
-	scheme := ""
-	host := ""
-	if err != nil || in.Scheme == "" {
-		log.Printf("Cannot parse QNAP login URL %v", err)
-
-		// try localhost and hope for the best
-		scheme = "http"
-		host = "localhost"
-	} else {
-		scheme = in.Scheme
-		host = in.Host
-	}
-
-	u := url.URL{
-		Scheme:   scheme,
-		Host:     host,
-		Path:     "/cgi-bin/authLogin.cgi",
-		RawQuery: query.Encode(),
-	}
-
-	return u.String()
-}
-
-func qnapAuthnQtoken(r *http.Request, user, token string) (string, *qnapAuthResponse, error) {
-	query := url.Values{
-		"qtoken": []string{token},
-		"user":   []string{user},
-	}
-	return qnapAuthnFinish(user, qnapAuthnURL(r.URL.String(), query))
-}
-
-func qnapAuthnSid(r *http.Request, user, sid string) (string, *qnapAuthResponse, error) {
-	query := url.Values{
-		"sid": []string{sid},
-	}
-	return qnapAuthnFinish(user, qnapAuthnURL(r.URL.String(), query))
-}
-
-func qnapAuthnFinish(user, url string) (string, *qnapAuthResponse, error) {
-	// QNAP Force HTTPS mode uses a self-signed certificate. Even importing
-	// the QNAP root CA isn't enough, the cert doesn't have a usable CN nor
-	// SAN. See https://github.com/tailscale/tailscale/issues/6903
-	tr := &http.Transport{
-		TLSClientConfig: &tls.Config{InsecureSkipVerify: true},
-	}
-	client := &http.Client{Transport: tr}
-	resp, err := client.Get(url)
-	if err != nil {
-		return "", nil, err
-	}
-	defer resp.Body.Close()
-	out, err := io.ReadAll(resp.Body)
-	if err != nil {
-		return "", nil, err
-	}
-	authResp := &qnapAuthResponse{}
-	if err := xml.Unmarshal(out, authResp); err != nil {
-		return "", nil, err
-	}
-	if authResp.AuthPassed == 0 {
-		return "", nil, fmt.Errorf("not authenticated")
-	}
-	return user, authResp, nil
-}
-
-func synoAuthn() (string, error) {
-	cmd := exec.Command("/usr/syno/synoman/webman/modules/authenticate.cgi")
-	out, err := cmd.CombinedOutput()
-	if err != nil {
-		return "", fmt.Errorf("auth: %v: %s", err, out)
-	}
-	return strings.TrimSpace(string(out)), nil
-}
-
-func authRedirect(w http.ResponseWriter, r *http.Request) bool {
-	if distro.Get() == distro.Synology {
-		return synoTokenRedirect(w, r)
-	}
-	return false
-}
-
-func synoTokenRedirect(w http.ResponseWriter, r *http.Request) bool {
-	if r.Header.Get("X-Syno-Token") != "" {
-		return false
-	}
-	if r.URL.Query().Get("SynoToken") != "" {
-		return false
-	}
-	if r.Method == "POST" && r.FormValue("SynoToken") != "" {
-		return false
-	}
-	// We need a SynoToken for authenticate.cgi.
-	// So we tell the client to get one.
-	_, _ = fmt.Fprint(w, synoTokenRedirectHTML)
-	return true
-}
-
-const synoTokenRedirectHTML = `<html><body>
-Redirecting with session token...
-<script>
-var serverURL = window.location.protocol + "//" + window.location.host;
-var req = new XMLHttpRequest();
-req.overrideMimeType("application/json");
-req.open("GET", serverURL + "/webman/login.cgi", true);
-req.onload = function() {
-	var jsonResponse = JSON.parse(req.responseText);
-	var token = jsonResponse["SynoToken"];
-	document.location.href = serverURL + "/webman/3rdparty/Tailscale/?SynoToken=" + token;
-};
-req.send(null);
-</script>
-</body></html>
-`
-
 // ServeHTTP processes all requests for the Tailscale web client.
 func (s *Server) ServeHTTP(w http.ResponseWriter, r *http.Request) {
-	if s.devMode {
-		if r.URL.Path == "/api/data" {
-			user, err := authorize(w, r)
-			if err != nil {
-				return
+	handler := s.serve
+
+	// if running in cgi mode, strip the cgi path prefix
+	if s.cgiMode {
+		prefix := s.cgiPath
+		if prefix == "" {
+			switch distro.Get() {
+			case distro.Synology:
+				prefix = synologyPrefix
+			case distro.QNAP:
+				prefix = qnapPrefix
 			}
-			switch r.Method {
-			case httpm.GET:
-				s.serveGetNodeDataJSON(w, r, user)
-			case httpm.POST:
-				s.servePostNodeUpdate(w, r)
-			default:
-				http.Error(w, "method not allowed", http.StatusMethodNotAllowed)
-			}
-			return
 		}
-		// When in dev mode, proxy to the Vite dev server.
+		if prefix != "" {
+			handler = enforcePrefix(prefix, handler)
+		}
+	}
+
+	handler(w, r)
+}
+
+// authorize checks if the request is authorized to access the web client for those platforms that support it.
+func authorize(w http.ResponseWriter, r *http.Request) (handled bool) {
+	if strings.HasPrefix(r.URL.Path, "/assets/") {
+		// don't require authorization for static assets
+		return false
+	}
+
+	switch distro.Get() {
+	case distro.Synology:
+		return authorizeSynology(w, r)
+	case distro.QNAP:
+		return authorizeQNAP(w, r)
+	}
+
+	return false
+}
+
+func (s *Server) serve(w http.ResponseWriter, r *http.Request) {
+	switch {
+	case authorize(w, r):
+		// Authenticate and authorize the request for platforms that support it.
+		// Return if the request was processed.
+		return
+	case strings.HasPrefix(r.URL.Path, "/api/"):
+		// Pass API requests through to the API handler.
+		s.apiHandler.ServeHTTP(w, r)
+		return
+	case s.devMode:
+		// When in dev mode, proxy non-api requests to the Vite dev server.
 		s.devProxy.ServeHTTP(w, r)
 		return
-	}
-
-	if authRedirect(w, r) {
-		return
-	}
-
-	user, err := authorize(w, r)
-	if err != nil {
-		return
-	}
-
-	switch {
-	case r.URL.Path == "/redirect" || r.URL.Path == "/redirect/":
-		io.WriteString(w, authenticationRedirectHTML)
-		return
-	case r.Method == "POST":
-		s.servePostNodeUpdate(w, r)
-		return
 	default:
-		s.serveGetNodeData(w, r, user)
+		// Otherwise, serve static files from the embedded filesystem.
+		s.lc.IncrementCounter(context.Background(), "web_client_page_load", 1)
+		staticfiles.ServeHTTP(w, r)
 		return
 	}
 }
 
+// serveAPI serves requests for the web client api.
+// It should only be called by Server.ServeHTTP, via Server.apiHandler,
+// which protects the handler using gorilla csrf.
+func (s *Server) serveAPI(w http.ResponseWriter, r *http.Request) {
+	w.Header().Set("X-CSRF-Token", csrf.Token(r))
+	path := strings.TrimPrefix(r.URL.Path, "/api")
+	switch {
+	case path == "/data":
+		switch r.Method {
+		case httpm.GET:
+			s.serveGetNodeDataJSON(w, r)
+		case httpm.POST:
+			s.servePostNodeUpdate(w, r)
+		default:
+			http.Error(w, "method not allowed", http.StatusMethodNotAllowed)
+		}
+		return
+	case strings.HasPrefix(path, "/local/"):
+		s.proxyRequestToLocalAPI(w, r)
+		return
+	}
+	http.Error(w, "invalid endpoint", http.StatusNotFound)
+}
+
 type nodeData struct {
 	Profile           tailcfg.UserProfile
-	SynologyUser      string
 	Status            string
 	DeviceName        string
 	IP                string
@@ -330,7 +214,7 @@ type nodeData struct {
 	IPNVersion        string
 }
 
-func (s *Server) getNodeData(ctx context.Context, user string) (*nodeData, error) {
+func (s *Server) getNodeData(ctx context.Context) (*nodeData, error) {
 	st, err := s.lc.Status(ctx)
 	if err != nil {
 		return nil, err
@@ -343,17 +227,16 @@ func (s *Server) getNodeData(ctx context.Context, user string) (*nodeData, error
 	deviceName := strings.Split(st.Self.DNSName, ".")[0]
 	versionShort := strings.Split(st.Version, "-")[0]
 	data := &nodeData{
-		SynologyUser: user,
-		Profile:      profile,
-		Status:       st.BackendState,
-		DeviceName:   deviceName,
-		LicensesURL:  licenses.LicensesURL(),
-		TUNMode:      st.TUN,
-		IsSynology:   distro.Get() == distro.Synology || envknob.Bool("TS_FAKE_SYNOLOGY"),
-		DSMVersion:   distro.DSMVersion(),
-		IsUnraid:     distro.Get() == distro.Unraid,
-		UnraidToken:  os.Getenv("UNRAID_CSRF_TOKEN"),
-		IPNVersion:   versionShort,
+		Profile:     profile,
+		Status:      st.BackendState,
+		DeviceName:  deviceName,
+		LicensesURL: licenses.LicensesURL(),
+		TUNMode:     st.TUN,
+		IsSynology:  distro.Get() == distro.Synology || envknob.Bool("TS_FAKE_SYNOLOGY"),
+		DSMVersion:  distro.DSMVersion(),
+		IsUnraid:    distro.Get() == distro.Unraid,
+		UnraidToken: os.Getenv("UNRAID_CSRF_TOKEN"),
+		IPNVersion:  versionShort,
 	}
 	exitNodeRouteV4 := netip.MustParsePrefix("0.0.0.0/0")
 	exitNodeRouteV6 := netip.MustParsePrefix("::/0")
@@ -373,22 +256,8 @@ func (s *Server) getNodeData(ctx context.Context, user string) (*nodeData, error
 	return data, nil
 }
 
-func (s *Server) serveGetNodeData(w http.ResponseWriter, r *http.Request, user string) {
-	data, err := s.getNodeData(r.Context(), user)
-	if err != nil {
-		http.Error(w, err.Error(), http.StatusInternalServerError)
-		return
-	}
-	buf := new(bytes.Buffer)
-	if err := tmpl.Execute(buf, *data); err != nil {
-		http.Error(w, err.Error(), http.StatusInternalServerError)
-		return
-	}
-	w.Write(buf.Bytes())
-}
-
-func (s *Server) serveGetNodeDataJSON(w http.ResponseWriter, r *http.Request, user string) {
-	data, err := s.getNodeData(r.Context(), user)
+func (s *Server) serveGetNodeDataJSON(w http.ResponseWriter, r *http.Request) {
+	data, err := s.getNodeData(r.Context())
 	if err != nil {
 		http.Error(w, err.Error(), http.StatusInternalServerError)
 		return
@@ -398,7 +267,6 @@ func (s *Server) serveGetNodeDataJSON(w http.ResponseWriter, r *http.Request, us
 		return
 	}
 	w.Header().Set("Content-Type", "application/json")
-	return
 }
 
 type nodeUpdate struct {
@@ -527,3 +395,104 @@ func (s *Server) tailscaleUp(ctx context.Context, st *ipnstate.Status, postData
 		}
 	}
 }
+
+// proxyRequestToLocalAPI proxies the web API request to the localapi.
+//
+// The web API request path is expected to exactly match a localapi path,
+// with prefix /api/local/ rather than /localapi/.
+//
+// If the localapi path is not included in localapiAllowlist,
+// the request is rejected.
+func (s *Server) proxyRequestToLocalAPI(w http.ResponseWriter, r *http.Request) {
+	path := strings.TrimPrefix(r.URL.Path, "/api/local")
+	if r.URL.Path == path { // missing prefix
+		http.Error(w, "invalid request", http.StatusBadRequest)
+		return
+	}
+	if !slices.Contains(localapiAllowlist, path) {
+		http.Error(w, fmt.Sprintf("%s not allowed from localapi proxy", path), http.StatusForbidden)
+		return
+	}
+
+	localAPIURL := "http://" + apitype.LocalAPIHost + "/localapi" + path
+	req, err := http.NewRequestWithContext(r.Context(), r.Method, localAPIURL, r.Body)
+	if err != nil {
+		http.Error(w, "failed to construct request", http.StatusInternalServerError)
+		return
+	}
+
+	// Make request to tailscaled localapi.
+	resp, err := s.lc.DoLocalRequest(req)
+	if err != nil {
+		http.Error(w, err.Error(), resp.StatusCode)
+		return
+	}
+	defer resp.Body.Close()
+
+	// Send response back to web frontend.
+	w.Header().Set("Content-Type", resp.Header.Get("Content-Type"))
+	w.WriteHeader(resp.StatusCode)
+	if _, err := io.Copy(w, resp.Body); err != nil {
+		http.Error(w, err.Error(), http.StatusInternalServerError)
+	}
+}
+
+// localapiAllowlist is an allowlist of localapi endpoints the
+// web client is allowed to proxy to the client's localapi.
+//
+// Rather than exposing all localapi endpoints over the proxy,
+// this limits to just the ones actually used from the web
+// client frontend.
+//
+// TODO(sonia,will): Shouldn't expand this beyond the existing
+// localapi endpoints until the larger web client auth story
+// is worked out (tailscale/corp#14335).
+var localapiAllowlist = []string{
+	"/v0/logout",
+}
+
+// csrfKey returns a key that can be used for CSRF protection.
+// If an error occurs during key creation, the error is logged and the active process terminated.
+// If the server is running in CGI mode, the key is cached to disk and reused between requests.
+// If an error occurs during key storage, the error is logged and the active process terminated.
+func (s *Server) csrfKey() []byte {
+	csrfFile := filepath.Join(os.TempDir(), "tailscale-web-csrf.key")
+
+	// if running in CGI mode, try to read from disk, but ignore errors
+	if s.cgiMode {
+		key, _ := os.ReadFile(csrfFile)
+		if len(key) == 32 {
+			return key
+		}
+	}
+
+	// create a new key
+	key := make([]byte, 32)
+	if _, err := rand.Read(key); err != nil {
+		log.Fatal("error generating CSRF key: %w", err)
+	}
+
+	// if running in CGI mode, try to write the newly created key to disk, and exit if it fails.
+	if s.cgiMode {
+		if err := os.WriteFile(csrfFile, key, 0600); err != nil {
+			log.Fatalf("unable to store CSRF key: %v", err)
+		}
+	}
+
+	return key
+}
+
+// enforcePrefix returns a HandlerFunc that enforces a given path prefix is used in requests,
+// then strips it before invoking h.
+// Unlike http.StripPrefix, it does not return a 404 if the prefix is not present.
+// Instead, it returns a redirect to the prefix path.
+func enforcePrefix(prefix string, h http.HandlerFunc) http.HandlerFunc {
+	return func(w http.ResponseWriter, r *http.Request) {
+		if !strings.HasPrefix(r.URL.Path, prefix) {
+			http.Redirect(w, r, prefix, http.StatusFound)
+			return
+		}
+		prefix = strings.TrimSuffix(prefix, "/")
+		http.StripPrefix(prefix, h).ServeHTTP(w, r)
+	}
+}

client/web/web.html

@@ -1,210 +0,0 @@
-<!doctype html>
-<html class="bg-gray-50">
-
-<head>
-	<meta charset="utf-8" />
-	<meta name="viewport" content="width=device-width, initial-scale=1" />
-	<link rel="shortcut icon"
-		href="data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAACAAAAAgCAQAAADZc7J/AAAABGdBTUEAALGPC/xhBQAAACBjSFJNAAB6JgAAgIQAAPoAAACA6AAAdTAAAOpgAAA6mAAAF3CculE8AAAAAmJLR0QA/4ePzL8AAAAHdElNRQflAx4QGA4EvmzDAAAA30lEQVRIx2NgGAWMCKa8JKM4A8Ovt88ekyLCDGOoyDBJMjExMbFy8zF8/EKsCAMDE8yAPyIwFps48SJIBpAL4AZwvoSx/r0lXgQpDN58EWL5x/7/H+vL20+JFxluQKVe5b3Ke5V+0kQQCamfoYKBg4GDwUKI8d0BYkWQkrLKewYBKPPDHUFiRaiZkBgmwhj/F5IgggyUJ6i8V3mv0kCayDAAeEsklXqGAgYGhgV3CnGrwVciYSYk0kokhgS44/JxqqFpiYSZbEgskd4dEBRk1GD4wdB5twKXmlHAwMDAAACdEZau06NQUwAAACV0RVh0ZGF0ZTpjcmVhdGUAMjAyMC0wNy0xNVQxNTo1Mzo0MCswMDowMCVXsDIAAAAldEVYdGRhdGU6bW9kaWZ5ADIwMjAtMDctMTVUMTU6NTM6NDArMDA6MDBUCgiOAAAAAElFTkSuQmCC" />
-	<title>Tailscale</title>
-	<style>{{template "web.css"}}</style>
-</head>
-
-<body class="py-14">
-<main class="container max-w-lg mx-auto mb-8 py-6 px-8 bg-white rounded-md shadow-2xl" style="width: 95%">
-	<header class="flex justify-between items-center min-width-0 py-2 mb-8">
-		<svg width="26" height="26" viewBox="0 0 23 23" title="Tailscale" fill="none" xmlns="http://www.w3.org/2000/svg"
-			class="flex-shrink-0 mr-4">
-			<circle opacity="0.2" cx="3.4" cy="3.25" r="2.7" fill="currentColor"></circle>
-			<circle cx="3.4" cy="11.3" r="2.7" fill="currentColor"></circle>
-			<circle opacity="0.2" cx="3.4" cy="19.5" r="2.7" fill="currentColor"></circle>
-			<circle cx="11.5" cy="11.3" r="2.7" fill="currentColor"></circle>
-			<circle cx="11.5" cy="19.5" r="2.7" fill="currentColor"></circle>
-			<circle opacity="0.2" cx="11.5" cy="3.25" r="2.7" fill="currentColor"></circle>
-			<circle opacity="0.2" cx="19.5" cy="3.25" r="2.7" fill="currentColor"></circle>
-			<circle cx="19.5" cy="11.3" r="2.7" fill="currentColor"></circle>
-			<circle opacity="0.2" cx="19.5" cy="19.5" r="2.7" fill="currentColor"></circle>
-		</svg>
-		<div class="flex items-center justify-end space-x-2 w-2/3">
-			{{ with .Profile }}
-			<div class="text-right w-full leading-4">
-				<h4 class="truncate leading-normal">{{.LoginName}}</h4>
-				<div class="text-xs text-gray-500 text-right">
-					<a href="#" class="hover:text-gray-700 js-loginButton">Switch account</a> | <a href="#"
-						class="hover:text-gray-700 js-loginButton">Reauthenticate</a> | <a href="#"
-						class="hover:text-gray-700 js-logoutButton">Logout</a>
-				</div>
-			</div>
-			{{ end }}
-			<div class="relative flex-shrink-0 w-8 h-8 rounded-full overflow-hidden">
-				{{ with .Profile.ProfilePicURL }}
-				<div class="w-8 h-8 flex pointer-events-none rounded-full bg-gray-200"
-					style="background-image: url('{{.}}'); background-size: cover;"></div>
-				{{ else }}
-				<div class="w-8 h-8 flex pointer-events-none rounded-full border border-gray-400 border-dashed"></div>
-				{{ end }}
-			</div>
-		</div>
-	</header>
-	{{ if .IP }}
-	<div
-		class="border border-gray-200 bg-gray-0 rounded-md p-2 pl-3 pr-3 width-full flex items-center justify-between">
-		<div class="flex items-center min-width-0">
-			<svg class="flex-shrink-0 text-gray-600 mr-3 ml-1" xmlns="http://www.w3.org/2000/svg" width="20" height="20"
-				viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round"
-				stroke-linejoin="round">
-				<rect x="2" y="2" width="20" height="8" rx="2" ry="2"></rect>
-				<rect x="2" y="14" width="20" height="8" rx="2" ry="2"></rect>
-				<line x1="6" y1="6" x2="6.01" y2="6"></line>
-				<line x1="6" y1="18" x2="6.01" y2="18"></line>
-			</svg>
-			<div>
-				<h4 class="font-semibold truncate mr-2">{{.DeviceName}}</h4>
-			</div>
-		</div>
-		<h5>{{.IP}}</h5>
-	</div>
-	<p class="mt-1 ml-1 mb-6 text-xs text-gray-600">
-		Debug info: Tailscale {{ .IPNVersion }}, tun={{.TUNMode}}{{ if .IsSynology }}, DSM{{ .DSMVersion}}
-			{{if not .TUNMode}}
-			(<a href="https://tailscale.com/kb/1152/synology-outbound/" class="link-underline text-gray-600" target="_blank"
-				aria-label="Configure outbound synology traffic"
-				rel="noopener noreferrer">outgoing access not configured</a>)
-			{{end}}
-		{{end}}
-	</p>
-	{{ end }}
-	{{ if or (eq .Status "NeedsLogin") (eq .Status "NoState") }}
-	{{ if .IP }}
-	<div class="mb-6">
-		<p class="text-gray-700">Your device's key has expired. Reauthenticate this device by logging in again, or <a
-				href="https://tailscale.com/kb/1028/key-expiry" class="link" target="_blank">learn more</a>.</p>
-	</div>
-	<a href="#" class="mb-4 js-loginButton" target="_blank">
-		<button class="button button-blue w-full">Reauthenticate</button>
-	</a>
-	{{ else }}
-	<div class="mb-6">
-		<h3 class="text-3xl font-semibold mb-3">Log in</h3>
-		<p class="text-gray-700">Get started by logging in to your Tailscale network. Or,&nbsp;learn&nbsp;more at <a
-				href="https://tailscale.com/" class="link" target="_blank">tailscale.com</a>.</p>
-	</div>
-	<a href="#" class="mb-4 js-loginButton" target="_blank">
-		<button class="button button-blue w-full">Log In</button>
-	</a>
-	{{ end }}
-	{{ else if eq .Status "NeedsMachineAuth" }}
-	<div class="mb-4">
-		This device is authorized, but needs approval from a network admin before it can connect to the network.
-	</div>
-	{{ else }}
-	<div class="mb-4">
-		<p>You are connected! Access this device over Tailscale using the device name or IP address above.</p>
-	</div>
-	<div class="mb-4">
-	<a href="#" class="mb-4 js-advertiseExitNode">
-		{{if .AdvertiseExitNode}}
-		<button class="button button-red button-medium" id="enabled">Stop advertising Exit Node</button>
-		{{else}}
-		<button class="button button-blue button-medium" id="enabled">Advertise as Exit Node</button>
-		{{end}}
-	</a>
-	</div>
-	{{ end }}
-</main>
-<footer class="container max-w-lg mx-auto text-center">
-	<a class="text-xs text-gray-500 hover:text-gray-600" href="{{ .LicensesURL }}">Open Source Licenses</a>
-</footer>
-<script>(function () {
-const advertiseExitNode = {{ .AdvertiseExitNode }};
-const isUnraid = {{ .IsUnraid }};
-const unraidCsrfToken = "{{ .UnraidToken }}";
-let fetchingUrl = false;
-var data = {
-	AdvertiseRoutes: "{{ .AdvertiseRoutes }}",
-	AdvertiseExitNode: advertiseExitNode,
-	Reauthenticate: false,
-	ForceLogout: false
-};
-
-function postData(e) {
-	e.preventDefault();
-
-	if (fetchingUrl) {
-		return;
-	}
-
-	fetchingUrl = true;
-	const urlParams = new URLSearchParams(window.location.search);
-	const token = urlParams.get("SynoToken");
-	const nextParams = new URLSearchParams({ up: true });
-	if (token) {
-		nextParams.set("SynoToken", token)
-	}
-	const nextUrl = new URL(window.location);
-	nextUrl.search = nextParams.toString()
-
-	let body = JSON.stringify(data);
-	let contentType = "application/json";
-
-	if (isUnraid) {
-		const params = new URLSearchParams();
-		params.append("csrf_token", unraidCsrfToken);
-		params.append("ts_data", JSON.stringify(data));
-
-		body = params.toString();
-		contentType = "application/x-www-form-urlencoded;charset=UTF-8";
-	}
-
-	const url = nextUrl.toString();
-	fetch(url, {
-		method: "POST",
-		headers: {
-			"Accept": "application/json",
-			"Content-Type": contentType,
-		},
-		body: body
-	}).then(res => res.json()).then(res => {
-		fetchingUrl = false;
-		const err = res["error"];
-		if (err) {
-			throw new Error(err);
-		}
-		const url = res["url"];
-		if (url) {
-			if(isUnraid) {
-				window.open(url, "_blank");
-			} else {
-				document.location.href = url;
-			}
-		} else {
-			location.reload();
-		}
-	}).catch(err => {
-		alert("Failed operation: " + err.message);
-	});
-}
-
-document.querySelectorAll(".js-loginButton").forEach(function (el){
-	el.addEventListener("click", function(e) {
-		data.Reauthenticate = true;
-		postData(e);
-	});
-})
-document.querySelectorAll(".js-logoutButton").forEach(function(el) {
-	el.addEventListener("click", function (e) {
-		data.ForceLogout = true;
-		postData(e);
-	});
-})
-document.querySelectorAll(".js-advertiseExitNode").forEach(function (el) {
-	el.addEventListener("click", function(e) {
-		data.AdvertiseExitNode = !advertiseExitNode;
-		postData(e);
-	});
-})
-
-})();</script>
-</body>
-
-</html>

client/web/web_test.go

@@ -4,8 +4,16 @@
 package web
 
 import (
+	"fmt"
+	"io"
+	"net/http"
+	"net/http/httptest"
 	"net/url"
+	"strings"
 	"testing"
+
+	"tailscale.com/client/tailscale"
+	"tailscale.com/net/memnet"
 )
 
 func TestQnapAuthnURL(t *testing.T) {
@@ -62,3 +70,62 @@ func TestQnapAuthnURL(t *testing.T) {
 		})
 	}
 }
+
+// TestServeAPI tests the web client api's handling of
+//  1. invalid endpoint errors
+//  2. localapi proxy allowlist
+func TestServeAPI(t *testing.T) {
+	lal := memnet.Listen("local-tailscaled.sock:80")
+	defer lal.Close()
+	// Serve dummy localapi. Just returns "success".
+	localapi := &http.Server{Handler: http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		fmt.Fprintf(w, "success")
+	})}
+	defer localapi.Close()
+
+	go localapi.Serve(lal)
+	s := &Server{lc: &tailscale.LocalClient{Dial: lal.Dial}}
+
+	tests := []struct {
+		name       string
+		reqPath    string
+		wantResp   string
+		wantStatus int
+	}{{
+		name:       "invalid_endpoint",
+		reqPath:    "/not-an-endpoint",
+		wantResp:   "invalid endpoint",
+		wantStatus: http.StatusNotFound,
+	}, {
+		name:       "not_in_localapi_allowlist",
+		reqPath:    "/local/v0/not-allowlisted",
+		wantResp:   "/v0/not-allowlisted not allowed from localapi proxy",
+		wantStatus: http.StatusForbidden,
+	}, {
+		name:       "in_localapi_allowlist",
+		reqPath:    "/local/v0/logout",
+		wantResp:   "success", // Successfully allowed to hit localapi.
+		wantStatus: http.StatusOK,
+	}}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			r := httptest.NewRequest("POST", "/api"+tt.reqPath, nil)
+			w := httptest.NewRecorder()
+
+			s.serveAPI(w, r)
+			res := w.Result()
+			defer res.Body.Close()
+			if gotStatus := res.StatusCode; tt.wantStatus != gotStatus {
+				t.Errorf("wrong status; want=%q, got=%q", tt.wantStatus, gotStatus)
+			}
+			body, err := io.ReadAll(res.Body)
+			if err != nil {
+				t.Fatal(err)
+			}
+			gotResp := strings.TrimSuffix(string(body), "\n") // trim trailing newline
+			if tt.wantResp != gotResp {
+				t.Errorf("wrong response; want=%q, got=%q", tt.wantResp, gotResp)
+			}
+		})
+	}
+}

client/web/yarn.lock

@@ -543,6 +543,13 @@
   resolved "https://registry.yarnpkg.com/@types/chai/-/chai-4.3.5.tgz#ae69bcbb1bebb68c4ac0b11e9d8ed04526b3562b"
   integrity sha512-mEo1sAde+UCE6b2hxn332f1g1E8WfYRu6p5SvTKr2ZKC1f7gFJXk4h5PyGP9Dt6gCaG8y8XhwnXWC6Iy2cmBng==
 
+"@types/classnames@^2.2.10":
+  version "2.3.1"
+  resolved "https://registry.yarnpkg.com/@types/classnames/-/classnames-2.3.1.tgz#3c2467aa0f1a93f1f021e3b9bcf938bd5dfdc0dd"
+  integrity sha512-zeOWb0JGBoVmlQoznvqXbE0tEC/HONsnoUNH19Hc96NFsTAwTXbTqb8FMYkru1F/iqp7a18Ws3nWJvtA1sHD1A==
+  dependencies:
+    classnames "*"
+
 "@types/estree@^1.0.0":
   version "1.0.1"
   resolved "https://registry.yarnpkg.com/@types/estree/-/estree-1.0.1.tgz#aa22750962f3bf0e79d753d3cc067f010c95f194"
@@ -798,6 +805,11 @@ chokidar@^3.5.3:
   optionalDependencies:
     fsevents "~2.3.2"
 
+classnames@*, classnames@^2.3.1:
+  version "2.3.2"
+  resolved "https://registry.yarnpkg.com/classnames/-/classnames-2.3.2.tgz#351d813bf0137fcc6a76a16b88208d2560a0d924"
+  integrity sha512-CSbhY4cFEJRe6/GQzIk5qXZ4Jeg5pcsP7b5peFSDpffpe1cqjASH/n9UTjBwOp6XpMSTwQ8Za2K5V02ueA7Tmw==
+
 color-convert@^1.9.0:
   version "1.9.3"
   resolved "https://registry.yarnpkg.com/color-convert/-/color-convert-1.9.3.tgz#bb71850690e1f136567de629d2d5471deda4c1e8"

clientupdate/clientupdate.go

@@ -7,15 +7,16 @@
 package clientupdate
 
 import (
+	"archive/tar"
 	"bufio"
 	"bytes"
+	"compress/gzip"
 	"context"
-	"crypto/sha256"
-	"encoding/hex"
 	"encoding/json"
 	"errors"
 	"fmt"
 	"io"
+	"maps"
 	"net/http"
 	"os"
 	"os/exec"
@@ -25,14 +26,10 @@ import (
 	"runtime"
 	"strconv"
 	"strings"
-	"time"
 
 	"github.com/google/uuid"
-	"tailscale.com/hostinfo"
-	"tailscale.com/net/tshttpproxy"
-	"tailscale.com/tailcfg"
+	"tailscale.com/clientupdate/distsign"
 	"tailscale.com/types/logger"
-	"tailscale.com/util/must"
 	"tailscale.com/util/winutil"
 	"tailscale.com/version"
 	"tailscale.com/version/distro"
@@ -63,14 +60,8 @@ func versionToTrack(v string) (string, error) {
 	return "unstable", nil
 }
 
-type updater struct {
-	UpdateArgs
-	track  string
-	update func() error
-}
-
-// UpdateArgs contains arguments needed to run an update.
-type UpdateArgs struct {
+// Arguments contains arguments needed to run an update.
+type Arguments struct {
 	// Version can be a specific version number or one of the predefined track
 	// constants:
 	//
@@ -82,7 +73,7 @@ type UpdateArgs struct {
 	// Leaving this empty is the same as using CurrentTrack.
 	Version string
 	// AppStore forces a local app store check, even if the current binary was
-	// not installed via an app store.
+	// not installed via an app store. TODO(cpalmer): Remove this.
 	AppStore bool
 	// Logf is a logger for update progress messages.
 	Logf logger.Logf
@@ -90,29 +81,36 @@ type UpdateArgs struct {
 	// if this new version should be installed. When Confirm returns false, the
 	// update is aborted.
 	Confirm func(newVer string) bool
+	// PkgsAddr is the address of the pkgs server to fetch updates from.
+	// Defaults to "https://pkgs.tailscale.com".
+	PkgsAddr string
 }
 
-func (args UpdateArgs) validate() error {
+func (args Arguments) validate() error {
 	if args.Confirm == nil {
-		return errors.New("missing Confirm callback in UpdateArgs")
+		return errors.New("missing Confirm callback in Arguments")
 	}
 	if args.Logf == nil {
-		return errors.New("missing Logf callback in UpdateArgs")
+		return errors.New("missing Logf callback in Arguments")
 	}
 	return nil
 }
 
-// Update runs a single update attempt using the platform-specific mechanism.
-//
-// On Windows, this copies the calling binary and re-executes it to apply the
-// update. The calling binary should handle an "update" subcommand and call
-// this function again for the re-executed binary to proceed.
-func Update(args UpdateArgs) error {
-	if err := args.validate(); err != nil {
-		return err
+type Updater struct {
+	Arguments
+	track string
+	// Update is a platform-specific method that updates the installation. May be
+	// nil (not all platforms support updates from within Tailscale).
+	Update func() error
+}
+
+func NewUpdater(args Arguments) (*Updater, error) {
+	up := Updater{
+		Arguments: args,
 	}
-	up := &updater{
-		UpdateArgs: args,
+	up.Update = up.getUpdateFunction()
+	if up.Update == nil {
+		return nil, errors.ErrUnsupported
 	}
 	switch up.Version {
 	case StableTrack, UnstableTrack:
@@ -127,56 +125,82 @@ func Update(args UpdateArgs) error {
 		var err error
 		up.track, err = versionToTrack(args.Version)
 		if err != nil {
-			return err
+			return nil, err
 		}
 	}
+	if args.PkgsAddr == "" {
+		args.PkgsAddr = "https://pkgs.tailscale.com"
+	}
+	return &up, nil
+}
+
+type updateFunction func() error
+
+func (up *Updater) getUpdateFunction() updateFunction {
 	switch runtime.GOOS {
 	case "windows":
-		up.update = up.updateWindows
+		return up.updateWindows
 	case "linux":
 		switch distro.Get() {
 		case distro.Synology:
-			up.update = up.updateSynology
+			return up.updateSynology
 		case distro.Debian: // includes Ubuntu
-			up.update = up.updateDebLike
+			return up.updateDebLike
 		case distro.Arch:
-			up.update = up.updateArchLike
+			return up.updateArchLike
 		case distro.Alpine:
-			up.update = up.updateAlpineLike
+			return up.updateAlpineLike
 		}
 		switch {
 		case haveExecutable("pacman"):
-			up.update = up.updateArchLike
+			return up.updateArchLike
 		case haveExecutable("apt-get"): // TODO(awly): add support for "apt"
 			// The distro.Debian switch case above should catch most apt-based
 			// systems, but add this fallback just in case.
-			up.update = up.updateDebLike
+			return up.updateDebLike
 		case haveExecutable("dnf"):
-			up.update = up.updateFedoraLike("dnf")
+			return up.updateFedoraLike("dnf")
 		case haveExecutable("yum"):
-			up.update = up.updateFedoraLike("yum")
+			return up.updateFedoraLike("yum")
 		case haveExecutable("apk"):
-			up.update = up.updateAlpineLike
+			return up.updateAlpineLike
+		}
+		// If nothing matched, fall back to tarball updates.
+		if up.Update == nil {
+			return up.updateLinuxBinary
 		}
 	case "darwin":
 		switch {
-		case !args.AppStore && !version.IsSandboxedMacOS():
-			return errors.ErrUnsupported
-		case !args.AppStore && strings.HasSuffix(os.Getenv("HOME"), "/io.tailscale.ipn.macsys/Data"):
-			up.update = up.updateMacSys
+		case !up.Arguments.AppStore && !version.IsSandboxedMacOS():
+			return nil
+		case !up.Arguments.AppStore && strings.HasSuffix(os.Getenv("HOME"), "/io.tailscale.ipn.macsys/Data"):
+			return up.updateMacSys
 		default:
-			up.update = up.updateMacAppStore
+			return up.updateMacAppStore
 		}
 	case "freebsd":
-		up.update = up.updateFreeBSD
+		return up.updateFreeBSD
 	}
-	if up.update == nil {
-		return errors.ErrUnsupported
-	}
-	return up.update()
+	return nil
 }
 
-func (up *updater) confirm(ver string) bool {
+// Update runs a single update attempt using the platform-specific mechanism.
+//
+// On Windows, this copies the calling binary and re-executes it to apply the
+// update. The calling binary should handle an "update" subcommand and call
+// this function again for the re-executed binary to proceed.
+func Update(args Arguments) error {
+	if err := args.validate(); err != nil {
+		return err
+	}
+	up, err := NewUpdater(args)
+	if err != nil {
+		return err
+	}
+	return up.Update()
+}
+
+func (up *Updater) confirm(ver string) bool {
 	if version.Short() == ver {
 		up.Logf("already running %v; no update needed", ver)
 		return false
@@ -187,14 +211,16 @@ func (up *updater) confirm(ver string) bool {
 	return true
 }
 
-func (up *updater) updateSynology() error {
+const synoinfoConfPath = "/etc/synoinfo.conf"
+
+func (up *Updater) updateSynology() error {
 	if up.Version != "" {
 		return errors.New("installing a specific version on Synology is not supported")
 	}
 
 	// Get the latest version and list of SPKs from pkgs.tailscale.com.
 	osName := fmt.Sprintf("dsm%d", distro.DSMVersion())
-	arch, err := synoArch(hostinfo.New())
+	arch, err := synoArch(runtime.GOARCH, synoinfoConfPath)
 	if err != nil {
 		return err
 	}
@@ -202,15 +228,12 @@ func (up *updater) updateSynology() error {
 	if err != nil {
 		return err
 	}
-	if latest.Version == "" {
-		return fmt.Errorf("no latest version found for %q track", up.track)
-	}
 	spkName := latest.SPKs[osName][arch]
 	if spkName == "" {
 		return fmt.Errorf("cannot find Synology package for os=%s arch=%s, please report a bug with your device model", osName, arch)
 	}
 
-	if !up.confirm(latest.Version) {
+	if !up.confirm(latest.SPKsVersion) {
 		return nil
 	}
 	if err := requireRoot(); err != nil {
@@ -222,10 +245,9 @@ func (up *updater) updateSynology() error {
 	if err != nil {
 		return err
 	}
-	url := fmt.Sprintf("https://pkgs.tailscale.com/%s/%s", up.track, spkName)
-	spkPath := filepath.Join(spkDir, path.Base(url))
-	// TODO(awly): we should sign SPKs and validate signatures here too.
-	if err := up.downloadURLToFile(url, spkPath); err != nil {
+	pkgsPath := fmt.Sprintf("%s/%s", up.track, spkName)
+	spkPath := filepath.Join(spkDir, path.Base(pkgsPath))
+	if err := up.downloadURLToFile(pkgsPath, spkPath); err != nil {
 		return err
 	}
 
@@ -245,52 +267,71 @@ func (up *updater) updateSynology() error {
 
 // synoArch returns the Synology CPU architecture matching one of the SPK
 // architectures served from pkgs.tailscale.com.
-func synoArch(hinfo *tailcfg.Hostinfo) (string, error) {
+func synoArch(goArch, synoinfoPath string) (string, error) {
 	// Most Synology boxes just use a different arch name from GOARCH.
 	arch := map[string]string{
 		"amd64": "x86_64",
 		"386":   "i686",
 		"arm64": "armv8",
-	}[hinfo.GoArch]
-	// Here's the fun part, some older ARM boxes require you to use SPKs
-	// specifically for their CPU.
-	//
-	// See https://github.com/SynoCommunity/spksrc/wiki/Synology-and-SynoCommunity-Package-Architectures
-	// for a complete list. Here, we override GOARCH for those older boxes that
-	// support at least DSM6.
-	//
-	// This is an artisanal hand-crafted list based on the wiki page. Some
-	// values may be wrong, since we don't have all those devices to actually
-	// test with.
-	switch hinfo.DeviceModel {
-	case "DS213air", "DS213", "DS413j",
-		"DS112", "DS112+", "DS212", "DS212+", "RS212", "RS812", "DS212j", "DS112j",
-		"DS111", "DS211", "DS211+", "DS411slim", "DS411", "RS411", "DS211j", "DS411j":
-		arch = "88f6281"
-	case "NVR1218", "NVR216", "VS960HD", "VS360HD":
-		arch = "hi3535"
-	case "DS1517", "DS1817", "DS416", "DS2015xs", "DS715", "DS1515", "DS215+":
-		arch = "alpine"
-	case "DS216se", "DS115j", "DS114", "DS214se", "DS414slim", "RS214", "DS14", "EDS14", "DS213j":
-		arch = "armada370"
-	case "DS115", "DS215j":
-		arch = "armada375"
-	case "DS419slim", "DS218j", "RS217", "DS116", "DS216j", "DS216", "DS416slim", "RS816", "DS416j":
-		arch = "armada38x"
-	case "RS815", "DS214", "DS214+", "DS414", "RS814":
-		arch = "armadaxp"
-	case "DS414j":
-		arch = "comcerto2k"
-	case "DS216play":
-		arch = "monaco"
-	}
+	}[goArch]
+
 	if arch == "" {
-		return "", fmt.Errorf("cannot determine CPU architecture for Synology model %q (Go arch %q), please report a bug at https://github.com/tailscale/tailscale/issues/new/choose", hinfo.DeviceModel, hinfo.GoArch)
+		// Here's the fun part, some older ARM boxes require you to use SPKs
+		// specifically for their CPU. See
+		// https://github.com/SynoCommunity/spksrc/wiki/Synology-and-SynoCommunity-Package-Architectures
+		// for a complete list.
+		//
+		// Some CPUs will map to neither this list nor the goArch map above, and we
+		// don't have SPKs for them.
+		cpu, err := parseSynoinfo(synoinfoPath)
+		if err != nil {
+			return "", fmt.Errorf("failed to get CPU architecture: %w", err)
+		}
+		switch cpu {
+		case "88f6281", "88f6282", "hi3535", "alpine", "armada370",
+			"armada375", "armada38x", "armadaxp", "comcerto2k", "monaco":
+			arch = cpu
+		default:
+			return "", fmt.Errorf("unsupported Synology CPU architecture %q (Go arch %q), please report a bug at https://github.com/tailscale/tailscale/issues/new/choose", cpu, goArch)
+		}
 	}
 	return arch, nil
 }
 
-func (up *updater) updateDebLike() error {
+func parseSynoinfo(path string) (string, error) {
+	f, err := os.Open(path)
+	if err != nil {
+		return "", err
+	}
+	defer f.Close()
+
+	// Look for a line like:
+	// unique="synology_88f6282_413j"
+	// Extract the CPU in the middle (88f6282 in the above example).
+	s := bufio.NewScanner(f)
+	for s.Scan() {
+		l := s.Text()
+		if !strings.HasPrefix(l, "unique=") {
+			continue
+		}
+		parts := strings.SplitN(l, "_", 3)
+		if len(parts) != 3 {
+			return "", fmt.Errorf(`malformed %q: found %q, expected format like 'unique="synology_$cpu_$model'`, path, l)
+		}
+		return parts[1], nil
+	}
+	return "", fmt.Errorf(`missing "unique=" field in %q`, path)
+}
+
+func (up *Updater) updateDebLike() error {
+	if err := requireRoot(); err != nil {
+		return err
+	}
+	if err := exec.Command("dpkg", "--status", "tailscale").Run(); err != nil && isExitError(err) {
+		// Tailscale was not installed via apt, update via tarball download
+		// instead.
+		return up.updateLinuxBinary()
+	}
 	ver, err := requestedTailscaleVersion(up.Version, up.track)
 	if err != nil {
 		return err
@@ -299,10 +340,6 @@ func (up *updater) updateDebLike() error {
 		return nil
 	}
 
-	if err := requireRoot(); err != nil {
-		return err
-	}
-
 	if updated, err := updateDebianAptSourcesList(up.track); err != nil {
 		return err
 	} else if updated {
@@ -392,61 +429,17 @@ func updateDebianAptSourcesListBytes(was []byte, dstTrack string) (newContent []
 	return buf.Bytes(), nil
 }
 
-func (up *updater) updateArchLike() (err error) {
-	if up.Version != "" {
-		return errors.New("installing a specific version on Arch-based distros is not supported")
+func (up *Updater) updateArchLike() error {
+	if err := exec.Command("pacman", "--query", "tailscale").Run(); err != nil && isExitError(err) {
+		// Tailscale was not installed via pacman, update via tarball download
+		// instead.
+		return up.updateLinuxBinary()
 	}
-	if err := requireRoot(); err != nil {
-		return err
-	}
-
-	defer func() {
-		if err != nil {
-			err = fmt.Errorf(`%w; you can try updating using "pacman --sync --refresh tailscale"`, err)
-		}
-	}()
-
-	out, err := exec.Command("pacman", "--sync", "--refresh", "--info", "tailscale").CombinedOutput()
-	if err != nil {
-		return fmt.Errorf("failed checking pacman for latest tailscale version: %w, output: %q", err, out)
-	}
-	ver, err := parsePacmanVersion(out)
-	if err != nil {
-		return err
-	}
-	if !up.confirm(ver) {
-		return nil
-	}
-
-	cmd := exec.Command("pacman", "--sync", "--noconfirm", "tailscale")
-	cmd.Stdout = os.Stdout
-	cmd.Stderr = os.Stderr
-	if err := cmd.Run(); err != nil {
-		return fmt.Errorf("failed tailscale update using pacman: %w", err)
-	}
-	return nil
-}
-
-func parsePacmanVersion(out []byte) (string, error) {
-	for _, line := range strings.Split(string(out), "\n") {
-		// The line we're looking for looks like this:
-		// Version         : 1.44.2-1
-		if !strings.HasPrefix(line, "Version") {
-			continue
-		}
-		parts := strings.SplitN(line, ":", 2)
-		if len(parts) != 2 {
-			return "", fmt.Errorf("version output from pacman is malformed: %q, cannot determine upgrade version", line)
-		}
-		ver := strings.TrimSpace(parts[1])
-		// Trim the Arch patch version.
-		ver = strings.Split(ver, "-")[0]
-		if ver == "" {
-			return "", fmt.Errorf("version output from pacman is malformed: %q, cannot determine upgrade version", line)
-		}
-		return ver, nil
-	}
-	return "", fmt.Errorf("could not find latest version of tailscale via pacman")
+	// Arch maintainer asked us not to implement "tailscale update" or
+	// auto-updates on Arch-based distros:
+	// https://github.com/tailscale/tailscale/issues/6995#issuecomment-1687080106
+	return errors.New(`individual package updates are not supported on Arch-based distros, only full-system updates are: https://wiki.archlinux.org/title/System_maintenance#Partial_upgrades_are_unsupported.
+you can use "pacman --sync --refresh --sysupgrade" or "pacman -Syu" to upgrade the system, including Tailscale.`)
 }
 
 const yumRepoConfigFile = "/etc/yum.repos.d/tailscale.repo"
@@ -454,11 +447,16 @@ const yumRepoConfigFile = "/etc/yum.repos.d/tailscale.repo"
 // updateFedoraLike updates tailscale on any distros in the Fedora family,
 // specifically anything that uses "dnf" or "yum" package managers. The actual
 // package manager is passed via packageManager.
-func (up *updater) updateFedoraLike(packageManager string) func() error {
+func (up *Updater) updateFedoraLike(packageManager string) func() error {
 	return func() (err error) {
 		if err := requireRoot(); err != nil {
 			return err
 		}
+		if err := exec.Command(packageManager, "info", "--installed", "tailscale").Run(); err != nil && isExitError(err) {
+			// Tailscale was not installed via yum/dnf, update via tarball
+			// download instead.
+			return up.updateLinuxBinary()
+		}
 		defer func() {
 			if err != nil {
 				err = fmt.Errorf(`%w; you can try updating using "%s upgrade tailscale"`, err, packageManager)
@@ -530,13 +528,18 @@ func updateYUMRepoTrack(repoFile, dstTrack string) (rewrote bool, err error) {
 	return true, os.WriteFile(repoFile, newContent.Bytes(), 0644)
 }
 
-func (up *updater) updateAlpineLike() (err error) {
+func (up *Updater) updateAlpineLike() (err error) {
 	if up.Version != "" {
 		return errors.New("installing a specific version on Alpine-based distros is not supported")
 	}
 	if err := requireRoot(); err != nil {
 		return err
 	}
+	if err := exec.Command("apk", "info", "--installed", "tailscale").Run(); err != nil && isExitError(err) {
+		// Tailscale was not installed via apk, update via tarball download
+		// instead.
+		return up.updateLinuxBinary()
+	}
 
 	defer func() {
 		if err != nil {
@@ -587,11 +590,11 @@ func parseAlpinePackageVersion(out []byte) (string, error) {
 	return "", errors.New("tailscale version not found in output")
 }
 
-func (up *updater) updateMacSys() error {
+func (up *Updater) updateMacSys() error {
 	return errors.New("NOTREACHED: On MacSys builds, `tailscale update` is handled in Swift to launch the GUI updater")
 }
 
-func (up *updater) updateMacAppStore() error {
+func (up *Updater) updateMacAppStore() error {
 	out, err := exec.Command("defaults", "read", "/Library/Preferences/com.apple.commerce.plist", "AutoUpdate").CombinedOutput()
 	if err != nil {
 		return fmt.Errorf("can't check App Store auto-update setting: %w, output: %q", err, string(out))
@@ -652,7 +655,7 @@ var (
 	markTempFileFunc   func(string) error // or nil on non-Windows
 )
 
-func (up *updater) updateWindows() error {
+func (up *Updater) updateWindows() error {
 	if msi := os.Getenv(winMSIEnv); msi != "" {
 		up.Logf("installing %v ...", msi)
 		if err := up.installMSI(msi); err != nil {
@@ -688,9 +691,9 @@ func (up *updater) updateWindows() error {
 	if err := os.MkdirAll(msiDir, 0700); err != nil {
 		return err
 	}
-	url := fmt.Sprintf("https://pkgs.tailscale.com/%s/tailscale-setup-%s-%s.msi", up.track, ver, arch)
-	msiTarget := filepath.Join(msiDir, path.Base(url))
-	if err := up.downloadURLToFile(url, msiTarget); err != nil {
+	pkgsPath := fmt.Sprintf("%s/tailscale-setup-%s-%s.msi", up.track, ver, arch)
+	msiTarget := filepath.Join(msiDir, path.Base(pkgsPath))
+	if err := up.downloadURLToFile(pkgsPath, msiTarget); err != nil {
 		return err
 	}
 
@@ -722,7 +725,7 @@ func (up *updater) updateWindows() error {
 	panic("unreachable")
 }
 
-func (up *updater) installMSI(msi string) error {
+func (up *Updater) installMSI(msi string) error {
 	var err error
 	for tries := 0; tries < 2; tries++ {
 		cmd := exec.Command("msiexec.exe", "/i", filepath.Base(msi), "/quiet", "/promptrestart", "/qn")
@@ -789,115 +792,26 @@ func makeSelfCopy() (tmpPathExe string, err error) {
 	return f2.Name(), f2.Close()
 }
 
-func (up *updater) downloadURLToFile(urlSrc, fileDst string) (ret error) {
-	tr := http.DefaultTransport.(*http.Transport).Clone()
-	tr.Proxy = tshttpproxy.ProxyFromEnvironment
-	defer tr.CloseIdleConnections()
-	c := &http.Client{Transport: tr}
-
-	quickCtx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
-	defer cancel()
-	headReq := must.Get(http.NewRequestWithContext(quickCtx, "HEAD", urlSrc, nil))
-
-	res, err := c.Do(headReq)
+func (up *Updater) downloadURLToFile(pathSrc, fileDst string) (ret error) {
+	c, err := distsign.NewClient(up.Logf, up.PkgsAddr)
 	if err != nil {
 		return err
 	}
-	if res.StatusCode != http.StatusOK {
-		return fmt.Errorf("HEAD %s: %v", urlSrc, res.Status)
-	}
-	if res.ContentLength <= 0 {
-		return fmt.Errorf("HEAD %s: unexpected Content-Length %v", urlSrc, res.ContentLength)
-	}
-	up.Logf("Download size: %v", res.ContentLength)
-
-	hashReq := must.Get(http.NewRequestWithContext(quickCtx, "GET", urlSrc+".sha256", nil))
-	hashRes, err := c.Do(hashReq)
-	if err != nil {
-		return err
-	}
-	hashHex, err := io.ReadAll(io.LimitReader(hashRes.Body, 100))
-	hashRes.Body.Close()
-	if res.StatusCode != http.StatusOK {
-		return fmt.Errorf("GET %s.sha256: %v", urlSrc, res.Status)
-	}
-	if err != nil {
-		return err
-	}
-	wantHash, err := hex.DecodeString(string(strings.TrimSpace(string(hashHex))))
-	if err != nil {
-		return err
-	}
-	hash := sha256.New()
-
-	dlReq := must.Get(http.NewRequestWithContext(context.Background(), "GET", urlSrc, nil))
-	dlRes, err := c.Do(dlReq)
-	if err != nil {
-		return err
-	}
-	// TODO(bradfitz): resume from existing partial file on disk
-	if dlRes.StatusCode != http.StatusOK {
-		return fmt.Errorf("GET %s: %v", urlSrc, dlRes.Status)
-	}
-
-	of, err := os.Create(fileDst)
-	if err != nil {
-		return err
-	}
-	defer func() {
-		if ret != nil {
-			of.Close()
-			// TODO(bradfitz): os.Remove(fileDst) too? or keep it to resume from/debug later.
-		}
-	}()
-	pw := &progressWriter{total: res.ContentLength, logf: up.Logf}
-	n, err := io.Copy(io.MultiWriter(hash, of, pw), io.LimitReader(dlRes.Body, res.ContentLength))
-	if err != nil {
-		return err
-	}
-	if n != res.ContentLength {
-		return fmt.Errorf("downloaded %v; want %v", n, res.ContentLength)
-	}
-	if err := of.Close(); err != nil {
-		return err
-	}
-	pw.print()
-
-	if !bytes.Equal(hash.Sum(nil), wantHash) {
-		return fmt.Errorf("SHA-256 of downloaded MSI didn't match expected value")
-	}
-	up.Logf("hash matched")
-
-	return nil
+	return c.Download(context.Background(), pathSrc, fileDst)
 }
 
-type progressWriter struct {
-	done      int64
-	total     int64
-	lastPrint time.Time
-	logf      logger.Logf
-}
-
-func (pw *progressWriter) Write(p []byte) (n int, err error) {
-	pw.done += int64(len(p))
-	if time.Since(pw.lastPrint) > 2*time.Second {
-		pw.print()
-	}
-	return len(p), nil
-}
-
-func (pw *progressWriter) print() {
-	pw.lastPrint = time.Now()
-	pw.logf("Downloaded %v/%v (%.1f%%)", pw.done, pw.total, float64(pw.done)/float64(pw.total)*100)
-}
-
-func (up *updater) updateFreeBSD() (err error) {
+func (up *Updater) updateFreeBSD() (err error) {
 	if up.Version != "" {
 		return errors.New("installing a specific version on FreeBSD is not supported")
 	}
 	if err := requireRoot(); err != nil {
 		return err
 	}
+	if err := exec.Command("pkg", "query", "%n", "tailscale").Run(); err != nil && isExitError(err) {
+		// Tailscale was not installed via pkg and we don't pre-compile
+		// binaries for it.
+		return errors.New("Tailscale was not installed via pkg, binary updates on FreeBSD are not supported; please reinstall Tailscale using pkg or update manually")
+	}
 
 	defer func() {
 		if err != nil {
@@ -927,6 +841,165 @@ func (up *updater) updateFreeBSD() (err error) {
 	return nil
 }
 
+func (up *Updater) updateLinuxBinary() error {
+	ver, err := requestedTailscaleVersion(up.Version, up.track)
+	if err != nil {
+		return err
+	}
+	if !up.confirm(ver) {
+		return nil
+	}
+	// Root is needed to overwrite binaries and restart systemd unit.
+	if err := requireRoot(); err != nil {
+		return err
+	}
+
+	dlPath, err := up.downloadLinuxTarball(ver)
+	if err != nil {
+		return err
+	}
+	up.Logf("Extracting %q", dlPath)
+	if err := up.unpackLinuxTarball(dlPath); err != nil {
+		return err
+	}
+	if err := os.Remove(dlPath); err != nil {
+		up.Logf("failed to clean up %q: %w", dlPath, err)
+	}
+	if err := restartSystemdUnit(context.Background()); err != nil {
+		if errors.Is(err, errors.ErrUnsupported) {
+			up.Logf("Tailscale binaries updated successfully.\nPlease restart tailscaled to finish the update.")
+		} else {
+			up.Logf("Tailscale binaries updated successfully, but failed to restart tailscaled: %s.\nPlease restart tailscaled to finish the update.", err)
+		}
+	} else {
+		up.Logf("Success")
+	}
+
+	return nil
+}
+
+func (up *Updater) downloadLinuxTarball(ver string) (string, error) {
+	dlDir, err := os.UserCacheDir()
+	if err != nil {
+		return "", err
+	}
+	dlDir = filepath.Join(dlDir, "tailscale-update")
+	if err := os.MkdirAll(dlDir, 0700); err != nil {
+		return "", err
+	}
+	pkgsPath := fmt.Sprintf("%s/tailscale_%s_%s.tgz", up.track, ver, runtime.GOARCH)
+	dlPath := filepath.Join(dlDir, path.Base(pkgsPath))
+	if err := up.downloadURLToFile(pkgsPath, dlPath); err != nil {
+		return "", err
+	}
+	return dlPath, nil
+}
+
+func (up *Updater) unpackLinuxTarball(path string) error {
+	tailscale, tailscaled, err := binaryPaths()
+	if err != nil {
+		return err
+	}
+	f, err := os.Open(path)
+	if err != nil {
+		return err
+	}
+	defer f.Close()
+	gr, err := gzip.NewReader(f)
+	if err != nil {
+		return err
+	}
+	defer gr.Close()
+	tr := tar.NewReader(gr)
+	files := make(map[string]int)
+	wantFiles := map[string]int{
+		"tailscale":  1,
+		"tailscaled": 1,
+	}
+	for {
+		th, err := tr.Next()
+		if err == io.EOF {
+			break
+		}
+		if err != nil {
+			return fmt.Errorf("failed extracting %q: %w", path, err)
+		}
+		// TODO(awly): try to also extract tailscaled.service. The tricky part
+		// is fixing up binary paths in that file if they differ from where
+		// local tailscale/tailscaled are installed. Also, this may not be a
+		// systemd distro.
+		switch filepath.Base(th.Name) {
+		case "tailscale":
+			files["tailscale"]++
+			if err := writeFile(tr, tailscale+".new", 0755); err != nil {
+				return fmt.Errorf("failed extracting the new tailscale binary from %q: %w", path, err)
+			}
+		case "tailscaled":
+			files["tailscaled"]++
+			if err := writeFile(tr, tailscaled+".new", 0755); err != nil {
+				return fmt.Errorf("failed extracting the new tailscaled binary from %q: %w", path, err)
+			}
+		}
+	}
+	if !maps.Equal(files, wantFiles) {
+		return fmt.Errorf("%q has missing or duplicate files: got %v, want %v", path, files, wantFiles)
+	}
+
+	// Only place the files in final locations after everything extracted correctly.
+	if err := os.Rename(tailscale+".new", tailscale); err != nil {
+		return err
+	}
+	up.Logf("Updated %s", tailscale)
+	if err := os.Rename(tailscaled+".new", tailscaled); err != nil {
+		return err
+	}
+	up.Logf("Updated %s", tailscaled)
+	return nil
+}
+
+func writeFile(r io.Reader, path string, perm os.FileMode) error {
+	if err := os.Remove(path); err != nil && !os.IsNotExist(err) {
+		return fmt.Errorf("failed to remove existing file at %q: %w", path, err)
+	}
+	f, err := os.OpenFile(path, os.O_WRONLY|os.O_CREATE|os.O_EXCL, perm)
+	if err != nil {
+		return err
+	}
+	defer f.Close()
+	if _, err := io.Copy(f, r); err != nil {
+		return err
+	}
+	return f.Close()
+}
+
+// Var allows overriding this in tests.
+var binaryPaths = func() (tailscale, tailscaled string, err error) {
+	// This can be either tailscale or tailscaled.
+	this, err := os.Executable()
+	if err != nil {
+		return "", "", err
+	}
+	otherName := "tailscaled"
+	if filepath.Base(this) == "tailscaled" {
+		otherName = "tailscale"
+	}
+	// Try to find the other binary in the same directory.
+	other := filepath.Join(filepath.Dir(this), otherName)
+	_, err = os.Stat(other)
+	if os.IsNotExist(err) {
+		// If it's not in the same directory, try to find it in $PATH.
+		other, err = exec.LookPath(otherName)
+	}
+	if err != nil {
+		return "", "", fmt.Errorf("cannot find %q in neither %q nor $PATH: %w", otherName, filepath.Dir(this), err)
+	}
+	if otherName == "tailscaled" {
+		return this, other, nil
+	} else {
+		return other, this, nil
+	}
+}
+
 func haveExecutable(name string) bool {
 	path, err := exec.LookPath(name)
 	return err == nil && path != ""
@@ -961,12 +1034,17 @@ func LatestTailscaleVersion(track string) (string, error) {
 }
 
 type trackPackages struct {
-	Version  string
-	Tarballs map[string]string
-	Exes     []string
-	MSIs     map[string]string
-	MacZips  map[string]string
-	SPKs     map[string]map[string]string
+	Version         string
+	Tarballs        map[string]string
+	TarballsVersion string
+	Exes            []string
+	ExesVersion     string
+	MSIs            map[string]string
+	MSIsVersion     string
+	MacZips         map[string]string
+	MacZipsVersion  string
+	SPKs            map[string]map[string]string
+	SPKsVersion     string
 }
 
 func latestPackages(track string) (*trackPackages, error) {
@@ -996,3 +1074,8 @@ func requireRoot() error {
 		return errors.New("must be root")
 	}
 }
+
+func isExitError(err error) bool {
+	var exitErr *exec.ExitError
+	return errors.As(err, &exitErr)
+}

clientupdate/clientupdate_test.go

@@ -4,12 +4,15 @@
 package clientupdate
 
 import (
+	"archive/tar"
+	"compress/gzip"
 	"fmt"
+	"io/fs"
+	"maps"
 	"os"
 	"path/filepath"
+	"strings"
 	"testing"
-
-	"tailscale.com/tailcfg"
 )
 
 func TestUpdateDebianAptSourcesListBytes(t *testing.T) {
@@ -159,108 +162,6 @@ func TestParseSoftwareupdateList(t *testing.T) {
 	}
 }
 
-func TestParsePacmanVersion(t *testing.T) {
-	tests := []struct {
-		desc    string
-		out     string
-		want    string
-		wantErr bool
-	}{
-		{
-			desc: "valid version",
-			out: `
-:: Synchronizing package databases...
- endeavouros is up to date
- core is up to date
- extra is up to date
- multilib is up to date
-Repository      : extra
-Name            : tailscale
-Version         : 1.44.2-1
-Description     : A mesh VPN that makes it easy to connect your devices, wherever they are.
-Architecture    : x86_64
-URL             : https://tailscale.com
-Licenses        : MIT
-Groups          : None
-Provides        : None
-Depends On      : glibc
-Optional Deps   : None
-Conflicts With  : None
-Replaces        : None
-Download Size   : 7.98 MiB
-Installed Size  : 32.47 MiB
-Packager        : Christian Heusel <gromit@archlinux.org>
-Build Date      : Tue 18 Jul 2023 12:28:37 PM PDT
-Validated By    : MD5 Sum  SHA-256 Sum  Signature
-`,
-			want: "1.44.2",
-		},
-		{
-			desc: "version without Arch patch number",
-			out: `
-... snip ...
-Name            : tailscale
-Version         : 1.44.2
-Description     : A mesh VPN that makes it easy to connect your devices, wherever they are.
-... snip ...
-`,
-			want: "1.44.2",
-		},
-		{
-			desc: "missing version",
-			out: `
-... snip ...
-Name            : tailscale
-Description     : A mesh VPN that makes it easy to connect your devices, wherever they are.
-... snip ...
-`,
-			wantErr: true,
-		},
-		{
-			desc: "empty version",
-			out: `
-... snip ...
-Name            : tailscale
-Version         :
-Description     : A mesh VPN that makes it easy to connect your devices, wherever they are.
-... snip ...
-`,
-			wantErr: true,
-		},
-		{
-			desc:    "empty input",
-			out:     "",
-			wantErr: true,
-		},
-		{
-			desc: "sneaky version in description",
-			out: `
-... snip ...
-Name            : tailscale
-Description     : A mesh VPN that makes it easy to connect your devices, wherever they are. Version : 1.2.3
-Version         : 1.44.2
-... snip ...
-`,
-			want: "1.44.2",
-		},
-	}
-
-	for _, tt := range tests {
-		t.Run(tt.desc, func(t *testing.T) {
-			got, err := parsePacmanVersion([]byte(tt.out))
-			if err == nil && tt.wantErr {
-				t.Fatalf("got nil error and version %q, want non-nil error", got)
-			}
-			if err != nil && !tt.wantErr {
-				t.Fatalf("got error: %q, want nil", err)
-			}
-			if got != tt.want {
-				t.Fatalf("got version: %q, want %q", got, tt.want)
-			}
-		})
-	}
-}
-
 func TestUpdateYUMRepoTrack(t *testing.T) {
 	tests := []struct {
 		desc    string
@@ -446,29 +347,38 @@ tailscale installed size:
 
 func TestSynoArch(t *testing.T) {
 	tests := []struct {
-		goarch  string
-		model   string
-		want    string
-		wantErr bool
+		goarch         string
+		synoinfoUnique string
+		want           string
+		wantErr        bool
 	}{
-		{goarch: "amd64", model: "DS224+", want: "x86_64"},
-		{goarch: "arm64", model: "DS124", want: "armv8"},
-		{goarch: "386", model: "DS415play", want: "i686"},
-		{goarch: "arm", model: "DS213air", want: "88f6281"},
-		{goarch: "arm", model: "NVR1218", want: "hi3535"},
-		{goarch: "arm", model: "DS1517", want: "alpine"},
-		{goarch: "arm", model: "DS216se", want: "armada370"},
-		{goarch: "arm", model: "DS115", want: "armada375"},
-		{goarch: "arm", model: "DS419slim", want: "armada38x"},
-		{goarch: "arm", model: "RS815", want: "armadaxp"},
-		{goarch: "arm", model: "DS414j", want: "comcerto2k"},
-		{goarch: "arm", model: "DS216play", want: "monaco"},
-		{goarch: "riscv64", model: "DS999", wantErr: true},
+		{goarch: "amd64", synoinfoUnique: "synology_x86_224", want: "x86_64"},
+		{goarch: "arm64", synoinfoUnique: "synology_armv8_124", want: "armv8"},
+		{goarch: "386", synoinfoUnique: "synology_i686_415play", want: "i686"},
+		{goarch: "arm", synoinfoUnique: "synology_88f6281_213air", want: "88f6281"},
+		{goarch: "arm", synoinfoUnique: "synology_88f6282_413j", want: "88f6282"},
+		{goarch: "arm", synoinfoUnique: "synology_hi3535_NVR1218", want: "hi3535"},
+		{goarch: "arm", synoinfoUnique: "synology_alpine_1517", want: "alpine"},
+		{goarch: "arm", synoinfoUnique: "synology_armada370_216se", want: "armada370"},
+		{goarch: "arm", synoinfoUnique: "synology_armada375_115", want: "armada375"},
+		{goarch: "arm", synoinfoUnique: "synology_armada38x_419slim", want: "armada38x"},
+		{goarch: "arm", synoinfoUnique: "synology_armadaxp_RS815", want: "armadaxp"},
+		{goarch: "arm", synoinfoUnique: "synology_comcerto2k_414j", want: "comcerto2k"},
+		{goarch: "arm", synoinfoUnique: "synology_monaco_216play", want: "monaco"},
+		{goarch: "ppc64", synoinfoUnique: "synology_qoriq_413", wantErr: true},
 	}
 
 	for _, tt := range tests {
-		t.Run(fmt.Sprintf("%s-%s", tt.goarch, tt.model), func(t *testing.T) {
-			got, err := synoArch(&tailcfg.Hostinfo{GoArch: tt.goarch, DeviceModel: tt.model})
+		t.Run(fmt.Sprintf("%s-%s", tt.goarch, tt.synoinfoUnique), func(t *testing.T) {
+			synoinfoConfPath := filepath.Join(t.TempDir(), "synoinfo.conf")
+			if err := os.WriteFile(
+				synoinfoConfPath,
+				[]byte(fmt.Sprintf("unique=%q\n", tt.synoinfoUnique)),
+				0600,
+			); err != nil {
+				t.Fatal(err)
+			}
+			got, err := synoArch(tt.goarch, synoinfoConfPath)
 			if err != nil {
 				if !tt.wantErr {
 					t.Fatalf("got unexpected error %v", err)
@@ -484,3 +394,370 @@ func TestSynoArch(t *testing.T) {
 		})
 	}
 }
+
+func TestParseSynoinfo(t *testing.T) {
+	tests := []struct {
+		desc    string
+		content string
+		want    string
+		wantErr bool
+	}{
+		{
+			desc: "double-quoted",
+			content: `
+company_title="Synology"
+unique="synology_88f6281_213air"
+`,
+			want: "88f6281",
+		},
+		{
+			desc: "single-quoted",
+			content: `
+company_title="Synology"
+unique='synology_88f6281_213air'
+`,
+			want: "88f6281",
+		},
+		{
+			desc: "unquoted",
+			content: `
+company_title="Synology"
+unique=synology_88f6281_213air
+`,
+			want: "88f6281",
+		},
+		{
+			desc: "missing unique",
+			content: `
+company_title="Synology"
+`,
+			wantErr: true,
+		},
+		{
+			desc: "empty unique",
+			content: `
+company_title="Synology"
+unique=
+`,
+			wantErr: true,
+		},
+		{
+			desc: "empty unique double-quoted",
+			content: `
+company_title="Synology"
+unique=""
+`,
+			wantErr: true,
+		},
+		{
+			desc: "empty unique single-quoted",
+			content: `
+company_title="Synology"
+unique=''
+`,
+			wantErr: true,
+		},
+		{
+			desc: "malformed unique",
+			content: `
+company_title="Synology"
+unique="synology_88f6281"
+`,
+			wantErr: true,
+		},
+		{
+			desc:    "empty file",
+			content: ``,
+			wantErr: true,
+		},
+		{
+			desc: "empty lines and comments",
+			content: `
+
+# In a file named synoinfo? Shocking!
+company_title="Synology"
+
+
+# unique= is_a_field_that_follows
+unique="synology_88f6281_213air"
+
+`,
+			want: "88f6281",
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.desc, func(t *testing.T) {
+			synoinfoConfPath := filepath.Join(t.TempDir(), "synoinfo.conf")
+			if err := os.WriteFile(synoinfoConfPath, []byte(tt.content), 0600); err != nil {
+				t.Fatal(err)
+			}
+			got, err := parseSynoinfo(synoinfoConfPath)
+			if err != nil {
+				if !tt.wantErr {
+					t.Fatalf("got unexpected error %v", err)
+				}
+				return
+			}
+			if tt.wantErr {
+				t.Fatalf("got %q, expected an error", got)
+			}
+			if got != tt.want {
+				t.Errorf("got %q, want %q", got, tt.want)
+			}
+		})
+	}
+}
+
+func TestUnpackLinuxTarball(t *testing.T) {
+	oldBinaryPaths := binaryPaths
+	t.Cleanup(func() { binaryPaths = oldBinaryPaths })
+
+	tests := []struct {
+		desc    string
+		tarball map[string]string
+		before  map[string]string
+		after   map[string]string
+		wantErr bool
+	}{
+		{
+			desc: "success",
+			before: map[string]string{
+				"tailscale":  "v1",
+				"tailscaled": "v1",
+			},
+			tarball: map[string]string{
+				"/usr/bin/tailscale":  "v2",
+				"/usr/bin/tailscaled": "v2",
+			},
+			after: map[string]string{
+				"tailscale":  "v2",
+				"tailscaled": "v2",
+			},
+		},
+		{
+			desc: "don't touch unrelated files",
+			before: map[string]string{
+				"tailscale":  "v1",
+				"tailscaled": "v1",
+				"foo":        "bar",
+			},
+			tarball: map[string]string{
+				"/usr/bin/tailscale":  "v2",
+				"/usr/bin/tailscaled": "v2",
+			},
+			after: map[string]string{
+				"tailscale":  "v2",
+				"tailscaled": "v2",
+				"foo":        "bar",
+			},
+		},
+		{
+			desc: "unmodified",
+			before: map[string]string{
+				"tailscale":  "v1",
+				"tailscaled": "v1",
+			},
+			tarball: map[string]string{
+				"/usr/bin/tailscale":  "v1",
+				"/usr/bin/tailscaled": "v1",
+			},
+			after: map[string]string{
+				"tailscale":  "v1",
+				"tailscaled": "v1",
+			},
+		},
+		{
+			desc: "ignore extra tarball files",
+			before: map[string]string{
+				"tailscale":  "v1",
+				"tailscaled": "v1",
+			},
+			tarball: map[string]string{
+				"/usr/bin/tailscale":          "v2",
+				"/usr/bin/tailscaled":         "v2",
+				"/systemd/tailscaled.service": "v2",
+			},
+			after: map[string]string{
+				"tailscale":  "v2",
+				"tailscaled": "v2",
+			},
+		},
+		{
+			desc: "tarball missing tailscaled",
+			before: map[string]string{
+				"tailscale":  "v1",
+				"tailscaled": "v1",
+			},
+			tarball: map[string]string{
+				"/usr/bin/tailscale": "v2",
+			},
+			after: map[string]string{
+				"tailscale":     "v1",
+				"tailscale.new": "v2",
+				"tailscaled":    "v1",
+			},
+			wantErr: true,
+		},
+		{
+			desc: "duplicate tailscale binary",
+			before: map[string]string{
+				"tailscale":  "v1",
+				"tailscaled": "v1",
+			},
+			tarball: map[string]string{
+				"/usr/bin/tailscale":  "v2",
+				"/usr/sbin/tailscale": "v2",
+				"/usr/bin/tailscaled": "v2",
+			},
+			after: map[string]string{
+				"tailscale":      "v1",
+				"tailscale.new":  "v2",
+				"tailscaled":     "v1",
+				"tailscaled.new": "v2",
+			},
+			wantErr: true,
+		},
+		{
+			desc: "empty archive",
+			before: map[string]string{
+				"tailscale":  "v1",
+				"tailscaled": "v1",
+			},
+			tarball: map[string]string{},
+			after: map[string]string{
+				"tailscale":  "v1",
+				"tailscaled": "v1",
+			},
+			wantErr: true,
+		},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.desc, func(t *testing.T) {
+			// Swap out binaryPaths function to point at dummy file paths.
+			tmp := t.TempDir()
+			tailscalePath := filepath.Join(tmp, "tailscale")
+			tailscaledPath := filepath.Join(tmp, "tailscaled")
+			binaryPaths = func() (string, string, error) {
+				return tailscalePath, tailscaledPath, nil
+			}
+			for name, content := range tt.before {
+				if err := os.WriteFile(filepath.Join(tmp, name), []byte(content), 0755); err != nil {
+					t.Fatal(err)
+				}
+			}
+			tarPath := filepath.Join(tmp, "tailscale.tgz")
+			genTarball(t, tarPath, tt.tarball)
+
+			up := &Updater{Arguments: Arguments{Logf: t.Logf}}
+			err := up.unpackLinuxTarball(tarPath)
+			if err != nil {
+				if !tt.wantErr {
+					t.Fatalf("unexpected error: %v", err)
+				}
+			} else if tt.wantErr {
+				t.Fatalf("unpack succeeded, expected an error")
+			}
+
+			gotAfter := make(map[string]string)
+			err = filepath.WalkDir(tmp, func(path string, d fs.DirEntry, err error) error {
+				if err != nil {
+					return err
+				}
+				if d.Type().IsDir() {
+					return nil
+				}
+				if path == tarPath {
+					return nil
+				}
+				content, err := os.ReadFile(path)
+				if err != nil {
+					return err
+				}
+				path = filepath.ToSlash(path)
+				base := filepath.ToSlash(tmp)
+				gotAfter[strings.TrimPrefix(path, base+"/")] = string(content)
+				return nil
+			})
+			if err != nil {
+				t.Fatal(err)
+			}
+
+			if !maps.Equal(gotAfter, tt.after) {
+				t.Errorf("files after unpack: %+v, want %+v", gotAfter, tt.after)
+			}
+		})
+	}
+}
+
+func genTarball(t *testing.T, path string, files map[string]string) {
+	f, err := os.Create(path)
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer f.Close()
+	gw := gzip.NewWriter(f)
+	defer gw.Close()
+	tw := tar.NewWriter(gw)
+	defer tw.Close()
+	for file, content := range files {
+		if err := tw.WriteHeader(&tar.Header{
+			Name: file,
+			Size: int64(len(content)),
+			Mode: 0755,
+		}); err != nil {
+			t.Fatal(err)
+		}
+		if _, err := tw.Write([]byte(content)); err != nil {
+			t.Fatal(err)
+		}
+	}
+}
+
+func TestWriteFileOverwrite(t *testing.T) {
+	path := filepath.Join(t.TempDir(), "test")
+	for i := 0; i < 2; i++ {
+		content := fmt.Sprintf("content %d", i)
+		if err := writeFile(strings.NewReader(content), path, 0600); err != nil {
+			t.Fatal(err)
+		}
+		got, err := os.ReadFile(path)
+		if err != nil {
+			t.Fatal(err)
+		}
+		if string(got) != content {
+			t.Errorf("got content: %q, want: %q", got, content)
+		}
+	}
+}
+
+func TestWriteFileSymlink(t *testing.T) {
+	// Test for a malicious symlink at the destination path.
+	// f2 points to f1 and writeFile(f2) should not end up overwriting f1.
+	tmp := t.TempDir()
+	f1 := filepath.Join(tmp, "f1")
+	if err := os.WriteFile(f1, []byte("old"), 0600); err != nil {
+		t.Fatal(err)
+	}
+	f2 := filepath.Join(tmp, "f2")
+	if err := os.Symlink(f1, f2); err != nil {
+		t.Fatal(err)
+	}
+
+	if err := writeFile(strings.NewReader("new"), f2, 0600); err != nil {
+		t.Errorf("writeFile(%q) failed: %v", f2, err)
+	}
+	want := map[string]string{
+		f1: "old",
+		f2: "new",
+	}
+	for f, content := range want {
+		got, err := os.ReadFile(f)
+		if err != nil {
+			t.Fatal(err)
+		}
+		if string(got) != content {
+			t.Errorf("%q: got content %q, want %q", f, got, content)
+		}
+	}
+}

clientupdate/distsign/distsign.go

@@ -0,0 +1,443 @@
+// Copyright (c) Tailscale Inc & AUTHORS
+// SPDX-License-Identifier: BSD-3-Clause
+
+// Package distsign implements signature and validation of arbitrary
+// distributable files.
+//
+// There are 3 parties in this exchange:
+//   - builder, which creates files, signs them with signing keys and publishes
+//     to server
+//   - server, which distributes public signing keys, files and signatures
+//   - client, which downloads files and signatures from server, and validates
+//     the signatures
+//
+// There are 2 types of keys:
+//   - signing keys, that sign individual distributable files on the builder
+//   - root keys, that sign signing keys and are kept offline
+//
+// root keys -(sign)-> signing keys -(sign)-> files
+//
+// All keys are asymmetric Ed25519 key pairs.
+//
+// The server serves static files under some known prefix. The kinds of files are:
+//   - distsign.pub - bundle of PEM-encoded public signing keys
+//   - distsign.pub.sig - signature of distsign.pub using one of the root keys
+//   - $file - any distributable file
+//   - $file.sig - signature of $file using any of the signing keys
+//
+// The root public keys are baked into the client software at compile time.
+// These keys are long-lived and prove the validity of current signing keys
+// from distsign.pub. To rotate root keys, a new client release must be
+// published, they are not rotated dynamically. There are multiple root keys in
+// different locations specifically to allow this rotation without using the
+// discarded root key for any new signatures.
+//
+// The signing public keys are fetched by the client dynamically before every
+// download and can be rotated more readily, assuming that most deployed
+// clients trust the root keys used to issue fresh signing keys.
+package distsign
+
+import (
+	"context"
+	"crypto/ed25519"
+	"crypto/rand"
+	"encoding/binary"
+	"encoding/pem"
+	"errors"
+	"fmt"
+	"hash"
+	"io"
+	"log"
+	"net/http"
+	"net/url"
+	"os"
+	"time"
+
+	"github.com/hdevalence/ed25519consensus"
+	"golang.org/x/crypto/blake2s"
+	"tailscale.com/net/tshttpproxy"
+	"tailscale.com/types/logger"
+	"tailscale.com/util/must"
+)
+
+const (
+	pemTypeRootPrivate    = "ROOT PRIVATE KEY"
+	pemTypeRootPublic     = "ROOT PUBLIC KEY"
+	pemTypeSigningPrivate = "SIGNING PRIVATE KEY"
+	pemTypeSigningPublic  = "SIGNING PUBLIC KEY"
+
+	downloadSizeLimit    = 1 << 29 // 512MB
+	signingKeysSizeLimit = 1 << 20 // 1MB
+	signatureSizeLimit   = ed25519.SignatureSize
+)
+
+// RootKey is a root key used to sign signing keys.
+type RootKey struct {
+	k ed25519.PrivateKey
+}
+
+// GenerateRootKey generates a new root key pair and encodes it as PEM.
+func GenerateRootKey() (priv, pub []byte, err error) {
+	pub, priv, err = ed25519.GenerateKey(rand.Reader)
+	if err != nil {
+		return nil, nil, err
+	}
+	return pem.EncodeToMemory(&pem.Block{
+			Type:  pemTypeRootPrivate,
+			Bytes: []byte(priv),
+		}), pem.EncodeToMemory(&pem.Block{
+			Type:  pemTypeRootPublic,
+			Bytes: []byte(pub),
+		}), nil
+}
+
+// ParseRootKey parses the PEM-encoded private root key. The key must be in the
+// same format as returned by GenerateRootKey.
+func ParseRootKey(privKey []byte) (*RootKey, error) {
+	k, err := parsePrivateKey(privKey, pemTypeRootPrivate)
+	if err != nil {
+		return nil, fmt.Errorf("failed to parse root key: %w", err)
+	}
+	return &RootKey{k: k}, nil
+}
+
+// SignSigningKeys signs the bundle of public signing keys. The bundle must be
+// a sequence of PEM blocks joined with newlines.
+func (r *RootKey) SignSigningKeys(pubBundle []byte) ([]byte, error) {
+	if _, err := ParseSigningKeyBundle(pubBundle); err != nil {
+		return nil, err
+	}
+	return ed25519.Sign(r.k, pubBundle), nil
+}
+
+// SigningKey is a signing key used to sign packages.
+type SigningKey struct {
+	k ed25519.PrivateKey
+}
+
+// GenerateSigningKey generates a new signing key pair and encodes it as PEM.
+func GenerateSigningKey() (priv, pub []byte, err error) {
+	pub, priv, err = ed25519.GenerateKey(rand.Reader)
+	if err != nil {
+		return nil, nil, err
+	}
+	return pem.EncodeToMemory(&pem.Block{
+			Type:  pemTypeSigningPrivate,
+			Bytes: []byte(priv),
+		}), pem.EncodeToMemory(&pem.Block{
+			Type:  pemTypeSigningPublic,
+			Bytes: []byte(pub),
+		}), nil
+}
+
+// ParseSigningKey parses the PEM-encoded private signing key. The key must be
+// in the same format as returned by GenerateSigningKey.
+func ParseSigningKey(privKey []byte) (*SigningKey, error) {
+	k, err := parsePrivateKey(privKey, pemTypeSigningPrivate)
+	if err != nil {
+		return nil, fmt.Errorf("failed to parse root key: %w", err)
+	}
+	return &SigningKey{k: k}, nil
+}
+
+// SignPackageHash signs the hash and the length of a package. Use PackageHash
+// to compute the inputs.
+func (s *SigningKey) SignPackageHash(hash []byte, len int64) ([]byte, error) {
+	if len <= 0 {
+		return nil, fmt.Errorf("package length must be positive, got %d", len)
+	}
+	msg := binary.LittleEndian.AppendUint64(hash, uint64(len))
+	return ed25519.Sign(s.k, msg), nil
+}
+
+// PackageHash is a hash.Hash that counts the number of bytes written. Use it
+// to get the hash and length inputs to SigningKey.SignPackageHash.
+type PackageHash struct {
+	hash.Hash
+	len int64
+}
+
+// NewPackageHash returns an initialized PackageHash using BLAKE2s.
+func NewPackageHash() *PackageHash {
+	h, err := blake2s.New256(nil)
+	if err != nil {
+		// Should never happen with a nil key passed to blake2s.
+		panic(err)
+	}
+	return &PackageHash{Hash: h}
+}
+
+func (ph *PackageHash) Write(b []byte) (int, error) {
+	ph.len += int64(len(b))
+	return ph.Hash.Write(b)
+}
+
+// Reset the PackageHash to its initial state.
+func (ph *PackageHash) Reset() {
+	ph.len = 0
+	ph.Hash.Reset()
+}
+
+// Len returns the total number of bytes written.
+func (ph *PackageHash) Len() int64 { return ph.len }
+
+// Client downloads and validates files from a distribution server.
+type Client struct {
+	logf     logger.Logf
+	roots    []ed25519.PublicKey
+	pkgsAddr *url.URL
+}
+
+// NewClient returns a new client for distribution server located at pkgsAddr,
+// and uses embedded root keys from the roots/ subdirectory of this package.
+func NewClient(logf logger.Logf, pkgsAddr string) (*Client, error) {
+	if logf == nil {
+		logf = log.Printf
+	}
+	u, err := url.Parse(pkgsAddr)
+	if err != nil {
+		return nil, fmt.Errorf("invalid pkgsAddr %q: %w", pkgsAddr, err)
+	}
+	return &Client{logf: logf, roots: roots(), pkgsAddr: u}, nil
+}
+
+func (c *Client) url(path string) string {
+	return c.pkgsAddr.JoinPath(path).String()
+}
+
+// Download fetches a file at path srcPath from pkgsAddr passed in NewClient.
+// The file is downloaded to dstPath and its signature is validated using the
+// embedded root keys. Download returns an error if anything goes wrong with
+// the actual file download or with signature validation.
+func (c *Client) Download(ctx context.Context, srcPath, dstPath string) error {
+	// Always fetch a fresh signing key.
+	sigPub, err := c.signingKeys()
+	if err != nil {
+		return err
+	}
+
+	srcURL := c.url(srcPath)
+	sigURL := srcURL + ".sig"
+
+	c.logf("Downloading %q", srcURL)
+	dstPathUnverified := dstPath + ".unverified"
+	hash, len, err := c.download(ctx, srcURL, dstPathUnverified, downloadSizeLimit)
+	if err != nil {
+		return err
+	}
+	c.logf("Downloading %q", sigURL)
+	sig, err := fetch(sigURL, signatureSizeLimit)
+	if err != nil {
+		// Best-effort clean up of downloaded package.
+		os.Remove(dstPathUnverified)
+		return err
+	}
+	msg := binary.LittleEndian.AppendUint64(hash, uint64(len))
+	if !VerifyAny(sigPub, msg, sig) {
+		// Best-effort clean up of downloaded package.
+		os.Remove(dstPathUnverified)
+		return fmt.Errorf("signature %q for file %q does not validate with the current release signing key; either you are under attack, or attempting to download an old version of Tailscale which was signed with an older signing key", sigURL, srcURL)
+	}
+	c.logf("Signature OK")
+
+	if err := os.Rename(dstPathUnverified, dstPath); err != nil {
+		return fmt.Errorf("failed to move %q to %q after signature validation", dstPathUnverified, dstPath)
+	}
+
+	return nil
+}
+
+// signingKeys fetches current signing keys from the server and validates them
+// against the roots. Should be called before validation of any downloaded file
+// to get the fresh keys.
+func (c *Client) signingKeys() ([]ed25519.PublicKey, error) {
+	keyURL := c.url("distsign.pub")
+	sigURL := keyURL + ".sig"
+	raw, err := fetch(keyURL, signingKeysSizeLimit)
+	if err != nil {
+		return nil, err
+	}
+	sig, err := fetch(sigURL, signatureSizeLimit)
+	if err != nil {
+		return nil, err
+	}
+	if !VerifyAny(c.roots, raw, sig) {
+		return nil, fmt.Errorf("signature %q for key %q does not validate with any known root key; either you are under attack, or running a very old version of Tailscale with outdated root keys", sigURL, keyURL)
+	}
+
+	keys, err := ParseSigningKeyBundle(raw)
+	if err != nil {
+		return nil, fmt.Errorf("cannot parse signing key bundle from %q: %w", keyURL, err)
+	}
+	return keys, nil
+}
+
+// fetch reads the response body from url into memory, up to limit bytes.
+func fetch(url string, limit int64) ([]byte, error) {
+	resp, err := http.Get(url)
+	if err != nil {
+		return nil, err
+	}
+	defer resp.Body.Close()
+
+	return io.ReadAll(io.LimitReader(resp.Body, limit))
+}
+
+// download writes the response body of url into a local file at dst, up to
+// limit bytes. On success, the returned value is a BLAKE2s hash of the file.
+func (c *Client) download(ctx context.Context, url, dst string, limit int64) ([]byte, int64, error) {
+	tr := http.DefaultTransport.(*http.Transport).Clone()
+	tr.Proxy = tshttpproxy.ProxyFromEnvironment
+	defer tr.CloseIdleConnections()
+	hc := &http.Client{Transport: tr}
+
+	quickCtx, cancel := context.WithTimeout(ctx, 30*time.Second)
+	defer cancel()
+	headReq := must.Get(http.NewRequestWithContext(quickCtx, http.MethodHead, url, nil))
+
+	res, err := hc.Do(headReq)
+	if err != nil {
+		return nil, 0, err
+	}
+	if res.StatusCode != http.StatusOK {
+		return nil, 0, fmt.Errorf("HEAD %q: %v", url, res.Status)
+	}
+	if res.ContentLength <= 0 {
+		return nil, 0, fmt.Errorf("HEAD %q: unexpected Content-Length %v", url, res.ContentLength)
+	}
+	c.logf("Download size: %v", res.ContentLength)
+
+	dlReq := must.Get(http.NewRequestWithContext(ctx, http.MethodGet, url, nil))
+	dlRes, err := hc.Do(dlReq)
+	if err != nil {
+		return nil, 0, err
+	}
+	defer dlRes.Body.Close()
+	// TODO(bradfitz): resume from existing partial file on disk
+	if dlRes.StatusCode != http.StatusOK {
+		return nil, 0, fmt.Errorf("GET %q: %v", url, dlRes.Status)
+	}
+
+	of, err := os.Create(dst)
+	if err != nil {
+		return nil, 0, err
+	}
+	defer of.Close()
+	pw := &progressWriter{total: res.ContentLength, logf: c.logf}
+	h := NewPackageHash()
+	n, err := io.Copy(io.MultiWriter(of, h, pw), io.LimitReader(dlRes.Body, limit))
+	if err != nil {
+		return nil, n, err
+	}
+	if n != res.ContentLength {
+		return nil, n, fmt.Errorf("GET %q: downloaded %v, want %v", url, n, res.ContentLength)
+	}
+	if err := dlRes.Body.Close(); err != nil {
+		return nil, n, err
+	}
+	if err := of.Close(); err != nil {
+		return nil, n, err
+	}
+	pw.print()
+
+	return h.Sum(nil), h.Len(), nil
+}
+
+type progressWriter struct {
+	done      int64
+	total     int64
+	lastPrint time.Time
+	logf      logger.Logf
+}
+
+func (pw *progressWriter) Write(p []byte) (n int, err error) {
+	pw.done += int64(len(p))
+	if time.Since(pw.lastPrint) > 2*time.Second {
+		pw.print()
+	}
+	return len(p), nil
+}
+
+func (pw *progressWriter) print() {
+	pw.lastPrint = time.Now()
+	pw.logf("Downloaded %v/%v (%.1f%%)", pw.done, pw.total, float64(pw.done)/float64(pw.total)*100)
+}
+
+func parsePrivateKey(data []byte, typeTag string) (ed25519.PrivateKey, error) {
+	b, rest := pem.Decode(data)
+	if b == nil {
+		return nil, errors.New("failed to decode PEM data")
+	}
+	if len(rest) > 0 {
+		return nil, errors.New("trailing PEM data")
+	}
+	if b.Type != typeTag {
+		return nil, fmt.Errorf("PEM type is %q, want %q", b.Type, typeTag)
+	}
+	if len(b.Bytes) != ed25519.PrivateKeySize {
+		return nil, errors.New("private key has incorrect length for an Ed25519 private key")
+	}
+	return ed25519.PrivateKey(b.Bytes), nil
+}
+
+// ParseSigningKeyBundle parses the bundle of PEM-encoded public signing keys.
+func ParseSigningKeyBundle(bundle []byte) ([]ed25519.PublicKey, error) {
+	return parsePublicKeyBundle(bundle, pemTypeSigningPublic)
+}
+
+// ParseRootKeyBundle parses the bundle of PEM-encoded public root keys.
+func ParseRootKeyBundle(bundle []byte) ([]ed25519.PublicKey, error) {
+	return parsePublicKeyBundle(bundle, pemTypeRootPublic)
+}
+
+func parsePublicKeyBundle(bundle []byte, typeTag string) ([]ed25519.PublicKey, error) {
+	var keys []ed25519.PublicKey
+	for len(bundle) > 0 {
+		pub, rest, err := parsePublicKey(bundle, typeTag)
+		if err != nil {
+			return nil, err
+		}
+		keys = append(keys, pub)
+		bundle = rest
+	}
+	if len(keys) == 0 {
+		return nil, errors.New("no signing keys found in the bundle")
+	}
+	return keys, nil
+}
+
+func parseSinglePublicKey(data []byte, typeTag string) (ed25519.PublicKey, error) {
+	pub, rest, err := parsePublicKey(data, typeTag)
+	if err != nil {
+		return nil, err
+	}
+	if len(rest) > 0 {
+		return nil, errors.New("trailing PEM data")
+	}
+	return pub, err
+}
+
+func parsePublicKey(data []byte, typeTag string) (pub ed25519.PublicKey, rest []byte, retErr error) {
+	b, rest := pem.Decode(data)
+	if b == nil {
+		return nil, nil, errors.New("failed to decode PEM data")
+	}
+	if b.Type != typeTag {
+		return nil, nil, fmt.Errorf("PEM type is %q, want %q", b.Type, typeTag)
+	}
+	if len(b.Bytes) != ed25519.PublicKeySize {
+		return nil, nil, errors.New("public key has incorrect length for an Ed25519 public key")
+	}
+	return ed25519.PublicKey(b.Bytes), rest, nil
+}
+
+// VerifyAny verifies whether sig is valid for msg using any of the keys.
+// VerifyAny will panic if any of the keys have the wrong size for Ed25519.
+func VerifyAny(keys []ed25519.PublicKey, msg, sig []byte) bool {
+	for _, k := range keys {
+		if ed25519consensus.Verify(k, msg, sig) {
+			return true
+		}
+	}
+	return false
+}

clientupdate/distsign/distsign_test.go

@@ -0,0 +1,470 @@
+// Copyright (c) Tailscale Inc & AUTHORS
+// SPDX-License-Identifier: BSD-3-Clause
+
+package distsign
+
+import (
+	"bytes"
+	"context"
+	"crypto/ed25519"
+	"net/http"
+	"net/http/httptest"
+	"net/url"
+	"os"
+	"path/filepath"
+	"strings"
+	"testing"
+
+	"golang.org/x/crypto/blake2s"
+)
+
+func TestDownload(t *testing.T) {
+	srv := newTestServer(t)
+	c := srv.client(t)
+
+	tests := []struct {
+		desc    string
+		before  func(*testing.T)
+		src     string
+		want    []byte
+		wantErr bool
+	}{
+		{
+			desc:    "missing file",
+			before:  func(*testing.T) {},
+			src:     "hello",
+			wantErr: true,
+		},
+		{
+			desc: "success",
+			before: func(*testing.T) {
+				srv.addSigned("hello", []byte("world"))
+			},
+			src:  "hello",
+			want: []byte("world"),
+		},
+		{
+			desc: "no signature",
+			before: func(*testing.T) {
+				srv.add("hello", []byte("world"))
+			},
+			src:     "hello",
+			wantErr: true,
+		},
+		{
+			desc: "bad signature",
+			before: func(*testing.T) {
+				srv.add("hello", []byte("world"))
+				srv.add("hello.sig", []byte("potato"))
+			},
+			src:     "hello",
+			wantErr: true,
+		},
+		{
+			desc: "signed with untrusted key",
+			before: func(t *testing.T) {
+				srv.add("hello", []byte("world"))
+				srv.add("hello.sig", newSigningKeyPair(t).sign([]byte("world")))
+			},
+			src:     "hello",
+			wantErr: true,
+		},
+		{
+			desc: "signed with root key",
+			before: func(t *testing.T) {
+				srv.add("hello", []byte("world"))
+				srv.add("hello.sig", ed25519.Sign(srv.roots[0].k, []byte("world")))
+			},
+			src:     "hello",
+			wantErr: true,
+		},
+		{
+			desc: "bad signing key signature",
+			before: func(t *testing.T) {
+				srv.add("distsign.pub.sig", []byte("potato"))
+				srv.addSigned("hello", []byte("world"))
+			},
+			src:     "hello",
+			wantErr: true,
+		},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.desc, func(t *testing.T) {
+			srv.reset()
+			tt.before(t)
+
+			dst := filepath.Join(t.TempDir(), tt.src)
+			t.Cleanup(func() {
+				os.Remove(dst)
+			})
+			err := c.Download(context.Background(), tt.src, dst)
+			if err != nil {
+				if tt.wantErr {
+					return
+				}
+				t.Fatalf("unexpected error from Download(%q): %v", tt.src, err)
+			}
+			if tt.wantErr {
+				t.Fatalf("Download(%q) succeeded, expected an error", tt.src)
+			}
+			got, err := os.ReadFile(dst)
+			if err != nil {
+				t.Fatal(err)
+			}
+			if !bytes.Equal(tt.want, got) {
+				t.Errorf("Download(%q): got %q, want %q", tt.src, got, tt.want)
+			}
+		})
+	}
+}
+
+func TestRotateRoot(t *testing.T) {
+	srv := newTestServer(t)
+	c1 := srv.client(t)
+	ctx := context.Background()
+
+	srv.addSigned("hello", []byte("world"))
+	if err := c1.Download(ctx, "hello", filepath.Join(t.TempDir(), "hello")); err != nil {
+		t.Fatalf("Download failed on a fresh server: %v", err)
+	}
+
+	// Remove first root and replace it with a new key.
+	srv.roots = append(srv.roots[1:], newRootKeyPair(t))
+
+	// Old client can still download files because it still trusts the old
+	// root key.
+	if err := c1.Download(ctx, "hello", filepath.Join(t.TempDir(), "hello")); err != nil {
+		t.Fatalf("Download failed after root rotation on old client: %v", err)
+	}
+	// New client should fail download because current signing key is signed by
+	// the revoked root that new client doesn't trust.
+	c2 := srv.client(t)
+	if err := c2.Download(ctx, "hello", filepath.Join(t.TempDir(), "hello")); err == nil {
+		t.Fatalf("Download succeeded on new client, but signing key is signed with revoked root key")
+	}
+	// Re-sign signing key with another valid root that client still trusts.
+	srv.resignSigningKeys()
+	// Both old and new clients should now be able to download.
+	//
+	// Note: we don't need to re-sign the "hello" file because signing key
+	// didn't change (only signing key's signature).
+	if err := c1.Download(ctx, "hello", filepath.Join(t.TempDir(), "hello")); err != nil {
+		t.Fatalf("Download failed after root rotation on old client with re-signed signing key: %v", err)
+	}
+	if err := c2.Download(ctx, "hello", filepath.Join(t.TempDir(), "hello")); err != nil {
+		t.Fatalf("Download failed after root rotation on new client with re-signed signing key: %v", err)
+	}
+}
+
+func TestRotateSigning(t *testing.T) {
+	srv := newTestServer(t)
+	c := srv.client(t)
+	ctx := context.Background()
+
+	srv.addSigned("hello", []byte("world"))
+	if err := c.Download(ctx, "hello", filepath.Join(t.TempDir(), "hello")); err != nil {
+		t.Fatalf("Download failed on a fresh server: %v", err)
+	}
+
+	// Replace signing key but don't publish it yet.
+	srv.sign = append(srv.sign, newSigningKeyPair(t))
+	if err := c.Download(ctx, "hello", filepath.Join(t.TempDir(), "hello")); err != nil {
+		t.Fatalf("Download failed after new signing key added but before publishing it: %v", err)
+	}
+
+	// Publish new signing key bundle with both keys.
+	srv.resignSigningKeys()
+	if err := c.Download(ctx, "hello", filepath.Join(t.TempDir(), "hello")); err != nil {
+		t.Fatalf("Download failed after new signing key was published: %v", err)
+	}
+
+	// Re-sign the "hello" file with new signing key.
+	srv.add("hello.sig", srv.sign[1].sign([]byte("world")))
+	if err := c.Download(ctx, "hello", filepath.Join(t.TempDir(), "hello")); err != nil {
+		t.Fatalf("Download failed after re-signing with new signing key: %v", err)
+	}
+
+	// Drop the old signing key.
+	srv.sign = srv.sign[1:]
+	srv.resignSigningKeys()
+	if err := c.Download(ctx, "hello", filepath.Join(t.TempDir(), "hello")); err != nil {
+		t.Fatalf("Download failed after removing old signing key: %v", err)
+	}
+
+	// Add another key and re-sign the file with it *before* publishing.
+	srv.sign = append(srv.sign, newSigningKeyPair(t))
+	srv.add("hello.sig", srv.sign[1].sign([]byte("world")))
+	if err := c.Download(ctx, "hello", filepath.Join(t.TempDir(), "hello")); err == nil {
+		t.Fatalf("Download succeeded when signed with a not-yet-published signing key")
+	}
+	// Fix this by publishing the new key.
+	srv.resignSigningKeys()
+	if err := c.Download(ctx, "hello", filepath.Join(t.TempDir(), "hello")); err != nil {
+		t.Fatalf("Download failed after publishing new signing key: %v", err)
+	}
+}
+
+func TestParseRootKey(t *testing.T) {
+	tests := []struct {
+		desc     string
+		generate func() ([]byte, []byte, error)
+		wantErr  bool
+	}{
+		{
+			desc:     "valid",
+			generate: GenerateRootKey,
+		},
+		{
+			desc:     "signing",
+			generate: GenerateSigningKey,
+			wantErr:  true,
+		},
+		{
+			desc:     "nil",
+			generate: func() ([]byte, []byte, error) { return nil, nil, nil },
+			wantErr:  true,
+		},
+		{
+			desc: "invalid PEM tag",
+			generate: func() ([]byte, []byte, error) {
+				priv, pub, err := GenerateRootKey()
+				priv = bytes.Replace(priv, []byte("ROOT "), nil, -1)
+				return priv, pub, err
+			},
+			wantErr: true,
+		},
+		{
+			desc:     "not PEM",
+			generate: func() ([]byte, []byte, error) { return []byte("s3cr3t"), nil, nil },
+			wantErr:  true,
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.desc, func(t *testing.T) {
+			priv, _, err := tt.generate()
+			if err != nil {
+				t.Fatal(err)
+			}
+			r, err := ParseRootKey(priv)
+			if err != nil {
+				if tt.wantErr {
+					return
+				}
+				t.Fatalf("unexpected error: %v", err)
+			}
+			if tt.wantErr {
+				t.Fatal("expected non-nil error")
+			}
+			if r == nil {
+				t.Errorf("got nil error and nil RootKey")
+			}
+		})
+	}
+}
+
+func TestParseSigningKey(t *testing.T) {
+	tests := []struct {
+		desc     string
+		generate func() ([]byte, []byte, error)
+		wantErr  bool
+	}{
+		{
+			desc:     "valid",
+			generate: GenerateSigningKey,
+		},
+		{
+			desc:     "root",
+			generate: GenerateRootKey,
+			wantErr:  true,
+		},
+		{
+			desc:     "nil",
+			generate: func() ([]byte, []byte, error) { return nil, nil, nil },
+			wantErr:  true,
+		},
+		{
+			desc: "invalid PEM tag",
+			generate: func() ([]byte, []byte, error) {
+				priv, pub, err := GenerateSigningKey()
+				priv = bytes.Replace(priv, []byte("SIGNING "), nil, -1)
+				return priv, pub, err
+			},
+			wantErr: true,
+		},
+		{
+			desc:     "not PEM",
+			generate: func() ([]byte, []byte, error) { return []byte("s3cr3t"), nil, nil },
+			wantErr:  true,
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.desc, func(t *testing.T) {
+			priv, _, err := tt.generate()
+			if err != nil {
+				t.Fatal(err)
+			}
+			r, err := ParseSigningKey(priv)
+			if err != nil {
+				if tt.wantErr {
+					return
+				}
+				t.Fatalf("unexpected error: %v", err)
+			}
+			if tt.wantErr {
+				t.Fatal("expected non-nil error")
+			}
+			if r == nil {
+				t.Errorf("got nil error and nil SigningKey")
+			}
+		})
+	}
+}
+
+type testServer struct {
+	roots []rootKeyPair
+	sign  []signingKeyPair
+	files map[string][]byte
+	srv   *httptest.Server
+}
+
+func newTestServer(t *testing.T) *testServer {
+	var roots []rootKeyPair
+	for i := 0; i < 3; i++ {
+		roots = append(roots, newRootKeyPair(t))
+	}
+
+	ts := &testServer{
+		roots: roots,
+		sign:  []signingKeyPair{newSigningKeyPair(t)},
+	}
+	ts.reset()
+	ts.srv = httptest.NewServer(ts)
+	t.Cleanup(ts.srv.Close)
+	return ts
+}
+
+func (s *testServer) client(t *testing.T) *Client {
+	roots := make([]ed25519.PublicKey, 0, len(s.roots))
+	for _, r := range s.roots {
+		pub, err := parseSinglePublicKey(r.pubRaw, pemTypeRootPublic)
+		if err != nil {
+			t.Fatalf("parsePublicKey: %v", err)
+		}
+		roots = append(roots, pub)
+	}
+	u, err := url.Parse(s.srv.URL)
+	if err != nil {
+		t.Fatal(err)
+	}
+	return &Client{
+		logf:     t.Logf,
+		roots:    roots,
+		pkgsAddr: u,
+	}
+}
+
+func (s *testServer) ServeHTTP(w http.ResponseWriter, r *http.Request) {
+	path := strings.TrimPrefix(r.URL.Path, "/")
+	data, ok := s.files[path]
+	if !ok {
+		http.NotFound(w, r)
+		return
+	}
+	w.Write(data)
+}
+
+func (s *testServer) addSigned(name string, data []byte) {
+	s.files[name] = data
+	s.files[name+".sig"] = s.sign[0].sign(data)
+}
+
+func (s *testServer) add(name string, data []byte) {
+	s.files[name] = data
+}
+
+func (s *testServer) reset() {
+	s.files = make(map[string][]byte)
+	s.resignSigningKeys()
+}
+
+func (s *testServer) resignSigningKeys() {
+	var pubs [][]byte
+	for _, k := range s.sign {
+		pubs = append(pubs, k.pubRaw)
+	}
+	bundle := bytes.Join(pubs, []byte("\n"))
+	sig := s.roots[0].sign(bundle)
+	s.files["distsign.pub"] = bundle
+	s.files["distsign.pub.sig"] = sig
+}
+
+type rootKeyPair struct {
+	*RootKey
+	keyPair
+}
+
+func newRootKeyPair(t *testing.T) rootKeyPair {
+	privRaw, pubRaw, err := GenerateRootKey()
+	if err != nil {
+		t.Fatalf("GenerateRootKey: %v", err)
+	}
+	kp := keyPair{
+		privRaw: privRaw,
+		pubRaw:  pubRaw,
+	}
+	priv, err := parsePrivateKey(kp.privRaw, pemTypeRootPrivate)
+	if err != nil {
+		t.Fatalf("parsePrivateKey: %v", err)
+	}
+	return rootKeyPair{
+		RootKey: &RootKey{k: priv},
+		keyPair: kp,
+	}
+}
+
+func (s rootKeyPair) sign(bundle []byte) []byte {
+	sig, err := s.SignSigningKeys(bundle)
+	if err != nil {
+		panic(err)
+	}
+	return sig
+}
+
+type signingKeyPair struct {
+	*SigningKey
+	keyPair
+}
+
+func newSigningKeyPair(t *testing.T) signingKeyPair {
+	privRaw, pubRaw, err := GenerateSigningKey()
+	if err != nil {
+		t.Fatalf("GenerateSigningKey: %v", err)
+	}
+	kp := keyPair{
+		privRaw: privRaw,
+		pubRaw:  pubRaw,
+	}
+	priv, err := parsePrivateKey(kp.privRaw, pemTypeSigningPrivate)
+	if err != nil {
+		t.Fatalf("parsePrivateKey: %v", err)
+	}
+	return signingKeyPair{
+		SigningKey: &SigningKey{k: priv},
+		keyPair:    kp,
+	}
+}
+
+func (s signingKeyPair) sign(blob []byte) []byte {
+	hash := blake2s.Sum256(blob)
+	sig, err := s.SignPackageHash(hash[:], int64(len(blob)))
+	if err != nil {
+		panic(err)
+	}
+	return sig
+}
+
+type keyPair struct {
+	privRaw []byte
+	pubRaw  []byte
+}

clientupdate/distsign/roots.go

@@ -0,0 +1,54 @@
+// Copyright (c) Tailscale Inc & AUTHORS
+// SPDX-License-Identifier: BSD-3-Clause
+
+package distsign
+
+import (
+	"crypto/ed25519"
+	"embed"
+	"errors"
+	"fmt"
+	"path"
+	"path/filepath"
+	"sync"
+)
+
+//go:embed roots
+var rootsFS embed.FS
+
+var roots = sync.OnceValue(func() []ed25519.PublicKey {
+	roots, err := parseRoots()
+	if err != nil {
+		panic(err)
+	}
+	return roots
+})
+
+func parseRoots() ([]ed25519.PublicKey, error) {
+	files, err := rootsFS.ReadDir("roots")
+	if err != nil {
+		return nil, err
+	}
+	var keys []ed25519.PublicKey
+	for _, f := range files {
+		if !f.Type().IsRegular() {
+			continue
+		}
+		if filepath.Ext(f.Name()) != ".pem" {
+			continue
+		}
+		raw, err := rootsFS.ReadFile(path.Join("roots", f.Name()))
+		if err != nil {
+			return nil, err
+		}
+		key, err := parseSinglePublicKey(raw, pemTypeRootPublic)
+		if err != nil {
+			return nil, fmt.Errorf("parsing root key %q: %w", f.Name(), err)
+		}
+		keys = append(keys, key)
+	}
+	if len(keys) == 0 {
+		return nil, errors.New("no embedded root keys, please check clientupdate/distsign/roots/")
+	}
+	return keys, nil
+}

clientupdate/distsign/roots/distsign-dev-root-pub.pem

@@ -0,0 +1,3 @@
+-----BEGIN ROOT PUBLIC KEY-----
+Muw5GkO5mASsJ7k6kS+svfuanr6XcW9I7fPGtyqOTeI=
+-----END ROOT PUBLIC KEY-----

clientupdate/distsign/roots_test.go

@@ -0,0 +1,16 @@
+// Copyright (c) Tailscale Inc & AUTHORS
+// SPDX-License-Identifier: BSD-3-Clause
+
+package distsign
+
+import "testing"
+
+func TestParseRoots(t *testing.T) {
+	roots, err := parseRoots()
+	if err != nil {
+		t.Fatal(err)
+	}
+	if len(roots) == 0 {
+		t.Error("parseRoots returned no root keys")
+	}
+}

clientupdate/systemd_linux.go

@@ -0,0 +1,37 @@
+// Copyright (c) Tailscale Inc & AUTHORS
+// SPDX-License-Identifier: BSD-3-Clause
+
+package clientupdate
+
+import (
+	"context"
+	"errors"
+	"fmt"
+
+	"github.com/coreos/go-systemd/v22/dbus"
+)
+
+func restartSystemdUnit(ctx context.Context) error {
+	c, err := dbus.NewWithContext(ctx)
+	if err != nil {
+		// Likely not a systemd-managed distro.
+		return errors.ErrUnsupported
+	}
+	defer c.Close()
+	if err := c.ReloadContext(ctx); err != nil {
+		return fmt.Errorf("failed to reload tailsacled.service: %w", err)
+	}
+	ch := make(chan string, 1)
+	if _, err := c.RestartUnitContext(ctx, "tailscaled.service", "replace", ch); err != nil {
+		return fmt.Errorf("failed to restart tailsacled.service: %w", err)
+	}
+	select {
+	case res := <-ch:
+		if res != "done" {
+			return fmt.Errorf("systemd service restart failed with result %q", res)
+		}
+	case <-ctx.Done():
+		return ctx.Err()
+	}
+	return nil
+}

clientupdate/systemd_other.go

@@ -0,0 +1,15 @@
+// Copyright (c) Tailscale Inc & AUTHORS
+// SPDX-License-Identifier: BSD-3-Clause
+
+//go:build !linux
+
+package clientupdate
+
+import (
+	"context"
+	"errors"
+)
+
+func restartSystemdUnit(ctx context.Context) error {
+	return errors.ErrUnsupported
+}

cmd/cloner/cloner.go

@@ -126,8 +126,8 @@ func gen(buf *bytes.Buffer, it *codegen.ImportTracker, typ *types.Named) {
 				writef("for i := range dst.%s {", fname)
 				if ptr, isPtr := ft.Elem().(*types.Pointer); isPtr {
 					if _, isBasic := ptr.Elem().Underlying().(*types.Basic); isBasic {
-						writef("\tx := *src.%s[i]", fname)
-						writef("\tdst.%s[i] = &x", fname)
+						it.Import("tailscale.com/types/ptr")
+						writef("\tdst.%s[i] = ptr.To(*src.%s[i])", fname, fname)
 					} else {
 						writef("\tdst.%s[i] = src.%s[i].Clone()", fname, fname)
 					}
@@ -145,41 +145,41 @@ func gen(buf *bytes.Buffer, it *codegen.ImportTracker, typ *types.Named) {
 				writef("dst.%s = src.%s.Clone()", fname, fname)
 				continue
 			}
-			n := it.QualifiedName(ft.Elem())
+			it.Import("tailscale.com/types/ptr")
 			writef("if dst.%s != nil {", fname)
-			writef("\tdst.%s = new(%s)", fname, n)
-			writef("\t*dst.%s = *src.%s", fname, fname)
+			writef("\tdst.%s = ptr.To(*src.%s)", fname, fname)
 			if codegen.ContainsPointers(ft.Elem()) {
 				writef("\t" + `panic("TODO pointers in pointers")`)
 			}
 			writef("}")
 		case *types.Map:
 			elem := ft.Elem()
-			writef("if dst.%s != nil {", fname)
-			writef("\tdst.%s = map[%s]%s{}", fname, it.QualifiedName(ft.Key()), it.QualifiedName(elem))
 			if sliceType, isSlice := elem.(*types.Slice); isSlice {
 				n := it.QualifiedName(sliceType.Elem())
+				writef("if dst.%s != nil {", fname)
+				writef("\tdst.%s = map[%s]%s{}", fname, it.QualifiedName(ft.Key()), it.QualifiedName(elem))
 				writef("\tfor k := range src.%s {", fname)
 				// use zero-length slice instead of nil to ensure
 				// the key is always copied.
 				writef("\t\tdst.%s[k] = append([]%s{}, src.%s[k]...)", fname, n, fname)
 				writef("\t}")
+				writef("}")
 			} else if codegen.ContainsPointers(elem) {
+				writef("if dst.%s != nil {", fname)
+				writef("\tdst.%s = map[%s]%s{}", fname, it.QualifiedName(ft.Key()), it.QualifiedName(elem))
 				writef("\tfor k, v := range src.%s {", fname)
 				switch elem.(type) {
 				case *types.Pointer:
 					writef("\t\tdst.%s[k] = v.Clone()", fname)
 				default:
-					writef("\t\tv2 := v.Clone()")
-					writef("\t\tdst.%s[k] = *v2", fname)
+					writef("\t\tdst.%s[k] = *(v.Clone())", fname)
 				}
 				writef("\t}")
+				writef("}")
 			} else {
-				writef("\tfor k, v := range src.%s {", fname)
-				writef("\t\tdst.%s[k] = v", fname)
-				writef("\t}")
+				it.Import("maps")
+				writef("\tdst.%s = maps.Clone(src.%s)", fname, fname)
 			}
-			writef("}")
 		default:
 			writef(`panic("TODO: %s (%T)")`, fname, ft)
 		}

cmd/containerboot/kube.go

@@ -7,9 +7,11 @@ package main
 
 import (
 	"context"
+	"encoding/json"
 	"fmt"
 	"log"
 	"net/http"
+	"net/netip"
 	"os"
 
 	"tailscale.com/kube"
@@ -32,7 +34,7 @@ func findKeyInKubeSecret(ctx context.Context, secretName string) (string, error)
 
 // storeDeviceInfo writes deviceID into the "device_id" data field of the kube
 // secret secretName.
-func storeDeviceInfo(ctx context.Context, secretName string, deviceID tailcfg.StableNodeID, fqdn string) error {
+func storeDeviceInfo(ctx context.Context, secretName string, deviceID tailcfg.StableNodeID, fqdn string, addresses []netip.Prefix) error {
 	// First check if the secret exists at all. Even if running on
 	// kubernetes, we do not necessarily store state in a k8s secret.
 	if _, err := kc.GetSecret(ctx, secretName); err != nil {
@@ -46,10 +48,20 @@ func storeDeviceInfo(ctx context.Context, secretName string, deviceID tailcfg.St
 		return err
 	}
 
+	var ips []string
+	for _, addr := range addresses {
+		ips = append(ips, addr.Addr().String())
+	}
+	deviceIPs, err := json.Marshal(ips)
+	if err != nil {
+		return err
+	}
+
 	m := &kube.Secret{
 		Data: map[string][]byte{
 			"device_id":   []byte(deviceID),
 			"device_fqdn": []byte(fqdn),
+			"device_ips":  deviceIPs,
 		},
 	}
 	return kc.StrategicMergePatchSecret(ctx, secretName, m, "tailscale-container")

cmd/containerboot/main.go

@@ -16,8 +16,11 @@
 //   - TS_ROUTES: subnet routes to advertise.
 //   - TS_DEST_IP: proxy all incoming Tailscale traffic to the given
 //     destination.
+//   - TS_TAILNET_TARGET_IP: proxy all incoming non-Tailscale traffic to the given
+//     destination.
 //   - TS_TAILSCALED_EXTRA_ARGS: extra arguments to 'tailscaled'.
-//   - TS_EXTRA_ARGS: extra arguments to 'tailscale up'.
+//   - TS_EXTRA_ARGS: extra arguments to 'tailscale login', these are not
+//     reset on restart.
 //   - TS_USERSPACE: run with userspace networking (the default)
 //     instead of kernel networking.
 //   - TS_STATE_DIR: the directory in which to store tailscaled
@@ -36,6 +39,11 @@
 //     logged in. If false (the default, for backwards
 //     compatibility), forcibly log in every time the
 //     container starts.
+//   - TS_SERVE_CONFIG: if specified, is the file path where the ipn.ServeConfig is located.
+//     It will be applied once tailscaled is up and running. If the file contains
+//     ${TS_CERT_DOMAIN}, it will be replaced with the value of the available FQDN.
+//     It cannot be used in conjunction with TS_DEST_IP. The file is watched for changes,
+//     and will be re-applied when it changes.
 //
 // When running on Kubernetes, containerboot defaults to storing state in the
 // "tailscale" kube secret. To store state on local disk instead, set
@@ -47,7 +55,9 @@
 package main
 
 import (
+	"bytes"
 	"context"
+	"encoding/json"
 	"errors"
 	"fmt"
 	"io/fs"
@@ -57,14 +67,18 @@ import (
 	"os/exec"
 	"os/signal"
 	"path/filepath"
+	"reflect"
 	"strconv"
 	"strings"
+	"sync/atomic"
 	"syscall"
 	"time"
 
+	"github.com/fsnotify/fsnotify"
 	"golang.org/x/sys/unix"
 	"tailscale.com/client/tailscale"
 	"tailscale.com/ipn"
+	"tailscale.com/types/ptr"
 	"tailscale.com/util/deephash"
 )
 
@@ -76,7 +90,9 @@ func main() {
 		AuthKey:         defaultEnvs([]string{"TS_AUTHKEY", "TS_AUTH_KEY"}, ""),
 		Hostname:        defaultEnv("TS_HOSTNAME", ""),
 		Routes:          defaultEnv("TS_ROUTES", ""),
+		ServeConfigPath: defaultEnv("TS_SERVE_CONFIG", ""),
 		ProxyTo:         defaultEnv("TS_DEST_IP", ""),
+		TailnetTargetIP: defaultEnv("TS_TAILNET_TARGET_IP", ""),
 		DaemonExtraArgs: defaultEnv("TS_TAILSCALED_EXTRA_ARGS", ""),
 		ExtraArgs:       defaultEnv("TS_EXTRA_ARGS", ""),
 		InKubernetes:    os.Getenv("KUBERNETES_SERVICE_HOST") != "",
@@ -95,12 +111,16 @@ func main() {
 		log.Fatal("TS_DEST_IP is not supported with TS_USERSPACE")
 	}
 
+	if cfg.TailnetTargetIP != "" && cfg.UserspaceMode {
+		log.Fatal("TS_TAILNET_TARGET_IP is not supported with TS_USERSPACE")
+	}
+
 	if !cfg.UserspaceMode {
 		if err := ensureTunFile(cfg.Root); err != nil {
 			log.Fatalf("Unable to create tuntap device file: %v", err)
 		}
-		if cfg.ProxyTo != "" || cfg.Routes != "" {
-			if err := ensureIPForwarding(cfg.Root, cfg.ProxyTo, cfg.Routes); err != nil {
+		if cfg.ProxyTo != "" || cfg.Routes != "" || cfg.TailnetTargetIP != "" {
+			if err := ensureIPForwarding(cfg.Root, cfg.ProxyTo, cfg.TailnetTargetIP, cfg.Routes); err != nil {
 				log.Printf("Failed to enable IP forwarding: %v", err)
 				log.Printf("To run tailscale as a proxy or router container, IP forwarding must be enabled.")
 				if cfg.InKubernetes {
@@ -119,18 +139,18 @@ func main() {
 	// Context is used for all setup stuff until we're in steady
 	// state, so that if something is hanging we eventually time out
 	// and crashloop the container.
-	ctx, cancel := context.WithTimeout(context.Background(), 60*time.Second)
+	bootCtx, cancel := context.WithTimeout(context.Background(), 60*time.Second)
 	defer cancel()
 
 	if cfg.InKubernetes && cfg.KubeSecret != "" {
-		canPatch, err := kc.CheckSecretPermissions(ctx, cfg.KubeSecret)
+		canPatch, err := kc.CheckSecretPermissions(bootCtx, cfg.KubeSecret)
 		if err != nil {
 			log.Fatalf("Some Kubernetes permissions are missing, please check your RBAC configuration: %v", err)
 		}
 		cfg.KubernetesCanPatch = canPatch
 
 		if cfg.AuthKey == "" {
-			key, err := findKeyInKubeSecret(ctx, cfg.KubeSecret)
+			key, err := findKeyInKubeSecret(bootCtx, cfg.KubeSecret)
 			if err != nil {
 				log.Fatalf("Getting authkey from kube secret: %v", err)
 			}
@@ -153,12 +173,12 @@ func main() {
 		}
 	}
 
-	client, daemonPid, err := startTailscaled(ctx, cfg)
+	client, daemonPid, err := startTailscaled(bootCtx, cfg)
 	if err != nil {
 		log.Fatalf("failed to bring up tailscale: %v", err)
 	}
 
-	w, err := client.WatchIPNBus(ctx, ipn.NotifyInitialNetMap|ipn.NotifyInitialPrefs|ipn.NotifyInitialState)
+	w, err := client.WatchIPNBus(bootCtx, ipn.NotifyInitialNetMap|ipn.NotifyInitialPrefs|ipn.NotifyInitialState)
 	if err != nil {
 		log.Fatalf("failed to watch tailscaled for updates: %v", err)
 	}
@@ -177,10 +197,10 @@ func main() {
 		}
 		didLogin = true
 		w.Close()
-		if err := tailscaleUp(ctx, cfg); err != nil {
+		if err := tailscaleLogin(bootCtx, cfg); err != nil {
 			return fmt.Errorf("failed to auth tailscale: %v", err)
 		}
-		w, err = client.WatchIPNBus(ctx, ipn.NotifyInitialNetMap|ipn.NotifyInitialState)
+		w, err = client.WatchIPNBus(bootCtx, ipn.NotifyInitialNetMap|ipn.NotifyInitialState)
 		if err != nil {
 			return fmt.Errorf("rewatching tailscaled for updates after auth: %v", err)
 		}
@@ -224,6 +244,20 @@ authLoop:
 
 	w.Close()
 
+	ctx, cancel := context.WithCancel(context.Background()) // no deadline now that we're in steady state
+	defer cancel()
+
+	// Now that we are authenticated, we can set/reset any of the
+	// settings that we need to.
+	if err := tailscaleSet(ctx, cfg); err != nil {
+		log.Fatalf("failed to auth tailscale: %v", err)
+	}
+	// Remove any serve config that may have been set by a previous
+	// run of containerboot.
+	if err := client.SetServeConfig(ctx, new(ipn.ServeConfig)); err != nil {
+		log.Fatalf("failed to unset serve config: %v", err)
+	}
+
 	if cfg.InKubernetes && cfg.KubeSecret != "" && cfg.KubernetesCanPatch && cfg.AuthOnce {
 		// We were told to only auth once, so any secret-bound
 		// authkey is no longer needed. We don't strictly need to
@@ -234,18 +268,24 @@ authLoop:
 		}
 	}
 
-	w, err = client.WatchIPNBus(context.Background(), ipn.NotifyInitialNetMap|ipn.NotifyInitialState)
+	w, err = client.WatchIPNBus(ctx, ipn.NotifyInitialNetMap|ipn.NotifyInitialState)
 	if err != nil {
 		log.Fatalf("rewatching tailscaled for updates after auth: %v", err)
 	}
 
 	var (
-		wantProxy         = cfg.ProxyTo != ""
+		wantProxy         = cfg.ProxyTo != "" || cfg.TailnetTargetIP != ""
 		wantDeviceInfo    = cfg.InKubernetes && cfg.KubeSecret != "" && cfg.KubernetesCanPatch
 		startupTasksDone  = false
 		currentIPs        deephash.Sum // tailscale IPs assigned to device
 		currentDeviceInfo deephash.Sum // device ID and fqdn
+
+		certDomain        = new(atomic.Pointer[string])
+		certDomainChanged = make(chan bool, 1)
 	)
+	if cfg.ServeConfigPath != "" {
+		go watchServeConfigChanges(ctx, cfg.ServeConfigPath, certDomainChanged, certDomain, client)
+	}
 	for {
 		n, err := w.Next()
 		if err != nil {
@@ -261,14 +301,35 @@ authLoop:
 			log.Fatalf("tailscaled left running state (now in state %q), exiting", *n.State)
 		}
 		if n.NetMap != nil {
-			if cfg.ProxyTo != "" && len(n.NetMap.Addresses) > 0 && deephash.Update(&currentIPs, &n.NetMap.Addresses) {
-				if err := installIPTablesRule(ctx, cfg.ProxyTo, n.NetMap.Addresses); err != nil {
-					log.Fatalf("installing proxy rules: %v", err)
+			addrs := n.NetMap.SelfNode.Addresses().AsSlice()
+			newCurrentIPs := deephash.Hash(&addrs)
+			ipsHaveChanged := newCurrentIPs != currentIPs
+			if cfg.ProxyTo != "" && len(addrs) > 0 && ipsHaveChanged {
+				log.Printf("Installing proxy rules")
+				if err := installIngressForwardingRule(ctx, cfg.ProxyTo, addrs); err != nil {
+					log.Fatalf("installing ingress proxy rules: %v", err)
 				}
 			}
-			deviceInfo := []any{n.NetMap.SelfNode.StableID, n.NetMap.SelfNode.Name}
+			if cfg.ServeConfigPath != "" && len(n.NetMap.DNS.CertDomains) > 0 {
+				cd := n.NetMap.DNS.CertDomains[0]
+				prev := certDomain.Swap(ptr.To(cd))
+				if prev == nil || *prev != cd {
+					select {
+					case certDomainChanged <- true:
+					default:
+					}
+				}
+			}
+			if cfg.TailnetTargetIP != "" && ipsHaveChanged && len(addrs) > 0 {
+				if err := installEgressForwardingRule(ctx, cfg.TailnetTargetIP, addrs); err != nil {
+					log.Fatalf("installing egress proxy rules: %v", err)
+				}
+			}
+			currentIPs = newCurrentIPs
+
+			deviceInfo := []any{n.NetMap.SelfNode.StableID(), n.NetMap.SelfNode.Name()}
 			if cfg.InKubernetes && cfg.KubernetesCanPatch && cfg.KubeSecret != "" && deephash.Update(&currentDeviceInfo, &deviceInfo) {
-				if err := storeDeviceInfo(ctx, cfg.KubeSecret, n.NetMap.SelfNode.StableID, n.NetMap.SelfNode.Name); err != nil {
+				if err := storeDeviceInfo(ctx, cfg.KubeSecret, n.NetMap.SelfNode.StableID(), n.NetMap.SelfNode.Name(), n.NetMap.SelfNode.Addresses().AsSlice()); err != nil {
 					log.Fatalf("storing device ID in kube secret: %v", err)
 				}
 			}
@@ -305,6 +366,79 @@ authLoop:
 	}
 }
 
+// watchServeConfigChanges watches path for changes, and when it sees one, reads
+// the serve config from it, replacing ${TS_CERT_DOMAIN} with certDomain, and
+// applies it to lc. It exits when ctx is canceled. cdChanged is a channel that
+// is written to when the certDomain changes, causing the serve config to be
+// re-read and applied.
+func watchServeConfigChanges(ctx context.Context, path string, cdChanged <-chan bool, certDomainAtomic *atomic.Pointer[string], lc *tailscale.LocalClient) {
+	if certDomainAtomic == nil {
+		panic("cd must not be nil")
+	}
+	var tickChan <-chan time.Time
+	w, err := fsnotify.NewWatcher()
+	if err != nil {
+		log.Printf("failed to create fsnotify watcher, timer-only mode: %v", err)
+		ticker := time.NewTicker(5 * time.Second)
+		defer ticker.Stop()
+		tickChan = ticker.C
+	} else {
+		defer w.Close()
+	}
+
+	if err := w.Add(filepath.Dir(path)); err != nil {
+		log.Fatalf("failed to add fsnotify watch: %v", err)
+	}
+	var certDomain string
+	var prevServeConfig *ipn.ServeConfig
+	for {
+		select {
+		case <-ctx.Done():
+			return
+		case <-cdChanged:
+			certDomain = *certDomainAtomic.Load()
+		case <-tickChan:
+		case <-w.Events:
+			// We can't do any reasonable filtering on the event because of how
+			// k8s handles these mounts. So just re-read the file and apply it
+			// if it's changed.
+		}
+		if certDomain == "" {
+			continue
+		}
+		sc, err := readServeConfig(path, certDomain)
+		if err != nil {
+			log.Fatalf("failed to read serve config: %v", err)
+		}
+		if prevServeConfig != nil && reflect.DeepEqual(sc, prevServeConfig) {
+			continue
+		}
+		log.Printf("Applying serve config")
+		if err := lc.SetServeConfig(ctx, sc); err != nil {
+			log.Fatalf("failed to set serve config: %v", err)
+		}
+		prevServeConfig = sc
+	}
+}
+
+// readServeConfig reads the ipn.ServeConfig from path, replacing
+// ${TS_CERT_DOMAIN} with certDomain.
+func readServeConfig(path, certDomain string) (*ipn.ServeConfig, error) {
+	if path == "" {
+		return nil, nil
+	}
+	j, err := os.ReadFile(path)
+	if err != nil {
+		return nil, err
+	}
+	j = bytes.ReplaceAll(j, []byte("${TS_CERT_DOMAIN}"), []byte(certDomain))
+	var sc ipn.ServeConfig
+	if err := json.Unmarshal(j, &sc); err != nil {
+		return nil, err
+	}
+	return &sc, nil
+}
+
 func startTailscaled(ctx context.Context, cfg *settings) (*tailscale.LocalClient, int, error) {
 	args := tailscaledArgs(cfg)
 	sigCh := make(chan os.Signal, 1)
@@ -385,32 +519,48 @@ func tailscaledArgs(cfg *settings) []string {
 	return args
 }
 
-// tailscaleUp uses cfg to run 'tailscale up'.
-func tailscaleUp(ctx context.Context, cfg *settings) error {
-	args := []string{"--socket=" + cfg.Socket, "up"}
+// tailscaleLogin uses cfg to run 'tailscale login' everytime containerboot
+// starts, or if TS_AUTH_ONCE is set, only the first time containerboot starts.
+func tailscaleLogin(ctx context.Context, cfg *settings) error {
+	args := []string{"--socket=" + cfg.Socket, "login"}
+	if cfg.AuthKey != "" {
+		args = append(args, "--authkey="+cfg.AuthKey)
+	}
+	if cfg.ExtraArgs != "" {
+		args = append(args, strings.Fields(cfg.ExtraArgs)...)
+	}
+	log.Printf("Running 'tailscale login'")
+	cmd := exec.CommandContext(ctx, "tailscale", args...)
+	cmd.Stdout = os.Stdout
+	cmd.Stderr = os.Stderr
+	if err := cmd.Run(); err != nil {
+		return fmt.Errorf("tailscale login failed: %v", err)
+	}
+	return nil
+}
+
+// tailscaleSet uses cfg to run 'tailscale set' to set any known configuration
+// options that are passed in via environment variables. This is run after the
+// node is in Running state.
+func tailscaleSet(ctx context.Context, cfg *settings) error {
+	args := []string{"--socket=" + cfg.Socket, "set"}
 	if cfg.AcceptDNS {
 		args = append(args, "--accept-dns=true")
 	} else {
 		args = append(args, "--accept-dns=false")
 	}
-	if cfg.AuthKey != "" {
-		args = append(args, "--authkey="+cfg.AuthKey)
-	}
 	if cfg.Routes != "" {
 		args = append(args, "--advertise-routes="+cfg.Routes)
 	}
 	if cfg.Hostname != "" {
 		args = append(args, "--hostname="+cfg.Hostname)
 	}
-	if cfg.ExtraArgs != "" {
-		args = append(args, strings.Fields(cfg.ExtraArgs)...)
-	}
-	log.Printf("Running 'tailscale up'")
+	log.Printf("Running 'tailscale set'")
 	cmd := exec.CommandContext(ctx, "tailscale", args...)
 	cmd.Stdout = os.Stdout
 	cmd.Stderr = os.Stderr
 	if err := cmd.Run(); err != nil {
-		return fmt.Errorf("tailscale up failed: %v", err)
+		return fmt.Errorf("tailscale set failed: %v", err)
 	}
 	return nil
 }
@@ -435,14 +585,25 @@ func ensureTunFile(root string) error {
 }
 
 // ensureIPForwarding enables IPv4/IPv6 forwarding for the container.
-func ensureIPForwarding(root, proxyTo, routes string) error {
+func ensureIPForwarding(root, clusterProxyTarget, tailnetTargetiP, routes string) error {
 	var (
 		v4Forwarding, v6Forwarding bool
 	)
-	if proxyTo != "" {
-		proxyIP, err := netip.ParseAddr(proxyTo)
+	if clusterProxyTarget != "" {
+		proxyIP, err := netip.ParseAddr(clusterProxyTarget)
 		if err != nil {
-			return fmt.Errorf("invalid proxy destination IP: %v", err)
+			return fmt.Errorf("invalid cluster destination IP: %v", err)
+		}
+		if proxyIP.Is4() {
+			v4Forwarding = true
+		} else {
+			v6Forwarding = true
+		}
+	}
+	if tailnetTargetiP != "" {
+		proxyIP, err := netip.ParseAddr(tailnetTargetiP)
+		if err != nil {
+			return fmt.Errorf("invalid tailnet destination IP: %v", err)
 		}
 		if proxyIP.Is4() {
 			v4Forwarding = true
@@ -492,7 +653,53 @@ func ensureIPForwarding(root, proxyTo, routes string) error {
 	return nil
 }
 
-func installIPTablesRule(ctx context.Context, dstStr string, tsIPs []netip.Prefix) error {
+func installEgressForwardingRule(ctx context.Context, dstStr string, tsIPs []netip.Prefix) error {
+	dst, err := netip.ParseAddr(dstStr)
+	if err != nil {
+		return err
+	}
+	argv0 := "iptables"
+	if dst.Is6() {
+		argv0 = "ip6tables"
+	}
+	var local string
+	for _, pfx := range tsIPs {
+		if !pfx.IsSingleIP() {
+			continue
+		}
+		if pfx.Addr().Is4() != dst.Is4() {
+			continue
+		}
+		local = pfx.Addr().String()
+		break
+	}
+	if local == "" {
+		return fmt.Errorf("no tailscale IP matching family of %s found in %v", dstStr, tsIPs)
+	}
+	// Technically, if the control server ever changes the IPs assigned to this
+	// node, we'll slowly accumulate iptables rules. This shouldn't happen, so
+	// for now we'll live with it.
+	// Set up a rule that ensures that all packets
+	// except for those received on tailscale0 interface is forwarded to
+	// destination address
+	cmdDNAT := exec.CommandContext(ctx, argv0, "-t", "nat", "-I", "PREROUTING", "1", "!", "-i", "tailscale0", "-j", "DNAT", "--to-destination", dstStr)
+	cmdDNAT.Stdout = os.Stdout
+	cmdDNAT.Stderr = os.Stderr
+	if err := cmdDNAT.Run(); err != nil {
+		return fmt.Errorf("executing iptables failed: %w", err)
+	}
+	// Set up a rule that ensures that all packets sent to the destination
+	// address will have the proxy's IP set as source IP
+	cmdSNAT := exec.CommandContext(ctx, argv0, "-t", "nat", "-I", "POSTROUTING", "1", "--destination", dstStr, "-j", "SNAT", "--to-source", local)
+	cmdSNAT.Stdout = os.Stdout
+	cmdSNAT.Stderr = os.Stderr
+	if err := cmdSNAT.Run(); err != nil {
+		return fmt.Errorf("setting up SNAT via iptables failed: %w", err)
+	}
+	return nil
+}
+
+func installIngressForwardingRule(ctx context.Context, dstStr string, tsIPs []netip.Prefix) error {
 	dst, err := netip.ParseAddr(dstStr)
 	if err != nil {
 		return err
@@ -529,10 +736,18 @@ func installIPTablesRule(ctx context.Context, dstStr string, tsIPs []netip.Prefi
 
 // settings is all the configuration for containerboot.
 type settings struct {
-	AuthKey            string
-	Hostname           string
-	Routes             string
-	ProxyTo            string
+	AuthKey  string
+	Hostname string
+	Routes   string
+	// ProxyTo is the destination IP to which all incoming
+	// Tailscale traffic should be proxied. If empty, no proxying
+	// is done. This is typically a locally reachable IP.
+	ProxyTo string
+	// TailnetTargetIP is the destination IP to which all incoming
+	// non-Tailscale traffic should be proxied. If empty, no
+	// proxying is done. This is typically a Tailscale IP.
+	TailnetTargetIP    string
+	ServeConfigPath    string
 	DaemonExtraArgs    string
 	ExtraArgs          string
 	InKubernetes       bool

cmd/containerboot/main_test.go

@@ -112,11 +112,11 @@ func TestContainerBoot(t *testing.T) {
 	runningNotify := &ipn.Notify{
 		State: ptr.To(ipn.Running),
 		NetMap: &netmap.NetworkMap{
-			SelfNode: &tailcfg.Node{
-				StableID: tailcfg.StableNodeID("myID"),
-				Name:     "test-node.test.ts.net",
-			},
-			Addresses: []netip.Prefix{netip.MustParsePrefix("100.64.0.1/32")},
+			SelfNode: (&tailcfg.Node{
+				StableID:  tailcfg.StableNodeID("myID"),
+				Name:      "test-node.test.ts.net",
+				Addresses: []netip.Prefix{netip.MustParsePrefix("100.64.0.1/32")},
+			}).View(),
 		},
 	}
 	tests := []struct {
@@ -134,11 +134,14 @@ func TestContainerBoot(t *testing.T) {
 				{
 					WantCmds: []string{
 						"/usr/bin/tailscaled --socket=/tmp/tailscaled.sock --state=mem: --statedir=/tmp --tun=userspace-networking",
-						"/usr/bin/tailscale --socket=/tmp/tailscaled.sock up --accept-dns=false",
+						"/usr/bin/tailscale --socket=/tmp/tailscaled.sock login",
 					},
 				},
 				{
 					Notify: runningNotify,
+					WantCmds: []string{
+						"/usr/bin/tailscale --socket=/tmp/tailscaled.sock set --accept-dns=false",
+					},
 				},
 			},
 		},
@@ -152,11 +155,14 @@ func TestContainerBoot(t *testing.T) {
 				{
 					WantCmds: []string{
 						"/usr/bin/tailscaled --socket=/tmp/tailscaled.sock --state=mem: --statedir=/tmp --tun=userspace-networking",
-						"/usr/bin/tailscale --socket=/tmp/tailscaled.sock up --accept-dns=false --authkey=tskey-key",
+						"/usr/bin/tailscale --socket=/tmp/tailscaled.sock login --authkey=tskey-key",
 					},
 				},
 				{
 					Notify: runningNotify,
+					WantCmds: []string{
+						"/usr/bin/tailscale --socket=/tmp/tailscaled.sock set --accept-dns=false",
+					},
 				},
 			},
 		},
@@ -170,11 +176,14 @@ func TestContainerBoot(t *testing.T) {
 				{
 					WantCmds: []string{
 						"/usr/bin/tailscaled --socket=/tmp/tailscaled.sock --state=mem: --statedir=/tmp --tun=userspace-networking",
-						"/usr/bin/tailscale --socket=/tmp/tailscaled.sock up --accept-dns=false --authkey=tskey-key",
+						"/usr/bin/tailscale --socket=/tmp/tailscaled.sock login --authkey=tskey-key",
 					},
 				},
 				{
 					Notify: runningNotify,
+					WantCmds: []string{
+						"/usr/bin/tailscale --socket=/tmp/tailscaled.sock set --accept-dns=false",
+					},
 				},
 			},
 		},
@@ -188,11 +197,14 @@ func TestContainerBoot(t *testing.T) {
 				{
 					WantCmds: []string{
 						"/usr/bin/tailscaled --socket=/tmp/tailscaled.sock --statedir=/tmp --tun=userspace-networking",
-						"/usr/bin/tailscale --socket=/tmp/tailscaled.sock up --accept-dns=false --authkey=tskey-key",
+						"/usr/bin/tailscale --socket=/tmp/tailscaled.sock login --authkey=tskey-key",
 					},
 				},
 				{
 					Notify: runningNotify,
+					WantCmds: []string{
+						"/usr/bin/tailscale --socket=/tmp/tailscaled.sock set --accept-dns=false",
+					},
 				},
 			},
 		},
@@ -206,7 +218,7 @@ func TestContainerBoot(t *testing.T) {
 				{
 					WantCmds: []string{
 						"/usr/bin/tailscaled --socket=/tmp/tailscaled.sock --state=mem: --statedir=/tmp --tun=userspace-networking",
-						"/usr/bin/tailscale --socket=/tmp/tailscaled.sock up --accept-dns=false --authkey=tskey-key --advertise-routes=1.2.3.0/24,10.20.30.0/24",
+						"/usr/bin/tailscale --socket=/tmp/tailscaled.sock login --authkey=tskey-key",
 					},
 				},
 				{
@@ -215,6 +227,9 @@ func TestContainerBoot(t *testing.T) {
 						"proc/sys/net/ipv4/ip_forward":          "0",
 						"proc/sys/net/ipv6/conf/all/forwarding": "0",
 					},
+					WantCmds: []string{
+						"/usr/bin/tailscale --socket=/tmp/tailscaled.sock set --accept-dns=false --advertise-routes=1.2.3.0/24,10.20.30.0/24",
+					},
 				},
 			},
 		},
@@ -229,7 +244,7 @@ func TestContainerBoot(t *testing.T) {
 				{
 					WantCmds: []string{
 						"/usr/bin/tailscaled --socket=/tmp/tailscaled.sock --state=mem: --statedir=/tmp",
-						"/usr/bin/tailscale --socket=/tmp/tailscaled.sock up --accept-dns=false --authkey=tskey-key --advertise-routes=1.2.3.0/24,10.20.30.0/24",
+						"/usr/bin/tailscale --socket=/tmp/tailscaled.sock login --authkey=tskey-key",
 					},
 				},
 				{
@@ -238,6 +253,9 @@ func TestContainerBoot(t *testing.T) {
 						"proc/sys/net/ipv4/ip_forward":          "1",
 						"proc/sys/net/ipv6/conf/all/forwarding": "0",
 					},
+					WantCmds: []string{
+						"/usr/bin/tailscale --socket=/tmp/tailscaled.sock set --accept-dns=false --advertise-routes=1.2.3.0/24,10.20.30.0/24",
+					},
 				},
 			},
 		},
@@ -252,7 +270,7 @@ func TestContainerBoot(t *testing.T) {
 				{
 					WantCmds: []string{
 						"/usr/bin/tailscaled --socket=/tmp/tailscaled.sock --state=mem: --statedir=/tmp",
-						"/usr/bin/tailscale --socket=/tmp/tailscaled.sock up --accept-dns=false --authkey=tskey-key --advertise-routes=::/64,1::/64",
+						"/usr/bin/tailscale --socket=/tmp/tailscaled.sock login --authkey=tskey-key",
 					},
 				},
 				{
@@ -261,6 +279,9 @@ func TestContainerBoot(t *testing.T) {
 						"proc/sys/net/ipv4/ip_forward":          "0",
 						"proc/sys/net/ipv6/conf/all/forwarding": "1",
 					},
+					WantCmds: []string{
+						"/usr/bin/tailscale --socket=/tmp/tailscaled.sock set --accept-dns=false --advertise-routes=::/64,1::/64",
+					},
 				},
 			},
 		},
@@ -275,7 +296,7 @@ func TestContainerBoot(t *testing.T) {
 				{
 					WantCmds: []string{
 						"/usr/bin/tailscaled --socket=/tmp/tailscaled.sock --state=mem: --statedir=/tmp",
-						"/usr/bin/tailscale --socket=/tmp/tailscaled.sock up --accept-dns=false --authkey=tskey-key --advertise-routes=::/64,1.2.3.0/24",
+						"/usr/bin/tailscale --socket=/tmp/tailscaled.sock login --authkey=tskey-key",
 					},
 				},
 				{
@@ -284,11 +305,14 @@ func TestContainerBoot(t *testing.T) {
 						"proc/sys/net/ipv4/ip_forward":          "1",
 						"proc/sys/net/ipv6/conf/all/forwarding": "1",
 					},
+					WantCmds: []string{
+						"/usr/bin/tailscale --socket=/tmp/tailscaled.sock set --accept-dns=false --advertise-routes=::/64,1.2.3.0/24",
+					},
 				},
 			},
 		},
 		{
-			Name: "proxy",
+			Name: "ingres proxy",
 			Env: map[string]string{
 				"TS_AUTHKEY":   "tskey-key",
 				"TS_DEST_IP":   "1.2.3.4",
@@ -298,17 +322,42 @@ func TestContainerBoot(t *testing.T) {
 				{
 					WantCmds: []string{
 						"/usr/bin/tailscaled --socket=/tmp/tailscaled.sock --state=mem: --statedir=/tmp",
-						"/usr/bin/tailscale --socket=/tmp/tailscaled.sock up --accept-dns=false --authkey=tskey-key",
+						"/usr/bin/tailscale --socket=/tmp/tailscaled.sock login --authkey=tskey-key",
 					},
 				},
 				{
 					Notify: runningNotify,
 					WantCmds: []string{
+						"/usr/bin/tailscale --socket=/tmp/tailscaled.sock set --accept-dns=false",
 						"/usr/bin/iptables -t nat -I PREROUTING 1 -d 100.64.0.1 -j DNAT --to-destination 1.2.3.4",
 					},
 				},
 			},
 		},
+		{
+			Name: "egress proxy",
+			Env: map[string]string{
+				"TS_AUTHKEY":           "tskey-key",
+				"TS_TAILNET_TARGET_IP": "100.99.99.99",
+				"TS_USERSPACE":         "false",
+			},
+			Phases: []phase{
+				{
+					WantCmds: []string{
+						"/usr/bin/tailscaled --socket=/tmp/tailscaled.sock --state=mem: --statedir=/tmp",
+						"/usr/bin/tailscale --socket=/tmp/tailscaled.sock login --authkey=tskey-key",
+					},
+				},
+				{
+					Notify: runningNotify,
+					WantCmds: []string{
+						"/usr/bin/tailscale --socket=/tmp/tailscaled.sock set --accept-dns=false",
+						"/usr/bin/iptables -t nat -I PREROUTING 1 ! -i tailscale0 -j DNAT --to-destination 100.99.99.99",
+						"/usr/bin/iptables -t nat -I POSTROUTING 1 --destination 100.99.99.99 -j SNAT --to-source 100.64.0.1",
+					},
+				},
+			},
+		},
 		{
 			Name: "authkey_once",
 			Env: map[string]string{
@@ -326,11 +375,14 @@ func TestContainerBoot(t *testing.T) {
 						State: ptr.To(ipn.NeedsLogin),
 					},
 					WantCmds: []string{
-						"/usr/bin/tailscale --socket=/tmp/tailscaled.sock up --accept-dns=false --authkey=tskey-key",
+						"/usr/bin/tailscale --socket=/tmp/tailscaled.sock login --authkey=tskey-key",
 					},
 				},
 				{
 					Notify: runningNotify,
+					WantCmds: []string{
+						"/usr/bin/tailscale --socket=/tmp/tailscaled.sock set --accept-dns=false",
+					},
 				},
 			},
 		},
@@ -347,7 +399,7 @@ func TestContainerBoot(t *testing.T) {
 				{
 					WantCmds: []string{
 						"/usr/bin/tailscaled --socket=/tmp/tailscaled.sock --state=kube:tailscale --statedir=/tmp --tun=userspace-networking",
-						"/usr/bin/tailscale --socket=/tmp/tailscaled.sock up --accept-dns=false --authkey=tskey-key",
+						"/usr/bin/tailscale --socket=/tmp/tailscaled.sock login --authkey=tskey-key",
 					},
 					WantKubeSecret: map[string]string{
 						"authkey": "tskey-key",
@@ -355,10 +407,14 @@ func TestContainerBoot(t *testing.T) {
 				},
 				{
 					Notify: runningNotify,
+					WantCmds: []string{
+						"/usr/bin/tailscale --socket=/tmp/tailscaled.sock set --accept-dns=false",
+					},
 					WantKubeSecret: map[string]string{
 						"authkey":     "tskey-key",
 						"device_fqdn": "test-node.test.ts.net",
 						"device_id":   "myID",
+						"device_ips":  `["100.64.0.1"]`,
 					},
 				},
 			},
@@ -378,12 +434,15 @@ func TestContainerBoot(t *testing.T) {
 				{
 					WantCmds: []string{
 						"/usr/bin/tailscaled --socket=/tmp/tailscaled.sock --statedir=/tmp --tun=userspace-networking",
-						"/usr/bin/tailscale --socket=/tmp/tailscaled.sock up --accept-dns=false --authkey=tskey-key",
+						"/usr/bin/tailscale --socket=/tmp/tailscaled.sock login --authkey=tskey-key",
 					},
 					WantKubeSecret: map[string]string{},
 				},
 				{
-					Notify:         runningNotify,
+					Notify: runningNotify,
+					WantCmds: []string{
+						"/usr/bin/tailscale --socket=/tmp/tailscaled.sock set --accept-dns=false",
+					},
 					WantKubeSecret: map[string]string{},
 				},
 			},
@@ -401,12 +460,15 @@ func TestContainerBoot(t *testing.T) {
 				{
 					WantCmds: []string{
 						"/usr/bin/tailscaled --socket=/tmp/tailscaled.sock --state=kube:tailscale --statedir=/tmp --tun=userspace-networking",
-						"/usr/bin/tailscale --socket=/tmp/tailscaled.sock up --accept-dns=false --authkey=tskey-key",
+						"/usr/bin/tailscale --socket=/tmp/tailscaled.sock login --authkey=tskey-key",
 					},
 					WantKubeSecret: map[string]string{},
 				},
 				{
-					Notify:         runningNotify,
+					Notify: runningNotify,
+					WantCmds: []string{
+						"/usr/bin/tailscale --socket=/tmp/tailscaled.sock set --accept-dns=false",
+					},
 					WantKubeSecret: map[string]string{},
 				},
 			},
@@ -436,7 +498,7 @@ func TestContainerBoot(t *testing.T) {
 						State: ptr.To(ipn.NeedsLogin),
 					},
 					WantCmds: []string{
-						"/usr/bin/tailscale --socket=/tmp/tailscaled.sock up --accept-dns=false --authkey=tskey-key",
+						"/usr/bin/tailscale --socket=/tmp/tailscaled.sock login --authkey=tskey-key",
 					},
 					WantKubeSecret: map[string]string{
 						"authkey": "tskey-key",
@@ -444,9 +506,13 @@ func TestContainerBoot(t *testing.T) {
 				},
 				{
 					Notify: runningNotify,
+					WantCmds: []string{
+						"/usr/bin/tailscale --socket=/tmp/tailscaled.sock set --accept-dns=false",
+					},
 					WantKubeSecret: map[string]string{
 						"device_fqdn": "test-node.test.ts.net",
 						"device_id":   "myID",
+						"device_ips":  `["100.64.0.1"]`,
 					},
 				},
 			},
@@ -464,7 +530,7 @@ func TestContainerBoot(t *testing.T) {
 				{
 					WantCmds: []string{
 						"/usr/bin/tailscaled --socket=/tmp/tailscaled.sock --state=kube:tailscale --statedir=/tmp --tun=userspace-networking",
-						"/usr/bin/tailscale --socket=/tmp/tailscaled.sock up --accept-dns=false --authkey=tskey-key",
+						"/usr/bin/tailscale --socket=/tmp/tailscaled.sock login --authkey=tskey-key",
 					},
 					WantKubeSecret: map[string]string{
 						"authkey": "tskey-key",
@@ -472,27 +538,32 @@ func TestContainerBoot(t *testing.T) {
 				},
 				{
 					Notify: runningNotify,
+					WantCmds: []string{
+						"/usr/bin/tailscale --socket=/tmp/tailscaled.sock set --accept-dns=false",
+					},
 					WantKubeSecret: map[string]string{
 						"authkey":     "tskey-key",
 						"device_fqdn": "test-node.test.ts.net",
 						"device_id":   "myID",
+						"device_ips":  `["100.64.0.1"]`,
 					},
 				},
 				{
 					Notify: &ipn.Notify{
 						State: ptr.To(ipn.Running),
 						NetMap: &netmap.NetworkMap{
-							SelfNode: &tailcfg.Node{
-								StableID: tailcfg.StableNodeID("newID"),
-								Name:     "new-name.test.ts.net",
-							},
-							Addresses: []netip.Prefix{netip.MustParsePrefix("100.64.0.1/32")},
+							SelfNode: (&tailcfg.Node{
+								StableID:  tailcfg.StableNodeID("newID"),
+								Name:      "new-name.test.ts.net",
+								Addresses: []netip.Prefix{netip.MustParsePrefix("100.64.0.1/32")},
+							}).View(),
 						},
 					},
 					WantKubeSecret: map[string]string{
 						"authkey":     "tskey-key",
 						"device_fqdn": "new-name.test.ts.net",
 						"device_id":   "newID",
+						"device_ips":  `["100.64.0.1"]`,
 					},
 				},
 			},
@@ -507,11 +578,14 @@ func TestContainerBoot(t *testing.T) {
 				{
 					WantCmds: []string{
 						"/usr/bin/tailscaled --socket=/tmp/tailscaled.sock --state=mem: --statedir=/tmp --tun=userspace-networking --socks5-server=localhost:1080 --outbound-http-proxy-listen=localhost:8080",
-						"/usr/bin/tailscale --socket=/tmp/tailscaled.sock up --accept-dns=false",
+						"/usr/bin/tailscale --socket=/tmp/tailscaled.sock login",
 					},
 				},
 				{
 					Notify: runningNotify,
+					WantCmds: []string{
+						"/usr/bin/tailscale --socket=/tmp/tailscaled.sock set --accept-dns=false",
+					},
 				},
 			},
 		},
@@ -524,11 +598,14 @@ func TestContainerBoot(t *testing.T) {
 				{
 					WantCmds: []string{
 						"/usr/bin/tailscaled --socket=/tmp/tailscaled.sock --state=mem: --statedir=/tmp --tun=userspace-networking",
-						"/usr/bin/tailscale --socket=/tmp/tailscaled.sock up --accept-dns=true",
+						"/usr/bin/tailscale --socket=/tmp/tailscaled.sock login",
 					},
 				},
 				{
 					Notify: runningNotify,
+					WantCmds: []string{
+						"/usr/bin/tailscale --socket=/tmp/tailscaled.sock set --accept-dns=true",
+					},
 				},
 			},
 		},
@@ -542,10 +619,14 @@ func TestContainerBoot(t *testing.T) {
 				{
 					WantCmds: []string{
 						"/usr/bin/tailscaled --socket=/tmp/tailscaled.sock --state=mem: --statedir=/tmp --tun=userspace-networking --experiments=widgets",
-						"/usr/bin/tailscale --socket=/tmp/tailscaled.sock up --accept-dns=false --widget=rotated",
+						"/usr/bin/tailscale --socket=/tmp/tailscaled.sock login --widget=rotated",
 					},
-				}, {
+				},
+				{
 					Notify: runningNotify,
+					WantCmds: []string{
+						"/usr/bin/tailscale --socket=/tmp/tailscaled.sock set --accept-dns=false",
+					},
 				},
 			},
 		},
@@ -558,10 +639,14 @@ func TestContainerBoot(t *testing.T) {
 				{
 					WantCmds: []string{
 						"/usr/bin/tailscaled --socket=/tmp/tailscaled.sock --state=mem: --statedir=/tmp --tun=userspace-networking",
-						"/usr/bin/tailscale --socket=/tmp/tailscaled.sock up --accept-dns=false --hostname=my-server",
+						"/usr/bin/tailscale --socket=/tmp/tailscaled.sock login",
 					},
-				}, {
+				},
+				{
 					Notify: runningNotify,
+					WantCmds: []string{
+						"/usr/bin/tailscale --socket=/tmp/tailscaled.sock set --accept-dns=false --hostname=my-server",
+					},
 				},
 			},
 		},
@@ -602,7 +687,7 @@ func TestContainerBoot(t *testing.T) {
 				t.Fatalf("starting containerboot: %v", err)
 			}
 			defer func() {
-				cmd.Process.Signal(unix.SIGTERM)
+				cmd.Process.Signal(unix.SIGKILL)
 				cmd.Process.Wait()
 			}()
 
@@ -790,10 +875,17 @@ func (l *localAPI) Notify(n *ipn.Notify) {
 }
 
 func (l *localAPI) ServeHTTP(w http.ResponseWriter, r *http.Request) {
-	if r.Method != "GET" {
-		panic(fmt.Sprintf("unsupported method %q", r.Method))
-	}
-	if r.URL.Path != "/localapi/v0/watch-ipn-bus" {
+	switch r.URL.Path {
+	case "/localapi/v0/serve-config":
+		if r.Method != "POST" {
+			panic(fmt.Sprintf("unsupported method %q", r.Method))
+		}
+		return
+	case "/localapi/v0/watch-ipn-bus":
+		if r.Method != "GET" {
+			panic(fmt.Sprintf("unsupported method %q", r.Method))
+		}
+	default:
 		panic(fmt.Sprintf("unsupported path %q", r.URL.Path))
 	}
 

cmd/derper/bootstrap_dns.go

@@ -25,6 +25,7 @@ var (
 	dnsCache            syncs.AtomicValue[dnsEntryMap]
 	dnsCacheBytes       syncs.AtomicValue[[]byte] // of JSON
 	unpublishedDNSCache syncs.AtomicValue[dnsEntryMap]
+	bootstrapLookupMap  syncs.Map[string, bool]
 )
 
 var (
@@ -35,6 +36,12 @@ var (
 	unpublishedDNSMisses = expvar.NewInt("counter_bootstrap_dns_unpublished_misses")
 )
 
+func init() {
+	expvar.Publish("counter_bootstrap_dns_queried_domains", expvar.Func(func() any {
+		return bootstrapLookupMap.Len()
+	}))
+}
+
 func refreshBootstrapDNSLoop() {
 	if *bootstrapDNS == "" && *unpublishedDNS == "" {
 		return
@@ -107,6 +114,7 @@ func handleBootstrapDNS(w http.ResponseWriter, r *http.Request) {
 
 	// Try answering a query from our hidden map first
 	if q := r.URL.Query().Get("q"); q != "" {
+		bootstrapLookupMap.Store(q, true)
 		if ips, ok := unpublishedDNSCache.Load()[q]; ok && len(ips) > 0 {
 			unpublishedDNSHits.Add(1)
 

cmd/derper/bootstrap_dns_test.go

@@ -98,6 +98,7 @@ func resetMetrics() {
 	publishedDNSMisses.Set(0)
 	unpublishedDNSHits.Set(0)
 	unpublishedDNSMisses.Set(0)
+	bootstrapLookupMap.Clear()
 }
 
 // Verify that we don't count an empty list in the unpublishedDNSCache as a
@@ -148,4 +149,17 @@ func TestUnpublishedDNSEmptyList(t *testing.T) {
 			t.Errorf("got misses=%d; want 0", v)
 		}
 	})
+
+}
+
+func TestLookupMetric(t *testing.T) {
+	d := []string{"a.io", "b.io", "c.io", "d.io", "e.io", "e.io", "e.io", "a.io"}
+	resetMetrics()
+	for _, q := range d {
+		_ = getBootstrapDNS(t, q)
+	}
+	// {"a.io": true, "b.io": true, "c.io": true, "d.io": true, "e.io": true}
+	if bootstrapLookupMap.Len() != 5 {
+		t.Errorf("bootstrapLookupMap.Len() want=5, got %v", bootstrapLookupMap.Len())
+	}
 }

cmd/derper/depaware.txt

@@ -13,6 +13,7 @@ tailscale.com/cmd/derper dependencies: (generated by github.com/tailscale/depawa
         github.com/beorn7/perks/quantile                             from github.com/prometheus/client_golang/prometheus
      💣 github.com/cespare/xxhash/v2                                 from github.com/prometheus/client_golang/prometheus
    L    github.com/coreos/go-iptables/iptables                       from tailscale.com/util/linuxfw
+   W 💣 github.com/dblohm7/wingoes                                   from tailscale.com/util/winutil
         github.com/fxamacker/cbor/v2                                 from tailscale.com/tka
         github.com/golang/groupcache/lru                             from tailscale.com/net/dnscache
         github.com/golang/protobuf/proto                             from github.com/matttproud/golang_protobuf_extensions/pbutil+
@@ -168,9 +169,6 @@ tailscale.com/cmd/derper dependencies: (generated by github.com/tailscale/depawa
         golang.org/x/crypto/nacl/box                                 from tailscale.com/types/key
         golang.org/x/crypto/nacl/secretbox                           from golang.org/x/crypto/nacl/box
         golang.org/x/crypto/salsa20/salsa                            from golang.org/x/crypto/nacl/box+
-        golang.org/x/exp/constraints                                 from golang.org/x/exp/slices
-        golang.org/x/exp/maps                                        from tailscale.com/types/views
-        golang.org/x/exp/slices                                      from tailscale.com/net/tsaddr+
    L    golang.org/x/net/bpf                                         from github.com/mdlayher/netlink+
         golang.org/x/net/dns/dnsmessage                              from net+
         golang.org/x/net/http/httpguts                               from net/http
@@ -193,6 +191,7 @@ tailscale.com/cmd/derper dependencies: (generated by github.com/tailscale/depawa
         golang.org/x/time/rate                                       from tailscale.com/cmd/derper+
         bufio                                                        from compress/flate+
         bytes                                                        from bufio+
+        cmp                                                          from slices
         compress/flate                                               from compress/gzip+
         compress/gzip                                                from internal/profile+
         container/list                                               from crypto/tls+
@@ -242,6 +241,7 @@ tailscale.com/cmd/derper dependencies: (generated by github.com/tailscale/depawa
         io/ioutil                                                    from github.com/mitchellh/go-ps+
         log                                                          from expvar+
         log/internal                                                 from log
+        maps                                                         from tailscale.com/types/views+
         math                                                         from compress/flate+
         math/big                                                     from crypto/dsa+
         math/bits                                                    from compress/flate+
@@ -269,6 +269,7 @@ tailscale.com/cmd/derper dependencies: (generated by github.com/tailscale/depawa
         runtime/metrics                                              from github.com/prometheus/client_golang/prometheus+
         runtime/pprof                                                from net/http/pprof
         runtime/trace                                                from net/http/pprof
+        slices                                                       from tailscale.com/ipn+
         sort                                                         from compress/flate+
         strconv                                                      from compress/flate+
         strings                                                      from bufio+

cmd/derper/websocket.go

@@ -50,7 +50,7 @@ func addWebSocketSupport(s *derp.Server, base http.Handler) http.Handler {
 			return
 		}
 		counterWebSocketAccepts.Add(1)
-		wc := wsconn.NetConn(r.Context(), c, websocket.MessageBinary)
+		wc := wsconn.NetConn(r.Context(), c, websocket.MessageBinary, r.RemoteAddr)
 		brw := bufio.NewReadWriter(bufio.NewReader(wc), bufio.NewWriter(wc))
 		s.Accept(r.Context(), wc, brw, r.RemoteAddr)
 	})

cmd/dist/dist.go

@@ -19,10 +19,10 @@ import (
 
 var synologyPackageCenter bool
 
-func getTargets(signers unixpkgs.Signers) ([]dist.Target, error) {
+func getTargets() ([]dist.Target, error) {
 	var ret []dist.Target
 
-	ret = append(ret, unixpkgs.Targets(signers)...)
+	ret = append(ret, unixpkgs.Targets(unixpkgs.Signers{})...)
 	// Synology packages can be built either for sideloading, or for
 	// distribution by Synology in their package center. When
 	// distributed through the package center, apps can request
@@ -33,7 +33,7 @@ func getTargets(signers unixpkgs.Signers) ([]dist.Target, error) {
 	// Since only we can provide packages to Synology for
 	// distribution, we default to building the "sideload" variant of
 	// packages that we distribute on pkgs.tailscale.com.
-	ret = append(ret, synology.Targets(synologyPackageCenter)...)
+	ret = append(ret, synology.Targets(synologyPackageCenter, nil)...)
 	return ret, nil
 }
 

cmd/k8s-operator/ingress.go

@@ -0,0 +1,264 @@
+// Copyright (c) Tailscale Inc & AUTHORS
+// SPDX-License-Identifier: BSD-3-Clause
+
+//go:build !plan9
+
+package main
+
+import (
+	"context"
+	"fmt"
+	"strings"
+	"sync"
+
+	"go.uber.org/zap"
+	"golang.org/x/exp/slices"
+	corev1 "k8s.io/api/core/v1"
+	networkingv1 "k8s.io/api/networking/v1"
+	apierrors "k8s.io/apimachinery/pkg/api/errors"
+	"k8s.io/apimachinery/pkg/types"
+	"k8s.io/client-go/tools/record"
+	"sigs.k8s.io/controller-runtime/pkg/client"
+	"sigs.k8s.io/controller-runtime/pkg/reconcile"
+	"tailscale.com/ipn"
+	"tailscale.com/types/opt"
+	"tailscale.com/util/clientmetric"
+	"tailscale.com/util/set"
+)
+
+type IngressReconciler struct {
+	client.Client
+
+	recorder record.EventRecorder
+	ssr      *tailscaleSTSReconciler
+	logger   *zap.SugaredLogger
+
+	mu sync.Mutex // protects following
+
+	// managedIngresses is a set of all ingress resources that we're currently
+	// managing. This is only used for metrics.
+	managedIngresses set.Slice[types.UID]
+}
+
+var (
+	// gaugeIngressResources tracks the number of ingress resources that we're
+	// currently managing.
+	gaugeIngressResources = clientmetric.NewGauge("k8s_ingress_resources")
+)
+
+func (a *IngressReconciler) Reconcile(ctx context.Context, req reconcile.Request) (_ reconcile.Result, err error) {
+	logger := a.logger.With("ingress-ns", req.Namespace, "ingress-name", req.Name)
+	logger.Debugf("starting reconcile")
+	defer logger.Debugf("reconcile finished")
+
+	ing := new(networkingv1.Ingress)
+	err = a.Get(ctx, req.NamespacedName, ing)
+	if apierrors.IsNotFound(err) {
+		// Request object not found, could have been deleted after reconcile request.
+		logger.Debugf("ingress not found, assuming it was deleted")
+		return reconcile.Result{}, nil
+	} else if err != nil {
+		return reconcile.Result{}, fmt.Errorf("failed to get ing: %w", err)
+	}
+	if !ing.DeletionTimestamp.IsZero() || !a.shouldExpose(ing) {
+		logger.Debugf("ingress is being deleted or should not be exposed, cleaning up")
+		return reconcile.Result{}, a.maybeCleanup(ctx, logger, ing)
+	}
+
+	return reconcile.Result{}, a.maybeProvision(ctx, logger, ing)
+}
+
+func (a *IngressReconciler) maybeCleanup(ctx context.Context, logger *zap.SugaredLogger, ing *networkingv1.Ingress) error {
+	ix := slices.Index(ing.Finalizers, FinalizerName)
+	if ix < 0 {
+		logger.Debugf("no finalizer, nothing to do")
+		a.mu.Lock()
+		defer a.mu.Unlock()
+		a.managedIngresses.Remove(ing.UID)
+		gaugeIngressResources.Set(int64(a.managedIngresses.Len()))
+		return nil
+	}
+
+	if done, err := a.ssr.Cleanup(ctx, logger, childResourceLabels(ing.Name, ing.Namespace, "ingress")); err != nil {
+		return fmt.Errorf("failed to cleanup: %w", err)
+	} else if !done {
+		logger.Debugf("cleanup not done yet, waiting for next reconcile")
+		return nil
+	}
+
+	ing.Finalizers = append(ing.Finalizers[:ix], ing.Finalizers[ix+1:]...)
+	if err := a.Update(ctx, ing); err != nil {
+		return fmt.Errorf("failed to remove finalizer: %w", err)
+	}
+
+	// Unlike most log entries in the reconcile loop, this will get printed
+	// exactly once at the very end of cleanup, because the final step of
+	// cleanup removes the tailscale finalizer, which will make all future
+	// reconciles exit early.
+	logger.Infof("unexposed ingress from tailnet")
+	a.mu.Lock()
+	defer a.mu.Unlock()
+	a.managedIngresses.Remove(ing.UID)
+	gaugeIngressResources.Set(int64(a.managedIngresses.Len()))
+	return nil
+}
+
+// maybeProvision ensures that ing is exposed over tailscale, taking any actions
+// necessary to reach that state.
+//
+// This function adds a finalizer to ing, ensuring that we can handle orderly
+// deprovisioning later.
+func (a *IngressReconciler) maybeProvision(ctx context.Context, logger *zap.SugaredLogger, ing *networkingv1.Ingress) error {
+	if !slices.Contains(ing.Finalizers, FinalizerName) {
+		// This log line is printed exactly once during initial provisioning,
+		// because once the finalizer is in place this block gets skipped. So,
+		// this is a nice place to tell the operator that the high level,
+		// multi-reconcile operation is underway.
+		logger.Infof("exposing ingress over tailscale")
+		ing.Finalizers = append(ing.Finalizers, FinalizerName)
+		if err := a.Update(ctx, ing); err != nil {
+			return fmt.Errorf("failed to add finalizer: %w", err)
+		}
+	}
+	a.mu.Lock()
+	a.managedIngresses.Add(ing.UID)
+	gaugeIngressResources.Set(int64(a.managedIngresses.Len()))
+	a.mu.Unlock()
+
+	if !a.ssr.IsHTTPSEnabledOnTailnet() {
+		a.recorder.Event(ing, corev1.EventTypeWarning, "HTTPSNotEnabled", "HTTPS is not enabled on the tailnet; ingress may not work")
+	}
+
+	// magic443 is a fake hostname that we can use to tell containerboot to swap
+	// out with the real hostname once it's known.
+	const magic443 = "${TS_CERT_DOMAIN}:443"
+	sc := &ipn.ServeConfig{
+		TCP: map[uint16]*ipn.TCPPortHandler{
+			443: {
+				HTTPS: true,
+			},
+		},
+		Web: map[ipn.HostPort]*ipn.WebServerConfig{
+			magic443: {
+				Handlers: map[string]*ipn.HTTPHandler{},
+			},
+		},
+	}
+	if opt.Bool(ing.Annotations[AnnotationFunnel]).EqualBool(true) {
+		sc.AllowFunnel = map[ipn.HostPort]bool{
+			magic443: true,
+		}
+	}
+
+	web := sc.Web[magic443]
+	addIngressBackend := func(b *networkingv1.IngressBackend, path string) {
+		if b == nil {
+			return
+		}
+		if b.Service == nil {
+			a.recorder.Eventf(ing, corev1.EventTypeWarning, "InvalidIngressBackend", "backend for path %q is missing service", path)
+			return
+		}
+		var svc corev1.Service
+		if err := a.Get(ctx, types.NamespacedName{Namespace: ing.Namespace, Name: b.Service.Name}, &svc); err != nil {
+			a.recorder.Eventf(ing, corev1.EventTypeWarning, "InvalidIngressBackend", "failed to get service %q for path %q: %v", b.Service.Name, path, err)
+			return
+		}
+		if svc.Spec.ClusterIP == "" || svc.Spec.ClusterIP == "None" {
+			a.recorder.Eventf(ing, corev1.EventTypeWarning, "InvalidIngressBackend", "backend for path %q has invalid ClusterIP", path)
+			return
+		}
+		var port int32
+		if b.Service.Port.Name != "" {
+			for _, p := range svc.Spec.Ports {
+				if p.Name == b.Service.Port.Name {
+					port = p.Port
+					break
+				}
+			}
+		} else {
+			port = b.Service.Port.Number
+		}
+		if port == 0 {
+			a.recorder.Eventf(ing, corev1.EventTypeWarning, "InvalidIngressBackend", "backend for path %q has invalid port", path)
+			return
+		}
+		proto := "http://"
+		if port == 443 || b.Service.Port.Name == "https" {
+			proto = "https+insecure://"
+		}
+		web.Handlers[path] = &ipn.HTTPHandler{
+			Proxy: proto + svc.Spec.ClusterIP + ":" + fmt.Sprint(port) + path,
+		}
+	}
+	addIngressBackend(ing.Spec.DefaultBackend, "/")
+	for _, rule := range ing.Spec.Rules {
+		if rule.Host != "" {
+			a.recorder.Eventf(ing, corev1.EventTypeWarning, "InvalidIngressBackend", "rule with host %q ignored, unsupported", rule.Host)
+			continue
+		}
+		for _, p := range rule.HTTP.Paths {
+			addIngressBackend(&p.Backend, p.Path)
+		}
+	}
+
+	crl := childResourceLabels(ing.Name, ing.Namespace, "ingress")
+	var tags []string
+	if tstr, ok := ing.Annotations[AnnotationTags]; ok {
+		tags = strings.Split(tstr, ",")
+	}
+	hostname := ing.Namespace + "-" + ing.Name + "-ingress"
+	if ing.Spec.TLS != nil && len(ing.Spec.TLS) > 0 && len(ing.Spec.TLS[0].Hosts) > 0 {
+		hostname, _, _ = strings.Cut(ing.Spec.TLS[0].Hosts[0], ".")
+	}
+
+	sts := &tailscaleSTSConfig{
+		Hostname:            hostname,
+		ParentResourceName:  ing.Name,
+		ParentResourceUID:   string(ing.UID),
+		ServeConfig:         sc,
+		Tags:                tags,
+		ChildResourceLabels: crl,
+	}
+
+	if _, err := a.ssr.Provision(ctx, logger, sts); err != nil {
+		return fmt.Errorf("failed to provision: %w", err)
+	}
+
+	_, tsHost, _, err := a.ssr.DeviceInfo(ctx, crl)
+	if err != nil {
+		return fmt.Errorf("failed to get device ID: %w", err)
+	}
+	if tsHost == "" {
+		logger.Debugf("no Tailscale hostname known yet, waiting for proxy pod to finish auth")
+		// No hostname yet. Wait for the proxy pod to auth.
+		ing.Status.LoadBalancer.Ingress = nil
+		if err := a.Status().Update(ctx, ing); err != nil {
+			return fmt.Errorf("failed to update ingress status: %w", err)
+		}
+		return nil
+	}
+
+	logger.Debugf("setting ingress hostname to %q", tsHost)
+	ing.Status.LoadBalancer.Ingress = []networkingv1.IngressLoadBalancerIngress{
+		{
+			Hostname: tsHost,
+			Ports: []networkingv1.IngressPortStatus{
+				{
+					Protocol: "TCP",
+					Port:     443,
+				},
+			},
+		},
+	}
+	if err := a.Status().Update(ctx, ing); err != nil {
+		return fmt.Errorf("failed to update ingress status: %w", err)
+	}
+	return nil
+}
+
+func (a *IngressReconciler) shouldExpose(ing *networkingv1.Ingress) bool {
+	return ing != nil &&
+		ing.Spec.IngressClassName != nil &&
+		*ing.Spec.IngressClassName == "tailscale"
+}

cmd/k8s-operator/manifests/operator.yaml

@@ -48,7 +48,10 @@ metadata:
   name: tailscale-operator
 rules:
 - apiGroups: [""]
-  resources: ["services", "services/status"]
+  resources: ["events", "services", "services/status"]
+  verbs: ["*"]
+- apiGroups: ["networking.k8s.io"]
+  resources: ["ingresses", "ingresses/status"]
   verbs: ["*"]
 ---
 apiVersion: rbac.authorization.k8s.io/v1

cmd/k8s-operator/manifests/proxy.yaml

@@ -2,7 +2,7 @@
 # at build time and then uses to construct Tailscale proxy pods.
 apiVersion: apps/v1
 kind: StatefulSet
-metadata:
+metadata: {}
 spec:
   replicas: 1
   template:

cmd/k8s-operator/manifests/userspace-proxy.yaml

@@ -0,0 +1,24 @@
+# This file is not a complete manifest, it's a skeleton that the operator embeds
+# at build time and then uses to construct Tailscale proxy pods.
+apiVersion: apps/v1
+kind: StatefulSet
+metadata: {}
+spec:
+  replicas: 1
+  template:
+    metadata:
+      deletionGracePeriodSeconds: 10
+    spec:
+      serviceAccountName: proxies
+      resources:
+        requests:
+          cpu: 1m
+          memory: 1Mi
+      containers:
+        - name: tailscale
+          imagePullPolicy: Always
+          env:
+            - name: TS_USERSPACE
+              value: "true"
+            - name: TS_AUTH_ONCE
+              value: "true"

cmd/k8s-operator/operator.go

@@ -1,16 +1,14 @@
 // Copyright (c) Tailscale Inc & AUTHORS
 // SPDX-License-Identifier: BSD-3-Clause
 
+//go:build !plan9
+
 // tailscale-operator provides a way to expose services running in a Kubernetes
 // cluster to your Tailnet.
 package main
 
 import (
 	"context"
-	"crypto/tls"
-	_ "embed"
-	"fmt"
-	"net/http"
 	"os"
 	"strings"
 	"time"
@@ -18,15 +16,12 @@ import (
 	"github.com/go-logr/zapr"
 	"go.uber.org/zap"
 	"go.uber.org/zap/zapcore"
-	"golang.org/x/exp/slices"
 	"golang.org/x/oauth2/clientcredentials"
 	appsv1 "k8s.io/api/apps/v1"
 	corev1 "k8s.io/api/core/v1"
-	apierrors "k8s.io/apimachinery/pkg/api/errors"
-	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
-	"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
+	networkingv1 "k8s.io/api/networking/v1"
 	"k8s.io/apimachinery/pkg/types"
-	"k8s.io/client-go/transport"
+	"k8s.io/client-go/rest"
 	"sigs.k8s.io/controller-runtime/pkg/builder"
 	"sigs.k8s.io/controller-runtime/pkg/cache"
 	"sigs.k8s.io/controller-runtime/pkg/client"
@@ -37,15 +32,12 @@ import (
 	"sigs.k8s.io/controller-runtime/pkg/manager"
 	"sigs.k8s.io/controller-runtime/pkg/manager/signals"
 	"sigs.k8s.io/controller-runtime/pkg/reconcile"
-	"sigs.k8s.io/yaml"
 	"tailscale.com/client/tailscale"
 	"tailscale.com/hostinfo"
 	"tailscale.com/ipn"
 	"tailscale.com/ipn/store/kubestore"
 	"tailscale.com/tsnet"
 	"tailscale.com/types/logger"
-	"tailscale.com/types/opt"
-	"tailscale.com/util/dnsname"
 	"tailscale.com/version"
 )
 
@@ -55,13 +47,8 @@ func main() {
 	tailscale.I_Acknowledge_This_API_Is_Unstable = true
 
 	var (
-		hostname           = defaultEnv("OPERATOR_HOSTNAME", "tailscale-operator")
-		kubeSecret         = defaultEnv("OPERATOR_SECRET", "")
-		operatorTags       = defaultEnv("OPERATOR_INITIAL_TAGS", "tag:k8s-operator")
 		tsNamespace        = defaultEnv("OPERATOR_NAMESPACE", "")
 		tslogging          = defaultEnv("OPERATOR_LOGGING", "info")
-		clientIDPath       = defaultEnv("CLIENT_ID_FILE", "")
-		clientSecretPath   = defaultEnv("CLIENT_SECRET_FILE", "")
 		image              = defaultEnv("PROXY_IMAGE", "tailscale/tailscale:latest")
 		priorityClassName  = defaultEnv("PROXY_PRIORITY_CLASS_NAME", "")
 		tags               = defaultEnv("PROXY_TAGS", "tag:k8s")
@@ -79,8 +66,29 @@ func main() {
 	}
 	zlog := kzap.NewRaw(opts...).Sugar()
 	logf.SetLogger(zapr.NewLogger(zlog.Desugar()))
-	startlog := zlog.Named("startup")
 
+	s, tsClient := initTSNet(zlog)
+	defer s.Close()
+	restConfig := config.GetConfigOrDie()
+	if shouldRunAuthProxy {
+		launchAuthProxy(zlog, restConfig, s)
+	}
+	startReconcilers(zlog, s, tsNamespace, restConfig, tsClient, image, priorityClassName, tags)
+}
+
+// initTSNet initializes the tsnet.Server and logs in to Tailscale. It uses the
+// CLIENT_ID_FILE and CLIENT_SECRET_FILE environment variables to authenticate
+// with Tailscale.
+func initTSNet(zlog *zap.SugaredLogger) (*tsnet.Server, *tailscale.Client) {
+	hostinfo.SetApp("k8s-operator")
+	var (
+		clientIDPath     = defaultEnv("CLIENT_ID_FILE", "")
+		clientSecretPath = defaultEnv("CLIENT_SECRET_FILE", "")
+		hostname         = defaultEnv("OPERATOR_HOSTNAME", "tailscale-operator")
+		kubeSecret       = defaultEnv("OPERATOR_SECRET", "")
+		operatorTags     = defaultEnv("OPERATOR_INITIAL_TAGS", "tag:k8s-operator")
+	)
+	startlog := zlog.Named("startup")
 	if clientIDPath == "" || clientSecretPath == "" {
 		startlog.Fatalf("CLIENT_ID_FILE and CLIENT_SECRET_FILE must be set")
 	}
@@ -100,12 +108,6 @@ func main() {
 	tsClient := tailscale.NewClient("-", nil)
 	tsClient.HTTPClient = credentials.Client(context.Background())
 
-	if shouldRunAuthProxy {
-		hostinfo.SetApp("k8s-operator-proxy")
-	} else {
-		hostinfo.SetApp("k8s-operator")
-	}
-
 	s := &tsnet.Server{
 		Hostname: hostname,
 		Logf:     zlog.Named("tailscaled").Debugf,
@@ -120,7 +122,6 @@ func main() {
 	if err := s.Start(); err != nil {
 		startlog.Fatalf("starting tailscale server: %v", err)
 	}
-	defer s.Close()
 	lc, err := s.LocalClient()
 	if err != nil {
 		startlog.Fatalf("getting local client: %v", err)
@@ -176,7 +177,16 @@ waitOnline:
 		}
 		time.Sleep(time.Second)
 	}
+	return s, tsClient
+}
 
+// startReconcilers starts the controller-runtime manager and registers the
+// ServiceReconciler.
+func startReconcilers(zlog *zap.SugaredLogger, s *tsnet.Server, tsNamespace string, restConfig *rest.Config, tsClient *tailscale.Client, image, priorityClassName, tags string) {
+	var (
+		isDefaultLoadBalancer = defaultBool("OPERATOR_DEFAULT_LOAD_BALANCER", false)
+	)
+	startlog := zlog.Named("startReconcilers")
 	// For secrets and statefulsets, we only get permission to touch the objects
 	// in the controller's own namespace. This cannot be expressed by
 	// .Watches(...) below, instead you have to add a per-type field selector to
@@ -186,7 +196,6 @@ waitOnline:
 	nsFilter := cache.ByObject{
 		Field: client.InNamespace(tsNamespace).AsSelector(),
 	}
-	restConfig := config.GetConfigOrDie()
 	mgr, err := manager.New(restConfig, manager.Options{
 		Cache: cache.Options{
 			ByObject: map[client.Object]cache.ByObject{
@@ -199,24 +208,11 @@ waitOnline:
 		startlog.Fatalf("could not create manager: %v", err)
 	}
 
-	sr := &ServiceReconciler{
-		Client:                 mgr.GetClient(),
-		tsClient:               tsClient,
-		defaultTags:            strings.Split(tags, ","),
-		operatorNamespace:      tsNamespace,
-		proxyImage:             image,
-		proxyPriorityClassName: priorityClassName,
-		logger:                 zlog.Named("service-reconciler"),
-	}
-
 	reconcileFilter := handler.EnqueueRequestsFromMapFunc(func(_ context.Context, o client.Object) []reconcile.Request {
 		ls := o.GetLabels()
 		if ls[LabelManaged] != "true" {
 			return nil
 		}
-		if ls[LabelParentType] != "svc" {
-			return nil
-		}
 		return []reconcile.Request{
 			{
 				NamespacedName: types.NamespacedName{
@@ -226,527 +222,52 @@ waitOnline:
 			},
 		}
 	})
+	eventRecorder := mgr.GetEventRecorderFor("tailscale-operator")
+	ssr := &tailscaleSTSReconciler{
+		Client:                 mgr.GetClient(),
+		tsnetServer:            s,
+		tsClient:               tsClient,
+		defaultTags:            strings.Split(tags, ","),
+		operatorNamespace:      tsNamespace,
+		proxyImage:             image,
+		proxyPriorityClassName: priorityClassName,
+	}
 	err = builder.
 		ControllerManagedBy(mgr).
 		For(&corev1.Service{}).
 		Watches(&appsv1.StatefulSet{}, reconcileFilter).
 		Watches(&corev1.Secret{}, reconcileFilter).
-		Complete(sr)
+		Complete(&ServiceReconciler{
+			ssr:                   ssr,
+			Client:                mgr.GetClient(),
+			logger:                zlog.Named("service-reconciler"),
+			isDefaultLoadBalancer: isDefaultLoadBalancer,
+		})
+	if err != nil {
+		startlog.Fatalf("could not create controller: %v", err)
+	}
+	err = builder.
+		ControllerManagedBy(mgr).
+		For(&networkingv1.Ingress{}).
+		Watches(&appsv1.StatefulSet{}, reconcileFilter).
+		Watches(&corev1.Secret{}, reconcileFilter).
+		Complete(&IngressReconciler{
+			ssr:      ssr,
+			recorder: eventRecorder,
+			Client:   mgr.GetClient(),
+			logger:   zlog.Named("ingress-reconciler"),
+		})
 	if err != nil {
 		startlog.Fatalf("could not create controller: %v", err)
 	}
 
 	startlog.Infof("Startup complete, operator running, version: %s", version.Long())
-	if shouldRunAuthProxy {
-		cfg, err := restConfig.TransportConfig()
-		if err != nil {
-			startlog.Fatalf("could not get rest.TransportConfig(): %v", err)
-		}
-
-		// Kubernetes uses SPDY for exec and port-forward, however SPDY is
-		// incompatible with HTTP/2; so disable HTTP/2 in the proxy.
-		tr := http.DefaultTransport.(*http.Transport).Clone()
-		tr.TLSClientConfig, err = transport.TLSConfigFor(cfg)
-		if err != nil {
-			startlog.Fatalf("could not get transport.TLSConfigFor(): %v", err)
-		}
-		tr.TLSNextProto = make(map[string]func(authority string, c *tls.Conn) http.RoundTripper)
-
-		rt, err := transport.HTTPWrappersForConfig(cfg, tr)
-		if err != nil {
-			startlog.Fatalf("could not get rest.TransportConfig(): %v", err)
-		}
-		go runAuthProxy(s, rt, zlog.Named("auth-proxy").Infof)
-	}
 	if err := mgr.Start(signals.SetupSignalHandler()); err != nil {
 		startlog.Fatalf("could not start manager: %v", err)
 	}
 }
 
-const (
-	LabelManaged         = "tailscale.com/managed"
-	LabelParentType      = "tailscale.com/parent-resource-type"
-	LabelParentName      = "tailscale.com/parent-resource"
-	LabelParentNamespace = "tailscale.com/parent-resource-ns"
-
-	FinalizerName = "tailscale.com/finalizer"
-
-	AnnotationExpose   = "tailscale.com/expose"
-	AnnotationTags     = "tailscale.com/tags"
-	AnnotationHostname = "tailscale.com/hostname"
-)
-
-// ServiceReconciler is a simple ControllerManagedBy example implementation.
-type ServiceReconciler struct {
-	client.Client
-	tsClient               tsClient
-	defaultTags            []string
-	operatorNamespace      string
-	proxyImage             string
-	proxyPriorityClassName string
-	logger                 *zap.SugaredLogger
-}
-
 type tsClient interface {
 	CreateKey(ctx context.Context, caps tailscale.KeyCapabilities) (string, *tailscale.Key, error)
-	DeleteDevice(ctx context.Context, id string) error
-}
-
-func childResourceLabels(parent *corev1.Service) map[string]string {
-	// You might wonder why we're using owner references, since they seem to be
-	// built for exactly this. Unfortunately, Kubernetes does not support
-	// cross-namespace ownership, by design. This means we cannot make the
-	// service being exposed the owner of the implementation details of the
-	// proxying. Instead, we have to do our own filtering and tracking with
-	// labels.
-	return map[string]string{
-		LabelManaged:         "true",
-		LabelParentName:      parent.GetName(),
-		LabelParentNamespace: parent.GetNamespace(),
-		LabelParentType:      "svc",
-	}
-}
-
-func (a *ServiceReconciler) Reconcile(ctx context.Context, req reconcile.Request) (_ reconcile.Result, err error) {
-	logger := a.logger.With("service-ns", req.Namespace, "service-name", req.Name)
-	logger.Debugf("starting reconcile")
-	defer logger.Debugf("reconcile finished")
-
-	svc := new(corev1.Service)
-	err = a.Get(ctx, req.NamespacedName, svc)
-	if apierrors.IsNotFound(err) {
-		// Request object not found, could have been deleted after reconcile request.
-		logger.Debugf("service not found, assuming it was deleted")
-		return reconcile.Result{}, nil
-	} else if err != nil {
-		return reconcile.Result{}, fmt.Errorf("failed to get svc: %w", err)
-	}
-	if !svc.DeletionTimestamp.IsZero() || !a.shouldExpose(svc) {
-		logger.Debugf("service is being deleted or should not be exposed, cleaning up")
-		return reconcile.Result{}, a.maybeCleanup(ctx, logger, svc)
-	}
-
-	return reconcile.Result{}, a.maybeProvision(ctx, logger, svc)
-}
-
-// maybeCleanup removes any existing resources related to serving svc over tailscale.
-//
-// This function is responsible for removing the finalizer from the service,
-// once all associated resources are gone.
-func (a *ServiceReconciler) maybeCleanup(ctx context.Context, logger *zap.SugaredLogger, svc *corev1.Service) error {
-	ix := slices.Index(svc.Finalizers, FinalizerName)
-	if ix < 0 {
-		logger.Debugf("no finalizer, nothing to do")
-		return nil
-	}
-
-	ml := childResourceLabels(svc)
-
-	// Need to delete the StatefulSet first, and delete it with foreground
-	// cascading deletion. That way, the pod that's writing to the Secret will
-	// stop running before we start looking at the Secret's contents, and
-	// assuming k8s ordering semantics don't mess with us, that should avoid
-	// tailscale device deletion races where we fail to notice a device that
-	// should be removed.
-	sts, err := getSingleObject[appsv1.StatefulSet](ctx, a.Client, a.operatorNamespace, ml)
-	if err != nil {
-		return fmt.Errorf("getting statefulset: %w", err)
-	}
-	if sts != nil {
-		if !sts.GetDeletionTimestamp().IsZero() {
-			// Deletion in progress, check again later. We'll get another
-			// notification when the deletion is complete.
-			logger.Debugf("waiting for statefulset %s/%s deletion", sts.GetNamespace(), sts.GetName())
-			return nil
-		}
-		err := a.DeleteAllOf(ctx, &appsv1.StatefulSet{}, client.InNamespace(a.operatorNamespace), client.MatchingLabels(ml), client.PropagationPolicy(metav1.DeletePropagationForeground))
-		if err != nil {
-			return fmt.Errorf("deleting statefulset: %w", err)
-		}
-		logger.Debugf("started deletion of statefulset %s/%s", sts.GetNamespace(), sts.GetName())
-		return nil
-	}
-
-	id, _, err := a.getDeviceInfo(ctx, svc)
-	if err != nil {
-		return fmt.Errorf("getting device info: %w", err)
-	}
-	if id != "" {
-		// TODO: handle case where the device is already deleted, but the secret
-		// is still around.
-		if err := a.tsClient.DeleteDevice(ctx, id); err != nil {
-			return fmt.Errorf("deleting device: %w", err)
-		}
-	}
-
-	types := []client.Object{
-		&corev1.Service{},
-		&corev1.Secret{},
-	}
-	for _, typ := range types {
-		if err := a.DeleteAllOf(ctx, typ, client.InNamespace(a.operatorNamespace), client.MatchingLabels(ml)); err != nil {
-			return err
-		}
-	}
-
-	svc.Finalizers = append(svc.Finalizers[:ix], svc.Finalizers[ix+1:]...)
-	if err := a.Update(ctx, svc); err != nil {
-		return fmt.Errorf("failed to remove finalizer: %w", err)
-	}
-
-	// Unlike most log entries in the reconcile loop, this will get printed
-	// exactly once at the very end of cleanup, because the final step of
-	// cleanup removes the tailscale finalizer, which will make all future
-	// reconciles exit early.
-	logger.Infof("unexposed service from tailnet")
-	return nil
-}
-
-// maybeProvision ensures that svc is exposed over tailscale, taking any actions
-// necessary to reach that state.
-//
-// This function adds a finalizer to svc, ensuring that we can handle orderly
-// deprovisioning later.
-func (a *ServiceReconciler) maybeProvision(ctx context.Context, logger *zap.SugaredLogger, svc *corev1.Service) error {
-	hostname, err := nameForService(svc)
-	if err != nil {
-		return err
-	}
-
-	if !slices.Contains(svc.Finalizers, FinalizerName) {
-		// This log line is printed exactly once during initial provisioning,
-		// because once the finalizer is in place this block gets skipped. So,
-		// this is a nice place to tell the operator that the high level,
-		// multi-reconcile operation is underway.
-		logger.Infof("exposing service over tailscale")
-		svc.Finalizers = append(svc.Finalizers, FinalizerName)
-		if err := a.Update(ctx, svc); err != nil {
-			return fmt.Errorf("failed to add finalizer: %w", err)
-		}
-	}
-
-	// Do full reconcile.
-	hsvc, err := a.reconcileHeadlessService(ctx, logger, svc)
-	if err != nil {
-		return fmt.Errorf("failed to reconcile headless service: %w", err)
-	}
-
-	tags := a.defaultTags
-	if tstr, ok := svc.Annotations[AnnotationTags]; ok {
-		tags = strings.Split(tstr, ",")
-	}
-	secretName, err := a.createOrGetSecret(ctx, logger, svc, hsvc, tags)
-	if err != nil {
-		return fmt.Errorf("failed to create or get API key secret: %w", err)
-	}
-	_, err = a.reconcileSTS(ctx, logger, svc, hsvc, secretName, hostname)
-	if err != nil {
-		return fmt.Errorf("failed to reconcile statefulset: %w", err)
-	}
-
-	if !a.hasLoadBalancerClass(svc) {
-		logger.Debugf("service is not a LoadBalancer, so not updating ingress")
-		return nil
-	}
-
-	_, tsHost, err := a.getDeviceInfo(ctx, svc)
-	if err != nil {
-		return fmt.Errorf("failed to get device ID: %w", err)
-	}
-	if tsHost == "" {
-		logger.Debugf("no Tailscale hostname known yet, waiting for proxy pod to finish auth")
-		// No hostname yet. Wait for the proxy pod to auth.
-		svc.Status.LoadBalancer.Ingress = nil
-		if err := a.Status().Update(ctx, svc); err != nil {
-			return fmt.Errorf("failed to update service status: %w", err)
-		}
-		return nil
-	}
-
-	logger.Debugf("setting ingress hostname to %q", tsHost)
-	svc.Status.LoadBalancer.Ingress = []corev1.LoadBalancerIngress{
-		{
-			Hostname: tsHost,
-		},
-	}
-	if err := a.Status().Update(ctx, svc); err != nil {
-		return fmt.Errorf("failed to update service status: %w", err)
-	}
-	return nil
-}
-
-func (a *ServiceReconciler) shouldExpose(svc *corev1.Service) bool {
-	// Headless services can't be exposed, since there is no ClusterIP to
-	// forward to.
-	if svc.Spec.ClusterIP == "" || svc.Spec.ClusterIP == "None" {
-		return false
-	}
-
-	return a.hasLoadBalancerClass(svc) || a.hasAnnotation(svc)
-}
-
-func (a *ServiceReconciler) hasLoadBalancerClass(svc *corev1.Service) bool {
-	return svc != nil &&
-		svc.Spec.Type == corev1.ServiceTypeLoadBalancer &&
-		svc.Spec.LoadBalancerClass != nil &&
-		*svc.Spec.LoadBalancerClass == "tailscale"
-}
-
-func (a *ServiceReconciler) hasAnnotation(svc *corev1.Service) bool {
-	return svc != nil &&
-		svc.Annotations[AnnotationExpose] == "true"
-}
-
-func (a *ServiceReconciler) reconcileHeadlessService(ctx context.Context, logger *zap.SugaredLogger, svc *corev1.Service) (*corev1.Service, error) {
-	hsvc := &corev1.Service{
-		ObjectMeta: metav1.ObjectMeta{
-			GenerateName: "ts-" + svc.Name + "-",
-			Namespace:    a.operatorNamespace,
-			Labels:       childResourceLabels(svc),
-		},
-		Spec: corev1.ServiceSpec{
-			ClusterIP: "None",
-			Selector: map[string]string{
-				"app": string(svc.UID),
-			},
-		},
-	}
-	logger.Debugf("reconciling headless service for StatefulSet")
-	return createOrUpdate(ctx, a.Client, a.operatorNamespace, hsvc, func(svc *corev1.Service) { svc.Spec = hsvc.Spec })
-}
-
-func (a *ServiceReconciler) createOrGetSecret(ctx context.Context, logger *zap.SugaredLogger, svc, hsvc *corev1.Service, tags []string) (string, error) {
-	secret := &corev1.Secret{
-		ObjectMeta: metav1.ObjectMeta{
-			// Hardcode a -0 suffix so that in future, if we support
-			// multiple StatefulSet replicas, we can provision -N for
-			// those.
-			Name:      hsvc.Name + "-0",
-			Namespace: a.operatorNamespace,
-			Labels:    childResourceLabels(svc),
-		},
-	}
-	if err := a.Get(ctx, client.ObjectKeyFromObject(secret), secret); err == nil {
-		logger.Debugf("secret %s/%s already exists", secret.GetNamespace(), secret.GetName())
-		return secret.Name, nil
-	} else if !apierrors.IsNotFound(err) {
-		return "", err
-	}
-
-	// Secret doesn't exist yet, create one. Initially it contains
-	// only the Tailscale authkey, but once Tailscale starts it'll
-	// also store the daemon state.
-	sts, err := getSingleObject[appsv1.StatefulSet](ctx, a.Client, a.operatorNamespace, childResourceLabels(svc))
-	if err != nil {
-		return "", err
-	}
-	if sts != nil {
-		// StatefulSet exists, so we have already created the secret.
-		// If the secret is missing, they should delete the StatefulSet.
-		logger.Errorf("Tailscale proxy secret doesn't exist, but the corresponding StatefulSet %s/%s already does. Something is wrong, please delete the StatefulSet.", sts.GetNamespace(), sts.GetName())
-		return "", nil
-	}
-	// Create API Key secret which is going to be used by the statefulset
-	// to authenticate with Tailscale.
-	logger.Debugf("creating authkey for new tailscale proxy")
-	authKey, err := a.newAuthKey(ctx, tags)
-	if err != nil {
-		return "", err
-	}
-
-	secret.StringData = map[string]string{
-		"authkey": authKey,
-	}
-	if err := a.Create(ctx, secret); err != nil {
-		return "", err
-	}
-	return secret.Name, nil
-}
-
-func (a *ServiceReconciler) getDeviceInfo(ctx context.Context, svc *corev1.Service) (id, hostname string, err error) {
-	sec, err := getSingleObject[corev1.Secret](ctx, a.Client, a.operatorNamespace, childResourceLabels(svc))
-	if err != nil {
-		return "", "", err
-	}
-	if sec == nil {
-		return "", "", nil
-	}
-	id = string(sec.Data["device_id"])
-	if id == "" {
-		return "", "", nil
-	}
-	// Kubernetes chokes on well-formed FQDNs with the trailing dot, so we have
-	// to remove it.
-	hostname = strings.TrimSuffix(string(sec.Data["device_fqdn"]), ".")
-	if hostname == "" {
-		return "", "", nil
-	}
-	return id, hostname, nil
-}
-
-func (a *ServiceReconciler) newAuthKey(ctx context.Context, tags []string) (string, error) {
-	caps := tailscale.KeyCapabilities{
-		Devices: tailscale.KeyDeviceCapabilities{
-			Create: tailscale.KeyDeviceCreateCapabilities{
-				Reusable:      false,
-				Preauthorized: true,
-				Tags:          tags,
-			},
-		},
-	}
-
-	key, _, err := a.tsClient.CreateKey(ctx, caps)
-	if err != nil {
-		return "", err
-	}
-	return key, nil
-}
-
-//go:embed manifests/proxy.yaml
-var proxyYaml []byte
-
-func (a *ServiceReconciler) reconcileSTS(ctx context.Context, logger *zap.SugaredLogger, parentSvc, headlessSvc *corev1.Service, authKeySecret, hostname string) (*appsv1.StatefulSet, error) {
-	var ss appsv1.StatefulSet
-	if err := yaml.Unmarshal(proxyYaml, &ss); err != nil {
-		return nil, fmt.Errorf("failed to unmarshal proxy spec: %w", err)
-	}
-	container := &ss.Spec.Template.Spec.Containers[0]
-	container.Image = a.proxyImage
-	container.Env = append(container.Env,
-		corev1.EnvVar{
-			Name:  "TS_DEST_IP",
-			Value: parentSvc.Spec.ClusterIP,
-		},
-		corev1.EnvVar{
-			Name:  "TS_KUBE_SECRET",
-			Value: authKeySecret,
-		},
-		corev1.EnvVar{
-			Name:  "TS_HOSTNAME",
-			Value: hostname,
-		})
-	ss.ObjectMeta = metav1.ObjectMeta{
-		Name:      headlessSvc.Name,
-		Namespace: a.operatorNamespace,
-		Labels:    childResourceLabels(parentSvc),
-	}
-	ss.Spec.ServiceName = headlessSvc.Name
-	ss.Spec.Selector = &metav1.LabelSelector{
-		MatchLabels: map[string]string{
-			"app": string(parentSvc.UID),
-		},
-	}
-	ss.Spec.Template.ObjectMeta.Labels = map[string]string{
-		"app": string(parentSvc.UID),
-	}
-	ss.Spec.Template.Spec.PriorityClassName = a.proxyPriorityClassName
-	logger.Debugf("reconciling statefulset %s/%s", ss.GetNamespace(), ss.GetName())
-	return createOrUpdate(ctx, a.Client, a.operatorNamespace, &ss, func(s *appsv1.StatefulSet) { s.Spec = ss.Spec })
-}
-
-// ptrObject is a type constraint for pointer types that implement
-// client.Object.
-type ptrObject[T any] interface {
-	client.Object
-	*T
-}
-
-// createOrUpdate adds obj to the k8s cluster, unless the object already exists,
-// in which case update is called to make changes to it. If update is nil, the
-// existing object is returned unmodified.
-//
-// obj is looked up by its Name and Namespace if Name is set, otherwise it's
-// looked up by labels.
-func createOrUpdate[T any, O ptrObject[T]](ctx context.Context, c client.Client, ns string, obj O, update func(O)) (O, error) {
-	var (
-		existing O
-		err      error
-	)
-	if obj.GetName() != "" {
-		existing = new(T)
-		existing.SetName(obj.GetName())
-		existing.SetNamespace(obj.GetNamespace())
-		err = c.Get(ctx, client.ObjectKeyFromObject(obj), existing)
-	} else {
-		existing, err = getSingleObject[T, O](ctx, c, ns, obj.GetLabels())
-	}
-	if err == nil && existing != nil {
-		if update != nil {
-			update(existing)
-			if err := c.Update(ctx, existing); err != nil {
-				return nil, err
-			}
-		}
-		return existing, nil
-	}
-	if err != nil && !apierrors.IsNotFound(err) {
-		return nil, fmt.Errorf("failed to get object: %w", err)
-	}
-	if err := c.Create(ctx, obj); err != nil {
-		return nil, err
-	}
-	return obj, nil
-}
-
-// getSingleObject searches for k8s objects of type T
-// (e.g. corev1.Service) with the given labels, and returns
-// it. Returns nil if no objects match the labels, and an error if
-// more than one object matches.
-func getSingleObject[T any, O ptrObject[T]](ctx context.Context, c client.Client, ns string, labels map[string]string) (O, error) {
-	ret := O(new(T))
-	kinds, _, err := c.Scheme().ObjectKinds(ret)
-	if err != nil {
-		return nil, err
-	}
-	if len(kinds) != 1 {
-		// TODO: the runtime package apparently has a "pick the best
-		// GVK" function somewhere that might be good enough?
-		return nil, fmt.Errorf("more than 1 GroupVersionKind for %T", ret)
-	}
-
-	gvk := kinds[0]
-	gvk.Kind += "List"
-	lst := unstructured.UnstructuredList{}
-	lst.SetGroupVersionKind(gvk)
-	if err := c.List(ctx, &lst, client.InNamespace(ns), client.MatchingLabels(labels)); err != nil {
-		return nil, err
-	}
-
-	if len(lst.Items) == 0 {
-		return nil, nil
-	}
-	if len(lst.Items) > 1 {
-		return nil, fmt.Errorf("found multiple matching %T objects", ret)
-	}
-	if err := c.Scheme().Convert(&lst.Items[0], ret, nil); err != nil {
-		return nil, err
-	}
-	return ret, nil
-}
-
-func defaultBool(envName string, defVal bool) bool {
-	vs := os.Getenv(envName)
-	if vs == "" {
-		return defVal
-	}
-	v, _ := opt.Bool(vs).Get()
-	return v
-}
-
-func defaultEnv(envName, defVal string) string {
-	v := os.Getenv(envName)
-	if v == "" {
-		return defVal
-	}
-	return v
-}
-
-func nameForService(svc *corev1.Service) (string, error) {
-	if h, ok := svc.Annotations[AnnotationHostname]; ok {
-		if err := dnsname.ValidLabel(h); err != nil {
-			return "", fmt.Errorf("invalid Tailscale hostname %q: %w", h, err)
-		}
-		return h, nil
-	}
-	return svc.Namespace + "-" + svc.Name, nil
+	DeleteDevice(ctx context.Context, nodeStableID string) error
 }

cmd/k8s-operator/operator_test.go

@@ -1,10 +1,13 @@
 // Copyright (c) Tailscale Inc & AUTHORS
 // SPDX-License-Identifier: BSD-3-Clause
 
+//go:build !plan9
+
 package main
 
 import (
 	"context"
+	"fmt"
 	"strings"
 	"sync"
 	"testing"
@@ -32,12 +35,15 @@ func TestLoadBalancerClass(t *testing.T) {
 		t.Fatal(err)
 	}
 	sr := &ServiceReconciler{
-		Client:            fc,
-		tsClient:          ft,
-		defaultTags:       []string{"tag:k8s"},
-		operatorNamespace: "operator-ns",
-		proxyImage:        "tailscale/tailscale",
-		logger:            zl.Sugar(),
+		Client: fc,
+		ssr: &tailscaleSTSReconciler{
+			Client:            fc,
+			tsClient:          ft,
+			defaultTags:       []string{"tag:k8s"},
+			operatorNamespace: "operator-ns",
+			proxyImage:        "tailscale/tailscale",
+		},
+		logger: zl.Sugar(),
 	}
 
 	// Create a service that we should manage, and check that the initial round
@@ -75,6 +81,7 @@ func TestLoadBalancerClass(t *testing.T) {
 		}
 		s.Data["device_id"] = []byte("ts-id-1234")
 		s.Data["device_fqdn"] = []byte("tailscale.device.name.")
+		s.Data["device_ips"] = []byte(`["100.99.98.97", "2c0a:8083:94d4:2012:3165:34a5:3616:5fdf"]`)
 	})
 	expectReconciled(t, sr, "default", "test")
 	want := &corev1.Service{
@@ -99,6 +106,9 @@ func TestLoadBalancerClass(t *testing.T) {
 					{
 						Hostname: "tailscale.device.name",
 					},
+					{
+						IP: "100.99.98.97",
+					},
 				},
 			},
 		},
@@ -144,6 +154,111 @@ func TestLoadBalancerClass(t *testing.T) {
 	}
 	expectEqual(t, fc, want)
 }
+func TestTailnetTargetIPAnnotation(t *testing.T) {
+	fc := fake.NewFakeClient()
+	ft := &fakeTSClient{}
+	zl, err := zap.NewDevelopment()
+	if err != nil {
+		t.Fatal(err)
+	}
+	tailnetTargetIP := "100.66.66.66"
+	sr := &ServiceReconciler{
+		Client: fc,
+		ssr: &tailscaleSTSReconciler{
+			Client:            fc,
+			tsClient:          ft,
+			defaultTags:       []string{"tag:k8s"},
+			operatorNamespace: "operator-ns",
+			proxyImage:        "tailscale/tailscale",
+		},
+		logger: zl.Sugar(),
+	}
+
+	// Create a service that we should manage, and check that the initial round
+	// of objects looks right.
+	mustCreate(t, fc, &corev1.Service{
+		ObjectMeta: metav1.ObjectMeta{
+			Name:      "test",
+			Namespace: "default",
+			// The apiserver is supposed to set the UID, but the fake client
+			// doesn't. So, set it explicitly because other code later depends
+			// on it being set.
+			UID: types.UID("1234-UID"),
+			Annotations: map[string]string{
+				AnnotationTailnetTargetIP: tailnetTargetIP,
+			},
+		},
+		Spec: corev1.ServiceSpec{
+			Type: corev1.ServiceTypeClusterIP,
+			Selector: map[string]string{
+				"foo": "bar",
+			},
+		},
+	})
+
+	expectReconciled(t, sr, "default", "test")
+
+	fullName, shortName := findGenName(t, fc, "default", "test")
+
+	expectEqual(t, fc, expectedSecret(fullName))
+	expectEqual(t, fc, expectedHeadlessService(shortName))
+	expectEqual(t, fc, expectedEgressSTS(shortName, fullName, tailnetTargetIP, "default-test", ""))
+	want := &corev1.Service{
+		TypeMeta: metav1.TypeMeta{
+			Kind:       "Service",
+			APIVersion: "v1",
+		},
+		ObjectMeta: metav1.ObjectMeta{
+			Name:       "test",
+			Namespace:  "default",
+			Finalizers: []string{"tailscale.com/finalizer"},
+			UID:        types.UID("1234-UID"),
+			Annotations: map[string]string{
+				AnnotationTailnetTargetIP: tailnetTargetIP,
+			},
+		},
+		Spec: corev1.ServiceSpec{
+			ExternalName: fmt.Sprintf("%s.operator-ns.svc", shortName),
+			Type:         corev1.ServiceTypeExternalName,
+			Selector:     nil,
+		},
+	}
+	expectEqual(t, fc, want)
+	expectEqual(t, fc, expectedSecret(fullName))
+	expectEqual(t, fc, expectedHeadlessService(shortName))
+	expectEqual(t, fc, expectedEgressSTS(shortName, fullName, tailnetTargetIP, "default-test", ""))
+
+	// Change the tailscale-target-ip annotation which should update the
+	// StatefulSet
+	tailnetTargetIP = "100.77.77.77"
+	mustUpdate(t, fc, "default", "test", func(s *corev1.Service) {
+		s.ObjectMeta.Annotations = map[string]string{
+			AnnotationTailnetTargetIP: tailnetTargetIP,
+		}
+	})
+
+	// Remove the tailscale-target-ip annotation which should make the
+	// operator clean up
+	mustUpdate(t, fc, "default", "test", func(s *corev1.Service) {
+		s.ObjectMeta.Annotations = map[string]string{}
+	})
+	expectReconciled(t, sr, "default", "test")
+
+	// // synchronous StatefulSet deletion triggers a requeue. But, the StatefulSet
+	// // didn't create any child resources since this is all faked, so the
+	// // deletion goes through immediately.
+	expectReconciled(t, sr, "default", "test")
+	expectMissing[appsv1.StatefulSet](t, fc, "operator-ns", shortName)
+	// // The deletion triggers another reconcile, to finish the cleanup.
+	expectReconciled(t, sr, "default", "test")
+	expectMissing[appsv1.StatefulSet](t, fc, "operator-ns", shortName)
+	expectMissing[corev1.Service](t, fc, "operator-ns", shortName)
+	expectMissing[corev1.Secret](t, fc, "operator-ns", fullName)
+
+	// At the moment we don't revert changes to the user created Service -
+	// we don't have a reliable way how to tell what it was before and also
+	// we don't really expect it to be re-used
+}
 
 func TestAnnotations(t *testing.T) {
 	fc := fake.NewFakeClient()
@@ -153,12 +268,15 @@ func TestAnnotations(t *testing.T) {
 		t.Fatal(err)
 	}
 	sr := &ServiceReconciler{
-		Client:            fc,
-		tsClient:          ft,
-		defaultTags:       []string{"tag:k8s"},
-		operatorNamespace: "operator-ns",
-		proxyImage:        "tailscale/tailscale",
-		logger:            zl.Sugar(),
+		Client: fc,
+		ssr: &tailscaleSTSReconciler{
+			Client:            fc,
+			tsClient:          ft,
+			defaultTags:       []string{"tag:k8s"},
+			operatorNamespace: "operator-ns",
+			proxyImage:        "tailscale/tailscale",
+		},
+		logger: zl.Sugar(),
 	}
 
 	// Create a service that we should manage, and check that the initial round
@@ -250,12 +368,15 @@ func TestAnnotationIntoLB(t *testing.T) {
 		t.Fatal(err)
 	}
 	sr := &ServiceReconciler{
-		Client:            fc,
-		tsClient:          ft,
-		defaultTags:       []string{"tag:k8s"},
-		operatorNamespace: "operator-ns",
-		proxyImage:        "tailscale/tailscale",
-		logger:            zl.Sugar(),
+		Client: fc,
+		ssr: &tailscaleSTSReconciler{
+			Client:            fc,
+			tsClient:          ft,
+			defaultTags:       []string{"tag:k8s"},
+			operatorNamespace: "operator-ns",
+			proxyImage:        "tailscale/tailscale",
+		},
+		logger: zl.Sugar(),
 	}
 
 	// Create a service that we should manage, and check that the initial round
@@ -295,6 +416,7 @@ func TestAnnotationIntoLB(t *testing.T) {
 		}
 		s.Data["device_id"] = []byte("ts-id-1234")
 		s.Data["device_fqdn"] = []byte("tailscale.device.name.")
+		s.Data["device_ips"] = []byte(`["100.99.98.97", "2c0a:8083:94d4:2012:3165:34a5:3616:5fdf"]`)
 	})
 	expectReconciled(t, sr, "default", "test")
 	want := &corev1.Service{
@@ -353,6 +475,9 @@ func TestAnnotationIntoLB(t *testing.T) {
 					{
 						Hostname: "tailscale.device.name",
 					},
+					{
+						IP: "100.99.98.97",
+					},
 				},
 			},
 		},
@@ -368,12 +493,15 @@ func TestLBIntoAnnotation(t *testing.T) {
 		t.Fatal(err)
 	}
 	sr := &ServiceReconciler{
-		Client:            fc,
-		tsClient:          ft,
-		defaultTags:       []string{"tag:k8s"},
-		operatorNamespace: "operator-ns",
-		proxyImage:        "tailscale/tailscale",
-		logger:            zl.Sugar(),
+		Client: fc,
+		ssr: &tailscaleSTSReconciler{
+			Client:            fc,
+			tsClient:          ft,
+			defaultTags:       []string{"tag:k8s"},
+			operatorNamespace: "operator-ns",
+			proxyImage:        "tailscale/tailscale",
+		},
+		logger: zl.Sugar(),
 	}
 
 	// Create a service that we should manage, and check that the initial round
@@ -411,6 +539,7 @@ func TestLBIntoAnnotation(t *testing.T) {
 		}
 		s.Data["device_id"] = []byte("ts-id-1234")
 		s.Data["device_fqdn"] = []byte("tailscale.device.name.")
+		s.Data["device_ips"] = []byte(`["100.99.98.97", "2c0a:8083:94d4:2012:3165:34a5:3616:5fdf"]`)
 	})
 	expectReconciled(t, sr, "default", "test")
 	want := &corev1.Service{
@@ -435,6 +564,9 @@ func TestLBIntoAnnotation(t *testing.T) {
 					{
 						Hostname: "tailscale.device.name",
 					},
+					{
+						IP: "100.99.98.97",
+					},
 				},
 			},
 		},
@@ -491,12 +623,15 @@ func TestCustomHostname(t *testing.T) {
 		t.Fatal(err)
 	}
 	sr := &ServiceReconciler{
-		Client:            fc,
-		tsClient:          ft,
-		defaultTags:       []string{"tag:k8s"},
-		operatorNamespace: "operator-ns",
-		proxyImage:        "tailscale/tailscale",
-		logger:            zl.Sugar(),
+		Client: fc,
+		ssr: &tailscaleSTSReconciler{
+			Client:            fc,
+			tsClient:          ft,
+			defaultTags:       []string{"tag:k8s"},
+			operatorNamespace: "operator-ns",
+			proxyImage:        "tailscale/tailscale",
+		},
+		logger: zl.Sugar(),
 	}
 
 	// Create a service that we should manage, and check that the initial round
@@ -593,13 +728,16 @@ func TestCustomPriorityClassName(t *testing.T) {
 		t.Fatal(err)
 	}
 	sr := &ServiceReconciler{
-		Client:                 fc,
-		tsClient:               ft,
-		defaultTags:            []string{"tag:k8s"},
-		operatorNamespace:      "operator-ns",
-		proxyImage:             "tailscale/tailscale",
-		proxyPriorityClassName: "tailscale-critical",
-		logger:                 zl.Sugar(),
+		Client: fc,
+		ssr: &tailscaleSTSReconciler{
+			Client:                 fc,
+			tsClient:               ft,
+			defaultTags:            []string{"tag:k8s"},
+			operatorNamespace:      "operator-ns",
+			proxyImage:             "tailscale/tailscale",
+			proxyPriorityClassName: "tailscale-critical",
+		},
+		logger: zl.Sugar(),
 	}
 
 	// Create a service that we should manage, and check that the initial round
@@ -630,6 +768,52 @@ func TestCustomPriorityClassName(t *testing.T) {
 	expectEqual(t, fc, expectedSTS(shortName, fullName, "custom-priority-class-name", "tailscale-critical"))
 }
 
+func TestDefaultLoadBalancer(t *testing.T) {
+	fc := fake.NewFakeClient()
+	ft := &fakeTSClient{}
+	zl, err := zap.NewDevelopment()
+	if err != nil {
+		t.Fatal(err)
+	}
+	sr := &ServiceReconciler{
+		Client: fc,
+		ssr: &tailscaleSTSReconciler{
+			Client:            fc,
+			tsClient:          ft,
+			defaultTags:       []string{"tag:k8s"},
+			operatorNamespace: "operator-ns",
+			proxyImage:        "tailscale/tailscale",
+		},
+		logger:                zl.Sugar(),
+		isDefaultLoadBalancer: true,
+	}
+
+	// Create a service that we should manage, and check that the initial round
+	// of objects looks right.
+	mustCreate(t, fc, &corev1.Service{
+		ObjectMeta: metav1.ObjectMeta{
+			Name:      "test",
+			Namespace: "default",
+			// The apiserver is supposed to set the UID, but the fake client
+			// doesn't. So, set it explicitly because other code later depends
+			// on it being set.
+			UID: types.UID("1234-UID"),
+		},
+		Spec: corev1.ServiceSpec{
+			ClusterIP: "10.20.30.40",
+			Type:      corev1.ServiceTypeLoadBalancer,
+		},
+	})
+
+	expectReconciled(t, sr, "default", "test")
+
+	fullName, shortName := findGenName(t, fc, "default", "test")
+
+	expectEqual(t, fc, expectedSecret(fullName))
+	expectEqual(t, fc, expectedHeadlessService(shortName))
+	expectEqual(t, fc, expectedSTS(shortName, fullName, "default-test", ""))
+}
+
 func expectedSecret(name string) *corev1.Secret {
 	return &corev1.Secret{
 		TypeMeta: metav1.TypeMeta{
@@ -702,6 +886,10 @@ func expectedSTS(stsName, secretName, hostname, priorityClassName string) *appsv
 			ServiceName: stsName,
 			Template: corev1.PodTemplateSpec{
 				ObjectMeta: metav1.ObjectMeta{
+					Annotations: map[string]string{
+						"tailscale.com/operator-last-set-hostname":   hostname,
+						"tailscale.com/operator-last-set-cluster-ip": "10.20.30.40",
+					},
 					DeletionGracePeriodSeconds: ptr.To[int64](10),
 					Labels:                     map[string]string{"app": "1234-UID"},
 				},
@@ -726,9 +914,78 @@ func expectedSTS(stsName, secretName, hostname, priorityClassName string) *appsv
 							Env: []corev1.EnvVar{
 								{Name: "TS_USERSPACE", Value: "false"},
 								{Name: "TS_AUTH_ONCE", Value: "true"},
-								{Name: "TS_DEST_IP", Value: "10.20.30.40"},
 								{Name: "TS_KUBE_SECRET", Value: secretName},
 								{Name: "TS_HOSTNAME", Value: hostname},
+								{Name: "TS_DEST_IP", Value: "10.20.30.40"},
+							},
+							SecurityContext: &corev1.SecurityContext{
+								Capabilities: &corev1.Capabilities{
+									Add: []corev1.Capability{"NET_ADMIN"},
+								},
+							},
+							ImagePullPolicy: "Always",
+						},
+					},
+				},
+			},
+		},
+	}
+}
+func expectedEgressSTS(stsName, secretName, tailnetTargetIP, hostname, priorityClassName string) *appsv1.StatefulSet {
+	return &appsv1.StatefulSet{
+		TypeMeta: metav1.TypeMeta{
+			Kind:       "StatefulSet",
+			APIVersion: "apps/v1",
+		},
+		ObjectMeta: metav1.ObjectMeta{
+			Name:      stsName,
+			Namespace: "operator-ns",
+			Labels: map[string]string{
+				"tailscale.com/managed":              "true",
+				"tailscale.com/parent-resource":      "test",
+				"tailscale.com/parent-resource-ns":   "default",
+				"tailscale.com/parent-resource-type": "svc",
+			},
+		},
+		Spec: appsv1.StatefulSetSpec{
+			Replicas: ptr.To[int32](1),
+			Selector: &metav1.LabelSelector{
+				MatchLabels: map[string]string{"app": "1234-UID"},
+			},
+			ServiceName: stsName,
+			Template: corev1.PodTemplateSpec{
+				ObjectMeta: metav1.ObjectMeta{
+					Annotations: map[string]string{
+						"tailscale.com/operator-last-set-hostname":             hostname,
+						"tailscale.com/operator-last-set-ts-tailnet-target-ip": tailnetTargetIP,
+					},
+					DeletionGracePeriodSeconds: ptr.To[int64](10),
+					Labels:                     map[string]string{"app": "1234-UID"},
+				},
+				Spec: corev1.PodSpec{
+					ServiceAccountName: "proxies",
+					PriorityClassName:  priorityClassName,
+					InitContainers: []corev1.Container{
+						{
+							Name:    "sysctler",
+							Image:   "busybox",
+							Command: []string{"/bin/sh"},
+							Args:    []string{"-c", "sysctl -w net.ipv4.ip_forward=1 net.ipv6.conf.all.forwarding=1"},
+							SecurityContext: &corev1.SecurityContext{
+								Privileged: ptr.To(true),
+							},
+						},
+					},
+					Containers: []corev1.Container{
+						{
+							Name:  "tailscale",
+							Image: "tailscale/tailscale",
+							Env: []corev1.EnvVar{
+								{Name: "TS_USERSPACE", Value: "false"},
+								{Name: "TS_AUTH_ONCE", Value: "true"},
+								{Name: "TS_KUBE_SECRET", Value: secretName},
+								{Name: "TS_HOSTNAME", Value: hostname},
+								{Name: "TS_TAILNET_TARGET_IP", Value: tailnetTargetIP},
 							},
 							SecurityContext: &corev1.SecurityContext{
 								Capabilities: &corev1.Capabilities{
@@ -756,6 +1013,9 @@ func findGenName(t *testing.T, client client.Client, ns, name string) (full, noS
 	if err != nil {
 		t.Fatalf("finding secret for %q: %v", name, err)
 	}
+	if s == nil {
+		t.Fatalf("no secret found for %q", name)
+	}
 	return s.GetName(), strings.TrimSuffix(s.GetName(), "-0")
 }
 

cmd/k8s-operator/proxy.go

@@ -1,6 +1,8 @@
 // Copyright (c) Tailscale Inc & AUTHORS
 // SPDX-License-Identifier: BSD-3-Clause
 
+//go:build !plan9
+
 package main
 
 import (
@@ -14,14 +16,62 @@ import (
 	"os"
 	"strings"
 
+	"go.uber.org/zap"
+	"k8s.io/client-go/rest"
+	"k8s.io/client-go/transport"
 	"tailscale.com/client/tailscale"
 	"tailscale.com/client/tailscale/apitype"
+	"tailscale.com/hostinfo"
+	"tailscale.com/tailcfg"
 	"tailscale.com/tsnet"
 	"tailscale.com/types/logger"
+	"tailscale.com/util/clientmetric"
+	"tailscale.com/util/set"
 )
 
 type whoIsKey struct{}
 
+// whoIsFromRequest returns the WhoIsResponse previously stashed by a call to
+// addWhoIsToRequest.
+func whoIsFromRequest(r *http.Request) *apitype.WhoIsResponse {
+	return r.Context().Value(whoIsKey{}).(*apitype.WhoIsResponse)
+}
+
+// addWhoIsToRequest stashes who in r's context, retrievable by a call to
+// whoIsFromRequest.
+func addWhoIsToRequest(r *http.Request, who *apitype.WhoIsResponse) *http.Request {
+	return r.WithContext(context.WithValue(r.Context(), whoIsKey{}, who))
+}
+
+var counterNumRequestsProxied = clientmetric.NewCounter("k8s_auth_proxy_requests_proxied")
+
+// launchAuthProxy launches the auth proxy, which is a small HTTP server that
+// authenticates requests using the Tailscale LocalAPI and then proxies them to
+// the kube-apiserver.
+func launchAuthProxy(zlog *zap.SugaredLogger, restConfig *rest.Config, s *tsnet.Server) {
+	hostinfo.SetApp("k8s-operator-proxy")
+	startlog := zlog.Named("launchAuthProxy")
+	cfg, err := restConfig.TransportConfig()
+	if err != nil {
+		startlog.Fatalf("could not get rest.TransportConfig(): %v", err)
+	}
+
+	// Kubernetes uses SPDY for exec and port-forward, however SPDY is
+	// incompatible with HTTP/2; so disable HTTP/2 in the proxy.
+	tr := http.DefaultTransport.(*http.Transport).Clone()
+	tr.TLSClientConfig, err = transport.TLSConfigFor(cfg)
+	if err != nil {
+		startlog.Fatalf("could not get transport.TLSConfigFor(): %v", err)
+	}
+	tr.TLSNextProto = make(map[string]func(authority string, c *tls.Conn) http.RoundTripper)
+
+	rt, err := transport.HTTPWrappersForConfig(cfg, tr)
+	if err != nil {
+		startlog.Fatalf("could not get rest.TransportConfig(): %v", err)
+	}
+	go runAuthProxy(s, rt, zlog.Named("auth-proxy").Infof)
+}
+
 // authProxy is an http.Handler that authenticates requests using the Tailscale
 // LocalAPI and then proxies them to the Kubernetes API.
 type authProxy struct {
@@ -37,8 +87,8 @@ func (h *authProxy) ServeHTTP(w http.ResponseWriter, r *http.Request) {
 		http.Error(w, "failed to authenticate caller", http.StatusInternalServerError)
 		return
 	}
-	r = r.WithContext(context.WithValue(r.Context(), whoIsKey{}, who))
-	h.rp.ServeHTTP(w, r)
+	counterNumRequestsProxied.Add(1)
+	h.rp.ServeHTTP(w, addWhoIsToRequest(r, who))
 }
 
 // runAuthProxy runs an HTTP server that authenticates requests using the
@@ -67,6 +117,10 @@ func runAuthProxy(s *tsnet.Server, rt http.RoundTripper, logf logger.Logf) {
 		lc:   lc,
 		rp: &httputil.ReverseProxy{
 			Director: func(r *http.Request) {
+				// Replace the URL with the Kubernetes APIServer.
+				r.URL.Scheme = u.Scheme
+				r.URL.Host = u.Host
+
 				// We want to proxy to the Kubernetes API, but we want to use
 				// the caller's identity to do so. We do this by impersonating
 				// the caller using the Kubernetes User Impersonation feature:
@@ -85,21 +139,9 @@ func runAuthProxy(s *tsnet.Server, rt http.RoundTripper, logf logger.Logf) {
 				}
 
 				// Now add the impersonation headers that we want.
-				who := r.Context().Value(whoIsKey{}).(*apitype.WhoIsResponse)
-				if who.Node.IsTagged() {
-					// Use the nodes FQDN as the username, and the nodes tags as the groups.
-					// "Impersonate-Group" requires "Impersonate-User" to be set.
-					r.Header.Set("Impersonate-User", strings.TrimSuffix(who.Node.Name, "."))
-					for _, tag := range who.Node.Tags {
-						r.Header.Add("Impersonate-Group", tag)
-					}
-				} else {
-					r.Header.Set("Impersonate-User", who.UserProfile.LoginName)
+				if err := addImpersonationHeaders(r); err != nil {
+					panic("failed to add impersonation headers: " + err.Error())
 				}
-
-				// Replace the URL with the Kubernetes APIServer.
-				r.URL.Scheme = u.Scheme
-				r.URL.Host = u.Host
 			},
 			Transport: rt,
 		},
@@ -118,3 +160,58 @@ func runAuthProxy(s *tsnet.Server, rt http.RoundTripper, logf logger.Logf) {
 		log.Fatalf("runAuthProxy: failed to serve %v", err)
 	}
 }
+
+const capabilityName = "https://tailscale.com/cap/kubernetes"
+
+type capRule struct {
+	// Impersonate is a list of rules that specify how to impersonate the caller
+	// when proxying to the Kubernetes API.
+	Impersonate *impersonateRule `json:"impersonate,omitempty"`
+}
+
+// TODO(maisem): move this to some well-known location so that it can be shared
+// with control.
+type impersonateRule struct {
+	Groups []string `json:"groups,omitempty"`
+}
+
+// addImpersonationHeaders adds the appropriate headers to r to impersonate the
+// caller when proxying to the Kubernetes API. It uses the WhoIsResponse stashed
+// in the context by the authProxy.
+func addImpersonationHeaders(r *http.Request) error {
+	who := whoIsFromRequest(r)
+	rules, err := tailcfg.UnmarshalCapJSON[capRule](who.CapMap, capabilityName)
+	if err != nil {
+		return fmt.Errorf("failed to unmarshal capability: %v", err)
+	}
+
+	var groupsAdded set.Slice[string]
+	for _, rule := range rules {
+		if rule.Impersonate == nil {
+			continue
+		}
+		for _, group := range rule.Impersonate.Groups {
+			if groupsAdded.Contains(group) {
+				continue
+			}
+			r.Header.Add("Impersonate-Group", group)
+			groupsAdded.Add(group)
+		}
+	}
+
+	if !who.Node.IsTagged() {
+		r.Header.Set("Impersonate-User", who.UserProfile.LoginName)
+		return nil
+	}
+	// "Impersonate-Group" requires "Impersonate-User" to be set, so we set it
+	// to the node FQDN for tagged nodes.
+	r.Header.Set("Impersonate-User", strings.TrimSuffix(who.Node.Name, "."))
+
+	// For legacy behavior (before caps), set the groups to the nodes tags.
+	if groupsAdded.Slice().Len() == 0 {
+		for _, tag := range who.Node.Tags {
+			r.Header.Add("Impersonate-Group", tag)
+		}
+	}
+	return nil
+}

cmd/k8s-operator/proxy_test.go

@@ -0,0 +1,109 @@
+// Copyright (c) Tailscale Inc & AUTHORS
+// SPDX-License-Identifier: BSD-3-Clause
+
+//go:build !plan9
+
+package main
+
+import (
+	"net/http"
+	"testing"
+
+	"github.com/google/go-cmp/cmp"
+	"tailscale.com/client/tailscale/apitype"
+	"tailscale.com/tailcfg"
+	"tailscale.com/util/must"
+)
+
+func TestImpersonationHeaders(t *testing.T) {
+	tests := []struct {
+		name     string
+		emailish string
+		tags     []string
+		capMap   tailcfg.PeerCapMap
+
+		wantHeaders http.Header
+	}{
+		{
+			name:     "user",
+			emailish: "foo@example.com",
+			wantHeaders: http.Header{
+				"Impersonate-User": {"foo@example.com"},
+			},
+		},
+		{
+			name:     "tagged",
+			emailish: "tagged-device",
+			tags:     []string{"tag:foo", "tag:bar"},
+			wantHeaders: http.Header{
+				"Impersonate-User":  {"node.ts.net"},
+				"Impersonate-Group": {"tag:foo", "tag:bar"},
+			},
+		},
+		{
+			name:     "user-with-cap",
+			emailish: "foo@example.com",
+			capMap: tailcfg.PeerCapMap{
+				capabilityName: {
+					[]byte(`{"impersonate":{"groups":["group1","group2"]}}`),
+					[]byte(`{"impersonate":{"groups":["group1","group3"]}}`), // One group is duplicated.
+					[]byte(`{"impersonate":{"groups":["group4"]}}`),
+					[]byte(`{"impersonate":{"groups":["group2"]}}`), // duplicate
+
+					// These should be ignored, but should parse correctly.
+					[]byte(`{}`),
+					[]byte(`{"impersonate":{}}`),
+					[]byte(`{"impersonate":{"groups":[]}}`),
+				},
+			},
+			wantHeaders: http.Header{
+				"Impersonate-Group": {"group1", "group2", "group3", "group4"},
+				"Impersonate-User":  {"foo@example.com"},
+			},
+		},
+		{
+			name:     "tagged-with-cap",
+			emailish: "tagged-device",
+			tags:     []string{"tag:foo", "tag:bar"},
+			capMap: tailcfg.PeerCapMap{
+				capabilityName: {
+					[]byte(`{"impersonate":{"groups":["group1"]}}`),
+				},
+			},
+			wantHeaders: http.Header{
+				"Impersonate-Group": {"group1"},
+				"Impersonate-User":  {"node.ts.net"},
+			},
+		},
+		{
+			name:     "bad-cap",
+			emailish: "tagged-device",
+			tags:     []string{"tag:foo", "tag:bar"},
+			capMap: tailcfg.PeerCapMap{
+				capabilityName: {
+					[]byte(`[]`),
+				},
+			},
+			wantHeaders: http.Header{},
+		},
+	}
+
+	for _, tc := range tests {
+		r := must.Get(http.NewRequest("GET", "https://op.ts.net/api/foo", nil))
+		r = addWhoIsToRequest(r, &apitype.WhoIsResponse{
+			Node: &tailcfg.Node{
+				Name: "node.ts.net",
+				Tags: tc.tags,
+			},
+			UserProfile: &tailcfg.UserProfile{
+				LoginName: tc.emailish,
+			},
+			CapMap: tc.capMap,
+		})
+		addImpersonationHeaders(r)
+
+		if d := cmp.Diff(tc.wantHeaders, r.Header); d != "" {
+			t.Errorf("unexpected header (-want +got):\n%s", d)
+		}
+	}
+}

cmd/k8s-operator/sts.go

@@ -0,0 +1,493 @@
+// Copyright (c) Tailscale Inc & AUTHORS
+// SPDX-License-Identifier: BSD-3-Clause
+
+//go:build !plan9
+
+package main
+
+import (
+	"context"
+	_ "embed"
+	"encoding/json"
+	"fmt"
+	"os"
+	"strings"
+
+	"go.uber.org/zap"
+	appsv1 "k8s.io/api/apps/v1"
+	corev1 "k8s.io/api/core/v1"
+	apierrors "k8s.io/apimachinery/pkg/api/errors"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
+	"sigs.k8s.io/controller-runtime/pkg/client"
+	"sigs.k8s.io/yaml"
+	"tailscale.com/client/tailscale"
+	"tailscale.com/ipn"
+	"tailscale.com/tailcfg"
+	"tailscale.com/tsnet"
+	"tailscale.com/types/opt"
+	"tailscale.com/util/dnsname"
+	"tailscale.com/util/mak"
+)
+
+const (
+	LabelManaged         = "tailscale.com/managed"
+	LabelParentType      = "tailscale.com/parent-resource-type"
+	LabelParentName      = "tailscale.com/parent-resource"
+	LabelParentNamespace = "tailscale.com/parent-resource-ns"
+
+	FinalizerName = "tailscale.com/finalizer"
+
+	// Annotations settable by users on services.
+	AnnotationExpose          = "tailscale.com/expose"
+	AnnotationTags            = "tailscale.com/tags"
+	AnnotationHostname        = "tailscale.com/hostname"
+	AnnotationTailnetTargetIP = "tailscale.com/ts-tailnet-target-ip"
+
+	// Annotations settable by users on ingresses.
+	AnnotationFunnel = "tailscale.com/funnel"
+
+	// Annotations set by the operator on pods to trigger restarts when the
+	// hostname or IP changes.
+	podAnnotationLastSetClusterIP       = "tailscale.com/operator-last-set-cluster-ip"
+	podAnnotationLastSetHostname        = "tailscale.com/operator-last-set-hostname"
+	podAnnotationLastSetTailnetTargetIP = "tailscale.com/operator-last-set-ts-tailnet-target-ip"
+)
+
+type tailscaleSTSConfig struct {
+	ParentResourceName  string
+	ParentResourceUID   string
+	ChildResourceLabels map[string]string
+
+	ServeConfig *ipn.ServeConfig
+	// Tailscale target in cluster we are setting up ingress for
+	ClusterTargetIP string
+
+	// Tailscale IP of a Tailscale service we are setting up egress for
+	TailnetTargetIP string
+
+	Hostname string
+	Tags     []string // if empty, use defaultTags
+}
+
+type tailscaleSTSReconciler struct {
+	client.Client
+	tsnetServer            *tsnet.Server
+	tsClient               tsClient
+	defaultTags            []string
+	operatorNamespace      string
+	proxyImage             string
+	proxyPriorityClassName string
+}
+
+// IsHTTPSEnabledOnTailnet reports whether HTTPS is enabled on the tailnet.
+func (a *tailscaleSTSReconciler) IsHTTPSEnabledOnTailnet() bool {
+	return len(a.tsnetServer.CertDomains()) > 0
+}
+
+// Provision ensures that the StatefulSet for the given service is running and
+// up to date.
+func (a *tailscaleSTSReconciler) Provision(ctx context.Context, logger *zap.SugaredLogger, sts *tailscaleSTSConfig) (*corev1.Service, error) {
+	// Do full reconcile.
+	hsvc, err := a.reconcileHeadlessService(ctx, logger, sts)
+	if err != nil {
+		return nil, fmt.Errorf("failed to reconcile headless service: %w", err)
+	}
+
+	secretName, err := a.createOrGetSecret(ctx, logger, sts, hsvc)
+	if err != nil {
+		return nil, fmt.Errorf("failed to create or get API key secret: %w", err)
+	}
+	_, err = a.reconcileSTS(ctx, logger, sts, hsvc, secretName)
+	if err != nil {
+		return nil, fmt.Errorf("failed to reconcile statefulset: %w", err)
+	}
+
+	return hsvc, nil
+}
+
+// Cleanup removes all resources associated that were created by Provision with
+// the given labels. It returns true when all resources have been removed,
+// otherwise it returns false and the caller should retry later.
+func (a *tailscaleSTSReconciler) Cleanup(ctx context.Context, logger *zap.SugaredLogger, labels map[string]string) (done bool, _ error) {
+	// Need to delete the StatefulSet first, and delete it with foreground
+	// cascading deletion. That way, the pod that's writing to the Secret will
+	// stop running before we start looking at the Secret's contents, and
+	// assuming k8s ordering semantics don't mess with us, that should avoid
+	// tailscale device deletion races where we fail to notice a device that
+	// should be removed.
+	sts, err := getSingleObject[appsv1.StatefulSet](ctx, a.Client, a.operatorNamespace, labels)
+	if err != nil {
+		return false, fmt.Errorf("getting statefulset: %w", err)
+	}
+	if sts != nil {
+		if !sts.GetDeletionTimestamp().IsZero() {
+			// Deletion in progress, check again later. We'll get another
+			// notification when the deletion is complete.
+			logger.Debugf("waiting for statefulset %s/%s deletion", sts.GetNamespace(), sts.GetName())
+			return false, nil
+		}
+		err := a.DeleteAllOf(ctx, &appsv1.StatefulSet{}, client.InNamespace(a.operatorNamespace), client.MatchingLabels(labels), client.PropagationPolicy(metav1.DeletePropagationForeground))
+		if err != nil {
+			return false, fmt.Errorf("deleting statefulset: %w", err)
+		}
+		logger.Debugf("started deletion of statefulset %s/%s", sts.GetNamespace(), sts.GetName())
+		return false, nil
+	}
+
+	id, _, _, err := a.DeviceInfo(ctx, labels)
+	if err != nil {
+		return false, fmt.Errorf("getting device info: %w", err)
+	}
+	if id != "" {
+		// TODO: handle case where the device is already deleted, but the secret
+		// is still around.
+		if err := a.tsClient.DeleteDevice(ctx, string(id)); err != nil {
+			return false, fmt.Errorf("deleting device: %w", err)
+		}
+	}
+
+	types := []client.Object{
+		&corev1.Service{},
+		&corev1.Secret{},
+	}
+	for _, typ := range types {
+		if err := a.DeleteAllOf(ctx, typ, client.InNamespace(a.operatorNamespace), client.MatchingLabels(labels)); err != nil {
+			return false, err
+		}
+	}
+	return true, nil
+}
+
+func (a *tailscaleSTSReconciler) reconcileHeadlessService(ctx context.Context, logger *zap.SugaredLogger, sts *tailscaleSTSConfig) (*corev1.Service, error) {
+	hsvc := &corev1.Service{
+		ObjectMeta: metav1.ObjectMeta{
+			GenerateName: "ts-" + sts.ParentResourceName + "-",
+			Namespace:    a.operatorNamespace,
+			Labels:       sts.ChildResourceLabels,
+		},
+		Spec: corev1.ServiceSpec{
+			ClusterIP: "None",
+			Selector: map[string]string{
+				"app": sts.ParentResourceUID,
+			},
+		},
+	}
+	logger.Debugf("reconciling headless service for StatefulSet")
+	return createOrUpdate(ctx, a.Client, a.operatorNamespace, hsvc, func(svc *corev1.Service) { svc.Spec = hsvc.Spec })
+}
+
+func (a *tailscaleSTSReconciler) createOrGetSecret(ctx context.Context, logger *zap.SugaredLogger, stsC *tailscaleSTSConfig, hsvc *corev1.Service) (string, error) {
+	secret := &corev1.Secret{
+		ObjectMeta: metav1.ObjectMeta{
+			// Hardcode a -0 suffix so that in future, if we support
+			// multiple StatefulSet replicas, we can provision -N for
+			// those.
+			Name:      hsvc.Name + "-0",
+			Namespace: a.operatorNamespace,
+			Labels:    stsC.ChildResourceLabels,
+		},
+	}
+	var orig *corev1.Secret // unmodified copy of secret
+	if err := a.Get(ctx, client.ObjectKeyFromObject(secret), secret); err == nil {
+		logger.Debugf("secret %s/%s already exists", secret.GetNamespace(), secret.GetName())
+		orig = secret.DeepCopy()
+	} else if !apierrors.IsNotFound(err) {
+		return "", err
+	}
+
+	if orig == nil {
+		// Secret doesn't exist yet, create one. Initially it contains
+		// only the Tailscale authkey, but once Tailscale starts it'll
+		// also store the daemon state.
+		sts, err := getSingleObject[appsv1.StatefulSet](ctx, a.Client, a.operatorNamespace, stsC.ChildResourceLabels)
+		if err != nil {
+			return "", err
+		}
+		if sts != nil {
+			// StatefulSet exists, so we have already created the secret.
+			// If the secret is missing, they should delete the StatefulSet.
+			logger.Errorf("Tailscale proxy secret doesn't exist, but the corresponding StatefulSet %s/%s already does. Something is wrong, please delete the StatefulSet.", sts.GetNamespace(), sts.GetName())
+			return "", nil
+		}
+		// Create API Key secret which is going to be used by the statefulset
+		// to authenticate with Tailscale.
+		logger.Debugf("creating authkey for new tailscale proxy")
+		tags := stsC.Tags
+		if len(tags) == 0 {
+			tags = a.defaultTags
+		}
+		authKey, err := a.newAuthKey(ctx, tags)
+		if err != nil {
+			return "", err
+		}
+
+		mak.Set(&secret.StringData, "authkey", authKey)
+	}
+	if stsC.ServeConfig != nil {
+		j, err := json.Marshal(stsC.ServeConfig)
+		if err != nil {
+			return "", err
+		}
+		mak.Set(&secret.StringData, "serve-config", string(j))
+	}
+	if orig != nil {
+		if err := a.Patch(ctx, secret, client.MergeFrom(orig)); err != nil {
+			return "", err
+		}
+	} else {
+		if err := a.Create(ctx, secret); err != nil {
+			return "", err
+		}
+	}
+	return secret.Name, nil
+}
+
+// DeviceInfo returns the device ID and hostname for the Tailscale device
+// associated with the given labels.
+func (a *tailscaleSTSReconciler) DeviceInfo(ctx context.Context, childLabels map[string]string) (id tailcfg.StableNodeID, hostname string, ips []string, err error) {
+	sec, err := getSingleObject[corev1.Secret](ctx, a.Client, a.operatorNamespace, childLabels)
+	if err != nil {
+		return "", "", nil, err
+	}
+	if sec == nil {
+		return "", "", nil, nil
+	}
+	id = tailcfg.StableNodeID(sec.Data["device_id"])
+	if id == "" {
+		return "", "", nil, nil
+	}
+	// Kubernetes chokes on well-formed FQDNs with the trailing dot, so we have
+	// to remove it.
+	hostname = strings.TrimSuffix(string(sec.Data["device_fqdn"]), ".")
+	if hostname == "" {
+		return "", "", nil, nil
+	}
+	if rawDeviceIPs, ok := sec.Data["device_ips"]; ok {
+		if err := json.Unmarshal(rawDeviceIPs, &ips); err != nil {
+			return "", "", nil, err
+		}
+	}
+
+	return id, hostname, ips, nil
+}
+
+func (a *tailscaleSTSReconciler) newAuthKey(ctx context.Context, tags []string) (string, error) {
+	caps := tailscale.KeyCapabilities{
+		Devices: tailscale.KeyDeviceCapabilities{
+			Create: tailscale.KeyDeviceCreateCapabilities{
+				Reusable:      false,
+				Preauthorized: true,
+				Tags:          tags,
+			},
+		},
+	}
+
+	key, _, err := a.tsClient.CreateKey(ctx, caps)
+	if err != nil {
+		return "", err
+	}
+	return key, nil
+}
+
+//go:embed manifests/proxy.yaml
+var proxyYaml []byte
+
+//go:embed manifests/userspace-proxy.yaml
+var userspaceProxyYaml []byte
+
+func (a *tailscaleSTSReconciler) reconcileSTS(ctx context.Context, logger *zap.SugaredLogger, sts *tailscaleSTSConfig, headlessSvc *corev1.Service, authKeySecret string) (*appsv1.StatefulSet, error) {
+	var ss appsv1.StatefulSet
+	if sts.ServeConfig != nil {
+		if err := yaml.Unmarshal(userspaceProxyYaml, &ss); err != nil {
+			return nil, fmt.Errorf("failed to unmarshal proxy spec: %w", err)
+		}
+	} else {
+		if err := yaml.Unmarshal(proxyYaml, &ss); err != nil {
+			return nil, fmt.Errorf("failed to unmarshal proxy spec: %w", err)
+		}
+	}
+	container := &ss.Spec.Template.Spec.Containers[0]
+	container.Image = a.proxyImage
+	container.Env = append(container.Env,
+		corev1.EnvVar{
+			Name:  "TS_KUBE_SECRET",
+			Value: authKeySecret,
+		},
+		corev1.EnvVar{
+			Name:  "TS_HOSTNAME",
+			Value: sts.Hostname,
+		})
+	if sts.ClusterTargetIP != "" {
+		container.Env = append(container.Env, corev1.EnvVar{
+			Name:  "TS_DEST_IP",
+			Value: sts.ClusterTargetIP,
+		})
+	} else if sts.TailnetTargetIP != "" {
+		container.Env = append(container.Env, corev1.EnvVar{
+			Name:  "TS_TAILNET_TARGET_IP",
+			Value: sts.TailnetTargetIP,
+		})
+
+	} else if sts.ServeConfig != nil {
+		container.Env = append(container.Env, corev1.EnvVar{
+			Name:  "TS_SERVE_CONFIG",
+			Value: "/etc/tailscaled/serve-config",
+		})
+		container.VolumeMounts = append(container.VolumeMounts, corev1.VolumeMount{
+			Name:      "serve-config",
+			ReadOnly:  true,
+			MountPath: "/etc/tailscaled",
+		})
+		ss.Spec.Template.Spec.Volumes = append(ss.Spec.Template.Spec.Volumes, corev1.Volume{
+			Name: "serve-config",
+			VolumeSource: corev1.VolumeSource{
+				Secret: &corev1.SecretVolumeSource{
+					SecretName: authKeySecret,
+					Items: []corev1.KeyToPath{{
+						Key:  "serve-config",
+						Path: "serve-config",
+					}},
+				},
+			},
+		})
+	}
+	ss.ObjectMeta = metav1.ObjectMeta{
+		Name:      headlessSvc.Name,
+		Namespace: a.operatorNamespace,
+		Labels:    sts.ChildResourceLabels,
+	}
+	ss.Spec.ServiceName = headlessSvc.Name
+	ss.Spec.Selector = &metav1.LabelSelector{
+		MatchLabels: map[string]string{
+			"app": sts.ParentResourceUID,
+		},
+	}
+
+	// containerboot currently doesn't have a way to re-read the hostname/ip as
+	// it is passed via an environment variable. So we need to restart the
+	// container when the value changes. We do this by adding an annotation to
+	// the pod template that contains the last value we set.
+	ss.Spec.Template.Annotations = map[string]string{
+		podAnnotationLastSetHostname: sts.Hostname,
+	}
+	if sts.ClusterTargetIP != "" {
+		ss.Spec.Template.Annotations[podAnnotationLastSetClusterIP] = sts.ClusterTargetIP
+	}
+	if sts.TailnetTargetIP != "" {
+		ss.Spec.Template.Annotations[podAnnotationLastSetTailnetTargetIP] = sts.TailnetTargetIP
+	}
+	ss.Spec.Template.Labels = map[string]string{
+		"app": sts.ParentResourceUID,
+	}
+	ss.Spec.Template.Spec.PriorityClassName = a.proxyPriorityClassName
+	logger.Debugf("reconciling statefulset %s/%s", ss.GetNamespace(), ss.GetName())
+	return createOrUpdate(ctx, a.Client, a.operatorNamespace, &ss, func(s *appsv1.StatefulSet) { s.Spec = ss.Spec })
+}
+
+// ptrObject is a type constraint for pointer types that implement
+// client.Object.
+type ptrObject[T any] interface {
+	client.Object
+	*T
+}
+
+// createOrUpdate adds obj to the k8s cluster, unless the object already exists,
+// in which case update is called to make changes to it. If update is nil, the
+// existing object is returned unmodified.
+//
+// obj is looked up by its Name and Namespace if Name is set, otherwise it's
+// looked up by labels.
+func createOrUpdate[T any, O ptrObject[T]](ctx context.Context, c client.Client, ns string, obj O, update func(O)) (O, error) {
+	var (
+		existing O
+		err      error
+	)
+	if obj.GetName() != "" {
+		existing = new(T)
+		existing.SetName(obj.GetName())
+		existing.SetNamespace(obj.GetNamespace())
+		err = c.Get(ctx, client.ObjectKeyFromObject(obj), existing)
+	} else {
+		existing, err = getSingleObject[T, O](ctx, c, ns, obj.GetLabels())
+	}
+	if err == nil && existing != nil {
+		if update != nil {
+			update(existing)
+			if err := c.Update(ctx, existing); err != nil {
+				return nil, err
+			}
+		}
+		return existing, nil
+	}
+	if err != nil && !apierrors.IsNotFound(err) {
+		return nil, fmt.Errorf("failed to get object: %w", err)
+	}
+	if err := c.Create(ctx, obj); err != nil {
+		return nil, err
+	}
+	return obj, nil
+}
+
+// getSingleObject searches for k8s objects of type T
+// (e.g. corev1.Service) with the given labels, and returns
+// it. Returns nil if no objects match the labels, and an error if
+// more than one object matches.
+func getSingleObject[T any, O ptrObject[T]](ctx context.Context, c client.Client, ns string, labels map[string]string) (O, error) {
+	ret := O(new(T))
+	kinds, _, err := c.Scheme().ObjectKinds(ret)
+	if err != nil {
+		return nil, err
+	}
+	if len(kinds) != 1 {
+		// TODO: the runtime package apparently has a "pick the best
+		// GVK" function somewhere that might be good enough?
+		return nil, fmt.Errorf("more than 1 GroupVersionKind for %T", ret)
+	}
+
+	gvk := kinds[0]
+	gvk.Kind += "List"
+	lst := unstructured.UnstructuredList{}
+	lst.SetGroupVersionKind(gvk)
+	if err := c.List(ctx, &lst, client.InNamespace(ns), client.MatchingLabels(labels)); err != nil {
+		return nil, err
+	}
+
+	if len(lst.Items) == 0 {
+		return nil, nil
+	}
+	if len(lst.Items) > 1 {
+		return nil, fmt.Errorf("found multiple matching %T objects", ret)
+	}
+	if err := c.Scheme().Convert(&lst.Items[0], ret, nil); err != nil {
+		return nil, err
+	}
+	return ret, nil
+}
+
+func defaultBool(envName string, defVal bool) bool {
+	vs := os.Getenv(envName)
+	if vs == "" {
+		return defVal
+	}
+	v, _ := opt.Bool(vs).Get()
+	return v
+}
+
+func defaultEnv(envName, defVal string) string {
+	v := os.Getenv(envName)
+	if v == "" {
+		return defVal
+	}
+	return v
+}
+
+func nameForService(svc *corev1.Service) (string, error) {
+	if h, ok := svc.Annotations[AnnotationHostname]; ok {
+		if err := dnsname.ValidLabel(h); err != nil {
+			return "", fmt.Errorf("invalid Tailscale hostname %q: %w", h, err)
+		}
+		return h, nil
+	}
+	return svc.Namespace + "-" + svc.Name, nil
+}

cmd/k8s-operator/svc.go

@@ -0,0 +1,268 @@
+// Copyright (c) Tailscale Inc & AUTHORS
+// SPDX-License-Identifier: BSD-3-Clause
+
+//go:build !plan9
+
+package main
+
+import (
+	"context"
+	"fmt"
+	"net/netip"
+	"strings"
+	"sync"
+
+	"go.uber.org/zap"
+	"golang.org/x/exp/slices"
+	corev1 "k8s.io/api/core/v1"
+	apierrors "k8s.io/apimachinery/pkg/api/errors"
+	"k8s.io/apimachinery/pkg/types"
+	"sigs.k8s.io/controller-runtime/pkg/client"
+	"sigs.k8s.io/controller-runtime/pkg/reconcile"
+	"tailscale.com/util/clientmetric"
+	"tailscale.com/util/set"
+)
+
+type ServiceReconciler struct {
+	client.Client
+	ssr                   *tailscaleSTSReconciler
+	logger                *zap.SugaredLogger
+	isDefaultLoadBalancer bool
+
+	mu sync.Mutex // protects following
+
+	// managedIngressProxies is a set of all ingress proxies that we're
+	// currently managing. This is only used for metrics.
+	managedIngressProxies set.Slice[types.UID]
+	// managedEgressProxies is a set of all egress proxies that we're currently
+	// managing. This is only used for metrics.
+	managedEgressProxies set.Slice[types.UID]
+}
+
+var (
+	// gaugeEgressProxies tracks the number of egress proxies that we're
+	// currently managing.
+	gaugeEgressProxies = clientmetric.NewGauge("k8s_egress_proxies")
+	// gaugeIngressProxies tracks the number of ingress proxies that we're
+	// currently managing.
+	gaugeIngressProxies = clientmetric.NewGauge("k8s_ingress_proxies")
+)
+
+func childResourceLabels(name, ns, typ string) map[string]string {
+	// You might wonder why we're using owner references, since they seem to be
+	// built for exactly this. Unfortunately, Kubernetes does not support
+	// cross-namespace ownership, by design. This means we cannot make the
+	// service being exposed the owner of the implementation details of the
+	// proxying. Instead, we have to do our own filtering and tracking with
+	// labels.
+	return map[string]string{
+		LabelManaged:         "true",
+		LabelParentName:      name,
+		LabelParentNamespace: ns,
+		LabelParentType:      typ,
+	}
+}
+
+func (a *ServiceReconciler) Reconcile(ctx context.Context, req reconcile.Request) (_ reconcile.Result, err error) {
+	logger := a.logger.With("service-ns", req.Namespace, "service-name", req.Name)
+	logger.Debugf("starting reconcile")
+	defer logger.Debugf("reconcile finished")
+
+	svc := new(corev1.Service)
+	err = a.Get(ctx, req.NamespacedName, svc)
+	if apierrors.IsNotFound(err) {
+		// Request object not found, could have been deleted after reconcile request.
+		logger.Debugf("service not found, assuming it was deleted")
+		return reconcile.Result{}, nil
+	} else if err != nil {
+		return reconcile.Result{}, fmt.Errorf("failed to get svc: %w", err)
+	}
+	if !svc.DeletionTimestamp.IsZero() || !a.shouldExpose(svc) && !a.hasTailnetTargetAnnotation(svc) {
+		logger.Debugf("service is being deleted or is (no longer) referring to Tailscale ingress/egress, ensuring any created resources are cleaned up")
+		return reconcile.Result{}, a.maybeCleanup(ctx, logger, svc)
+	}
+
+	return reconcile.Result{}, a.maybeProvision(ctx, logger, svc)
+}
+
+// maybeCleanup removes any existing resources related to serving svc over tailscale.
+//
+// This function is responsible for removing the finalizer from the service,
+// once all associated resources are gone.
+func (a *ServiceReconciler) maybeCleanup(ctx context.Context, logger *zap.SugaredLogger, svc *corev1.Service) error {
+	ix := slices.Index(svc.Finalizers, FinalizerName)
+	if ix < 0 {
+		logger.Debugf("no finalizer, nothing to do")
+		a.mu.Lock()
+		defer a.mu.Unlock()
+		a.managedIngressProxies.Remove(svc.UID)
+		a.managedEgressProxies.Remove(svc.UID)
+		gaugeIngressProxies.Set(int64(a.managedIngressProxies.Len()))
+		gaugeEgressProxies.Set(int64(a.managedEgressProxies.Len()))
+		return nil
+	}
+
+	if done, err := a.ssr.Cleanup(ctx, logger, childResourceLabels(svc.Name, svc.Namespace, "svc")); err != nil {
+		return fmt.Errorf("failed to cleanup: %w", err)
+	} else if !done {
+		logger.Debugf("cleanup not done yet, waiting for next reconcile")
+		return nil
+	}
+
+	svc.Finalizers = append(svc.Finalizers[:ix], svc.Finalizers[ix+1:]...)
+	if err := a.Update(ctx, svc); err != nil {
+		return fmt.Errorf("failed to remove finalizer: %w", err)
+	}
+
+	// Unlike most log entries in the reconcile loop, this will get printed
+	// exactly once at the very end of cleanup, because the final step of
+	// cleanup removes the tailscale finalizer, which will make all future
+	// reconciles exit early.
+	logger.Infof("unexposed service from tailnet")
+
+	a.mu.Lock()
+	defer a.mu.Unlock()
+	a.managedIngressProxies.Remove(svc.UID)
+	a.managedEgressProxies.Remove(svc.UID)
+	gaugeIngressProxies.Set(int64(a.managedIngressProxies.Len()))
+	gaugeEgressProxies.Set(int64(a.managedEgressProxies.Len()))
+	return nil
+}
+
+// maybeProvision ensures that svc is exposed over tailscale, taking any actions
+// necessary to reach that state.
+//
+// This function adds a finalizer to svc, ensuring that we can handle orderly
+// deprovisioning later.
+func (a *ServiceReconciler) maybeProvision(ctx context.Context, logger *zap.SugaredLogger, svc *corev1.Service) error {
+	hostname, err := nameForService(svc)
+	if err != nil {
+		return err
+	}
+
+	if !slices.Contains(svc.Finalizers, FinalizerName) {
+		// This log line is printed exactly once during initial provisioning,
+		// because once the finalizer is in place this block gets skipped. So,
+		// this is a nice place to tell the operator that the high level,
+		// multi-reconcile operation is underway.
+		logger.Infof("exposing service over tailscale")
+		svc.Finalizers = append(svc.Finalizers, FinalizerName)
+		if err := a.Update(ctx, svc); err != nil {
+			return fmt.Errorf("failed to add finalizer: %w", err)
+		}
+	}
+	crl := childResourceLabels(svc.Name, svc.Namespace, "svc")
+	var tags []string
+	if tstr, ok := svc.Annotations[AnnotationTags]; ok {
+		tags = strings.Split(tstr, ",")
+	}
+
+	sts := &tailscaleSTSConfig{
+		ParentResourceName:  svc.Name,
+		ParentResourceUID:   string(svc.UID),
+		Hostname:            hostname,
+		Tags:                tags,
+		ChildResourceLabels: crl,
+	}
+
+	a.mu.Lock()
+	if a.shouldExpose(svc) {
+		sts.ClusterTargetIP = svc.Spec.ClusterIP
+		a.managedIngressProxies.Add(svc.UID)
+		gaugeIngressProxies.Set(int64(a.managedIngressProxies.Len()))
+	} else if a.hasTailnetTargetAnnotation(svc) {
+		sts.TailnetTargetIP = svc.Annotations[AnnotationTailnetTargetIP]
+		a.managedEgressProxies.Add(svc.UID)
+		gaugeEgressProxies.Set(int64(a.managedEgressProxies.Len()))
+	}
+	a.mu.Unlock()
+
+	var hsvc *corev1.Service
+	if hsvc, err = a.ssr.Provision(ctx, logger, sts); err != nil {
+		return fmt.Errorf("failed to provision: %w", err)
+	}
+
+	if a.hasTailnetTargetAnnotation(svc) {
+		headlessSvcName := hsvc.Name + "." + hsvc.Namespace + ".svc"
+		if svc.Spec.ExternalName != headlessSvcName || svc.Spec.Type != corev1.ServiceTypeExternalName {
+			svc.Spec.ExternalName = headlessSvcName
+			svc.Spec.Selector = nil
+			svc.Spec.Type = corev1.ServiceTypeExternalName
+			if err := a.Update(ctx, svc); err != nil {
+				return fmt.Errorf("failed to update service: %w", err)
+			}
+		}
+		return nil
+	}
+
+	if !a.hasLoadBalancerClass(svc) {
+		logger.Debugf("service is not a LoadBalancer, so not updating ingress")
+		return nil
+	}
+
+	_, tsHost, tsIPs, err := a.ssr.DeviceInfo(ctx, crl)
+	if err != nil {
+		return fmt.Errorf("failed to get device ID: %w", err)
+	}
+	if tsHost == "" {
+		logger.Debugf("no Tailscale hostname known yet, waiting for proxy pod to finish auth")
+		// No hostname yet. Wait for the proxy pod to auth.
+		svc.Status.LoadBalancer.Ingress = nil
+		if err := a.Status().Update(ctx, svc); err != nil {
+			return fmt.Errorf("failed to update service status: %w", err)
+		}
+		return nil
+	}
+
+	logger.Debugf("setting ingress to %q, %s", tsHost, strings.Join(tsIPs, ", "))
+	ingress := []corev1.LoadBalancerIngress{
+		{Hostname: tsHost},
+	}
+	clusterIPAddr, err := netip.ParseAddr(svc.Spec.ClusterIP)
+	if err != nil {
+		return fmt.Errorf("failed to parse cluster IP: %w", err)
+	}
+	for _, ip := range tsIPs {
+		addr, err := netip.ParseAddr(ip)
+		if err != nil {
+			continue
+		}
+		if addr.Is4() == clusterIPAddr.Is4() { // only add addresses of the same family
+			ingress = append(ingress, corev1.LoadBalancerIngress{IP: ip})
+		}
+	}
+	svc.Status.LoadBalancer.Ingress = ingress
+	if err := a.Status().Update(ctx, svc); err != nil {
+		return fmt.Errorf("failed to update service status: %w", err)
+	}
+	return nil
+}
+
+func (a *ServiceReconciler) shouldExpose(svc *corev1.Service) bool {
+	// Headless services can't be exposed, since there is no ClusterIP to
+	// forward to.
+	if svc.Spec.ClusterIP == "" || svc.Spec.ClusterIP == "None" {
+		return false
+	}
+
+	return a.hasLoadBalancerClass(svc) || a.hasExposeAnnotation(svc)
+}
+
+func (a *ServiceReconciler) hasLoadBalancerClass(svc *corev1.Service) bool {
+	return svc != nil &&
+		svc.Spec.Type == corev1.ServiceTypeLoadBalancer &&
+		(svc.Spec.LoadBalancerClass != nil && *svc.Spec.LoadBalancerClass == "tailscale" ||
+			svc.Spec.LoadBalancerClass == nil && a.isDefaultLoadBalancer)
+}
+
+// hasExposeAnnotation reports whether Service has the tailscale.com/expose
+// annotation set
+func (a *ServiceReconciler) hasExposeAnnotation(svc *corev1.Service) bool {
+	return svc != nil && svc.Annotations[AnnotationExpose] == "true"
+}
+
+// hasTailnetTargetAnnotation reports whether Service has a
+// tailscale.com/ts-tailnet-target-ip annotation set
+func (a *ServiceReconciler) hasTailnetTargetAnnotation(svc *corev1.Service) bool {
+	return svc != nil && svc.Annotations[AnnotationTailnetTargetIP] != ""
+}

cmd/netlogfmt/main.go

@@ -35,14 +35,13 @@ import (
 	"net/http"
 	"net/netip"
 	"os"
+	"slices"
 	"strconv"
 	"strings"
 	"time"
 
 	"github.com/dsnet/try"
 	jsonv2 "github.com/go-json-experiment/json"
-	"golang.org/x/exp/maps"
-	"golang.org/x/exp/slices"
 	"tailscale.com/types/logid"
 	"tailscale.com/types/netlogtype"
 	"tailscale.com/util/cmpx"
@@ -315,8 +314,8 @@ func mustMakeNamesByAddr() map[netip.Addr]string {
 	namesByAddr := make(map[netip.Addr]string)
 retry:
 	for i := 0; i < 10; i++ {
-		maps.Clear(seen)
-		maps.Clear(namesByAddr)
+		clear(seen)
+		clear(namesByAddr)
 		for _, d := range m.Devices {
 			name := fieldPrefix(d.Name, i)
 			if seen[name] {

cmd/sniproxy/.gitignore

@@ -0,0 +1 @@
+sniproxy

cmd/sniproxy/sniproxy.go

@@ -3,46 +3,84 @@
 
 // The sniproxy is an outbound SNI proxy. It receives TLS connections over
 // Tailscale on one or more TCP ports and sends them out to the same SNI
-// hostname & port on the internet. It only does TCP.
+// hostname & port on the internet. It can optionally forward one or more
+// TCP ports to a specific destination. It only does TCP.
 package main
 
 import (
 	"context"
+	"errors"
+	"expvar"
 	"flag"
+	"fmt"
 	"log"
 	"net"
 	"net/http"
+	"os"
+	"strconv"
 	"strings"
 	"time"
 
+	"github.com/peterbourgon/ff/v3"
 	"golang.org/x/net/dns/dnsmessage"
 	"inet.af/tcpproxy"
 	"tailscale.com/client/tailscale"
 	"tailscale.com/hostinfo"
+	"tailscale.com/metrics"
 	"tailscale.com/net/netutil"
 	"tailscale.com/tsnet"
+	"tailscale.com/tsweb"
 	"tailscale.com/types/nettype"
 	"tailscale.com/util/clientmetric"
 )
 
-var (
-	ports        = flag.String("ports", "443", "comma-separated list of ports to proxy")
-	wgPort       = flag.Int("wg-listen-port", 0, "UDP port to listen on for WireGuard and peer-to-peer traffic; 0 means automatically select")
-	promoteHTTPS = flag.Bool("promote-https", true, "promote HTTP to HTTPS")
-)
-
 var tsMBox = dnsmessage.MustNewName("support.tailscale.com.")
 
-var (
-	numSessions    = clientmetric.NewCounter("sniproxy_sessions")
-	numBadAddrPort = clientmetric.NewCounter("sniproxy_bad_addrport")
-	dnsResponses   = clientmetric.NewCounter("sniproxy_dns_responses")
-	dnsFailures    = clientmetric.NewCounter("sniproxy_dns_failed")
-	httpPromoted   = clientmetric.NewCounter("sniproxy_http_promoted")
-)
+// portForward is the state for a single port forwarding entry, as passed to the --forward flag.
+type portForward struct {
+	Port        int
+	Proto       string
+	Destination string
+}
+
+// parseForward takes a proto/port/destination tuple as an input, as would be passed
+// to the --forward command line flag, and returns a *portForward struct of those parameters.
+func parseForward(value string) (*portForward, error) {
+	parts := strings.Split(value, "/")
+	if len(parts) != 3 {
+		return nil, errors.New("cannot parse: " + value)
+	}
+
+	proto := parts[0]
+	if proto != "tcp" {
+		return nil, errors.New("unsupported forwarding protocol: " + proto)
+	}
+	port, err := strconv.ParseUint(parts[1], 10, 16)
+	if err != nil {
+		return nil, errors.New("bad forwarding port: " + parts[1])
+	}
+	host := parts[2]
+	if host == "" {
+		return nil, errors.New("bad destination: " + value)
+	}
+
+	return &portForward{Port: int(port), Proto: proto, Destination: host}, nil
+}
 
 func main() {
-	flag.Parse()
+	fs := flag.NewFlagSet("sniproxy", flag.ContinueOnError)
+	var (
+		ports        = fs.String("ports", "443", "comma-separated list of ports to proxy")
+		forwards     = fs.String("forwards", "", "comma-separated list of ports to transparently forward, protocol/number/destination. For example, --forwards=tcp/22/github.com,tcp/5432/sql.example.com")
+		wgPort       = fs.Int("wg-listen-port", 0, "UDP port to listen on for WireGuard and peer-to-peer traffic; 0 means automatically select")
+		promoteHTTPS = fs.Bool("promote-https", true, "promote HTTP to HTTPS")
+		debugPort    = fs.Int("debug-port", 8893, "Listening port for debug/metrics endpoint")
+	)
+
+	err := ff.Parse(fs, os.Args[1:], ff.WithEnvVarPrefix("TS_APPC"))
+	if err != nil {
+		log.Fatal("ff.Parse")
+	}
 	if *ports == "" {
 		log.Fatal("no ports")
 	}
@@ -58,6 +96,7 @@ func main() {
 		log.Fatal(err)
 	}
 	s.lc = lc
+	s.initMetrics()
 
 	for _, portStr := range strings.Split(*ports, ",") {
 		ln, err := s.ts.Listen("tcp", ":"+portStr)
@@ -68,6 +107,33 @@ func main() {
 		go s.serve(ln)
 	}
 
+	for _, forwStr := range strings.Split(*forwards, ",") {
+		if forwStr == "" {
+			continue
+		}
+		forw, err := parseForward(forwStr)
+		if err != nil {
+			log.Fatal(err)
+		}
+
+		ln, err := s.ts.Listen("tcp", ":"+strconv.Itoa(forw.Port))
+		if err != nil {
+			log.Fatal(err)
+		}
+		log.Printf("Serving on port %d to %s...", forw.Port, forw.Destination)
+
+		// Add an entry to the expvar LabelMap for Prometheus metrics,
+		// and create a clientmetric to report that same value.
+		service := portNumberToName(forw)
+		s.numTCPsessions.SetInt64(service, 0)
+		metric := fmt.Sprintf("sniproxy_tcp_sessions_%s", service)
+		clientmetric.NewCounterFunc(metric, func() int64 {
+			return s.numTCPsessions.Get(service).Value()
+		})
+
+		go s.forward(ln, forw)
+	}
+
 	ln, err := s.ts.Listen("udp", ":53")
 	if err != nil {
 		log.Fatal(err)
@@ -83,12 +149,31 @@ func main() {
 		go s.promoteHTTPS(ln)
 	}
 
+	if *debugPort != 0 {
+		mux := http.NewServeMux()
+		tsweb.Debugger(mux)
+		dln, err := s.ts.Listen("tcp", fmt.Sprintf(":%d", *debugPort))
+		if err != nil {
+			log.Fatal(err)
+		}
+		go func() {
+			log.Fatal(http.Serve(dln, mux))
+		}()
+	}
+
 	select {}
 }
 
 type server struct {
 	ts tsnet.Server
 	lc *tailscale.LocalClient
+
+	numTLSsessions expvar.Int
+	numTCPsessions *metrics.LabelMap
+	numBadAddrPort expvar.Int
+	dnsResponses   expvar.Int
+	dnsFailures    expvar.Int
+	httpPromoted   expvar.Int
 }
 
 func (s *server) serve(ln net.Listener) {
@@ -101,6 +186,16 @@ func (s *server) serve(ln net.Listener) {
 	}
 }
 
+func (s *server) forward(ln net.Listener, forw *portForward) {
+	for {
+		c, err := ln.Accept()
+		if err != nil {
+			log.Fatal(err)
+		}
+		go s.forwardConn(c, forw)
+	}
+}
+
 func (s *server) serveDNS(ln net.Listener) {
 	for {
 		c, err := ln.Accept()
@@ -118,7 +213,7 @@ func (s *server) serveDNSConn(c nettype.ConnPacketConn) {
 	n, err := c.Read(buf)
 	if err != nil {
 		log.Printf("c.Read failed: %v\n ", err)
-		dnsFailures.Add(1)
+		s.dnsFailures.Add(1)
 		return
 	}
 
@@ -126,25 +221,25 @@ func (s *server) serveDNSConn(c nettype.ConnPacketConn) {
 	err = msg.Unpack(buf[:n])
 	if err != nil {
 		log.Printf("dnsmessage unpack failed: %v\n ", err)
-		dnsFailures.Add(1)
+		s.dnsFailures.Add(1)
 		return
 	}
 
 	buf, err = s.dnsResponse(&msg)
 	if err != nil {
 		log.Printf("s.dnsResponse failed: %v\n", err)
-		dnsFailures.Add(1)
+		s.dnsFailures.Add(1)
 		return
 	}
 
 	_, err = c.Write(buf)
 	if err != nil {
 		log.Printf("c.Write failed: %v\n", err)
-		dnsFailures.Add(1)
+		s.dnsFailures.Add(1)
 		return
 	}
 
-	dnsResponses.Add(1)
+	s.dnsResponses.Add(1)
 }
 
 func (s *server) serveConn(c net.Conn) {
@@ -152,7 +247,7 @@ func (s *server) serveConn(c net.Conn) {
 	_, port, err := net.SplitHostPort(addrPortStr)
 	if err != nil {
 		log.Printf("bogus addrPort %q", addrPortStr)
-		numBadAddrPort.Add(1)
+		s.numBadAddrPort.Add(1)
 		c.Close()
 		return
 	}
@@ -165,7 +260,7 @@ func (s *server) serveConn(c net.Conn) {
 		return netutil.NewOneConnListener(c, nil), nil
 	}
 	p.AddSNIRouteFunc(addrPortStr, func(ctx context.Context, sniName string) (t tcpproxy.Target, ok bool) {
-		numSessions.Add(1)
+		s.numTLSsessions.Add(1)
 		return &tcpproxy.DialProxy{
 			Addr:        net.JoinHostPort(sniName, port),
 			DialContext: dialer.DialContext,
@@ -174,6 +269,49 @@ func (s *server) serveConn(c net.Conn) {
 	p.Start()
 }
 
+// portNumberToName returns a human-readable name for several port numbers commonly forwarded,
+// and "tcp###" for everything else. It is used for metric label names.
+func portNumberToName(forw *portForward) string {
+	switch forw.Port {
+	case 22:
+		return "ssh"
+	case 1433:
+		return "sqlserver"
+	case 3306:
+		return "mysql"
+	case 3389:
+		return "rdp"
+	case 5432:
+		return "postgres"
+	default:
+		return fmt.Sprintf("%s%d", forw.Proto, forw.Port)
+	}
+}
+
+// forwardConn sets up a forwarder for a TCP connection. It does not inspect of the data
+// like the SNI forwarding does, it merely forwards all data to the destination specified
+// in the --forward=tcp/22/github.com argument.
+func (s *server) forwardConn(c net.Conn, forw *portForward) {
+	addrPortStr := c.LocalAddr().String()
+
+	var dialer net.Dialer
+	dialer.Timeout = 30 * time.Second
+
+	var p tcpproxy.Proxy
+	p.ListenFunc = func(net, laddr string) (net.Listener, error) {
+		return netutil.NewOneConnListener(c, nil), nil
+	}
+
+	dial := &tcpproxy.DialProxy{
+		Addr:        fmt.Sprintf("%s:%d", forw.Destination, forw.Port),
+		DialContext: dialer.DialContext,
+	}
+
+	p.AddRoute(addrPortStr, dial)
+	s.numTCPsessions.Add(portNumberToName(forw), 1)
+	p.Start()
+}
+
 func (s *server) dnsResponse(req *dnsmessage.Message) (buf []byte, err error) {
 	resp := dnsmessage.NewBuilder(buf,
 		dnsmessage.Header{
@@ -235,8 +373,36 @@ func (s *server) dnsResponse(req *dnsmessage.Message) (buf []byte, err error) {
 
 func (s *server) promoteHTTPS(ln net.Listener) {
 	err := http.Serve(ln, http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
-		httpPromoted.Add(1)
+		s.httpPromoted.Add(1)
 		http.Redirect(w, r, "https://"+r.Host+r.RequestURI, http.StatusFound)
 	}))
 	log.Fatalf("promoteHTTPS http.Serve: %v", err)
 }
+
+// initMetrics sets up local prometheus metrics, and creates clientmetrics to report those
+// same counters.
+func (s *server) initMetrics() {
+	stats := new(metrics.Set)
+
+	stats.Set("tls_sessions", &s.numTLSsessions)
+	clientmetric.NewCounterFunc("sniproxy_tls_sessions", s.numTLSsessions.Value)
+
+	s.numTCPsessions = &metrics.LabelMap{Label: "proto"}
+	stats.Set("tcp_sessions", s.numTCPsessions)
+	// clientmetric doesn't have a good way to implement a Map type.
+	// We create clientmetrics dynamically when parsing the --forwards argument
+
+	stats.Set("bad_addrport", &s.numBadAddrPort)
+	clientmetric.NewCounterFunc("sniproxy_bad_addrport", s.numBadAddrPort.Value)
+
+	stats.Set("dns_responses", &s.dnsResponses)
+	clientmetric.NewCounterFunc("sniproxy_dns_responses", s.dnsResponses.Value)
+
+	stats.Set("dns_failed", &s.dnsFailures)
+	clientmetric.NewCounterFunc("sniproxy_dns_failed", s.dnsFailures.Value)
+
+	stats.Set("http_promoted", &s.httpPromoted)
+	clientmetric.NewCounterFunc("sniproxy_http_promoted", s.httpPromoted.Value)
+
+	expvar.Publish("sniproxy", stats)
+}

cmd/sniproxy/sniproxy_test.go

@@ -0,0 +1,37 @@
+// Copyright (c) Tailscale Inc & AUTHORS
+// SPDX-License-Identifier: BSD-3-Clause
+
+package main
+
+import (
+	"strings"
+	"testing"
+
+	"github.com/google/go-cmp/cmp"
+)
+
+func TestPortForwardingArguments(t *testing.T) {
+	tests := []struct {
+		in      string
+		wanterr string
+		want    *portForward
+	}{
+		{"", "", nil},
+		{"bad port specifier", "cannot parse", nil},
+		{"tcp/xyz/example.com", "bad forwarding port", nil},
+		{"tcp//example.com", "bad forwarding port", nil},
+		{"tcp/2112/", "bad destination", nil},
+		{"udp/53/example.com", "unsupported forwarding protocol", nil},
+		{"tcp/22/github.com", "", &portForward{Proto: "tcp", Port: 22, Destination: "github.com"}},
+	}
+	for _, tt := range tests {
+		got, goterr := parseForward(tt.in)
+		if tt.wanterr != "" {
+			if !strings.Contains(goterr.Error(), tt.wanterr) {
+				t.Errorf("f(%q).err = %v; want %v", tt.in, goterr, tt.wanterr)
+			}
+		} else if diff := cmp.Diff(got, tt.want); diff != "" {
+			t.Errorf("Parsed forward (-got, +want):\n%s", diff)
+		}
+	}
+}

cmd/ssh-auth-none-demo/ssh-auth-none-demo.go

@@ -19,7 +19,6 @@ import (
 	"flag"
 	"fmt"
 	"io"
-	"io/ioutil"
 	"log"
 	"os"
 	"path/filepath"
@@ -149,7 +148,7 @@ func getHostKeys(dir string) (ret []ssh.Signer, err error) {
 
 func hostKeyFileOrCreate(keyDir, typ string) ([]byte, error) {
 	path := filepath.Join(keyDir, "ssh_host_"+typ+"_key")
-	v, err := ioutil.ReadFile(path)
+	v, err := os.ReadFile(path)
 	if err == nil {
 		return v, nil
 	}

cmd/sync-containers/main.go

@@ -1,6 +1,8 @@
 // Copyright (c) Tailscale Inc & AUTHORS
 // SPDX-License-Identifier: BSD-3-Clause
 
+//go:build !plan9
+
 // The sync-containers command synchronizes container image tags from one
 // registry to another.
 //

cmd/tailscale/cli/cli.go

@@ -14,12 +14,12 @@ import (
 	"log"
 	"os"
 	"runtime"
+	"slices"
 	"strings"
 	"sync"
 	"text/tabwriter"
 
 	"github.com/peterbourgon/ff/v3/ffcli"
-	"golang.org/x/exp/slices"
 	"tailscale.com/client/tailscale"
 	"tailscale.com/envknob"
 	"tailscale.com/paths"
@@ -120,7 +120,7 @@ change in the future.
 			pingCmd,
 			ncCmd,
 			sshCmd,
-			funnelCmd,
+			funnelCmd(),
 			serveCmd,
 			versionCmd,
 			webCmd,

cmd/tailscale/cli/cli_test.go

@@ -21,6 +21,7 @@ import (
 	"tailscale.com/tailcfg"
 	"tailscale.com/tka"
 	"tailscale.com/tstest"
+	"tailscale.com/types/logger"
 	"tailscale.com/types/persist"
 	"tailscale.com/types/preftype"
 	"tailscale.com/util/cmpx"
@@ -1151,18 +1152,13 @@ func TestUpdatePrefs(t *testing.T) {
 				justEditMP.Prefs = ipn.Prefs{} // uninteresting
 			}
 			if !reflect.DeepEqual(justEditMP, tt.wantJustEditMP) {
-				t.Logf("justEditMP != wantJustEditMP; following diff omits the Prefs field, which was \n%v", asJSON(oldEditPrefs))
+				t.Logf("justEditMP != wantJustEditMP; following diff omits the Prefs field, which was \n%v", logger.AsJSON(oldEditPrefs))
 				t.Fatalf("justEditMP: %v\n\n: ", cmp.Diff(justEditMP, tt.wantJustEditMP, cmpIP))
 			}
 		})
 	}
 }
 
-func asJSON(v any) string {
-	b, _ := json.MarshalIndent(v, "", "\t")
-	return string(b)
-}
-
 var cmpIP = cmp.Comparer(func(a, b netip.Addr) bool {
 	return a == b
 })

cmd/tailscale/cli/configure-kube.go

@@ -11,10 +11,10 @@ import (
 	"fmt"
 	"os"
 	"path/filepath"
+	"slices"
 	"strings"
 
 	"github.com/peterbourgon/ff/v3/ffcli"
-	"golang.org/x/exp/slices"
 	"k8s.io/client-go/util/homedir"
 	"sigs.k8s.io/yaml"
 	"tailscale.com/version"

cmd/tailscale/cli/debug.go

@@ -28,6 +28,7 @@ import (
 
 	"github.com/peterbourgon/ff/v3/ffcli"
 	"golang.org/x/net/http/httpproxy"
+	"tailscale.com/client/tailscale"
 	"tailscale.com/client/tailscale/apitype"
 	"tailscale.com/control/controlhttp"
 	"tailscale.com/hostinfo"
@@ -219,7 +220,9 @@ var debugCmd = &ffcli.Command{
 				fs := newFlagSet("portmap")
 				fs.DurationVar(&debugPortmapArgs.duration, "duration", 5*time.Second, "timeout for port mapping")
 				fs.StringVar(&debugPortmapArgs.ty, "type", "", `portmap debug type (one of "", "pmp", "pcp", or "upnp")`)
-				fs.StringVar(&debugPortmapArgs.gwSelf, "gw-self", "", `override gateway and self IP (format: "gatewayIP/selfIP")`)
+				fs.StringVar(&debugPortmapArgs.gatewayAddr, "gateway-addr", "", `override gateway IP (must also pass --self-addr)`)
+				fs.StringVar(&debugPortmapArgs.selfAddr, "self-addr", "", `override self IP (must also pass --gateway-addr)`)
+				fs.BoolVar(&debugPortmapArgs.logHTTP, "log-http", false, `print all HTTP requests and responses to the log`)
 				return fs
 			})(),
 		},
@@ -818,17 +821,34 @@ func runCapture(ctx context.Context, args []string) error {
 }
 
 var debugPortmapArgs struct {
-	duration time.Duration
-	gwSelf   string
-	ty       string
+	duration    time.Duration
+	gatewayAddr string
+	selfAddr    string
+	ty          string
+	logHTTP     bool
 }
 
 func debugPortmap(ctx context.Context, args []string) error {
-	rc, err := localClient.DebugPortmap(ctx,
-		debugPortmapArgs.duration,
-		debugPortmapArgs.ty,
-		debugPortmapArgs.gwSelf,
-	)
+	opts := &tailscale.DebugPortmapOpts{
+		Duration: debugPortmapArgs.duration,
+		Type:     debugPortmapArgs.ty,
+		LogHTTP:  debugPortmapArgs.logHTTP,
+	}
+	if (debugPortmapArgs.gatewayAddr != "") != (debugPortmapArgs.selfAddr != "") {
+		return fmt.Errorf("if one of --gateway-addr and --self-addr is provided, the other must be as well")
+	}
+	if debugPortmapArgs.gatewayAddr != "" {
+		var err error
+		opts.GatewayAddr, err = netip.ParseAddr(debugPortmapArgs.gatewayAddr)
+		if err != nil {
+			return fmt.Errorf("invalid --gateway-addr: %w", err)
+		}
+		opts.SelfAddr, err = netip.ParseAddr(debugPortmapArgs.selfAddr)
+		if err != nil {
+			return fmt.Errorf("invalid --self-addr: %w", err)
+		}
+	}
+	rc, err := localClient.DebugPortmap(ctx, opts)
 	if err != nil {
 		return err
 	}

cmd/tailscale/cli/exitnode.go

@@ -8,14 +8,13 @@ import (
 	"errors"
 	"flag"
 	"fmt"
-
 	"os"
+	"slices"
 	"strings"
 	"text/tabwriter"
 
 	"github.com/peterbourgon/ff/v3/ffcli"
-	"golang.org/x/exp/maps"
-	"golang.org/x/exp/slices"
+	xmaps "golang.org/x/exp/maps"
 	"tailscale.com/ipn/ipnstate"
 	"tailscale.com/tailcfg"
 	"tailscale.com/util/cmpx"
@@ -182,7 +181,7 @@ func filterFormatAndSortExitNodes(peers []*ipnstate.PeerStatus, filterBy string)
 	}
 
 	filteredExitNodes := filteredExitNodes{
-		Countries: maps.Values(countries),
+		Countries: xmaps.Values(countries),
 	}
 
 	for _, country := range filteredExitNodes.Countries {

cmd/tailscale/cli/funnel.go

@@ -9,18 +9,27 @@ import (
 	"fmt"
 	"net"
 	"os"
+	"slices"
 	"strconv"
 	"strings"
 
 	"github.com/peterbourgon/ff/v3/ffcli"
-	"golang.org/x/exp/slices"
 	"tailscale.com/ipn"
 	"tailscale.com/ipn/ipnstate"
 	"tailscale.com/tailcfg"
 	"tailscale.com/util/mak"
 )
 
-var funnelCmd = newFunnelCommand(&serveEnv{lc: &localClient})
+var funnelCmd = func() *ffcli.Command {
+	se := &serveEnv{lc: &localClient}
+	// This flag is used to switch to an in-development
+	// implementation of the tailscale funnel command.
+	// See https://github.com/tailscale/tailscale/issues/7844
+	if os.Getenv("TAILSCALE_FUNNEL_DEV") == "on" {
+		return newFunnelDevCommand(se)
+	}
+	return newFunnelCommand(se)
+}
 
 // newFunnelCommand returns a new "funnel" subcommand using e as its environment.
 // The funnel subcommand is used to turn on/off the Funnel service.
@@ -83,7 +92,7 @@ func (e *serveEnv) runFunnel(ctx context.Context, args []string) error {
 	if sc == nil {
 		sc = new(ipn.ServeConfig)
 	}
-	st, err := e.getLocalClientStatus(ctx)
+	st, err := e.getLocalClientStatusWithoutPeers(ctx)
 	if err != nil {
 		return fmt.Errorf("getting client status: %w", err)
 	}
@@ -146,7 +155,7 @@ func (e *serveEnv) verifyFunnelEnabled(ctx context.Context, st *ipnstate.Status,
 		return nil // already enabled
 	}
 	enableErr := e.enableFeatureInteractive(ctx, "funnel", hasFunnelAttrs)
-	st, statusErr := e.getLocalClientStatus(ctx) // get updated status; interactive flow may block
+	st, statusErr := e.getLocalClientStatusWithoutPeers(ctx) // get updated status; interactive flow may block
 	switch {
 	case statusErr != nil:
 		return fmt.Errorf("getting client status: %w", statusErr)

cmd/tailscale/cli/funnel_dev.go

@@ -0,0 +1,112 @@
+// Copyright (c) Tailscale Inc & AUTHORS
+// SPDX-License-Identifier: BSD-3-Clause
+
+package cli
+
+import (
+	"context"
+	"flag"
+	"fmt"
+	"io"
+	"os"
+	"strconv"
+	"strings"
+
+	"github.com/peterbourgon/ff/v3/ffcli"
+	"tailscale.com/ipn"
+)
+
+// newFunnelDevCommand returns a new "funnel" subcommand using e as its environment.
+// The funnel subcommand is used to turn on/off the Funnel service.
+// Funnel is off by default.
+// Funnel allows you to publish a 'tailscale serve' server publicly,
+// open to the entire internet.
+// newFunnelCommand shares the same serveEnv as the "serve" subcommand.
+// See newServeCommand and serve.go for more details.
+func newFunnelDevCommand(e *serveEnv) *ffcli.Command {
+	return &ffcli.Command{
+		Name:      "funnel",
+		ShortHelp: "Turn on/off Funnel service",
+		ShortUsage: strings.Join([]string{
+			"funnel <port>",
+			"funnel status [--json]",
+		}, "\n  "),
+		LongHelp: strings.Join([]string{
+			"Funnel allows you to expose your local",
+			"server publicly to the entire internet.",
+			"Note that it only supports https servers at this point.",
+			"This command is in development and is unsupported",
+		}, "\n"),
+		Exec:      e.runFunnelDev,
+		UsageFunc: usageFunc,
+		Subcommands: []*ffcli.Command{
+			{
+				Name:      "status",
+				Exec:      e.runServeStatus,
+				ShortHelp: "show current serve/Funnel status",
+				FlagSet: e.newFlags("funnel-status", func(fs *flag.FlagSet) {
+					fs.BoolVar(&e.json, "json", false, "output JSON")
+				}),
+				UsageFunc: usageFunc,
+			},
+		},
+	}
+}
+
+// runFunnelDev is the entry point for the "tailscale funnel" subcommand and
+// manages turning on/off Funnel. Funnel is off by default.
+//
+// Note: funnel is only supported on single DNS name for now. (2023-08-18)
+func (e *serveEnv) runFunnelDev(ctx context.Context, args []string) error {
+	if len(args) != 1 {
+		return flag.ErrHelp
+	}
+	var source string
+	port64, err := strconv.ParseUint(args[0], 10, 16)
+	if err == nil {
+		source = fmt.Sprintf("http://127.0.0.1:%d", port64)
+	} else {
+		source, err = expandProxyTarget(args[0])
+	}
+	if err != nil {
+		return err
+	}
+
+	st, err := e.getLocalClientStatusWithoutPeers(ctx)
+	if err != nil {
+		return fmt.Errorf("getting client status: %w", err)
+	}
+
+	if err := e.verifyFunnelEnabled(ctx, st, 443); err != nil {
+		return err
+	}
+
+	dnsName := strings.TrimSuffix(st.Self.DNSName, ".")
+	hp := ipn.HostPort(dnsName + ":443") // TODO(marwan-at-work): support the 2 other ports
+
+	// In the streaming case, the process stays running in the
+	// foreground and prints out connections to the HostPort.
+	//
+	// The local backend handles updating the ServeConfig as
+	// necessary, then restores it to its original state once
+	// the process's context is closed or the client turns off
+	// Tailscale.
+	return e.streamServe(ctx, ipn.ServeStreamRequest{
+		HostPort:   hp,
+		Source:     source,
+		MountPoint: "/", // TODO(marwan-at-work): support multiple mount points
+	})
+}
+
+func (e *serveEnv) streamServe(ctx context.Context, req ipn.ServeStreamRequest) error {
+	stream, err := e.lc.StreamServe(ctx, req)
+	if err != nil {
+		return err
+	}
+	defer stream.Close()
+
+	fmt.Fprintf(os.Stderr, "Funnel started on \"https://%s\".\n", strings.TrimSuffix(string(req.HostPort), ":443"))
+	fmt.Fprintf(os.Stderr, "Press Ctrl-C to stop Funnel.\n\n")
+	_, err = io.Copy(os.Stdout, stream)
+	return err
+}

cmd/tailscale/cli/netcheck.go

@@ -21,6 +21,7 @@ import (
 	"tailscale.com/net/netcheck"
 	"tailscale.com/net/netmon"
 	"tailscale.com/net/portmapper"
+	"tailscale.com/net/tlsdial"
 	"tailscale.com/tailcfg"
 	"tailscale.com/types/logger"
 )
@@ -76,7 +77,8 @@ func runNetcheck(ctx context.Context, args []string) error {
 		log.Printf("No DERP map from tailscaled; using default.")
 	}
 	if err != nil || noRegions {
-		dm, err = prodDERPMap(ctx, http.DefaultClient)
+		hc := &http.Client{Transport: tlsdial.NewTransport()}
+		dm, err = prodDERPMap(ctx, hc)
 		if err != nil {
 			return err
 		}

cmd/tailscale/cli/serve.go

@@ -18,12 +18,12 @@ import (
 	"path/filepath"
 	"reflect"
 	"runtime"
+	"slices"
 	"sort"
 	"strconv"
 	"strings"
 
 	"github.com/peterbourgon/ff/v3/ffcli"
-	"golang.org/x/exp/slices"
 	"tailscale.com/client/tailscale"
 	"tailscale.com/ipn"
 	"tailscale.com/ipn/ipnstate"
@@ -129,12 +129,13 @@ func (e *serveEnv) newFlags(name string, setup func(fs *flag.FlagSet)) *flag.Fla
 //
 // The purpose of this interface is to allow tests to provide a mock.
 type localServeClient interface {
-	Status(context.Context) (*ipnstate.Status, error)
+	StatusWithoutPeers(context.Context) (*ipnstate.Status, error)
 	GetServeConfig(context.Context) (*ipn.ServeConfig, error)
 	SetServeConfig(context.Context, *ipn.ServeConfig) error
 	QueryFeature(ctx context.Context, feature string) (*tailcfg.QueryFeatureResponse, error)
 	WatchIPNBus(ctx context.Context, mask ipn.NotifyWatchOpt) (*tailscale.IPNBusWatcher, error)
 	IncrementCounter(ctx context.Context, name string, delta int) error
+	StreamServe(ctx context.Context, req ipn.ServeStreamRequest) (io.ReadCloser, error) // TODO: testing :)
 }
 
 // serveEnv is the environment the serve command runs within. All I/O should be
@@ -158,19 +159,21 @@ type serveEnv struct {
 // The trailing dot is removed.
 // Returns an error if local client status fails.
 func (e *serveEnv) getSelfDNSName(ctx context.Context) (string, error) {
-	st, err := e.getLocalClientStatus(ctx)
+	st, err := e.getLocalClientStatusWithoutPeers(ctx)
 	if err != nil {
 		return "", fmt.Errorf("getting client status: %w", err)
 	}
 	return strings.TrimSuffix(st.Self.DNSName, "."), nil
 }
 
-// getLocalClientStatus returns the Status of the local client.
+// getLocalClientStatusWithoutPeers returns the Status of the local client
+// without any peers in the response.
+//
 // Returns error if unable to reach tailscaled or if self node is nil.
 //
 // Exits if status is not running or starting.
-func (e *serveEnv) getLocalClientStatus(ctx context.Context) (*ipnstate.Status, error) {
-	st, err := e.lc.Status(ctx)
+func (e *serveEnv) getLocalClientStatusWithoutPeers(ctx context.Context) (*ipnstate.Status, error) {
+	st, err := e.lc.StatusWithoutPeers(ctx)
 	if err != nil {
 		return nil, fixTailscaledConnectError(err)
 	}
@@ -641,7 +644,7 @@ func (e *serveEnv) runServeStatus(ctx context.Context, args []string) error {
 		printf("No serve config\n")
 		return nil
 	}
-	st, err := e.getLocalClientStatus(ctx)
+	st, err := e.getLocalClientStatusWithoutPeers(ctx)
 	if err != nil {
 		return err
 	}
@@ -849,8 +852,8 @@ func (e *serveEnv) enableFeatureInteractive(ctx context.Context, feature string,
 			e.lc.IncrementCounter(ctx, fmt.Sprintf("%s_enablement_lost_connection", feature), 1)
 			return err
 		}
-		if nm := n.NetMap; nm != nil && nm.SelfNode != nil {
-			if hasRequiredCapabilities(nm.SelfNode.Capabilities) {
+		if nm := n.NetMap; nm != nil && nm.SelfNode.Valid() {
+			if hasRequiredCapabilities(nm.SelfNode.Capabilities().AsSlice()) {
 				e.lc.IncrementCounter(ctx, fmt.Sprintf("%s_enabled", feature), 1)
 				fmt.Fprintln(os.Stdout, "Success.")
 				return nil

cmd/tailscale/cli/serve_test.go

@@ -9,6 +9,7 @@ import (
 	"errors"
 	"flag"
 	"fmt"
+	"io"
 	"os"
 	"path/filepath"
 	"reflect"
@@ -21,6 +22,7 @@ import (
 	"tailscale.com/ipn"
 	"tailscale.com/ipn/ipnstate"
 	"tailscale.com/tailcfg"
+	"tailscale.com/types/logger"
 )
 
 func TestCleanMountPoint(t *testing.T) {
@@ -736,8 +738,8 @@ func TestServeConfigMutations(t *testing.T) {
 			got = lc.config
 		}
 		if !reflect.DeepEqual(got, st.want) {
-			t.Fatalf("[%d] %v: bad state. got:\n%s\n\nwant:\n%s\n",
-				i, st.command, asJSON(got), asJSON(st.want))
+			t.Fatalf("[%d] %v: bad state. got:\n%v\n\nwant:\n%v\n",
+				i, st.command, logger.AsJSON(got), logger.AsJSON(st.want))
 			// NOTE: asJSON will omit empty fields, which might make
 			// result in bad state got/want diffs being the same, even
 			// though the actual state is different. Use below to debug:
@@ -810,7 +812,7 @@ func TestVerifyFunnelEnabled(t *testing.T) {
 				defer func() { fakeStatus.Self.Capabilities = oldCaps }() // reset after test
 				fakeStatus.Self.Capabilities = tt.caps
 			}
-			st, err := e.getLocalClientStatus(ctx)
+			st, err := e.getLocalClientStatusWithoutPeers(ctx)
 			if err != nil {
 				t.Fatal(err)
 			}
@@ -861,7 +863,7 @@ var fakeStatus = &ipnstate.Status{
 	},
 }
 
-func (lc *fakeLocalServeClient) Status(ctx context.Context) (*ipnstate.Status, error) {
+func (lc *fakeLocalServeClient) StatusWithoutPeers(ctx context.Context) (*ipnstate.Status, error) {
 	return fakeStatus, nil
 }
 
@@ -900,6 +902,11 @@ func (lc *fakeLocalServeClient) IncrementCounter(ctx context.Context, name strin
 	return nil // unused in tests
 }
 
+func (lc *fakeLocalServeClient) StreamServe(ctx context.Context, req ipn.ServeStreamRequest) (io.ReadCloser, error) {
+	// TODO: testing :)
+	return nil, nil
+}
+
 // exactError returns an error checker that wants exactly the provided want error.
 // If optName is non-empty, it's used in the error message.
 func exactErr(want error, optName ...string) func(error) string {

cmd/tailscale/cli/set.go

@@ -15,6 +15,7 @@ import (
 	"tailscale.com/net/netutil"
 	"tailscale.com/net/tsaddr"
 	"tailscale.com/safesocket"
+	"tailscale.com/types/views"
 )
 
 var setCmd = &ffcli.Command{
@@ -171,7 +172,7 @@ func calcAdvertiseRoutesForSet(advertiseExitNodeSet, advertiseRoutesSet bool, cu
 		if alreadyAdvertisesExitNode == setArgs.advertiseDefaultRoute {
 			return curPrefs.AdvertiseRoutes, nil
 		}
-		routes = tsaddr.FilterPrefixesCopy(curPrefs.AdvertiseRoutes, func(p netip.Prefix) bool {
+		routes = tsaddr.FilterPrefixesCopy(views.SliceOf(curPrefs.AdvertiseRoutes), func(p netip.Prefix) bool {
 			return p.Bits() != 0
 		})
 		if setArgs.advertiseDefaultRoute {

cmd/tailscale/cli/ssh_unix.go

@@ -1,7 +1,7 @@
 // Copyright (c) Tailscale Inc & AUTHORS
 // SPDX-License-Identifier: BSD-3-Clause
 
-//go:build !js && !windows
+//go:build !wasm && !windows && !plan9
 
 package cli
 

cmd/tailscale/cli/status.go

@@ -23,6 +23,7 @@ import (
 	"tailscale.com/ipn"
 	"tailscale.com/ipn/ipnstate"
 	"tailscale.com/net/interfaces"
+	"tailscale.com/util/cmpx"
 	"tailscale.com/util/dnsname"
 )
 
@@ -308,12 +309,20 @@ func dnsOrQuoteHostname(st *ipnstate.Status, ps *ipnstate.PeerStatus) string {
 }
 
 func ownerLogin(st *ipnstate.Status, ps *ipnstate.PeerStatus) string {
-	if ps.UserID.IsZero() {
+	// We prioritize showing the name of the sharer as the owner of a node if
+	// it's different from the node's user. This is less surprising: if user B
+	// from a company shares user's C node from the same company with user A who
+	// don't know user C, user A might be surprised to see user C listed in
+	// their netmap. We've historically (2021-01..2023-08) always shown the
+	// sharer's name in the UI. Perhaps we want to show both here? But the CLI's
+	// a bit space constrained.
+	uid := cmpx.Or(ps.AltSharerUserID, ps.UserID)
+	if uid.IsZero() {
 		return "-"
 	}
-	u, ok := st.User[ps.UserID]
+	u, ok := st.User[uid]
 	if !ok {
-		return fmt.Sprint(ps.UserID)
+		return fmt.Sprint(uid)
 	}
 	if i := strings.Index(u.LoginName, "@"); i != -1 {
 		return u.LoginName[:i+1]

cmd/tailscale/cli/update.go

@@ -60,7 +60,7 @@ func runUpdate(ctx context.Context, args []string) error {
 	if updateArgs.track != "" {
 		ver = updateArgs.track
 	}
-	err := clientupdate.Update(clientupdate.UpdateArgs{
+	err := clientupdate.Update(clientupdate.Arguments{
 		Version:  ver,
 		AppStore: updateArgs.appStore,
 		Logf:     func(format string, args ...any) { fmt.Printf(format+"\n", args...) },

cmd/tailscale/cli/web.go

@@ -78,7 +78,11 @@ func runWeb(ctx context.Context, args []string) error {
 		return fmt.Errorf("too many non-flag arguments: %q", args)
 	}
 
-	webServer, cleanup := web.NewServer(webArgs.dev, nil)
+	webServer, cleanup := web.NewServer(ctx, web.ServerOpts{
+		DevMode:     webArgs.dev,
+		CGIMode:     webArgs.cgi,
+		LocalClient: &localClient,
+	})
 	defer cleanup()
 
 	if webArgs.cgi {

cmd/tailscale/depaware.txt

@@ -11,9 +11,11 @@ tailscale.com/cmd/tailscale dependencies: (generated by github.com/tailscale/dep
    W    github.com/alexbrainman/sspi/internal/common                 from github.com/alexbrainman/sspi/negotiate
    W 💣 github.com/alexbrainman/sspi/negotiate                       from tailscale.com/net/tshttpproxy
    L    github.com/coreos/go-iptables/iptables                       from tailscale.com/util/linuxfw
-   W 💣 github.com/dblohm7/wingoes                                   from tailscale.com/util/winutil/authenticode
+   L    github.com/coreos/go-systemd/v22/dbus                        from tailscale.com/clientupdate
+   W 💣 github.com/dblohm7/wingoes                                   from tailscale.com/util/winutil/authenticode+
    W 💣 github.com/dblohm7/wingoes/pe                                from tailscale.com/util/winutil/authenticode
         github.com/fxamacker/cbor/v2                                 from tailscale.com/tka
+   L 💣 github.com/godbus/dbus/v5                                    from github.com/coreos/go-systemd/v22/dbus
         github.com/golang/groupcache/lru                             from tailscale.com/net/dnscache
    L    github.com/google/nftables                                   from tailscale.com/util/linuxfw
    L 💣 github.com/google/nftables/alignedbuff                       from github.com/google/nftables/xt
@@ -22,7 +24,9 @@ tailscale.com/cmd/tailscale dependencies: (generated by github.com/tailscale/dep
    L    github.com/google/nftables/internal/parseexprfunc            from github.com/google/nftables+
    L    github.com/google/nftables/xt                                from github.com/google/nftables/expr+
         github.com/google/uuid                                       from tailscale.com/util/quarantine+
-        github.com/hdevalence/ed25519consensus                       from tailscale.com/tka
+        github.com/gorilla/csrf                                      from tailscale.com/client/web
+        github.com/gorilla/securecookie                              from github.com/gorilla/csrf
+        github.com/hdevalence/ed25519consensus                       from tailscale.com/tka+
    L    github.com/josharian/native                                  from github.com/mdlayher/netlink+
    L 💣 github.com/jsimonetti/rtnetlink                              from tailscale.com/net/interfaces+
    L    github.com/jsimonetti/rtnetlink/internal/unix                from github.com/jsimonetti/rtnetlink
@@ -38,6 +42,7 @@ tailscale.com/cmd/tailscale dependencies: (generated by github.com/tailscale/dep
      💣 github.com/mitchellh/go-ps                                   from tailscale.com/cmd/tailscale/cli+
         github.com/peterbourgon/ff/v3                                from github.com/peterbourgon/ff/v3/ffcli
         github.com/peterbourgon/ff/v3/ffcli                          from tailscale.com/cmd/tailscale/cli
+        github.com/pkg/errors                                        from github.com/gorilla/csrf
         github.com/skip2/go-qrcode                                   from tailscale.com/cmd/tailscale/cli
         github.com/skip2/go-qrcode/bitset                            from github.com/skip2/go-qrcode+
         github.com/skip2/go-qrcode/reedsolomon                       from github.com/skip2/go-qrcode
@@ -70,6 +75,7 @@ tailscale.com/cmd/tailscale dependencies: (generated by github.com/tailscale/dep
         tailscale.com/client/tailscale/apitype                       from tailscale.com/cmd/tailscale/cli+
         tailscale.com/client/web                                     from tailscale.com/cmd/tailscale/cli
         tailscale.com/clientupdate                                   from tailscale.com/cmd/tailscale/cli
+        tailscale.com/clientupdate/distsign                          from tailscale.com/clientupdate
         tailscale.com/cmd/tailscale/cli                              from tailscale.com/cmd/tailscale
         tailscale.com/control/controlbase                            from tailscale.com/control/controlhttp
         tailscale.com/control/controlhttp                            from tailscale.com/cmd/tailscale/cli
@@ -168,9 +174,8 @@ tailscale.com/cmd/tailscale dependencies: (generated by github.com/tailscale/dep
         golang.org/x/crypto/nacl/secretbox                           from golang.org/x/crypto/nacl/box
         golang.org/x/crypto/pbkdf2                                   from software.sslmate.com/src/go-pkcs12
         golang.org/x/crypto/salsa20/salsa                            from golang.org/x/crypto/nacl/box+
-        golang.org/x/exp/constraints                                 from golang.org/x/exp/slices+
-        golang.org/x/exp/maps                                        from tailscale.com/types/views+
-        golang.org/x/exp/slices                                      from tailscale.com/net/tsaddr+
+   W    golang.org/x/exp/constraints                                 from github.com/dblohm7/wingoes/pe
+        golang.org/x/exp/maps                                        from tailscale.com/cmd/tailscale/cli
         golang.org/x/net/bpf                                         from github.com/mdlayher/netlink+
         golang.org/x/net/dns/dnsmessage                              from net+
         golang.org/x/net/http/httpguts                               from net/http+
@@ -197,10 +202,12 @@ tailscale.com/cmd/tailscale dependencies: (generated by github.com/tailscale/dep
         golang.org/x/text/unicode/bidi                               from golang.org/x/net/idna+
         golang.org/x/text/unicode/norm                               from golang.org/x/net/idna
         golang.org/x/time/rate                                       from tailscale.com/cmd/tailscale/cli+
+        archive/tar                                                  from tailscale.com/clientupdate
         bufio                                                        from compress/flate+
         bytes                                                        from bufio+
+        cmp                                                          from slices
         compress/flate                                               from compress/gzip+
-        compress/gzip                                                from net/http
+        compress/gzip                                                from net/http+
         compress/zlib                                                from image/png+
         container/list                                               from crypto/tls+
         context                                                      from crypto/tls+
@@ -234,6 +241,7 @@ tailscale.com/cmd/tailscale dependencies: (generated by github.com/tailscale/dep
         encoding/base32                                              from tailscale.com/tka+
         encoding/base64                                              from encoding/json+
         encoding/binary                                              from compress/gzip+
+        encoding/gob                                                 from github.com/gorilla/securecookie
         encoding/hex                                                 from crypto/x509+
         encoding/json                                                from expvar+
         encoding/pem                                                 from crypto/tls+
@@ -247,7 +255,7 @@ tailscale.com/cmd/tailscale dependencies: (generated by github.com/tailscale/dep
         hash/crc32                                                   from compress/gzip+
         hash/maphash                                                 from go4.org/mem
         html                                                         from tailscale.com/ipn/ipnstate+
-        html/template                                                from tailscale.com/client/web
+        html/template                                                from github.com/gorilla/csrf
         image                                                        from github.com/skip2/go-qrcode+
         image/color                                                  from github.com/skip2/go-qrcode+
         image/png                                                    from github.com/skip2/go-qrcode
@@ -256,6 +264,7 @@ tailscale.com/cmd/tailscale dependencies: (generated by github.com/tailscale/dep
         io/ioutil                                                    from golang.org/x/sys/cpu+
         log                                                          from expvar+
         log/internal                                                 from log
+        maps                                                         from tailscale.com/types/views+
         math                                                         from compress/flate+
         math/big                                                     from crypto/dsa+
         math/bits                                                    from compress/flate+
@@ -282,6 +291,7 @@ tailscale.com/cmd/tailscale dependencies: (generated by github.com/tailscale/dep
         regexp                                                       from github.com/tailscale/goupnp/httpu+
         regexp/syntax                                                from regexp
         runtime/debug                                                from tailscale.com/util/singleflight+
+        slices                                                       from tailscale.com/cmd/tailscale/cli+
         sort                                                         from compress/flate+
         strconv                                                      from compress/flate+
         strings                                                      from bufio+

cmd/tailscaled/debug.go

@@ -82,13 +82,13 @@ func runMonitor(ctx context.Context, loop bool) error {
 	}
 	defer mon.Close()
 
-	mon.RegisterChangeCallback(func(changed bool, st *interfaces.State) {
-		if !changed {
-			log.Printf("Network monitor fired; no change")
+	mon.RegisterChangeCallback(func(delta *netmon.ChangeDelta) {
+		if !delta.Major {
+			log.Printf("Network monitor fired; not a major change")
 			return
 		}
 		log.Printf("Network monitor fired. New state:")
-		dump(st)
+		dump(delta.New)
 	})
 	if loop {
 		log.Printf("Starting link change monitor; initial state:")

cmd/tailscaled/depaware.txt

@@ -76,9 +76,11 @@ tailscale.com/cmd/tailscaled dependencies: (generated by github.com/tailscale/de
    L    github.com/aws/smithy-go/transport/http/internal/io          from github.com/aws/smithy-go/transport/http
    L    github.com/aws/smithy-go/waiter                              from github.com/aws/aws-sdk-go-v2/service/ssm
    L    github.com/coreos/go-iptables/iptables                       from tailscale.com/util/linuxfw
+   L    github.com/coreos/go-systemd/v22/dbus                        from tailscale.com/clientupdate
   LD 💣 github.com/creack/pty                                        from tailscale.com/ssh/tailssh
    W 💣 github.com/dblohm7/wingoes                                   from github.com/dblohm7/wingoes/com+
-   W 💣 github.com/dblohm7/wingoes/com                               from tailscale.com/cmd/tailscaled
+   W 💣 github.com/dblohm7/wingoes/com                               from tailscale.com/cmd/tailscaled+
+   W 💣 github.com/dblohm7/wingoes/com/automation                    from tailscale.com/util/osdiag/internal/wsc
    W    github.com/dblohm7/wingoes/internal                          from github.com/dblohm7/wingoes/com
    W 💣 github.com/dblohm7/wingoes/pe                                from tailscale.com/util/osdiag+
         github.com/fxamacker/cbor/v2                                 from tailscale.com/tka
@@ -93,7 +95,8 @@ tailscale.com/cmd/tailscaled dependencies: (generated by github.com/tailscale/de
    L    github.com/google/nftables/expr                              from github.com/google/nftables+
    L    github.com/google/nftables/internal/parseexprfunc            from github.com/google/nftables+
    L    github.com/google/nftables/xt                                from github.com/google/nftables/expr+
-        github.com/hdevalence/ed25519consensus                       from tailscale.com/tka
+        github.com/google/uuid                                       from tailscale.com/ipn/ipnlocal+
+        github.com/hdevalence/ed25519consensus                       from tailscale.com/tka+
    L 💣 github.com/illarion/gonotify                                 from tailscale.com/net/dns
    L    github.com/insomniacslk/dhcp/dhcpv4                          from tailscale.com/net/tstun
    L    github.com/insomniacslk/dhcp/iana                            from github.com/insomniacslk/dhcp/dhcpv4
@@ -214,6 +217,8 @@ tailscale.com/cmd/tailscaled dependencies: (generated by github.com/tailscale/de
   LD    tailscale.com/chirp                                          from tailscale.com/cmd/tailscaled
         tailscale.com/client/tailscale                               from tailscale.com/derp
         tailscale.com/client/tailscale/apitype                       from tailscale.com/ipn/ipnlocal+
+        tailscale.com/clientupdate                                   from tailscale.com/ipn/ipnlocal
+        tailscale.com/clientupdate/distsign                          from tailscale.com/clientupdate
         tailscale.com/cmd/tailscaled/childproc                       from tailscale.com/ssh/tailssh+
         tailscale.com/control/controlbase                            from tailscale.com/control/controlclient+
         tailscale.com/control/controlclient                          from tailscale.com/ipn/ipnlocal+
@@ -242,7 +247,6 @@ tailscale.com/cmd/tailscaled dependencies: (generated by github.com/tailscale/de
         tailscale.com/ipn/store/mem                                  from tailscale.com/ipn/store+
    L    tailscale.com/kube                                           from tailscale.com/ipn/store/kubestore
         tailscale.com/log/filelogger                                 from tailscale.com/logpolicy
-        tailscale.com/log/logheap                                    from tailscale.com/control/controlclient
         tailscale.com/log/sockstatlog                                from tailscale.com/ipn/ipnlocal
         tailscale.com/logpolicy                                      from tailscale.com/cmd/tailscaled+
         tailscale.com/logtail                                        from tailscale.com/control/controlclient+
@@ -301,7 +305,7 @@ tailscale.com/cmd/tailscaled dependencies: (generated by github.com/tailscale/de
         tailscale.com/tstime/rate                                    from tailscale.com/wgengine/filter+
         tailscale.com/tsweb/varz                                     from tailscale.com/cmd/tailscaled
         tailscale.com/types/dnstype                                  from tailscale.com/ipn/ipnlocal+
-        tailscale.com/types/empty                                    from tailscale.com/control/controlclient+
+        tailscale.com/types/empty                                    from tailscale.com/ipn+
         tailscale.com/types/flagtype                                 from tailscale.com/cmd/tailscaled
         tailscale.com/types/ipproto                                  from tailscale.com/net/flowtrack+
         tailscale.com/types/key                                      from tailscale.com/control/controlbase+
@@ -325,7 +329,7 @@ tailscale.com/cmd/tailscaled dependencies: (generated by github.com/tailscale/de
      💣 tailscale.com/util/deephash                                  from tailscale.com/ipn/ipnlocal+
    L 💣 tailscale.com/util/dirwalk                                   from tailscale.com/metrics+
         tailscale.com/util/dnsname                                   from tailscale.com/hostinfo+
-        tailscale.com/util/goroutines                                from tailscale.com/control/controlclient+
+        tailscale.com/util/goroutines                                from tailscale.com/ipn/ipnlocal
         tailscale.com/util/groupmember                               from tailscale.com/ipn/ipnauth
      💣 tailscale.com/util/hashx                                     from tailscale.com/util/deephash
         tailscale.com/util/httpm                                     from tailscale.com/client/tailscale+
@@ -333,8 +337,9 @@ tailscale.com/cmd/tailscaled dependencies: (generated by github.com/tailscale/de
    L    tailscale.com/util/linuxfw                                   from tailscale.com/net/netns+
         tailscale.com/util/mak                                       from tailscale.com/control/controlclient+
         tailscale.com/util/multierr                                  from tailscale.com/control/controlclient+
-        tailscale.com/util/must                                      from tailscale.com/logpolicy
+        tailscale.com/util/must                                      from tailscale.com/logpolicy+
      💣 tailscale.com/util/osdiag                                    from tailscale.com/cmd/tailscaled+
+   W 💣 tailscale.com/util/osdiag/internal/wsc                       from tailscale.com/util/osdiag
         tailscale.com/util/osshare                                   from tailscale.com/ipn/ipnlocal+
    W    tailscale.com/util/pidowner                                  from tailscale.com/ipn/ipnauth
         tailscale.com/util/racebuild                                 from tailscale.com/logpolicy
@@ -347,7 +352,7 @@ tailscale.com/cmd/tailscaled dependencies: (generated by github.com/tailscale/de
         tailscale.com/util/testenv                                   from tailscale.com/ipn/ipnlocal+
         tailscale.com/util/uniq                                      from tailscale.com/wgengine/magicsock+
      💣 tailscale.com/util/winutil                                   from tailscale.com/control/controlclient+
-   W 💣 tailscale.com/util/winutil/authenticode                      from tailscale.com/util/osdiag
+   W 💣 tailscale.com/util/winutil/authenticode                      from tailscale.com/util/osdiag+
    W    tailscale.com/util/winutil/policy                            from tailscale.com/ipn/ipnlocal
         tailscale.com/version                                        from tailscale.com/derp+
         tailscale.com/version/distro                                 from tailscale.com/hostinfo+
@@ -380,9 +385,8 @@ tailscale.com/cmd/tailscaled dependencies: (generated by github.com/tailscale/de
         golang.org/x/crypto/poly1305                                 from github.com/tailscale/golang-x-crypto/ssh+
         golang.org/x/crypto/salsa20/salsa                            from golang.org/x/crypto/nacl/box+
   LD    golang.org/x/crypto/ssh                                      from tailscale.com/ssh/tailssh+
-        golang.org/x/exp/constraints                                 from golang.org/x/exp/slices+
-        golang.org/x/exp/maps                                        from tailscale.com/wgengine+
-        golang.org/x/exp/slices                                      from tailscale.com/ipn/ipnlocal+
+        golang.org/x/exp/constraints                                 from github.com/dblohm7/wingoes/pe+
+        golang.org/x/exp/maps                                        from tailscale.com/wgengine/magicsock
         golang.org/x/net/bpf                                         from github.com/mdlayher/genetlink+
         golang.org/x/net/dns/dnsmessage                              from net+
         golang.org/x/net/http/httpguts                               from golang.org/x/net/http2+
@@ -410,6 +414,7 @@ tailscale.com/cmd/tailscaled dependencies: (generated by github.com/tailscale/de
         golang.org/x/text/unicode/bidi                               from golang.org/x/net/idna+
         golang.org/x/text/unicode/norm                               from golang.org/x/net/idna
         golang.org/x/time/rate                                       from gvisor.dev/gvisor/pkg/tcpip/stack+
+        archive/tar                                                  from tailscale.com/clientupdate
         bufio                                                        from compress/flate+
         bytes                                                        from bufio+
         cmp                                                          from slices
@@ -440,6 +445,7 @@ tailscale.com/cmd/tailscaled dependencies: (generated by github.com/tailscale/de
         crypto/tls                                                   from github.com/tcnksm/go-httpstat+
         crypto/x509                                                  from crypto/tls+
         crypto/x509/pkix                                             from crypto/x509+
+        database/sql/driver                                          from github.com/google/uuid
    W    debug/dwarf                                                  from debug/pe
    W    debug/pe                                                     from github.com/dblohm7/wingoes/pe
         embed                                                        from tailscale.com+
@@ -468,6 +474,7 @@ tailscale.com/cmd/tailscaled dependencies: (generated by github.com/tailscale/de
         log                                                          from expvar+
         log/internal                                                 from log
   LD    log/syslog                                                   from tailscale.com/ssh/tailssh
+        maps                                                         from tailscale.com/types/views+
         math                                                         from compress/flate+
         math/big                                                     from crypto/dsa+
         math/bits                                                    from compress/flate+
@@ -495,9 +502,9 @@ tailscale.com/cmd/tailscaled dependencies: (generated by github.com/tailscale/de
         regexp                                                       from github.com/coreos/go-iptables/iptables+
         regexp/syntax                                                from regexp
         runtime/debug                                                from github.com/klauspost/compress/zstd+
-        runtime/pprof                                                from tailscale.com/log/logheap+
+        runtime/pprof                                                from net/http/pprof+
         runtime/trace                                                from net/http/pprof
-        slices                                                       from tailscale.com/wgengine/magicsock
+        slices                                                       from tailscale.com/wgengine/magicsock+
         sort                                                         from compress/flate+
         strconv                                                      from compress/flate+
         strings                                                      from bufio+

cmd/tailscaled/sigpipe.go

@@ -0,0 +1,12 @@
+// Copyright (c) Tailscale Inc & AUTHORS
+// SPDX-License-Identifier: BSD-3-Clause
+
+//go:build go1.21 && !plan9
+
+package main
+
+import "syscall"
+
+func init() {
+	sigPipe = syscall.SIGPIPE
+}

cmd/tailscaled/tailscaled.go

@@ -1,7 +1,7 @@
 // Copyright (c) Tailscale Inc & AUTHORS
 // SPDX-License-Identifier: BSD-3-Clause
 
-//go:build go1.19
+//go:build go1.21
 
 // The tailscaled program is the Tailscale client daemon. It's configured
 // and controlled via the tailscale CLI program.
@@ -76,6 +76,8 @@ func defaultTunName() string {
 		// "utun" is recognized by wireguard-go/tun/tun_darwin.go
 		// as a magic value that uses/creates any free number.
 		return "utun"
+	case "plan9":
+		return "userspace-networking"
 	case "linux":
 		switch distro.Get() {
 		case distro.Synology:
@@ -394,6 +396,8 @@ func run() error {
 	return startIPNServer(context.Background(), logf, pol.PublicID, sys)
 }
 
+var sigPipe os.Signal // set by sigpipe.go
+
 func startIPNServer(ctx context.Context, logf logger.Logf, logID logid.PublicID, sys *tsd.System) error {
 	ln, err := safesocket.Listen(args.socketpath)
 	if err != nil {
@@ -409,7 +413,9 @@ func startIPNServer(ctx context.Context, logf logger.Logf, logID logid.PublicID,
 	// SIGPIPE sometimes gets generated when CLIs disconnect from
 	// tailscaled. The default action is to terminate the process, we
 	// want to keep running.
-	signal.Ignore(syscall.SIGPIPE)
+	if sigPipe != nil {
+		signal.Ignore(sigPipe)
+	}
 	go func() {
 		select {
 		case s := <-interrupt:

cmd/testwrapper/testwrapper.go

@@ -22,7 +22,7 @@ import (
 	"strings"
 	"time"
 
-	"golang.org/x/exp/maps"
+	xmaps "golang.org/x/exp/maps"
 	"tailscale.com/cmd/testwrapper/flakytest"
 )
 
@@ -232,7 +232,7 @@ func main() {
 		var thisRun *nextRun
 		thisRun, toRun = toRun[0], toRun[1:]
 
-		if thisRun.attempt >= maxAttempts {
+		if thisRun.attempt > maxAttempts {
 			fmt.Println("max attempts reached")
 			os.Exit(1)
 		}
@@ -247,6 +247,12 @@ func main() {
 			go runTests(ctx, thisRun.attempt, pt, otherArgs, ch)
 			for tr := range ch {
 				if tr.pkgFinished {
+					if tr.outcome == "fail" && len(toRetry[tr.name.pkg]) == 0 {
+						// If a package fails and we don't have any tests to
+						// retry, then we should fail. This typically happens
+						// when a package times out.
+						failed = true
+					}
 					printPkgOutcome(tr.name.pkg, tr.outcome, thisRun.attempt)
 					continue
 				}
@@ -270,7 +276,7 @@ func main() {
 		if len(toRetry) == 0 {
 			continue
 		}
-		pkgs := maps.Keys(toRetry)
+		pkgs := xmaps.Keys(toRetry)
 		sort.Strings(pkgs)
 		nextRun := &nextRun{
 			attempt: thisRun.attempt + 1,

cmd/tsconnect/build-pkg.go

@@ -1,6 +1,8 @@
 // Copyright (c) Tailscale Inc & AUTHORS
 // SPDX-License-Identifier: BSD-3-Clause
 
+//go:build !plan9
+
 package main
 
 import (

cmd/tsconnect/build.go

@@ -1,6 +1,8 @@
 // Copyright (c) Tailscale Inc & AUTHORS
 // SPDX-License-Identifier: BSD-3-Clause
 
+//go:build !plan9
+
 package main
 
 import (

cmd/tsconnect/common.go

@@ -1,6 +1,8 @@
 // Copyright (c) Tailscale Inc & AUTHORS
 // SPDX-License-Identifier: BSD-3-Clause
 
+//go:build !plan9
+
 package main
 
 import (
@@ -12,11 +14,11 @@ import (
 	"path"
 	"path/filepath"
 	"runtime"
+	"slices"
 	"strconv"
 	"time"
 
 	esbuild "github.com/evanw/esbuild/pkg/api"
-	"golang.org/x/exp/slices"
 )
 
 const (

cmd/tsconnect/dev-pkg.go

@@ -1,6 +1,8 @@
 // Copyright (c) Tailscale Inc & AUTHORS
 // SPDX-License-Identifier: BSD-3-Clause
 
+//go:build !plan9
+
 package main
 
 import (

cmd/tsconnect/dev.go

@@ -1,6 +1,8 @@
 // Copyright (c) Tailscale Inc & AUTHORS
 // SPDX-License-Identifier: BSD-3-Clause
 
+//go:build !plan9
+
 package main
 
 import (

cmd/tsconnect/serve.go

@@ -1,6 +1,8 @@
 // Copyright (c) Tailscale Inc & AUTHORS
 // SPDX-License-Identifier: BSD-3-Clause
 
+//go:build !plan9
+
 package main
 
 import (

cmd/tsconnect/tsconnect.go

@@ -1,6 +1,8 @@
 // Copyright (c) Tailscale Inc & AUTHORS
 // SPDX-License-Identifier: BSD-3-Clause
 
+//go:build !plan9
+
 // The tsconnect command builds and serves the static site that is generated for
 // the Tailscale Connect JS/WASM client. Can be run in 3 modes:
 //   - dev: builds the site and serves it. JS and CSS changes can be picked up

cmd/tsconnect/wasm/wasm_js.go

@@ -257,24 +257,28 @@ func (i *jsIPN) run(jsCallbacks js.Value) {
 					},
 					MachineStatus: jsMachineStatus[nm.MachineStatus],
 				},
-				Peers: mapSlice(nm.Peers, func(p *tailcfg.Node) jsNetMapPeerNode {
-					name := p.Name
+				Peers: mapSlice(nm.Peers, func(p tailcfg.NodeView) jsNetMapPeerNode {
+					name := p.Name()
 					if name == "" {
 						// In practice this should only happen for Hello.
-						name = p.Hostinfo.Hostname()
+						name = p.Hostinfo().Hostname()
+					}
+					addrs := make([]string, p.Addresses().Len())
+					for i := range p.Addresses().LenIter() {
+						addrs[i] = p.Addresses().At(i).Addr().String()
 					}
 					return jsNetMapPeerNode{
 						jsNetMapNode: jsNetMapNode{
 							Name:       name,
-							Addresses:  mapSlice(p.Addresses, func(a netip.Prefix) string { return a.Addr().String() }),
-							MachineKey: p.Machine.String(),
-							NodeKey:    p.Key.String(),
+							Addresses:  addrs,
+							MachineKey: p.Machine().String(),
+							NodeKey:    p.Key().String(),
 						},
-						Online:              p.Online,
-						TailscaleSSHEnabled: p.Hostinfo.TailscaleSSHEnabled(),
+						Online:              p.Online(),
+						TailscaleSSHEnabled: p.Hostinfo().TailscaleSSHEnabled(),
 					}
 				}),
-				LockedOut: nm.TKAEnabled && len(nm.SelfNode.KeySignature) == 0,
+				LockedOut: nm.TKAEnabled && nm.SelfNode.KeySignature().Len() == 0,
 			}
 			if jsonNetMap, err := json.Marshal(jsNetMap); err == nil {
 				jsCallbacks.Call("notifyNetMap", string(jsonNetMap))
@@ -322,7 +326,11 @@ func (i *jsIPN) logout() {
 	if i.lb.State() == ipn.NoState {
 		log.Printf("Backend not running")
 	}
-	go i.lb.Logout()
+	go func() {
+		ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
+		defer cancel()
+		i.lb.Logout(ctx)
+	}()
 }
 
 func (i *jsIPN) ssh(host, username string, termConfig js.Value) map[string]any {

cmd/viewer/tests/tests_clone.go

@@ -6,7 +6,10 @@
 package tests
 
 import (
+	"maps"
 	"net/netip"
+
+	"tailscale.com/types/ptr"
 )
 
 // Clone makes a deep copy of StructWithPtrs.
@@ -18,12 +21,10 @@ func (src *StructWithPtrs) Clone() *StructWithPtrs {
 	dst := new(StructWithPtrs)
 	*dst = *src
 	if dst.Value != nil {
-		dst.Value = new(StructWithoutPtrs)
-		*dst.Value = *src.Value
+		dst.Value = ptr.To(*src.Value)
 	}
 	if dst.Int != nil {
-		dst.Int = new(int)
-		*dst.Int = *src.Int
+		dst.Int = ptr.To(*src.Int)
 	}
 	return dst
 }
@@ -60,12 +61,7 @@ func (src *Map) Clone() *Map {
 	}
 	dst := new(Map)
 	*dst = *src
-	if dst.Int != nil {
-		dst.Int = map[string]int{}
-		for k, v := range src.Int {
-			dst.Int[k] = v
-		}
-	}
+	dst.Int = maps.Clone(src.Int)
 	if dst.SliceInt != nil {
 		dst.SliceInt = map[string][]int{}
 		for k := range src.SliceInt {
@@ -84,12 +80,7 @@ func (src *Map) Clone() *Map {
 			dst.StructPtrWithoutPtr[k] = v.Clone()
 		}
 	}
-	if dst.StructWithoutPtr != nil {
-		dst.StructWithoutPtr = map[string]StructWithoutPtrs{}
-		for k, v := range src.StructWithoutPtr {
-			dst.StructWithoutPtr[k] = v
-		}
-	}
+	dst.StructWithoutPtr = maps.Clone(src.StructWithoutPtr)
 	if dst.SlicesWithPtrs != nil {
 		dst.SlicesWithPtrs = map[string][]*StructWithPtrs{}
 		for k := range src.SlicesWithPtrs {
@@ -102,35 +93,19 @@ func (src *Map) Clone() *Map {
 			dst.SlicesWithoutPtrs[k] = append([]*StructWithoutPtrs{}, src.SlicesWithoutPtrs[k]...)
 		}
 	}
-	if dst.StructWithoutPtrKey != nil {
-		dst.StructWithoutPtrKey = map[StructWithoutPtrs]int{}
-		for k, v := range src.StructWithoutPtrKey {
-			dst.StructWithoutPtrKey[k] = v
-		}
-	}
+	dst.StructWithoutPtrKey = maps.Clone(src.StructWithoutPtrKey)
 	if dst.SliceIntPtr != nil {
 		dst.SliceIntPtr = map[string][]*int{}
 		for k := range src.SliceIntPtr {
 			dst.SliceIntPtr[k] = append([]*int{}, src.SliceIntPtr[k]...)
 		}
 	}
-	if dst.PointerKey != nil {
-		dst.PointerKey = map[*string]int{}
-		for k, v := range src.PointerKey {
-			dst.PointerKey[k] = v
-		}
-	}
-	if dst.StructWithPtrKey != nil {
-		dst.StructWithPtrKey = map[StructWithPtrs]int{}
-		for k, v := range src.StructWithPtrKey {
-			dst.StructWithPtrKey[k] = v
-		}
-	}
+	dst.PointerKey = maps.Clone(src.PointerKey)
+	dst.StructWithPtrKey = maps.Clone(src.StructWithPtrKey)
 	if dst.StructWithPtr != nil {
 		dst.StructWithPtr = map[string]StructWithPtrs{}
 		for k, v := range src.StructWithPtr {
-			v2 := v.Clone()
-			dst.StructWithPtr[k] = *v2
+			dst.StructWithPtr[k] = *(v.Clone())
 		}
 	}
 	return dst
@@ -175,8 +150,7 @@ func (src *StructWithSlices) Clone() *StructWithSlices {
 	}
 	dst.Ints = make([]*int, len(src.Ints))
 	for i := range dst.Ints {
-		x := *src.Ints[i]
-		dst.Ints[i] = &x
+		dst.Ints[i] = ptr.To(*src.Ints[i])
 	}
 	dst.Slice = append(src.Slice[:0:0], src.Slice...)
 	dst.Prefixes = append(src.Prefixes[:0:0], src.Prefixes...)

cmd/viewer/tests/tests_view.go

@@ -10,7 +10,6 @@ import (
 	"errors"
 	"net/netip"
 
-	"go4.org/mem"
 	"tailscale.com/types/views"
 )
 
@@ -309,10 +308,10 @@ func (v StructWithSlicesView) StructPointers() views.SliceView[*StructWithPtrs,
 func (v StructWithSlicesView) Structs() StructWithPtrs    { panic("unsupported") }
 func (v StructWithSlicesView) Ints() *int                 { panic("unsupported") }
 func (v StructWithSlicesView) Slice() views.Slice[string] { return views.SliceOf(v.ж.Slice) }
-func (v StructWithSlicesView) Prefixes() views.IPPrefixSlice {
-	return views.IPPrefixSliceOf(v.ж.Prefixes)
+func (v StructWithSlicesView) Prefixes() views.Slice[netip.Prefix] {
+	return views.SliceOf(v.ж.Prefixes)
 }
-func (v StructWithSlicesView) Data() mem.RO { return mem.B(v.ж.Data) }
+func (v StructWithSlicesView) Data() views.ByteSlice[[]byte] { return views.ByteSliceOf(v.ж.Data) }
 
 // A compilation failure here means this code must be regenerated, with the command at the top of this file.
 var _StructWithSlicesViewNeedsRegeneration = StructWithSlices(struct {

cmd/viewer/viewer.go

@@ -67,9 +67,7 @@ func (v *{{.ViewName}}) UnmarshalJSON(b []byte) error {
 {{end}}
 {{define "valueField"}}func (v {{.ViewName}}) {{.FieldName}}() {{.FieldType}} { return v.ж.{{.FieldName}} }
 {{end}}
-{{define "byteSliceField"}}func (v {{.ViewName}}) {{.FieldName}}() mem.RO { return mem.B(v.ж.{{.FieldName}}) }
-{{end}}
-{{define "ipPrefixSliceField"}}func (v {{.ViewName}}) {{.FieldName}}() views.IPPrefixSlice { return views.IPPrefixSliceOf(v.ж.{{.FieldName}}) }
+{{define "byteSliceField"}}func (v {{.ViewName}}) {{.FieldName}}() views.ByteSlice[{{.FieldType}}] { return views.ByteSliceOf(v.ж.{{.FieldName}}) }
 {{end}}
 {{define "sliceField"}}func (v {{.ViewName}}) {{.FieldName}}() views.Slice[{{.FieldType}}] { return views.SliceOf(v.ж.{{.FieldName}}) }
 {{end}}
@@ -171,15 +169,12 @@ func genView(buf *bytes.Buffer, it *codegen.ImportTracker, typ *types.Named, thi
 		case *types.Slice:
 			slice := underlying
 			elem := slice.Elem()
-			args.FieldType = it.QualifiedName(elem)
 			switch elem.String() {
 			case "byte":
-				it.Import("go4.org/mem")
+				args.FieldType = it.QualifiedName(fieldType)
 				writeTemplate("byteSliceField")
-			case "inet.af/netip.Prefix", "net/netip.Prefix":
-				it.Import("tailscale.com/types/views")
-				writeTemplate("ipPrefixSliceField")
 			default:
+				args.FieldType = it.QualifiedName(elem)
 				it.Import("tailscale.com/types/views")
 				shallow, deep, base := requiresCloning(elem)
 				if deep {

control/controlclient/auto.go

@@ -17,12 +17,10 @@ import (
 	"tailscale.com/net/sockstats"
 	"tailscale.com/tailcfg"
 	"tailscale.com/tstime"
-	"tailscale.com/types/empty"
 	"tailscale.com/types/key"
 	"tailscale.com/types/logger"
 	"tailscale.com/types/netmap"
 	"tailscale.com/types/persist"
-	"tailscale.com/types/ptr"
 	"tailscale.com/types/structs"
 )
 
@@ -47,24 +45,24 @@ func (g *LoginGoal) sendLogoutError(err error) {
 
 var _ Client = (*Auto)(nil)
 
-// waitUnpause waits until the client is unpaused then returns. It only
-// returns an error if the client is closed.
-func (c *Auto) waitUnpause(routineLogName string) error {
+// waitUnpause waits until either the client is unpaused or the Auto client is
+// shut down. It reports whether the client should keep running (i.e. it's not
+// closed).
+func (c *Auto) waitUnpause(routineLogName string) (keepRunning bool) {
 	c.mu.Lock()
 	if !c.paused {
-		c.mu.Unlock()
-		return nil
+		defer c.mu.Unlock()
+		return !c.closed
 	}
 	unpaused := c.unpausedChanLocked()
 	c.mu.Unlock()
+
 	c.logf("%s: awaiting unpause", routineLogName)
-	select {
-	case <-unpaused:
+	v := <-unpaused
+	if v {
 		c.logf("%s: unpaused", routineLogName)
-		return nil
-	case <-c.quit:
-		return errors.New("quit")
 	}
+	return v
 }
 
 // updateRoutine is responsible for informing the server of worthy changes to
@@ -78,7 +76,7 @@ func (c *Auto) updateRoutine() {
 	var lastUpdateGenInformed updateGen
 
 	for {
-		if err := c.waitUnpause("updateRoutine"); err != nil {
+		if !c.waitUnpause("updateRoutine") {
 			c.logf("updateRoutine: exiting")
 			return
 		}
@@ -88,19 +86,11 @@ func (c *Auto) updateRoutine() {
 		needUpdate := gen > 0 && gen != lastUpdateGenInformed && c.loggedIn
 		c.mu.Unlock()
 
-		if needUpdate {
-			select {
-			case <-c.quit:
-				c.logf("updateRoutine: exiting")
-				return
-			default:
-			}
-		} else {
+		if !needUpdate {
 			// Nothing to do, wait for a signal.
 			select {
-			case <-c.quit:
-				c.logf("updateRoutine: exiting")
-				return
+			case <-ctx.Done():
+				continue
 			case <-c.updateCh:
 				continue
 			}
@@ -138,36 +128,34 @@ type updateGen int64
 // Auto connects to a tailcontrol server for a node.
 // It's a concrete implementation of the Client interface.
 type Auto struct {
-	direct     *Direct // our interface to the server APIs
-	clock      tstime.Clock
-	logf       logger.Logf
-	expiry     *time.Time
-	closed     bool
-	updateCh   chan struct{} // readable when we should inform the server of a change
-	newMapCh   chan struct{} // readable when we must restart a map request
-	statusFunc func(Status)  // called to update Client status; always non-nil
+	direct   *Direct // our interface to the server APIs
+	clock    tstime.Clock
+	logf     logger.Logf
+	closed   bool
+	updateCh chan struct{} // readable when we should inform the server of a change
+	observer Observer      // called to update Client status; always non-nil
 
 	unregisterHealthWatch func()
 
 	mu sync.Mutex // mutex guards the following fields
 
+	expiry time.Time
+
 	// lastUpdateGen is the gen of last update we had an update worth sending to
 	// the server.
 	lastUpdateGen updateGen
 
 	paused         bool // whether we should stop making HTTP requests
-	unpauseWaiters []chan struct{}
+	unpauseWaiters []chan bool
 	loggedIn       bool       // true if currently logged in
 	loginGoal      *LoginGoal // non-nil if some login activity is desired
 	synced         bool       // true if our netmap is up-to-date
-	inSendStatus   int        // number of sendStatus calls currently in progress
 	state          State
 
 	authCtx    context.Context // context used for auth requests
 	mapCtx     context.Context // context used for netmap and update requests
 	authCancel func()          // cancel authCtx
 	mapCancel  func()          // cancel mapCtx
-	quit       chan struct{}   // when closed, goroutines should all exit
 	authDone   chan struct{}   // when closed, authRoutine is done
 	mapDone    chan struct{}   // when closed, mapRoutine is done
 	updateDone chan struct{}   // when closed, updateRoutine is done
@@ -194,8 +182,8 @@ func NewNoStart(opts Options) (_ *Auto, err error) {
 		}
 	}()
 
-	if opts.Status == nil {
-		return nil, errors.New("missing required Options.Status")
+	if opts.Observer == nil {
+		return nil, errors.New("missing required Options.Observer")
 	}
 	if opts.Logf == nil {
 		opts.Logf = func(fmt string, args ...any) {}
@@ -208,12 +196,10 @@ func NewNoStart(opts Options) (_ *Auto, err error) {
 		clock:      opts.Clock,
 		logf:       opts.Logf,
 		updateCh:   make(chan struct{}, 1),
-		newMapCh:   make(chan struct{}, 1),
-		quit:       make(chan struct{}),
 		authDone:   make(chan struct{}),
 		mapDone:    make(chan struct{}),
 		updateDone: make(chan struct{}),
-		statusFunc: opts.Status,
+		observer:   opts.Observer,
 	}
 	c.authCtx, c.authCancel = context.WithCancel(context.Background())
 	c.authCtx = sockstats.WithSockStats(c.authCtx, sockstats.LabelControlClientAuto, opts.Logf)
@@ -232,7 +218,7 @@ func NewNoStart(opts Options) (_ *Auto, err error) {
 func (c *Auto) SetPaused(paused bool) {
 	c.mu.Lock()
 	defer c.mu.Unlock()
-	if paused == c.paused {
+	if paused == c.paused || c.closed {
 		return
 	}
 	c.logf("setPaused(%v)", paused)
@@ -240,10 +226,10 @@ func (c *Auto) SetPaused(paused bool) {
 	if paused {
 		// Only cancel the map routine. (The auth routine isn't expensive
 		// so it's fine to keep it running.)
-		c.cancelMapLocked()
+		c.cancelMapCtxLocked()
 	} else {
 		for _, ch := range c.unpauseWaiters {
-			close(ch)
+			ch <- true
 		}
 		c.unpauseWaiters = nil
 	}
@@ -280,9 +266,16 @@ func (c *Auto) updateControl() {
 	}
 }
 
-func (c *Auto) cancelAuth() {
+// cancelAuthCtx cancels the existing auth goroutine's context
+// & creates a new one, causing it to restart.
+func (c *Auto) cancelAuthCtx() {
 	c.mu.Lock()
 	defer c.mu.Unlock()
+	c.cancelAuthCtxLocked()
+}
+
+// cancelAuthCtxLocked is like cancelAuthCtx, but assumes the caller holds c.mu.
+func (c *Auto) cancelAuthCtxLocked() {
 	if c.authCancel != nil {
 		c.authCancel()
 	}
@@ -292,8 +285,16 @@ func (c *Auto) cancelAuth() {
 	}
 }
 
-// cancelMapLocked is like cancelMap, but assumes the caller holds c.mu.
-func (c *Auto) cancelMapLocked() {
+// cancelMapCtx cancels the context for the existing mapPoll and liteUpdates
+// goroutines and creates a new one, causing them to restart.
+func (c *Auto) cancelMapCtx() {
+	c.mu.Lock()
+	defer c.mu.Unlock()
+	c.cancelMapCtxLocked()
+}
+
+// cancelMapCtxLocked is like cancelMapCtx, but assumes the caller holds c.mu.
+func (c *Auto) cancelMapCtxLocked() {
 	if c.mapCancel != nil {
 		c.mapCancel()
 	}
@@ -303,32 +304,15 @@ func (c *Auto) cancelMapLocked() {
 	}
 }
 
-// cancelMap cancels the existing mapPoll and liteUpdates.
-func (c *Auto) cancelMap() {
-	c.mu.Lock()
-	defer c.mu.Unlock()
-	c.cancelMapLocked()
-}
-
 // restartMap cancels the existing mapPoll and liteUpdates, and then starts a
 // new one.
 func (c *Auto) restartMap() {
 	c.mu.Lock()
-	c.cancelMapLocked()
+	c.cancelMapCtxLocked()
 	synced := c.synced
 	c.mu.Unlock()
 
 	c.logf("[v1] restartMap: synced=%v", synced)
-
-	select {
-	case c.newMapCh <- struct{}{}:
-		c.logf("[v1] restartMap: wrote to channel")
-	default:
-		// if channel write failed, then there was already
-		// an outstanding newMapCh request. One is enough,
-		// since it'll always use the latest endpoints.
-		c.logf("[v1] restartMap: channel was full")
-	}
 	c.updateControl()
 }
 
@@ -337,6 +321,10 @@ func (c *Auto) authRoutine() {
 	bo := backoff.NewBackoff("authRoutine", c.logf, 30*time.Second)
 
 	for {
+		if !c.waitUnpause("authRoutine") {
+			c.logf("authRoutine: exiting")
+			return
+		}
 		c.mu.Lock()
 		goal := c.loginGoal
 		ctx := c.authCtx
@@ -347,13 +335,6 @@ func (c *Auto) authRoutine() {
 		}
 		c.mu.Unlock()
 
-		select {
-		case <-c.quit:
-			c.logf("[v1] authRoutine: quit")
-			return
-		default:
-		}
-
 		report := func(err error, msg string) {
 			c.logf("[v1] %s: %v", msg, err)
 			// don't send status updates for context errors,
@@ -456,26 +437,26 @@ func (c *Auto) authRoutine() {
 	}
 }
 
-// Expiry returns the credential expiration time, or the zero time if
-// the expiration time isn't known. Used in tests only.
-func (c *Auto) Expiry() *time.Time {
+// ExpiryForTests returns the credential expiration time, or the zero value if
+// the expiration time isn't known. It's used in tests only.
+func (c *Auto) ExpiryForTests() time.Time {
 	c.mu.Lock()
 	defer c.mu.Unlock()
 	return c.expiry
 }
 
-// Direct returns the underlying direct client object. Used in tests
-// only.
-func (c *Auto) Direct() *Direct {
+// DirectForTest returns the underlying direct client object.
+// It's used in tests only.
+func (c *Auto) DirectForTest() *Direct {
 	return c.direct
 }
 
-// unpausedChanLocked returns a new channel that is closed when the
-// current Auto pause is unpaused.
+// unpausedChanLocked returns a new channel that gets sent
+// either a true when unpaused or false on Auto.Shutdown.
 //
 // c.mu must be held
-func (c *Auto) unpausedChanLocked() <-chan struct{} {
-	unpaused := make(chan struct{})
+func (c *Auto) unpausedChanLocked() <-chan bool {
+	unpaused := make(chan bool, 1)
 	c.unpauseWaiters = append(c.unpauseWaiters, unpaused)
 	return unpaused
 }
@@ -488,7 +469,6 @@ type mapRoutineState struct {
 
 func (mrs mapRoutineState) UpdateFullNetmap(nm *netmap.NetworkMap) {
 	c := mrs.c
-	health.SetInPollNetMap(true)
 
 	c.mu.Lock()
 	ctx := c.mapCtx
@@ -496,7 +476,7 @@ func (mrs mapRoutineState) UpdateFullNetmap(nm *netmap.NetworkMap) {
 	if c.loggedIn {
 		c.state = StateSynchronized
 	}
-	c.expiry = ptr.To(nm.Expiry)
+	c.expiry = nm.Expiry
 	stillAuthed := c.loggedIn
 	c.logf("[v1] mapRoutine: netmap received: %s", c.state)
 	c.mu.Unlock()
@@ -518,7 +498,7 @@ func (c *Auto) mapRoutine() {
 	}
 
 	for {
-		if err := c.waitUnpause("mapRoutine"); err != nil {
+		if !c.waitUnpause("mapRoutine") {
 			c.logf("mapRoutine: exiting")
 			return
 		}
@@ -529,13 +509,6 @@ func (c *Auto) mapRoutine() {
 		ctx := c.mapCtx
 		c.mu.Unlock()
 
-		select {
-		case <-c.quit:
-			c.logf("mapRoutine: quit")
-			return
-		default:
-		}
-
 		report := func(err error, msg string) {
 			c.logf("[v1] %s: %v", msg, err)
 			err = fmt.Errorf("%s: %w", msg, err)
@@ -553,36 +526,33 @@ func (c *Auto) mapRoutine() {
 			// c.state is set by authRoutine()
 			c.mu.Unlock()
 
-			select {
-			case <-ctx.Done():
-				c.logf("[v1] mapRoutine: context done.")
-			case <-c.newMapCh:
-				c.logf("[v1] mapRoutine: new map needed while idle.")
-			}
-		} else {
-			health.SetInPollNetMap(false)
+			<-ctx.Done()
+			c.logf("[v1] mapRoutine: context done.")
+			continue
 
-			err := c.direct.PollNetMap(ctx, mrs)
+		}
+		health.SetOutOfPollNetMap()
 
-			health.SetInPollNetMap(false)
-			c.mu.Lock()
-			c.synced = false
-			if c.state == StateSynchronized {
-				c.state = StateAuthenticated
-			}
-			paused := c.paused
-			c.mu.Unlock()
+		err := c.direct.PollNetMap(ctx, mrs)
 
-			if paused {
-				mrs.bo.BackOff(ctx, nil)
-				c.logf("mapRoutine: paused")
-				continue
-			}
+		health.SetOutOfPollNetMap()
+		c.mu.Lock()
+		c.synced = false
+		if c.state == StateSynchronized {
+			c.state = StateAuthenticated
+		}
+		paused := c.paused
+		c.mu.Unlock()
 
-			report(err, "PollNetMap")
-			mrs.bo.BackOff(ctx, err)
+		if paused {
+			mrs.bo.BackOff(ctx, nil)
+			c.logf("mapRoutine: paused")
 			continue
 		}
+
+		report(err, "PollNetMap")
+		mrs.bo.BackOff(ctx, err)
+		continue
 	}
 }
 
@@ -640,40 +610,29 @@ func (c *Auto) sendStatus(who string, err error, url string, nm *netmap.NetworkM
 	state := c.state
 	loggedIn := c.loggedIn
 	synced := c.synced
-	c.inSendStatus++
 	c.mu.Unlock()
 
 	c.logf("[v1] sendStatus: %s: %v", who, state)
 
-	var p *persist.PersistView
-	var loginFin, logoutFin *empty.Message
-	if state == StateAuthenticated {
-		loginFin = new(empty.Message)
-	}
-	if state == StateNotAuthenticated {
-		logoutFin = new(empty.Message)
-	}
+	var p persist.PersistView
 	if nm != nil && loggedIn && synced {
-		p = ptr.To(c.direct.GetPersist())
+		p = c.direct.GetPersist()
 	} else {
 		// don't send netmap status, as it's misleading when we're
 		// not logged in.
 		nm = nil
 	}
 	new := Status{
-		LoginFinished:  loginFin,
-		LogoutFinished: logoutFin,
-		URL:            url,
-		Persist:        p,
-		NetMap:         nm,
-		State:          state,
-		Err:            err,
+		URL:     url,
+		Persist: p,
+		NetMap:  nm,
+		Err:     err,
+		state:   state,
 	}
-	c.statusFunc(new)
 
-	c.mu.Lock()
-	c.inSendStatus--
-	c.mu.Unlock()
+	// Launch a new goroutine to avoid blocking the caller while the observer
+	// does its thing, which may result in a call back into the client.
+	go c.observer.SetControlClientStatus(new)
 }
 
 func (c *Auto) Login(t *tailcfg.Oauth2Token, flags LoginFlags) {
@@ -687,18 +646,7 @@ func (c *Auto) Login(t *tailcfg.Oauth2Token, flags LoginFlags) {
 	}
 	c.mu.Unlock()
 
-	c.cancelAuth()
-}
-
-func (c *Auto) StartLogout() {
-	c.logf("client.StartLogout()")
-
-	c.mu.Lock()
-	c.loginGoal = &LoginGoal{
-		wantLoggedIn: false,
-	}
-	c.mu.Unlock()
-	c.cancelAuth()
+	c.cancelAuthCtx()
 }
 
 func (c *Auto) Logout(ctx context.Context) error {
@@ -712,7 +660,8 @@ func (c *Auto) Logout(ctx context.Context) error {
 		loggedOutResult: errc,
 	}
 	c.mu.Unlock()
-	c.cancelAuth()
+	c.cancelAuthCtx()
+	c.cancelMapCtx()
 
 	timer, timerChannel := c.clock.NewTimer(10 * time.Second)
 	defer timer.Stop()
@@ -745,21 +694,23 @@ func (c *Auto) Shutdown() {
 	c.logf("client.Shutdown()")
 
 	c.mu.Lock()
-	inSendStatus := c.inSendStatus
 	closed := c.closed
 	direct := c.direct
 	if !closed {
 		c.closed = true
+		c.cancelAuthCtxLocked()
+		c.cancelMapCtxLocked()
+		for _, ch := range c.unpauseWaiters {
+			ch <- false
+		}
+		c.unpauseWaiters = nil
 	}
 	c.mu.Unlock()
 
-	c.logf("client.Shutdown: inSendStatus=%v", inSendStatus)
+	c.logf("client.Shutdown")
 	if !closed {
 		c.unregisterHealthWatch()
-		close(c.quit)
-		c.cancelAuth()
 		<-c.authDone
-		c.cancelMap()
 		<-c.mapDone
 		<-c.updateDone
 		if direct != nil {

control/controlclient/client.go

@@ -34,10 +34,6 @@ type Client interface {
 	// LoginFinished flag (on success) or an auth URL (if further
 	// interaction is needed).
 	Login(*tailcfg.Oauth2Token, LoginFlags)
-	// StartLogout starts an asynchronous logout process.
-	// When it finishes, the Status callback will be called while
-	// AuthCantContinue()==true.
-	StartLogout()
 	// Logout starts a synchronous logout process. It doesn't return
 	// until the logout operation has been completed.
 	Logout(context.Context) error

control/controlclient/controlclient_test.go

@@ -6,8 +6,6 @@ package controlclient
 import (
 	"reflect"
 	"testing"
-
-	"tailscale.com/types/empty"
 )
 
 func fieldsOf(t reflect.Type) (fields []string) {
@@ -21,7 +19,7 @@ func fieldsOf(t reflect.Type) (fields []string) {
 
 func TestStatusEqual(t *testing.T) {
 	// Verify that the Equal method stays in sync with reality
-	equalHandles := []string{"LoginFinished", "LogoutFinished", "Err", "URL", "NetMap", "State", "Persist"}
+	equalHandles := []string{"Err", "URL", "NetMap", "Persist", "state"}
 	if have := fieldsOf(reflect.TypeOf(Status{})); !reflect.DeepEqual(have, equalHandles) {
 		t.Errorf("Status.Equal check might be out of sync\nfields: %q\nhandled: %q\n",
 			have, equalHandles)
@@ -52,18 +50,13 @@ func TestStatusEqual(t *testing.T) {
 			true,
 		},
 		{
-			&Status{State: StateNew},
-			&Status{State: StateNew},
+			&Status{state: StateNew},
+			&Status{state: StateNew},
 			true,
 		},
 		{
-			&Status{State: StateNew},
-			&Status{State: StateAuthenticated},
-			false,
-		},
-		{
-			&Status{LoginFinished: nil},
-			&Status{LoginFinished: new(empty.Message)},
+			&Status{state: StateNew},
+			&Status{state: StateAuthenticated},
 			false,
 		},
 	}

control/controlclient/debug.go

@@ -1,40 +0,0 @@
-// Copyright (c) Tailscale Inc & AUTHORS
-// SPDX-License-Identifier: BSD-3-Clause
-
-package controlclient
-
-import (
-	"bytes"
-	"compress/gzip"
-	"context"
-	"log"
-	"net/http"
-	"time"
-
-	"tailscale.com/util/goroutines"
-)
-
-func dumpGoroutinesToURL(c *http.Client, targetURL string) {
-	ctx, cancel := context.WithTimeout(context.Background(), 15*time.Second)
-	defer cancel()
-
-	zbuf := new(bytes.Buffer)
-	zw := gzip.NewWriter(zbuf)
-	zw.Write(goroutines.ScrubbedGoroutineDump(true))
-	zw.Close()
-
-	req, err := http.NewRequestWithContext(ctx, "PUT", targetURL, zbuf)
-	if err != nil {
-		log.Printf("dumpGoroutinesToURL: %v", err)
-		return
-	}
-	req.Header.Set("Content-Encoding", "gzip")
-	t0 := time.Now()
-	_, err = c.Do(req)
-	d := time.Since(t0).Round(time.Millisecond)
-	if err != nil {
-		log.Printf("dumpGoroutinesToURL error: %v to %v (after %v)", err, targetURL, d)
-	} else {
-		log.Printf("dumpGoroutinesToURL complete to %v (after %v)", targetURL, d)
-	}
-}

control/controlclient/direct.go

@@ -22,8 +22,10 @@ import (
 	"os"
 	"reflect"
 	"runtime"
+	"slices"
 	"strings"
 	"sync"
+	"sync/atomic"
 	"time"
 
 	"go4.org/mem"
@@ -32,7 +34,6 @@ import (
 	"tailscale.com/health"
 	"tailscale.com/hostinfo"
 	"tailscale.com/ipn/ipnstate"
-	"tailscale.com/log/logheap"
 	"tailscale.com/logtail"
 	"tailscale.com/net/dnscache"
 	"tailscale.com/net/dnsfallback"
@@ -61,26 +62,25 @@ import (
 
 // Direct is the client that connects to a tailcontrol server for a node.
 type Direct struct {
-	httpc                  *http.Client // HTTP client used to talk to tailcontrol
-	dialer                 *tsdial.Dialer
-	dnsCache               *dnscache.Resolver
-	serverURL              string // URL of the tailcontrol server
-	clock                  tstime.Clock
-	lastPrintMap           time.Time
-	newDecompressor        func() (Decompressor, error)
-	keepAlive              bool
-	logf                   logger.Logf
-	netMon                 *netmon.Monitor // or nil
-	discoPubKey            key.DiscoPublic
-	getMachinePrivKey      func() (key.MachinePrivate, error)
-	debugFlags             []string
-	keepSharerAndUserSplit bool
-	skipIPForwardingCheck  bool
-	pinger                 Pinger
-	popBrowser             func(url string)             // or nil
-	c2nHandler             http.Handler                 // or nil
-	onClientVersion        func(*tailcfg.ClientVersion) // or nil
-	onControlTime          func(time.Time)              // or nil
+	httpc                 *http.Client // HTTP client used to talk to tailcontrol
+	dialer                *tsdial.Dialer
+	dnsCache              *dnscache.Resolver
+	serverURL             string // URL of the tailcontrol server
+	clock                 tstime.Clock
+	lastPrintMap          time.Time
+	newDecompressor       func() (Decompressor, error)
+	keepAlive             bool
+	logf                  logger.Logf
+	netMon                *netmon.Monitor // or nil
+	discoPubKey           key.DiscoPublic
+	getMachinePrivKey     func() (key.MachinePrivate, error)
+	debugFlags            []string
+	skipIPForwardingCheck bool
+	pinger                Pinger
+	popBrowser            func(url string)             // or nil
+	c2nHandler            http.Handler                 // or nil
+	onClientVersion       func(*tailcfg.ClientVersion) // or nil
+	onControlTime         func(time.Time)              // or nil
 
 	dialPlan ControlDialPlanner // can be nil
 
@@ -94,7 +94,7 @@ type Direct struct {
 	persist      persist.PersistView
 	authKey      string
 	tryingNewKey key.NodePrivate
-	expiry       *time.Time
+	expiry       time.Time         // or zero value if none/unknown
 	hostinfo     *tailcfg.Hostinfo // always non-nil
 	netinfo      *tailcfg.NetInfo
 	endpoints    []tailcfg.Endpoint
@@ -102,6 +102,12 @@ type Direct struct {
 	lastPingURL  string // last PingRequest.URL received, for dup suppression
 }
 
+// Observer is implemented by users of the control client (such as LocalBackend)
+// to get notified of changes in the control client's status.
+type Observer interface {
+	SetControlClientStatus(Status)
+}
+
 type Options struct {
 	Persist              persist.Persist                    // initial persistent data
 	GetMachinePrivateKey func() (key.MachinePrivate, error) // returns the machine key to use
@@ -123,12 +129,9 @@ type Options struct {
 	Dialer               *tsdial.Dialer               // non-nil
 	C2NHandler           http.Handler                 // or nil
 
-	// Status is called when there's a change in status.
-	Status func(Status)
-
-	// KeepSharerAndUserSplit controls whether the client
-	// understands Node.Sharer. If false, the Sharer is mapped to the User.
-	KeepSharerAndUserSplit bool
+	// Observer is called when there's a change in status to report
+	// from the control client.
+	Observer Observer
 
 	// SkipIPForwardingCheck declares that the host's IP
 	// forwarding works and should not be double-checked by the
@@ -244,28 +247,27 @@ func NewDirect(opts Options) (*Direct, error) {
 	}
 
 	c := &Direct{
-		httpc:                  httpc,
-		getMachinePrivKey:      opts.GetMachinePrivateKey,
-		serverURL:              opts.ServerURL,
-		clock:                  opts.Clock,
-		logf:                   opts.Logf,
-		newDecompressor:        opts.NewDecompressor,
-		keepAlive:              opts.KeepAlive,
-		persist:                opts.Persist.View(),
-		authKey:                opts.AuthKey,
-		discoPubKey:            opts.DiscoPublicKey,
-		debugFlags:             opts.DebugFlags,
-		keepSharerAndUserSplit: opts.KeepSharerAndUserSplit,
-		netMon:                 opts.NetMon,
-		skipIPForwardingCheck:  opts.SkipIPForwardingCheck,
-		pinger:                 opts.Pinger,
-		popBrowser:             opts.PopBrowserURL,
-		onClientVersion:        opts.OnClientVersion,
-		onControlTime:          opts.OnControlTime,
-		c2nHandler:             opts.C2NHandler,
-		dialer:                 opts.Dialer,
-		dnsCache:               dnsCache,
-		dialPlan:               opts.DialPlan,
+		httpc:                 httpc,
+		getMachinePrivKey:     opts.GetMachinePrivateKey,
+		serverURL:             opts.ServerURL,
+		clock:                 opts.Clock,
+		logf:                  opts.Logf,
+		newDecompressor:       opts.NewDecompressor,
+		keepAlive:             opts.KeepAlive,
+		persist:               opts.Persist.View(),
+		authKey:               opts.AuthKey,
+		discoPubKey:           opts.DiscoPublicKey,
+		debugFlags:            opts.DebugFlags,
+		netMon:                opts.NetMon,
+		skipIPForwardingCheck: opts.SkipIPForwardingCheck,
+		pinger:                opts.Pinger,
+		popBrowser:            opts.PopBrowserURL,
+		onClientVersion:       opts.OnClientVersion,
+		onControlTime:         opts.OnControlTime,
+		c2nHandler:            opts.C2NHandler,
+		dialer:                opts.Dialer,
+		dnsCache:              dnsCache,
+		dialPlan:              opts.DialPlan,
 	}
 	if opts.Hostinfo == nil {
 		c.SetHostinfo(hostinfo.New())
@@ -444,7 +446,7 @@ func (c *Direct) doLogin(ctx context.Context, opt loginOpt) (mustRegen bool, new
 	authKey, isWrapped, wrappedSig, wrappedKey := decodeWrappedAuthkey(c.authKey, c.logf)
 	hi := c.hostInfoLocked()
 	backendLogID := hi.BackendLogID
-	expired := c.expiry != nil && !c.expiry.IsZero() && c.expiry.Before(c.clock.Now())
+	expired := !c.expiry.IsZero() && c.expiry.Before(c.clock.Now())
 	c.mu.Unlock()
 
 	machinePrivKey, err := c.getMachinePrivKey()
@@ -735,18 +737,6 @@ func resignNKS(priv key.NLPrivate, nodeKey key.NodePublic, oldNKS tkatype.Marsha
 	return newSig.Serialize(), nil
 }
 
-func sameEndpoints(a, b []tailcfg.Endpoint) bool {
-	if len(a) != len(b) {
-		return false
-	}
-	for i := range a {
-		if a[i] != b[i] {
-			return false
-		}
-	}
-	return true
-}
-
 // newEndpoints acquires c.mu and sets the local port and endpoints and reports
 // whether they've changed.
 //
@@ -756,15 +746,11 @@ func (c *Direct) newEndpoints(endpoints []tailcfg.Endpoint) (changed bool) {
 	defer c.mu.Unlock()
 
 	// Nothing new?
-	if sameEndpoints(c.endpoints, endpoints) {
+	if slices.Equal(c.endpoints, endpoints) {
 		return false // unchanged
 	}
-	var epStrs []string
-	for _, ep := range endpoints {
-		epStrs = append(epStrs, ep.Addr.String())
-	}
-	c.logf("[v2] client.newEndpoints(%v)", epStrs)
-	c.endpoints = append(c.endpoints[:0], endpoints...)
+	c.logf("[v2] client.newEndpoints(%v)", endpoints)
+	c.endpoints = slices.Clone(endpoints)
 	return true // changed
 }
 
@@ -811,10 +797,10 @@ func (c *Direct) SendUpdate(ctx context.Context) error {
 	return c.sendMapRequest(ctx, false, nil)
 }
 
-// If we go more than pollTimeout without hearing from the server,
+// If we go more than watchdogTimeout without hearing from the server,
 // end the long poll. We should be receiving a keep alive ping
 // every minute.
-const pollTimeout = 120 * time.Second
+const watchdogTimeout = 120 * time.Second
 
 // sendMapRequest makes a /map request to download the network map, calling cb
 // with each new netmap. If isStreaming, it will poll forever and only returns
@@ -961,40 +947,48 @@ func (c *Direct) sendMapRequest(ctx context.Context, isStreaming bool, nu Netmap
 		return nil
 	}
 
-	timeout, timeoutChannel := c.clock.NewTimer(pollTimeout)
-	timeoutReset := make(chan struct{})
-	pollDone := make(chan struct{})
-	defer close(pollDone)
-	go func() {
-		for {
-			select {
-			case <-pollDone:
-				vlogf("netmap: ending timeout goroutine")
-				return
-			case <-timeoutChannel:
-				c.logf("map response long-poll timed out!")
-				cancel()
-				return
-			case <-timeoutReset:
-				if !timeout.Stop() {
-					select {
-					case <-timeoutChannel:
-					case <-pollDone:
-						vlogf("netmap: ending timeout goroutine")
-						return
-					}
-				}
-				vlogf("netmap: reset timeout timer")
-				timeout.Reset(pollTimeout)
-			}
-		}
-	}()
+	var mapResIdx int // 0 for first message, then 1+ for deltas
 
-	sess := newMapSession(persist.PrivateNodeKey())
+	sess := newMapSession(persist.PrivateNodeKey(), nu)
+	defer sess.Close()
+	sess.cancel = cancel
 	sess.logf = c.logf
 	sess.vlogf = vlogf
+	sess.altClock = c.clock
 	sess.machinePubKey = machinePubKey
-	sess.keepSharerAndUserSplit = c.keepSharerAndUserSplit
+	sess.onDebug = c.handleDebugMessage
+	sess.onConciseNetMapSummary = func(summary string) {
+		// Occasionally print the netmap header.
+		// This is handy for debugging, and our logs processing
+		// pipeline depends on it. (TODO: Remove this dependency.)
+		now := c.clock.Now()
+		if now.Sub(c.lastPrintMap) < 5*time.Minute {
+			return
+		}
+		c.lastPrintMap = now
+		c.logf("[v1] new network map[%d]:\n%s", mapResIdx, summary)
+	}
+	sess.onSelfNodeChanged = func(nm *netmap.NetworkMap) {
+		c.mu.Lock()
+		defer c.mu.Unlock()
+		// If we are the ones who last updated persist, then we can update it
+		// again. Otherwise, we should not touch it. Also, it's only worth
+		// change it if the Node info changed.
+		if persist == c.persist {
+			newPersist := persist.AsStruct()
+			newPersist.NodeID = nm.SelfNode.StableID()
+			newPersist.UserProfile = nm.UserProfiles[nm.User()]
+
+			c.persist = newPersist.View()
+			persist = c.persist
+		}
+		c.expiry = nm.Expiry
+	}
+	sess.StartWatchdog()
+
+	// gotNonKeepAliveMessage is whether we've yet received a MapResponse message without
+	// KeepAlive set.
+	var gotNonKeepAliveMessage bool
 
 	// If allowStream, then the server will use an HTTP long poll to
 	// return incremental results. There is always one response right
@@ -1003,8 +997,8 @@ func (c *Direct) sendMapRequest(ctx context.Context, isStreaming bool, nu Netmap
 	// the same format before just closing the connection.
 	// We can use this same read loop either way.
 	var msg []byte
-	for i := 0; i == 0 || isStreaming; i++ {
-		vlogf("netmap: starting size read after %v (poll %v)", time.Since(t0).Round(time.Millisecond), i)
+	for ; mapResIdx == 0 || isStreaming; mapResIdx++ {
+		vlogf("netmap: starting size read after %v (poll %v)", time.Since(t0).Round(time.Millisecond), mapResIdx)
 		var siz [4]byte
 		if _, err := io.ReadFull(res.Body, siz[:]); err != nil {
 			vlogf("netmap: size read error after %v: %v", time.Since(t0).Round(time.Millisecond), err)
@@ -1068,7 +1062,7 @@ func (c *Direct) sendMapRequest(ctx context.Context, isStreaming bool, nu Netmap
 		}
 
 		select {
-		case timeoutReset <- struct{}{}:
+		case sess.watchdogReset <- struct{}{}:
 			vlogf("netmap: sent timer reset")
 		case <-ctx.Done():
 			c.logf("[v1] netmap: not resetting timer; context done: %v", ctx.Err())
@@ -1080,79 +1074,19 @@ func (c *Direct) sendMapRequest(ctx context.Context, isStreaming bool, nu Netmap
 		}
 
 		metricMapResponseMap.Add(1)
-		if i > 0 {
+		if gotNonKeepAliveMessage {
+			// If we've already seen a non-keep-alive message, this is a delta update.
 			metricMapResponseMapDelta.Add(1)
+		} else if resp.Node == nil {
+			// The very first non-keep-alive message should have Node populated.
+			c.logf("initial MapResponse lacked Node")
+			return errors.New("initial MapResponse lacked node")
 		}
+		gotNonKeepAliveMessage = true
 
-		hasDebug := resp.Debug != nil
-		// being conservative here, if Debug not present set to False
-		controlknobs.SetDisableUPnP(hasDebug && resp.Debug.DisableUPnP.EqualBool(true))
-		if hasDebug {
-			if code := resp.Debug.Exit; code != nil {
-				c.logf("exiting process with status %v per controlplane", *code)
-				os.Exit(*code)
-			}
-			if resp.Debug.DisableLogTail {
-				logtail.Disable()
-				envknob.SetNoLogsNoSupport()
-			}
-			if resp.Debug.LogHeapPprof {
-				go logheap.LogHeap(resp.Debug.LogHeapURL)
-			}
-			if resp.Debug.GoroutineDumpURL != "" {
-				go dumpGoroutinesToURL(c.httpc, resp.Debug.GoroutineDumpURL)
-			}
-			if sleep := time.Duration(resp.Debug.SleepSeconds * float64(time.Second)); sleep > 0 {
-				if err := sleepAsRequested(ctx, c.logf, timeoutReset, sleep, c.clock); err != nil {
-					return err
-				}
-			}
+		if err := sess.HandleNonKeepAliveMapResponse(ctx, &resp); err != nil {
+			return err
 		}
-
-		nm := sess.netmapForResponse(&resp)
-		if nm.SelfNode == nil {
-			c.logf("MapResponse lacked node")
-			return errors.New("MapResponse lacked node")
-		}
-
-		if d := nm.Debug; d != nil {
-			controlUseDERPRoute.Store(d.DERPRoute)
-			controlTrimWGConfig.Store(d.TrimWGConfig)
-		}
-
-		if DevKnob.StripEndpoints() {
-			for _, p := range resp.Peers {
-				p.Endpoints = nil
-			}
-		}
-		if DevKnob.StripCaps() {
-			nm.SelfNode.Capabilities = nil
-		}
-
-		// Occasionally print the netmap header.
-		// This is handy for debugging, and our logs processing
-		// pipeline depends on it. (TODO: Remove this dependency.)
-		// Code elsewhere prints netmap diffs every time they are received.
-		now := c.clock.Now()
-		if now.Sub(c.lastPrintMap) >= 5*time.Minute {
-			c.lastPrintMap = now
-			c.logf("[v1] new network map[%d]:\n%s", i, nm.VeryConcise())
-		}
-		newPersist := persist.AsStruct()
-		newPersist.NodeID = nm.SelfNode.StableID
-		newPersist.UserProfile = nm.UserProfiles[nm.User]
-
-		c.mu.Lock()
-		// If we are the ones who last updated persist, then we can update it
-		// again. Otherwise, we should not touch it.
-		if persist == c.persist {
-			c.persist = newPersist.View()
-			persist = c.persist
-		}
-		c.expiry = &nm.Expiry
-		c.mu.Unlock()
-
-		nu.UpdateFullNetmap(nm)
 	}
 	if ctx.Err() != nil {
 		return ctx.Err()
@@ -1160,6 +1094,45 @@ func (c *Direct) sendMapRequest(ctx context.Context, isStreaming bool, nu Netmap
 	return nil
 }
 
+func (c *Direct) handleDebugMessage(ctx context.Context, debug *tailcfg.Debug, watchdogReset chan<- struct{}) error {
+	if code := debug.Exit; code != nil {
+		c.logf("exiting process with status %v per controlplane", *code)
+		os.Exit(*code)
+	}
+	if debug.DisableLogTail {
+		logtail.Disable()
+		envknob.SetNoLogsNoSupport()
+	}
+	if sleep := time.Duration(debug.SleepSeconds * float64(time.Second)); sleep > 0 {
+		if err := sleepAsRequested(ctx, c.logf, watchdogReset, sleep, c.clock); err != nil {
+			return err
+		}
+	}
+	return nil
+}
+
+// initDisplayNames mutates any tailcfg.Nodes in resp to populate their display names,
+// calling InitDisplayNames on each.
+//
+// The magicDNSSuffix used is based on selfNode.
+func initDisplayNames(selfNode tailcfg.NodeView, resp *tailcfg.MapResponse) {
+	if resp.Node == nil && len(resp.Peers) == 0 && len(resp.PeersChanged) == 0 {
+		// Fast path for a common case (delta updates). No need to compute
+		// magicDNSSuffix.
+		return
+	}
+	magicDNSSuffix := netmap.MagicDNSSuffixOfNodeName(selfNode.Name())
+	if resp.Node != nil {
+		resp.Node.InitDisplayNames(magicDNSSuffix)
+	}
+	for _, n := range resp.Peers {
+		n.InitDisplayNames(magicDNSSuffix)
+	}
+	for _, n := range resp.PeersChanged {
+		n.InitDisplayNames(magicDNSSuffix)
+	}
+}
+
 // decode JSON decodes the res.Body into v. If serverNoiseKey is not specified,
 // it uses the serverKey and mkey to decode the message from the NaCl-crypto-box.
 func decode(res *http.Response, v any, serverKey, serverNoiseKey key.MachinePublic, mkey key.MachinePrivate) error {
@@ -1322,22 +1295,66 @@ func initDevKnob() devKnobs {
 
 var clock tstime.Clock = tstime.StdClock{}
 
-// opt.Bool configs from control.
+// config from control.
 var (
-	controlUseDERPRoute syncs.AtomicValue[opt.Bool]
-	controlTrimWGConfig syncs.AtomicValue[opt.Bool]
+	controlDisableDRPO         atomic.Bool
+	controlKeepFullWGConfig    atomic.Bool
+	controlRandomizeClientPort atomic.Bool
+	controlOneCGNAT            syncs.AtomicValue[opt.Bool]
 )
 
-// DERPRouteFlag reports the last reported value from control for whether
-// DERP route optimization (Issue 150) should be enabled.
-func DERPRouteFlag() opt.Bool {
-	return controlUseDERPRoute.Load()
+// DisableDRPO reports whether control says to disable the
+// DERP route optimization (Issue 150).
+func DisableDRPO() bool {
+	return controlDisableDRPO.Load()
 }
 
-// TrimWGConfig reports the last reported value from control for whether
-// we should do lazy wireguard configuration.
-func TrimWGConfig() opt.Bool {
-	return controlTrimWGConfig.Load()
+// KeepFullWGConfig reports whether control says we should disable the lazy
+// wireguard programming and instead give it the full netmap always.
+func KeepFullWGConfig() bool {
+	return controlKeepFullWGConfig.Load()
+}
+
+// RandomizeClientPort reports whether control says we should randomize
+// the client port.
+func RandomizeClientPort() bool {
+	return controlRandomizeClientPort.Load()
+}
+
+// ControlOneCGNATSetting returns control's OneCGNAT setting, if any.
+func ControlOneCGNATSetting() opt.Bool {
+	return controlOneCGNAT.Load()
+}
+
+func setControlKnobsFromNodeAttrs(selfNodeAttrs []string) {
+	var (
+		keepFullWG          bool
+		disableDRPO         bool
+		disableUPnP         bool
+		randomizeClientPort bool
+		oneCGNAT            opt.Bool
+	)
+	for _, attr := range selfNodeAttrs {
+		switch attr {
+		case tailcfg.NodeAttrDebugDisableWGTrim:
+			keepFullWG = true
+		case tailcfg.NodeAttrDebugDisableDRPO:
+			disableDRPO = true
+		case tailcfg.NodeAttrDisableUPnP:
+			disableUPnP = true
+		case tailcfg.NodeAttrRandomizeClientPort:
+			randomizeClientPort = true
+		case tailcfg.NodeAttrOneCGNATEnable:
+			oneCGNAT.Set(true)
+		case tailcfg.NodeAttrOneCGNATDisable:
+			oneCGNAT.Set(false)
+		}
+	}
+	controlKeepFullWGConfig.Store(keepFullWG)
+	controlDisableDRPO.Store(disableDRPO)
+	controlknobs.SetDisableUPnP(disableUPnP)
+	controlRandomizeClientPort.Store(randomizeClientPort)
+	controlOneCGNAT.Store(oneCGNAT)
 }
 
 // ipForwardingBroken reports whether the system's IP forwarding is disabled
@@ -1482,7 +1499,11 @@ func answerC2NPing(logf logger.Logf, c2nHandler http.Handler, c *http.Client, pr
 	}
 }
 
-func sleepAsRequested(ctx context.Context, logf logger.Logf, timeoutReset chan<- struct{}, d time.Duration, clock tstime.Clock) error {
+// sleepAsRequest implements the sleep for a tailcfg.Debug message requesting
+// that the client sleep. The complication is that while we're sleeping (if for
+// a long time), we need to periodically reset the watchdog timer before it
+// expires.
+func sleepAsRequested(ctx context.Context, logf logger.Logf, watchdogReset chan<- struct{}, d time.Duration, clock tstime.Clock) error {
 	const maxSleep = 5 * time.Minute
 	if d > maxSleep {
 		logf("sleeping for %v, capped from server-requested %v ...", maxSleep, d)
@@ -1491,7 +1512,7 @@ func sleepAsRequested(ctx context.Context, logf logger.Logf, timeoutReset chan<-
 		logf("sleeping for server-requested %v ...", d)
 	}
 
-	ticker, tickerChannel := clock.NewTicker(pollTimeout / 2)
+	ticker, tickerChannel := clock.NewTicker(watchdogTimeout / 2)
 	defer ticker.Stop()
 	timer, timerChannel := clock.NewTimer(d)
 	defer timer.Stop()
@@ -1503,7 +1524,7 @@ func sleepAsRequested(ctx context.Context, logf logger.Logf, timeoutReset chan<-
 			return nil
 		case <-tickerChannel:
 			select {
-			case timeoutReset <- struct{}{}:
+			case watchdogReset <- struct{}{}:
 			case <-timerChannel:
 				return nil
 			case <-ctx.Done():

control/controlclient/map.go

@@ -4,18 +4,27 @@
 package controlclient
 
 import (
+	"context"
+	"encoding/json"
 	"fmt"
-	"log"
+	"net"
 	"net/netip"
+	"reflect"
+	"slices"
 	"sort"
+	"strconv"
+	"sync"
 
 	"tailscale.com/envknob"
 	"tailscale.com/tailcfg"
+	"tailscale.com/tstime"
 	"tailscale.com/types/key"
 	"tailscale.com/types/logger"
 	"tailscale.com/types/netmap"
-	"tailscale.com/types/opt"
+	"tailscale.com/types/ptr"
 	"tailscale.com/types/views"
+	"tailscale.com/util/clientmetric"
+	"tailscale.com/util/cmpx"
 	"tailscale.com/wgengine/filter"
 )
 
@@ -29,14 +38,41 @@ import (
 // one MapRequest).
 type mapSession struct {
 	// Immutable fields.
-	privateNodeKey         key.NodePrivate
-	logf                   logger.Logf
-	vlogf                  logger.Logf
-	machinePubKey          key.MachinePublic
-	keepSharerAndUserSplit bool // see Options.KeepSharerAndUserSplit
+	nu             NetmapUpdater // called on changes (in addition to the optional hooks below)
+	privateNodeKey key.NodePrivate
+	publicNodeKey  key.NodePublic
+	logf           logger.Logf
+	vlogf          logger.Logf
+	machinePubKey  key.MachinePublic
+	altClock       tstime.Clock       // if nil, regular time is used
+	cancel         context.CancelFunc // always non-nil, shuts down caller's base long poll context
+	watchdogReset  chan struct{}      // send to request that the long poll activity watchdog timeout be reset
+
+	// sessionAliveCtx is a Background-based context that's alive for the
+	// duration of the mapSession that we own the lifetime of. It's closed by
+	// sessionAliveCtxClose.
+	sessionAliveCtx      context.Context
+	sessionAliveCtxClose context.CancelFunc // closes sessionAliveCtx
+
+	// Optional hooks, set once before use.
+
+	// onDebug specifies what to do with a *tailcfg.Debug message.
+	// If the watchdogReset chan is nil, it's not used. Otherwise it can be sent to
+	// to request that the long poll activity watchdog timeout be reset.
+	onDebug func(_ context.Context, _ *tailcfg.Debug, watchdogReset chan<- struct{}) error
+
+	// onConciseNetMapSummary, if non-nil, is called with the Netmap.VeryConcise summary
+	// whenever a map response is received.
+	onConciseNetMapSummary func(string)
+
+	// onSelfNodeChanged is called before the NetmapUpdater if the self node was
+	// changed.
+	onSelfNodeChanged func(*netmap.NetworkMap)
 
 	// Fields storing state over the course of multiple MapResponses.
-	lastNode               *tailcfg.Node
+	lastNode               tailcfg.NodeView
+	peers                  map[tailcfg.NodeID]*tailcfg.NodeView // pointer to view (oddly). same pointers as sortedPeers.
+	sortedPeers            []*tailcfg.NodeView                  // same pointers as peers, but sorted by Node.ID
 	lastDNSConfig          *tailcfg.DNSConfig
 	lastDERPMap            *tailcfg.DERPMap
 	lastUserProfile        map[tailcfg.UserID]tailcfg.UserProfile
@@ -44,51 +80,156 @@ type mapSession struct {
 	lastParsedPacketFilter []filter.Match
 	lastSSHPolicy          *tailcfg.SSHPolicy
 	collectServices        bool
-	previousPeers          []*tailcfg.Node // for delta-purposes
 	lastDomain             string
 	lastDomainAuditLogID   string
 	lastHealth             []string
 	lastPopBrowserURL      string
 	stickyDebug            tailcfg.Debug // accumulated opt.Bool values
 	lastTKAInfo            *tailcfg.TKAInfo
-
-	// netMapBuilding is non-nil during a netmapForResponse call,
-	// containing the value to be returned, once fully populated.
-	netMapBuilding *netmap.NetworkMap
+	lastNetmapSummary      string // from NetworkMap.VeryConcise
 }
 
-func newMapSession(privateNodeKey key.NodePrivate) *mapSession {
+// newMapSession returns a mostly unconfigured new mapSession.
+//
+// Modify its optional fields on the returned value before use.
+//
+// It must have its Close method called to release resources.
+func newMapSession(privateNodeKey key.NodePrivate, nu NetmapUpdater) *mapSession {
 	ms := &mapSession{
+		nu:              nu,
 		privateNodeKey:  privateNodeKey,
-		logf:            logger.Discard,
-		vlogf:           logger.Discard,
+		publicNodeKey:   privateNodeKey.Public(),
 		lastDNSConfig:   new(tailcfg.DNSConfig),
 		lastUserProfile: map[tailcfg.UserID]tailcfg.UserProfile{},
+		watchdogReset:   make(chan struct{}),
+
+		// Non-nil no-op defaults, to be optionally overridden by the caller.
+		logf:                   logger.Discard,
+		vlogf:                  logger.Discard,
+		cancel:                 func() {},
+		onDebug:                func(context.Context, *tailcfg.Debug, chan<- struct{}) error { return nil },
+		onConciseNetMapSummary: func(string) {},
+		onSelfNodeChanged:      func(*netmap.NetworkMap) {},
 	}
+	ms.sessionAliveCtx, ms.sessionAliveCtxClose = context.WithCancel(context.Background())
 	return ms
 }
 
-func (ms *mapSession) addUserProfile(userID tailcfg.UserID) {
-	nm := ms.netMapBuilding
-	if _, dup := nm.UserProfiles[userID]; dup {
-		// Already populated it from a previous peer.
-		return
-	}
-	if up, ok := ms.lastUserProfile[userID]; ok {
-		nm.UserProfiles[userID] = up
-	}
+func (ms *mapSession) clock() tstime.Clock {
+	return cmpx.Or[tstime.Clock](ms.altClock, tstime.StdClock{})
 }
 
-// netmapForResponse returns a fully populated NetworkMap from a full
-// or incremental MapResponse within the session, filling in omitted
-// information from prior MapResponse values.
-func (ms *mapSession) netmapForResponse(resp *tailcfg.MapResponse) *netmap.NetworkMap {
-	undeltaPeers(resp, ms.previousPeers)
+// StartWatchdog starts the session's watchdog timer.
+// If there's no activity in too long, it tears down the connection.
+// Call Close to release these resources.
+func (ms *mapSession) StartWatchdog() {
+	timer, timedOutChan := ms.clock().NewTimer(watchdogTimeout)
+	go func() {
+		defer timer.Stop()
+		for {
+			select {
+			case <-ms.sessionAliveCtx.Done():
+				ms.vlogf("netmap: ending timeout goroutine")
+				return
+			case <-timedOutChan:
+				ms.logf("map response long-poll timed out!")
+				ms.cancel()
+				return
+			case <-ms.watchdogReset:
+				if !timer.Stop() {
+					select {
+					case <-timedOutChan:
+					case <-ms.sessionAliveCtx.Done():
+						ms.vlogf("netmap: ending timeout goroutine")
+						return
+					}
+				}
+				ms.vlogf("netmap: reset timeout timer")
+				timer.Reset(watchdogTimeout)
+			}
+		}
+	}()
+}
+
+func (ms *mapSession) Close() {
+	ms.sessionAliveCtxClose()
+}
+
+// HandleNonKeepAliveMapResponse handles a non-KeepAlive MapResponse (full or
+// incremental).
+//
+// All fields that are valid on a KeepAlive MapResponse have already been
+// handled.
+//
+// TODO(bradfitz): make this handle all fields later. For now (2023-08-20) this
+// is [re]factoring progress enough.
+func (ms *mapSession) HandleNonKeepAliveMapResponse(ctx context.Context, resp *tailcfg.MapResponse) error {
+	if debug := resp.Debug; debug != nil {
+		if err := ms.onDebug(ctx, debug, ms.watchdogReset); err != nil {
+			return err
+		}
+	}
+
+	if DevKnob.StripEndpoints() {
+		for _, p := range resp.Peers {
+			p.Endpoints = nil
+		}
+		for _, p := range resp.PeersChanged {
+			p.Endpoints = nil
+		}
+	}
+
+	// For responses that mutate the self node, check for updated nodeAttrs.
+	if resp.Node != nil {
+		if DevKnob.StripCaps() {
+			resp.Node.Capabilities = nil
+		}
+		setControlKnobsFromNodeAttrs(resp.Node.Capabilities)
+	}
+
+	// Call Node.InitDisplayNames on any changed nodes.
+	initDisplayNames(cmpx.Or(resp.Node.View(), ms.lastNode), resp)
+
+	ms.patchifyPeersChanged(resp)
+
+	ms.updateStateFromResponse(resp)
+
+	nm := ms.netmap()
+
+	ms.lastNetmapSummary = nm.VeryConcise()
+	ms.onConciseNetMapSummary(ms.lastNetmapSummary)
+
+	// If the self node changed, we might need to update persist.
+	if resp.Node != nil {
+		ms.onSelfNodeChanged(nm)
+	}
+
+	ms.nu.UpdateFullNetmap(nm)
+	return nil
+}
+
+// updateStats are some stats from updateStateFromResponse, primarily for
+// testing. It's meant to be cheap enough to always compute, though. It doesn't
+// allocate.
+type updateStats struct {
+	allNew  bool
+	added   int
+	removed int
+	changed int
+}
+
+// updateStateFromResponse updates ms from res. It takes ownership of res.
+func (ms *mapSession) updateStateFromResponse(resp *tailcfg.MapResponse) {
+	ms.updatePeersStateFromResponse(resp)
+
+	if resp.Node != nil {
+		ms.lastNode = resp.Node.View()
+	}
 
-	ms.previousPeers = cloneNodes(resp.Peers) // defensive/lazy clone, since this escapes to who knows where
 	for _, up := range resp.UserProfiles {
 		ms.lastUserProfile[up.ID] = up
 	}
+	// TODO(bradfitz): clean up old user profiles? maybe not worth it.
 
 	if dm := resp.DERPMap; dm != nil {
 		ms.vlogf("netmap: new map contains DERP map")
@@ -144,34 +285,409 @@ func (ms *mapSession) netmapForResponse(resp *tailcfg.MapResponse) *netmap.Netwo
 	if resp.TKAInfo != nil {
 		ms.lastTKAInfo = resp.TKAInfo
 	}
+}
 
-	debug := resp.Debug
-	if debug != nil {
-		if debug.RandomizeClientPort {
-			debug.SetRandomizeClientPort.Set(true)
+var (
+	patchDERPRegion   = clientmetric.NewCounter("controlclient_patch_derp")
+	patchEndpoints    = clientmetric.NewCounter("controlclient_patch_endpoints")
+	patchCap          = clientmetric.NewCounter("controlclient_patch_capver")
+	patchKey          = clientmetric.NewCounter("controlclient_patch_key")
+	patchDiscoKey     = clientmetric.NewCounter("controlclient_patch_discokey")
+	patchOnline       = clientmetric.NewCounter("controlclient_patch_online")
+	patchLastSeen     = clientmetric.NewCounter("controlclient_patch_lastseen")
+	patchKeyExpiry    = clientmetric.NewCounter("controlclient_patch_keyexpiry")
+	patchCapabilities = clientmetric.NewCounter("controlclient_patch_capabilities")
+	patchKeySignature = clientmetric.NewCounter("controlclient_patch_keysig")
+
+	patchifiedPeer      = clientmetric.NewCounter("controlclient_patchified_peer")
+	patchifiedPeerEqual = clientmetric.NewCounter("controlclient_patchified_peer_equal")
+)
+
+// updatePeersStateFromResponseres updates ms.peers and ms.sortedPeers from res. It takes ownership of res.
+func (ms *mapSession) updatePeersStateFromResponse(resp *tailcfg.MapResponse) (stats updateStats) {
+	defer func() {
+		if stats.removed > 0 || stats.added > 0 {
+			ms.rebuildSorted()
 		}
-		if debug.ForceBackgroundSTUN {
-			debug.SetForceBackgroundSTUN.Set(true)
-		}
-		copyDebugOptBools(&ms.stickyDebug, debug)
-	} else if ms.stickyDebug != (tailcfg.Debug{}) {
-		debug = new(tailcfg.Debug)
+	}()
+
+	if ms.peers == nil {
+		ms.peers = make(map[tailcfg.NodeID]*tailcfg.NodeView)
 	}
-	if debug != nil {
-		copyDebugOptBools(debug, &ms.stickyDebug)
-		if !debug.ForceBackgroundSTUN {
-			debug.ForceBackgroundSTUN, _ = ms.stickyDebug.SetForceBackgroundSTUN.Get()
+
+	if len(resp.Peers) > 0 {
+		// Not delta encoded.
+		stats.allNew = true
+		keep := make(map[tailcfg.NodeID]bool, len(resp.Peers))
+		for _, n := range resp.Peers {
+			keep[n.ID] = true
+			if vp, ok := ms.peers[n.ID]; ok {
+				stats.changed++
+				*vp = n.View()
+			} else {
+				stats.added++
+				ms.peers[n.ID] = ptr.To(n.View())
+			}
 		}
-		if !debug.RandomizeClientPort {
-			debug.RandomizeClientPort, _ = ms.stickyDebug.SetRandomizeClientPort.Get()
+		for id := range ms.peers {
+			if !keep[id] {
+				stats.removed++
+				delete(ms.peers, id)
+			}
 		}
+		// Peers precludes all other delta operations so just return.
+		return
+	}
+
+	for _, id := range resp.PeersRemoved {
+		if _, ok := ms.peers[id]; ok {
+			delete(ms.peers, id)
+			stats.removed++
+		}
+	}
+
+	for _, n := range resp.PeersChanged {
+		if vp, ok := ms.peers[n.ID]; ok {
+			stats.changed++
+			*vp = n.View()
+		} else {
+			stats.added++
+			ms.peers[n.ID] = ptr.To(n.View())
+		}
+	}
+
+	for nodeID, seen := range resp.PeerSeenChange {
+		if vp, ok := ms.peers[nodeID]; ok {
+			mut := vp.AsStruct()
+			if seen {
+				mut.LastSeen = ptr.To(clock.Now())
+			} else {
+				mut.LastSeen = nil
+			}
+			*vp = mut.View()
+			stats.changed++
+		}
+	}
+
+	for nodeID, online := range resp.OnlineChange {
+		if vp, ok := ms.peers[nodeID]; ok {
+			mut := vp.AsStruct()
+			mut.Online = ptr.To(online)
+			*vp = mut.View()
+			stats.changed++
+		}
+	}
+
+	for _, pc := range resp.PeersChangedPatch {
+		vp, ok := ms.peers[pc.NodeID]
+		if !ok {
+			continue
+		}
+		stats.changed++
+		mut := vp.AsStruct()
+		if pc.DERPRegion != 0 {
+			mut.DERP = fmt.Sprintf("%s:%v", tailcfg.DerpMagicIP, pc.DERPRegion)
+			patchDERPRegion.Add(1)
+		}
+		if pc.Cap != 0 {
+			mut.Cap = pc.Cap
+			patchCap.Add(1)
+		}
+		if pc.Endpoints != nil {
+			mut.Endpoints = pc.Endpoints
+			patchEndpoints.Add(1)
+		}
+		if pc.Key != nil {
+			mut.Key = *pc.Key
+			patchKey.Add(1)
+		}
+		if pc.DiscoKey != nil {
+			mut.DiscoKey = *pc.DiscoKey
+			patchDiscoKey.Add(1)
+		}
+		if v := pc.Online; v != nil {
+			mut.Online = ptr.To(*v)
+			patchOnline.Add(1)
+		}
+		if v := pc.LastSeen; v != nil {
+			mut.LastSeen = ptr.To(*v)
+			patchLastSeen.Add(1)
+		}
+		if v := pc.KeyExpiry; v != nil {
+			mut.KeyExpiry = *v
+			patchKeyExpiry.Add(1)
+		}
+		if v := pc.Capabilities; v != nil {
+			mut.Capabilities = *v
+			patchCapabilities.Add(1)
+		}
+		if v := pc.KeySignature; v != nil {
+			mut.KeySignature = v
+			patchKeySignature.Add(1)
+		}
+		*vp = mut.View()
+	}
+
+	return
+}
+
+// rebuildSorted rebuilds ms.sortedPeers from ms.peers. It should be called
+// after any additions or removals from peers.
+func (ms *mapSession) rebuildSorted() {
+	if ms.sortedPeers == nil {
+		ms.sortedPeers = make([]*tailcfg.NodeView, 0, len(ms.peers))
+	} else {
+		if len(ms.sortedPeers) > len(ms.peers) {
+			clear(ms.sortedPeers[len(ms.peers):])
+		}
+		ms.sortedPeers = ms.sortedPeers[:0]
+	}
+	for _, p := range ms.peers {
+		ms.sortedPeers = append(ms.sortedPeers, p)
+	}
+	sort.Slice(ms.sortedPeers, func(i, j int) bool {
+		return ms.sortedPeers[i].ID() < ms.sortedPeers[j].ID()
+	})
+}
+
+func (ms *mapSession) addUserProfile(nm *netmap.NetworkMap, userID tailcfg.UserID) {
+	if userID == 0 {
+		return
+	}
+	if _, dup := nm.UserProfiles[userID]; dup {
+		// Already populated it from a previous peer.
+		return
+	}
+	if up, ok := ms.lastUserProfile[userID]; ok {
+		nm.UserProfiles[userID] = up
+	}
+}
+
+var debugPatchifyPeer = envknob.RegisterBool("TS_DEBUG_PATCHIFY_PEER")
+
+// patchifyPeersChanged mutates resp to promote PeersChanged entries to PeersChangedPatch
+// when possible.
+func (ms *mapSession) patchifyPeersChanged(resp *tailcfg.MapResponse) {
+	filtered := resp.PeersChanged[:0]
+	for _, n := range resp.PeersChanged {
+		if p, ok := ms.patchifyPeer(n); ok {
+			patchifiedPeer.Add(1)
+			if debugPatchifyPeer() {
+				patchj, _ := json.Marshal(p)
+				ms.logf("debug: patchifyPeer[ID=%v]: %s", n.ID, patchj)
+			}
+			if p != nil {
+				resp.PeersChangedPatch = append(resp.PeersChangedPatch, p)
+			} else {
+				patchifiedPeerEqual.Add(1)
+			}
+		} else {
+			filtered = append(filtered, n)
+		}
+	}
+	resp.PeersChanged = filtered
+	if len(resp.PeersChanged) == 0 {
+		resp.PeersChanged = nil
+	}
+}
+
+var nodeFields = sync.OnceValue(getNodeFields)
+
+// getNodeFields returns the fails of tailcfg.Node.
+func getNodeFields() []string {
+	rt := reflect.TypeOf((*tailcfg.Node)(nil)).Elem()
+	ret := make([]string, rt.NumField())
+	for i := 0; i < rt.NumField(); i++ {
+		ret[i] = rt.Field(i).Name
+	}
+	return ret
+}
+
+// patchifyPeer returns a *tailcfg.PeerChange of the session's existing copy of
+// the n.ID Node to n.
+//
+// It returns ok=false if a patch can't be made, (V, ok) on a delta, or (nil,
+// true) if all the fields were identical (a zero change).
+func (ms *mapSession) patchifyPeer(n *tailcfg.Node) (_ *tailcfg.PeerChange, ok bool) {
+	was, ok := ms.peers[n.ID]
+	if !ok {
+		return nil, false
+	}
+	return peerChangeDiff(*was, n)
+}
+
+// peerChangeDiff returns the difference from 'was' to 'n', if possible.
+//
+// It returns (nil, true) if the fields were identical.
+func peerChangeDiff(was tailcfg.NodeView, n *tailcfg.Node) (_ *tailcfg.PeerChange, ok bool) {
+	var ret *tailcfg.PeerChange
+	pc := func() *tailcfg.PeerChange {
+		if ret == nil {
+			ret = new(tailcfg.PeerChange)
+		}
+		return ret
+	}
+	for _, field := range nodeFields() {
+		switch field {
+		default:
+			// The whole point of using reflect in this function is to panic
+			// here in tests if we forget to handle a new field.
+			panic("unhandled field: " + field)
+		case "computedHostIfDifferent", "ComputedName", "ComputedNameWithHost":
+			// Caller's responsibility to have populated these.
+			continue
+		case "DataPlaneAuditLogID":
+			//  Not sent for peers.
+		case "ID":
+			if was.ID() != n.ID {
+				return nil, false
+			}
+		case "StableID":
+			if was.StableID() != n.StableID {
+				return nil, false
+			}
+		case "Name":
+			if was.Name() != n.Name {
+				return nil, false
+			}
+		case "User":
+			if was.User() != n.User {
+				return nil, false
+			}
+		case "Sharer":
+			if was.Sharer() != n.Sharer {
+				return nil, false
+			}
+		case "Key":
+			if was.Key() != n.Key {
+				pc().Key = ptr.To(n.Key)
+			}
+		case "KeyExpiry":
+			if !was.KeyExpiry().Equal(n.KeyExpiry) {
+				pc().KeyExpiry = ptr.To(n.KeyExpiry)
+			}
+		case "KeySignature":
+			if !was.KeySignature().Equal(n.KeySignature) {
+				pc().KeySignature = slices.Clone(n.KeySignature)
+			}
+		case "Machine":
+			if was.Machine() != n.Machine {
+				return nil, false
+			}
+		case "DiscoKey":
+			if was.DiscoKey() != n.DiscoKey {
+				pc().DiscoKey = ptr.To(n.DiscoKey)
+			}
+		case "Addresses":
+			if !views.SliceEqual(was.Addresses(), views.SliceOf(n.Addresses)) {
+				return nil, false
+			}
+		case "AllowedIPs":
+			if !views.SliceEqual(was.AllowedIPs(), views.SliceOf(n.AllowedIPs)) {
+				return nil, false
+			}
+		case "Endpoints":
+			if !views.SliceEqual(was.Endpoints(), views.SliceOf(n.Endpoints)) {
+				pc().Endpoints = slices.Clone(n.Endpoints)
+			}
+		case "DERP":
+			if was.DERP() != n.DERP {
+				ip, portStr, err := net.SplitHostPort(n.DERP)
+				if err != nil || ip != "127.3.3.40" {
+					return nil, false
+				}
+				port, err := strconv.Atoi(portStr)
+				if err != nil || port < 1 || port > 65535 {
+					return nil, false
+				}
+				pc().DERPRegion = port
+			}
+		case "Hostinfo":
+			if !was.Hostinfo().Valid() && !n.Hostinfo.Valid() {
+				continue
+			}
+			if !was.Hostinfo().Valid() || !n.Hostinfo.Valid() {
+				return nil, false
+			}
+			if !was.Hostinfo().Equal(n.Hostinfo) {
+				return nil, false
+			}
+		case "Created":
+			if !was.Created().Equal(n.Created) {
+				return nil, false
+			}
+		case "Cap":
+			if was.Cap() != n.Cap {
+				pc().Cap = n.Cap
+			}
+		case "Tags":
+			if !views.SliceEqual(was.Tags(), views.SliceOf(n.Tags)) {
+				return nil, false
+			}
+		case "PrimaryRoutes":
+			if !views.SliceEqual(was.PrimaryRoutes(), views.SliceOf(n.PrimaryRoutes)) {
+				return nil, false
+			}
+		case "Online":
+			wasOnline := was.Online()
+			if n.Online != nil && wasOnline != nil && *n.Online != *wasOnline {
+				pc().Online = ptr.To(*n.Online)
+			}
+		case "LastSeen":
+			wasSeen := was.LastSeen()
+			if n.LastSeen != nil && wasSeen != nil && !wasSeen.Equal(*n.LastSeen) {
+				pc().LastSeen = ptr.To(*n.LastSeen)
+			}
+		case "MachineAuthorized":
+			if was.MachineAuthorized() != n.MachineAuthorized {
+				return nil, false
+			}
+		case "Capabilities":
+			if !views.SliceEqual(was.Capabilities(), views.SliceOf(n.Capabilities)) {
+				pc().Capabilities = ptr.To(n.Capabilities)
+			}
+		case "UnsignedPeerAPIOnly":
+			if was.UnsignedPeerAPIOnly() != n.UnsignedPeerAPIOnly {
+				return nil, false
+			}
+		case "IsWireGuardOnly":
+			if was.IsWireGuardOnly() != n.IsWireGuardOnly {
+				return nil, false
+			}
+		case "Expired":
+			if was.Expired() != n.Expired {
+				return nil, false
+			}
+		case "SelfNodeV4MasqAddrForThisPeer":
+			va, vb := was.SelfNodeV4MasqAddrForThisPeer(), n.SelfNodeV4MasqAddrForThisPeer
+			if va == nil && vb == nil {
+				continue
+			}
+			if va == nil || vb == nil || *va != *vb {
+				return nil, false
+			}
+		}
+	}
+	if ret != nil {
+		ret.NodeID = n.ID
+	}
+	return ret, true
+}
+
+// netmap returns a fully populated NetworkMap from the last state seen from
+// a call to updateStateFromResponse, filling in omitted
+// information from prior MapResponse values.
+func (ms *mapSession) netmap() *netmap.NetworkMap {
+	peerViews := make([]tailcfg.NodeView, len(ms.sortedPeers))
+	for i, vp := range ms.sortedPeers {
+		peerViews[i] = *vp
 	}
 
 	nm := &netmap.NetworkMap{
-		NodeKey:           ms.privateNodeKey.Public(),
+		NodeKey:           ms.publicNodeKey,
 		PrivateKey:        ms.privateNodeKey,
 		MachineKey:        ms.machinePubKey,
-		Peers:             resp.Peers,
+		Peers:             peerViews,
 		UserProfiles:      make(map[tailcfg.UserID]tailcfg.UserProfile),
 		Domain:            ms.lastDomain,
 		DomainAuditLogID:  ms.lastDomainAuditLogID,
@@ -181,11 +697,9 @@ func (ms *mapSession) netmapForResponse(resp *tailcfg.MapResponse) *netmap.Netwo
 		SSHPolicy:         ms.lastSSHPolicy,
 		CollectServices:   ms.collectServices,
 		DERPMap:           ms.lastDERPMap,
-		Debug:             debug,
 		ControlHealth:     ms.lastHealth,
 		TKAEnabled:        ms.lastTKAInfo != nil && !ms.lastTKAInfo.Disabled,
 	}
-	ms.netMapBuilding = nm
 
 	if ms.lastTKAInfo != nil && ms.lastTKAInfo.Head != "" {
 		if err := nm.TKAHead.UnmarshalText([]byte(ms.lastTKAInfo.Head)); err != nil {
@@ -194,186 +708,29 @@ func (ms *mapSession) netmapForResponse(resp *tailcfg.MapResponse) *netmap.Netwo
 		}
 	}
 
-	if resp.Node != nil {
-		ms.lastNode = resp.Node
-	}
-	if node := ms.lastNode.Clone(); node != nil {
+	if node := ms.lastNode; node.Valid() {
 		nm.SelfNode = node
-		nm.Expiry = node.KeyExpiry
-		nm.Name = node.Name
-		nm.Addresses = filterSelfAddresses(node.Addresses)
-		nm.User = node.User
-		if node.Hostinfo.Valid() {
-			nm.Hostinfo = *node.Hostinfo.AsStruct()
-		}
-		if node.MachineAuthorized {
+		nm.Expiry = node.KeyExpiry()
+		nm.Name = node.Name()
+		nm.Addresses = filterSelfAddresses(node.Addresses().AsSlice())
+		if node.MachineAuthorized() {
 			nm.MachineStatus = tailcfg.MachineAuthorized
 		} else {
 			nm.MachineStatus = tailcfg.MachineUnauthorized
 		}
 	}
 
-	ms.addUserProfile(nm.User)
-	magicDNSSuffix := nm.MagicDNSSuffix()
-	if nm.SelfNode != nil {
-		nm.SelfNode.InitDisplayNames(magicDNSSuffix)
-	}
-	for _, peer := range resp.Peers {
-		peer.InitDisplayNames(magicDNSSuffix)
-		if !peer.Sharer.IsZero() {
-			if ms.keepSharerAndUserSplit {
-				ms.addUserProfile(peer.Sharer)
-			} else {
-				peer.User = peer.Sharer
-			}
-		}
-		ms.addUserProfile(peer.User)
+	ms.addUserProfile(nm, nm.User())
+	for _, peer := range peerViews {
+		ms.addUserProfile(nm, peer.Sharer())
+		ms.addUserProfile(nm, peer.User())
 	}
 	if DevKnob.ForceProxyDNS() {
 		nm.DNS.Proxied = true
 	}
-	ms.netMapBuilding = nil
 	return nm
 }
 
-// undeltaPeers updates mapRes.Peers to be complete based on the
-// provided previous peer list and the PeersRemoved and PeersChanged
-// fields in mapRes, as well as the PeerSeenChange and OnlineChange
-// maps.
-//
-// It then also nils out the delta fields.
-func undeltaPeers(mapRes *tailcfg.MapResponse, prev []*tailcfg.Node) {
-	if len(mapRes.Peers) > 0 {
-		// Not delta encoded.
-		if !nodesSorted(mapRes.Peers) {
-			log.Printf("netmap: undeltaPeers: MapResponse.Peers not sorted; sorting")
-			sortNodes(mapRes.Peers)
-		}
-		return
-	}
-
-	var removed map[tailcfg.NodeID]bool
-	if pr := mapRes.PeersRemoved; len(pr) > 0 {
-		removed = make(map[tailcfg.NodeID]bool, len(pr))
-		for _, id := range pr {
-			removed[id] = true
-		}
-	}
-	changed := mapRes.PeersChanged
-
-	if !nodesSorted(changed) {
-		log.Printf("netmap: undeltaPeers: MapResponse.PeersChanged not sorted; sorting")
-		sortNodes(changed)
-	}
-	if !nodesSorted(prev) {
-		// Internal error (unrelated to the network) if we get here.
-		log.Printf("netmap: undeltaPeers: [unexpected] prev not sorted; sorting")
-		sortNodes(prev)
-	}
-
-	newFull := prev
-	if len(removed) > 0 || len(changed) > 0 {
-		newFull = make([]*tailcfg.Node, 0, len(prev)-len(removed))
-		for len(prev) > 0 && len(changed) > 0 {
-			pID := prev[0].ID
-			cID := changed[0].ID
-			if removed[pID] {
-				prev = prev[1:]
-				continue
-			}
-			switch {
-			case pID < cID:
-				newFull = append(newFull, prev[0])
-				prev = prev[1:]
-			case pID == cID:
-				newFull = append(newFull, changed[0])
-				prev, changed = prev[1:], changed[1:]
-			case cID < pID:
-				newFull = append(newFull, changed[0])
-				changed = changed[1:]
-			}
-		}
-		newFull = append(newFull, changed...)
-		for _, n := range prev {
-			if !removed[n.ID] {
-				newFull = append(newFull, n)
-			}
-		}
-		sortNodes(newFull)
-	}
-
-	if len(mapRes.PeerSeenChange) != 0 || len(mapRes.OnlineChange) != 0 || len(mapRes.PeersChangedPatch) != 0 {
-		peerByID := make(map[tailcfg.NodeID]*tailcfg.Node, len(newFull))
-		for _, n := range newFull {
-			peerByID[n.ID] = n
-		}
-		now := clock.Now()
-		for nodeID, seen := range mapRes.PeerSeenChange {
-			if n, ok := peerByID[nodeID]; ok {
-				if seen {
-					n.LastSeen = &now
-				} else {
-					n.LastSeen = nil
-				}
-			}
-		}
-		for nodeID, online := range mapRes.OnlineChange {
-			if n, ok := peerByID[nodeID]; ok {
-				online := online
-				n.Online = &online
-			}
-		}
-		for _, ec := range mapRes.PeersChangedPatch {
-			if n, ok := peerByID[ec.NodeID]; ok {
-				if ec.DERPRegion != 0 {
-					n.DERP = fmt.Sprintf("%s:%v", tailcfg.DerpMagicIP, ec.DERPRegion)
-				}
-				if ec.Cap != 0 {
-					n.Cap = ec.Cap
-				}
-				if ec.Endpoints != nil {
-					n.Endpoints = ec.Endpoints
-				}
-				if ec.Key != nil {
-					n.Key = *ec.Key
-				}
-				if ec.DiscoKey != nil {
-					n.DiscoKey = *ec.DiscoKey
-				}
-				if v := ec.Online; v != nil {
-					n.Online = ptrCopy(v)
-				}
-				if v := ec.LastSeen; v != nil {
-					n.LastSeen = ptrCopy(v)
-				}
-				if v := ec.KeyExpiry; v != nil {
-					n.KeyExpiry = *v
-				}
-				if v := ec.Capabilities; v != nil {
-					n.Capabilities = *v
-				}
-				if v := ec.KeySignature; v != nil {
-					n.KeySignature = v
-				}
-			}
-		}
-	}
-
-	mapRes.Peers = newFull
-	mapRes.PeersChanged = nil
-	mapRes.PeersRemoved = nil
-}
-
-// ptrCopy returns a pointer to a newly allocated shallow copy of *v.
-func ptrCopy[T any](v *T) *T {
-	if v == nil {
-		return nil
-	}
-	ret := new(T)
-	*ret = *v
-	return ret
-}
-
 func nodesSorted(v []*tailcfg.Node) bool {
 	for i, n := range v {
 		if i > 0 && n.ID <= v[i-1].ID {
@@ -413,18 +770,3 @@ func filterSelfAddresses(in []netip.Prefix) (ret []netip.Prefix) {
 		return ret
 	}
 }
-
-func copyDebugOptBools(dst, src *tailcfg.Debug) {
-	copy := func(v *opt.Bool, s opt.Bool) {
-		if s != "" {
-			*v = s
-		}
-	}
-	copy(&dst.DERPRoute, src.DERPRoute)
-	copy(&dst.DisableSubnetsIfPAC, src.DisableSubnetsIfPAC)
-	copy(&dst.DisableUPnP, src.DisableUPnP)
-	copy(&dst.OneCGNATRoute, src.OneCGNATRoute)
-	copy(&dst.SetForceBackgroundSTUN, src.SetForceBackgroundSTUN)
-	copy(&dst.SetRandomizeClientPort, src.SetRandomizeClientPort)
-	copy(&dst.TrimWGConfig, src.TrimWGConfig)
-}

control/controlclient/map_test.go

@@ -4,25 +4,30 @@
 package controlclient
 
 import (
+	"context"
 	"encoding/json"
 	"fmt"
+	"net/netip"
 	"reflect"
 	"strings"
+	"sync/atomic"
 	"testing"
 	"time"
 
+	"github.com/google/go-cmp/cmp"
 	"go4.org/mem"
 	"tailscale.com/tailcfg"
 	"tailscale.com/tstest"
 	"tailscale.com/tstime"
 	"tailscale.com/types/key"
+	"tailscale.com/types/logger"
 	"tailscale.com/types/netmap"
-	"tailscale.com/types/opt"
 	"tailscale.com/types/ptr"
+	"tailscale.com/util/mak"
 	"tailscale.com/util/must"
 )
 
-func TestUndeltaPeers(t *testing.T) {
+func TestUpdatePeersStateFromResponse(t *testing.T) {
 	var curTime time.Time
 
 	online := func(v bool) func(*tailcfg.Node) {
@@ -54,11 +59,12 @@ func TestUndeltaPeers(t *testing.T) {
 	}
 	peers := func(nv ...*tailcfg.Node) []*tailcfg.Node { return nv }
 	tests := []struct {
-		name    string
-		mapRes  *tailcfg.MapResponse
-		curTime time.Time
-		prev    []*tailcfg.Node
-		want    []*tailcfg.Node
+		name      string
+		mapRes    *tailcfg.MapResponse
+		curTime   time.Time
+		prev      []*tailcfg.Node
+		want      []*tailcfg.Node
+		wantStats updateStats
 	}{
 		{
 			name: "full_peers",
@@ -66,6 +72,10 @@ func TestUndeltaPeers(t *testing.T) {
 				Peers: peers(n(1, "foo"), n(2, "bar")),
 			},
 			want: peers(n(1, "foo"), n(2, "bar")),
+			wantStats: updateStats{
+				allNew: true,
+				added:  2,
+			},
 		},
 		{
 			name: "full_peers_ignores_deltas",
@@ -74,6 +84,10 @@ func TestUndeltaPeers(t *testing.T) {
 				PeersRemoved: []tailcfg.NodeID{2},
 			},
 			want: peers(n(1, "foo"), n(2, "bar")),
+			wantStats: updateStats{
+				allNew: true,
+				added:  2,
+			},
 		},
 		{
 			name: "add_and_update",
@@ -82,14 +96,21 @@ func TestUndeltaPeers(t *testing.T) {
 				PeersChanged: peers(n(0, "zero"), n(2, "bar2"), n(3, "three")),
 			},
 			want: peers(n(0, "zero"), n(1, "foo"), n(2, "bar2"), n(3, "three")),
+			wantStats: updateStats{
+				added:   2, // added IDs 0 and 3
+				changed: 1, // changed ID 2
+			},
 		},
 		{
 			name: "remove",
 			prev: peers(n(1, "foo"), n(2, "bar")),
 			mapRes: &tailcfg.MapResponse{
-				PeersRemoved: []tailcfg.NodeID{1},
+				PeersRemoved: []tailcfg.NodeID{1, 3, 4},
 			},
 			want: peers(n(2, "bar")),
+			wantStats: updateStats{
+				removed: 1, // ID 1
+			},
 		},
 		{
 			name: "add_and_remove",
@@ -99,6 +120,10 @@ func TestUndeltaPeers(t *testing.T) {
 				PeersRemoved: []tailcfg.NodeID{2},
 			},
 			want: peers(n(1, "foo2")),
+			wantStats: updateStats{
+				changed: 1,
+				removed: 1,
+			},
 		},
 		{
 			name:   "unchanged",
@@ -111,13 +136,15 @@ func TestUndeltaPeers(t *testing.T) {
 			prev: peers(n(1, "foo"), n(2, "bar")),
 			mapRes: &tailcfg.MapResponse{
 				OnlineChange: map[tailcfg.NodeID]bool{
-					1: true,
+					1:   true,
+					404: true,
 				},
 			},
 			want: peers(
 				n(1, "foo", online(true)),
 				n(2, "bar"),
 			),
+			wantStats: updateStats{changed: 1},
 		},
 		{
 			name: "online_change_offline",
@@ -132,6 +159,7 @@ func TestUndeltaPeers(t *testing.T) {
 				n(1, "foo", online(false)),
 				n(2, "bar", online(true)),
 			),
+			wantStats: updateStats{changed: 2},
 		},
 		{
 			name:    "peer_seen_at",
@@ -147,6 +175,7 @@ func TestUndeltaPeers(t *testing.T) {
 				n(1, "foo"),
 				n(2, "bar", seenAt(time.Unix(123, 0))),
 			),
+			wantStats: updateStats{changed: 2},
 		},
 		{
 			name: "ep_change_derp",
@@ -157,7 +186,8 @@ func TestUndeltaPeers(t *testing.T) {
 					DERPRegion: 4,
 				}},
 			},
-			want: peers(n(1, "foo", withDERP("127.3.3.40:4"))),
+			want:      peers(n(1, "foo", withDERP("127.3.3.40:4"))),
+			wantStats: updateStats{changed: 1},
 		},
 		{
 			name: "ep_change_udp",
@@ -168,10 +198,11 @@ func TestUndeltaPeers(t *testing.T) {
 					Endpoints: []string{"1.2.3.4:56"},
 				}},
 			},
-			want: peers(n(1, "foo", withEP("1.2.3.4:56"))),
+			want:      peers(n(1, "foo", withEP("1.2.3.4:56"))),
+			wantStats: updateStats{changed: 1},
 		},
 		{
-			name: "ep_change_udp",
+			name: "ep_change_udp_2",
 			prev: peers(n(1, "foo", withDERP("127.3.3.40:3"), withEP("1.2.3.4:111"))),
 			mapRes: &tailcfg.MapResponse{
 				PeersChangedPatch: []*tailcfg.PeerChange{{
@@ -179,7 +210,8 @@ func TestUndeltaPeers(t *testing.T) {
 					Endpoints: []string{"1.2.3.4:56"},
 				}},
 			},
-			want: peers(n(1, "foo", withDERP("127.3.3.40:3"), withEP("1.2.3.4:56"))),
+			want:      peers(n(1, "foo", withDERP("127.3.3.40:3"), withEP("1.2.3.4:56"))),
+			wantStats: updateStats{changed: 1},
 		},
 		{
 			name: "ep_change_both",
@@ -191,7 +223,8 @@ func TestUndeltaPeers(t *testing.T) {
 					Endpoints:  []string{"1.2.3.4:56"},
 				}},
 			},
-			want: peers(n(1, "foo", withDERP("127.3.3.40:2"), withEP("1.2.3.4:56"))),
+			want:      peers(n(1, "foo", withDERP("127.3.3.40:2"), withEP("1.2.3.4:56"))),
+			wantStats: updateStats{changed: 1},
 		},
 		{
 			name: "change_key",
@@ -206,6 +239,7 @@ func TestUndeltaPeers(t *testing.T) {
 				Name: "foo",
 				Key:  key.NodePublicFromRaw32(mem.B(append(make([]byte, 31), 'A'))),
 			}),
+			wantStats: updateStats{changed: 1},
 		},
 		{
 			name: "change_key_signature",
@@ -215,11 +249,13 @@ func TestUndeltaPeers(t *testing.T) {
 					NodeID:       1,
 					KeySignature: []byte{3, 4},
 				}},
-			}, want: peers(&tailcfg.Node{
+			},
+			want: peers(&tailcfg.Node{
 				ID:           1,
 				Name:         "foo",
 				KeySignature: []byte{3, 4},
 			}),
+			wantStats: updateStats{changed: 1},
 		},
 		{
 			name: "change_disco_key",
@@ -229,11 +265,13 @@ func TestUndeltaPeers(t *testing.T) {
 					NodeID:   1,
 					DiscoKey: ptr.To(key.DiscoPublicFromRaw32(mem.B(append(make([]byte, 31), 'A')))),
 				}},
-			}, want: peers(&tailcfg.Node{
+			},
+			want: peers(&tailcfg.Node{
 				ID:       1,
 				Name:     "foo",
 				DiscoKey: key.DiscoPublicFromRaw32(mem.B(append(make([]byte, 31), 'A'))),
 			}),
+			wantStats: updateStats{changed: 1},
 		},
 		{
 			name: "change_online",
@@ -243,11 +281,13 @@ func TestUndeltaPeers(t *testing.T) {
 					NodeID: 1,
 					Online: ptr.To(true),
 				}},
-			}, want: peers(&tailcfg.Node{
+			},
+			want: peers(&tailcfg.Node{
 				ID:     1,
 				Name:   "foo",
 				Online: ptr.To(true),
 			}),
+			wantStats: updateStats{changed: 1},
 		},
 		{
 			name: "change_last_seen",
@@ -257,11 +297,13 @@ func TestUndeltaPeers(t *testing.T) {
 					NodeID:   1,
 					LastSeen: ptr.To(time.Unix(123, 0).UTC()),
 				}},
-			}, want: peers(&tailcfg.Node{
+			},
+			want: peers(&tailcfg.Node{
 				ID:       1,
 				Name:     "foo",
 				LastSeen: ptr.To(time.Unix(123, 0).UTC()),
 			}),
+			wantStats: updateStats{changed: 1},
 		},
 		{
 			name: "change_key_expiry",
@@ -271,11 +313,13 @@ func TestUndeltaPeers(t *testing.T) {
 					NodeID:    1,
 					KeyExpiry: ptr.To(time.Unix(123, 0).UTC()),
 				}},
-			}, want: peers(&tailcfg.Node{
+			},
+			want: peers(&tailcfg.Node{
 				ID:        1,
 				Name:      "foo",
 				KeyExpiry: time.Unix(123, 0).UTC(),
 			}),
+			wantStats: updateStats{changed: 1},
 		},
 		{
 			name: "change_capabilities",
@@ -285,11 +329,13 @@ func TestUndeltaPeers(t *testing.T) {
 					NodeID:       1,
 					Capabilities: ptr.To([]string{"foo"}),
 				}},
-			}, want: peers(&tailcfg.Node{
+			},
+			want: peers(&tailcfg.Node{
 				ID:           1,
 				Name:         "foo",
 				Capabilities: []string{"foo"},
 			}),
+			wantStats: updateStats{changed: 1},
 		}}
 
 	for _, tt := range tests {
@@ -298,9 +344,23 @@ func TestUndeltaPeers(t *testing.T) {
 				curTime = tt.curTime
 				tstest.Replace(t, &clock, tstime.Clock(tstest.NewClock(tstest.ClockOpts{Start: curTime})))
 			}
-			undeltaPeers(tt.mapRes, tt.prev)
-			if !reflect.DeepEqual(tt.mapRes.Peers, tt.want) {
-				t.Errorf("wrong results\n got: %s\nwant: %s", formatNodes(tt.mapRes.Peers), formatNodes(tt.want))
+			ms := newTestMapSession(t, nil)
+			for _, n := range tt.prev {
+				mak.Set(&ms.peers, n.ID, ptr.To(n.View()))
+			}
+			ms.rebuildSorted()
+
+			gotStats := ms.updatePeersStateFromResponse(tt.mapRes)
+
+			got := make([]*tailcfg.Node, len(ms.sortedPeers))
+			for i, vp := range ms.sortedPeers {
+				got[i] = vp.AsStruct()
+			}
+			if gotStats != tt.wantStats {
+				t.Errorf("got stats = %+v; want %+v", gotStats, tt.wantStats)
+			}
+			if !reflect.DeepEqual(got, tt.want) {
+				t.Errorf("wrong results\n got: %s\nwant: %s", formatNodes(got), formatNodes(tt.want))
 			}
 		})
 	}
@@ -331,12 +391,18 @@ func formatNodes(nodes []*tailcfg.Node) string {
 	return sb.String()
 }
 
-func newTestMapSession(t *testing.T) *mapSession {
-	ms := newMapSession(key.NewNode())
+func newTestMapSession(t testing.TB, nu NetmapUpdater) *mapSession {
+	ms := newMapSession(key.NewNode(), nu)
+	t.Cleanup(ms.Close)
 	ms.logf = t.Logf
 	return ms
 }
 
+func (ms *mapSession) netmapForResponse(res *tailcfg.MapResponse) *netmap.NetworkMap {
+	ms.updateStateFromResponse(res)
+	return ms.netmap()
+}
+
 func TestNetmapForResponse(t *testing.T) {
 	t.Run("implicit_packetfilter", func(t *testing.T) {
 		somePacketFilter := []tailcfg.FilterRule{
@@ -347,7 +413,7 @@ func TestNetmapForResponse(t *testing.T) {
 				},
 			},
 		}
-		ms := newTestMapSession(t)
+		ms := newTestMapSession(t, nil)
 		nm1 := ms.netmapForResponse(&tailcfg.MapResponse{
 			Node:         new(tailcfg.Node),
 			PacketFilter: somePacketFilter,
@@ -368,7 +434,7 @@ func TestNetmapForResponse(t *testing.T) {
 	})
 	t.Run("implicit_dnsconfig", func(t *testing.T) {
 		someDNSConfig := &tailcfg.DNSConfig{Domains: []string{"foo", "bar"}}
-		ms := newTestMapSession(t)
+		ms := newTestMapSession(t, nil)
 		nm1 := ms.netmapForResponse(&tailcfg.MapResponse{
 			Node:      new(tailcfg.Node),
 			DNSConfig: someDNSConfig,
@@ -385,7 +451,7 @@ func TestNetmapForResponse(t *testing.T) {
 		}
 	})
 	t.Run("collect_services", func(t *testing.T) {
-		ms := newTestMapSession(t)
+		ms := newTestMapSession(t, nil)
 		var nm *netmap.NetworkMap
 		wantCollect := func(v bool) {
 			t.Helper()
@@ -418,7 +484,7 @@ func TestNetmapForResponse(t *testing.T) {
 		wantCollect(true)
 	})
 	t.Run("implicit_domain", func(t *testing.T) {
-		ms := newTestMapSession(t)
+		ms := newTestMapSession(t, nil)
 		var nm *netmap.NetworkMap
 		want := func(v string) {
 			t.Helper()
@@ -441,17 +507,19 @@ func TestNetmapForResponse(t *testing.T) {
 		someNode := &tailcfg.Node{
 			Name: "foo",
 		}
-		wantNode := &tailcfg.Node{
+		wantNode := (&tailcfg.Node{
 			Name:                 "foo",
 			ComputedName:         "foo",
 			ComputedNameWithHost: "foo",
-		}
-		ms := newTestMapSession(t)
-
-		nm1 := ms.netmapForResponse(&tailcfg.MapResponse{
+		}).View()
+		ms := newTestMapSession(t, nil)
+		mapRes := &tailcfg.MapResponse{
 			Node: someNode,
-		})
-		if nm1.SelfNode == nil {
+		}
+		initDisplayNames(mapRes.Node.View(), mapRes)
+		ms.updateStateFromResponse(mapRes)
+		nm1 := ms.netmap()
+		if !nm1.SelfNode.Valid() {
 			t.Fatal("nil Node in 1st netmap")
 		}
 		if !reflect.DeepEqual(nm1.SelfNode, wantNode) {
@@ -459,8 +527,9 @@ func TestNetmapForResponse(t *testing.T) {
 			t.Errorf("Node mismatch in 1st netmap; got: %s", j)
 		}
 
-		nm2 := ms.netmapForResponse(&tailcfg.MapResponse{})
-		if nm2.SelfNode == nil {
+		ms.updateStateFromResponse(&tailcfg.MapResponse{})
+		nm2 := ms.netmap()
+		if !nm2.SelfNode.Valid() {
 			t.Fatal("nil Node in 1st netmap")
 		}
 		if !reflect.DeepEqual(nm2.SelfNode, wantNode) {
@@ -470,155 +539,6 @@ func TestNetmapForResponse(t *testing.T) {
 	})
 }
 
-// TestDeltaDebug tests that tailcfg.Debug values can be omitted in MapResponses
-// entirely or have their opt.Bool values unspecified between MapResponses in a
-// session and that should mean no change. (as of capver 37). But two Debug
-// fields existed prior to capver 37 that weren't opt.Bool; we test that we both
-// still accept the non-opt.Bool form from control for RandomizeClientPort and
-// ForceBackgroundSTUN and also accept the new form, keeping the old form in
-// sync.
-func TestDeltaDebug(t *testing.T) {
-	type step struct {
-		got  *tailcfg.Debug
-		want *tailcfg.Debug
-	}
-	tests := []struct {
-		name  string
-		steps []step
-	}{
-		{
-			name: "nothing-to-nothing",
-			steps: []step{
-				{nil, nil},
-				{nil, nil},
-			},
-		},
-		{
-			name: "sticky-with-old-style-randomize-client-port",
-			steps: []step{
-				{
-					&tailcfg.Debug{RandomizeClientPort: true},
-					&tailcfg.Debug{
-						RandomizeClientPort:    true,
-						SetRandomizeClientPort: "true",
-					},
-				},
-				{
-					nil, // not sent by server
-					&tailcfg.Debug{
-						RandomizeClientPort:    true,
-						SetRandomizeClientPort: "true",
-					},
-				},
-			},
-		},
-		{
-			name: "sticky-with-new-style-randomize-client-port",
-			steps: []step{
-				{
-					&tailcfg.Debug{SetRandomizeClientPort: "true"},
-					&tailcfg.Debug{
-						RandomizeClientPort:    true,
-						SetRandomizeClientPort: "true",
-					},
-				},
-				{
-					nil, // not sent by server
-					&tailcfg.Debug{
-						RandomizeClientPort:    true,
-						SetRandomizeClientPort: "true",
-					},
-				},
-			},
-		},
-		{
-			name: "opt-bool-sticky-changing-over-time",
-			steps: []step{
-				{nil, nil},
-				{nil, nil},
-				{
-					&tailcfg.Debug{OneCGNATRoute: "true"},
-					&tailcfg.Debug{OneCGNATRoute: "true"},
-				},
-				{
-					nil,
-					&tailcfg.Debug{OneCGNATRoute: "true"},
-				},
-				{
-					&tailcfg.Debug{OneCGNATRoute: "false"},
-					&tailcfg.Debug{OneCGNATRoute: "false"},
-				},
-				{
-					nil,
-					&tailcfg.Debug{OneCGNATRoute: "false"},
-				},
-			},
-		},
-		{
-			name: "legacy-ForceBackgroundSTUN",
-			steps: []step{
-				{
-					&tailcfg.Debug{ForceBackgroundSTUN: true},
-					&tailcfg.Debug{ForceBackgroundSTUN: true, SetForceBackgroundSTUN: "true"},
-				},
-			},
-		},
-		{
-			name: "opt-bool-SetForceBackgroundSTUN",
-			steps: []step{
-				{
-					&tailcfg.Debug{SetForceBackgroundSTUN: "true"},
-					&tailcfg.Debug{ForceBackgroundSTUN: true, SetForceBackgroundSTUN: "true"},
-				},
-			},
-		},
-		{
-			name: "server-reset-to-default",
-			steps: []step{
-				{
-					&tailcfg.Debug{SetForceBackgroundSTUN: "true"},
-					&tailcfg.Debug{ForceBackgroundSTUN: true, SetForceBackgroundSTUN: "true"},
-				},
-				{
-					&tailcfg.Debug{SetForceBackgroundSTUN: "unset"},
-					&tailcfg.Debug{ForceBackgroundSTUN: false, SetForceBackgroundSTUN: "unset"},
-				},
-			},
-		},
-	}
-	for _, tt := range tests {
-		t.Run(tt.name, func(t *testing.T) {
-			ms := newTestMapSession(t)
-			for stepi, s := range tt.steps {
-				nm := ms.netmapForResponse(&tailcfg.MapResponse{Debug: s.got})
-				if !reflect.DeepEqual(nm.Debug, s.want) {
-					t.Errorf("unexpected result at step index %v; got: %s", stepi, must.Get(json.Marshal(nm.Debug)))
-				}
-			}
-		})
-	}
-}
-
-// Verifies that copyDebugOptBools doesn't missing any opt.Bools.
-func TestCopyDebugOptBools(t *testing.T) {
-	rt := reflect.TypeOf(tailcfg.Debug{})
-	for i := 0; i < rt.NumField(); i++ {
-		sf := rt.Field(i)
-		if sf.Type != reflect.TypeOf(opt.Bool("")) {
-			continue
-		}
-		var src, dst tailcfg.Debug
-		reflect.ValueOf(&src).Elem().Field(i).Set(reflect.ValueOf(opt.Bool("true")))
-		if src == (tailcfg.Debug{}) {
-			t.Fatalf("failed to set field %v", sf.Name)
-		}
-		copyDebugOptBools(&dst, &src)
-		if src != dst {
-			t.Fatalf("copyDebugOptBools didn't copy field %v", sf.Name)
-		}
-	}
-}
-
 func TestDeltaDERPMap(t *testing.T) {
 	regions1 := map[int]*tailcfg.DERPRegion{
 		1: {
@@ -713,11 +633,269 @@ func TestDeltaDERPMap(t *testing.T) {
 	}
 	for _, tt := range tests {
 		t.Run(tt.name, func(t *testing.T) {
-			ms := newTestMapSession(t)
+			ms := newTestMapSession(t, nil)
 			for stepi, s := range tt.steps {
 				nm := ms.netmapForResponse(&tailcfg.MapResponse{DERPMap: s.got})
 				if !reflect.DeepEqual(nm.DERPMap, s.want) {
-					t.Errorf("unexpected result at step index %v; got: %s", stepi, must.Get(json.Marshal(nm.DERPMap)))
+					t.Errorf("unexpected result at step index %v; got: %s", stepi, logger.AsJSON(nm.DERPMap))
+				}
+			}
+		})
+	}
+}
+
+func TestPeerChangeDiff(t *testing.T) {
+	tests := []struct {
+		name      string
+		a, b      *tailcfg.Node
+		want      *tailcfg.PeerChange // nil means want ok=false, unless wantEqual is set
+		wantEqual bool                // means test wants (nil, true)
+	}{
+		{
+			name:      "eq",
+			a:         &tailcfg.Node{ID: 1},
+			b:         &tailcfg.Node{ID: 1},
+			wantEqual: true,
+		},
+		{
+			name: "patch-derp",
+			a:    &tailcfg.Node{ID: 1, DERP: "127.3.3.40:1"},
+			b:    &tailcfg.Node{ID: 1, DERP: "127.3.3.40:2"},
+			want: &tailcfg.PeerChange{NodeID: 1, DERPRegion: 2},
+		},
+		{
+			name: "patch-endpoints",
+			a:    &tailcfg.Node{ID: 1, Endpoints: []string{"10.0.0.1:1"}},
+			b:    &tailcfg.Node{ID: 1, Endpoints: []string{"10.0.0.2:2"}},
+			want: &tailcfg.PeerChange{NodeID: 1, Endpoints: []string{"10.0.0.2:2"}},
+		},
+		{
+			name: "patch-cap",
+			a:    &tailcfg.Node{ID: 1, Cap: 1},
+			b:    &tailcfg.Node{ID: 1, Cap: 2},
+			want: &tailcfg.PeerChange{NodeID: 1, Cap: 2},
+		},
+		{
+			name: "patch-lastseen",
+			a:    &tailcfg.Node{ID: 1, LastSeen: ptr.To(time.Unix(1, 0))},
+			b:    &tailcfg.Node{ID: 1, LastSeen: ptr.To(time.Unix(2, 0))},
+			want: &tailcfg.PeerChange{NodeID: 1, LastSeen: ptr.To(time.Unix(2, 0))},
+		},
+		{
+			name: "patch-capabilities-to-nonempty",
+			a:    &tailcfg.Node{ID: 1, Capabilities: []string{"foo"}},
+			b:    &tailcfg.Node{ID: 1, Capabilities: []string{"bar"}},
+			want: &tailcfg.PeerChange{NodeID: 1, Capabilities: ptr.To([]string{"bar"})},
+		},
+		{
+			name: "patch-capabilities-to-empty",
+			a:    &tailcfg.Node{ID: 1, Capabilities: []string{"foo"}},
+			b:    &tailcfg.Node{ID: 1},
+			want: &tailcfg.PeerChange{NodeID: 1, Capabilities: ptr.To([]string(nil))},
+		},
+		{
+			name: "patch-online-to-true",
+			a:    &tailcfg.Node{ID: 1, Online: ptr.To(false)},
+			b:    &tailcfg.Node{ID: 1, Online: ptr.To(true)},
+			want: &tailcfg.PeerChange{NodeID: 1, Online: ptr.To(true)},
+		},
+		{
+			name: "patch-online-to-false",
+			a:    &tailcfg.Node{ID: 1, Online: ptr.To(true)},
+			b:    &tailcfg.Node{ID: 1, Online: ptr.To(false)},
+			want: &tailcfg.PeerChange{NodeID: 1, Online: ptr.To(false)},
+		},
+		{
+			name: "mix-patchable-and-not",
+			a:    &tailcfg.Node{ID: 1, Cap: 1},
+			b:    &tailcfg.Node{ID: 1, Cap: 2, StableID: "foo"},
+			want: nil,
+		},
+		{
+			name: "miss-change-stableid",
+			a:    &tailcfg.Node{ID: 1},
+			b:    &tailcfg.Node{ID: 1, StableID: "diff"},
+			want: nil,
+		},
+		{
+			name: "miss-change-id",
+			a:    &tailcfg.Node{ID: 1},
+			b:    &tailcfg.Node{ID: 2},
+			want: nil,
+		},
+		{
+			name: "miss-change-name",
+			a:    &tailcfg.Node{ID: 1, Name: "foo"},
+			b:    &tailcfg.Node{ID: 1, Name: "bar"},
+			want: nil,
+		},
+		{
+			name: "miss-change-user",
+			a:    &tailcfg.Node{ID: 1, User: 1},
+			b:    &tailcfg.Node{ID: 1, User: 2},
+			want: nil,
+		}}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			pc, ok := peerChangeDiff(tt.a.View(), tt.b)
+			if tt.wantEqual {
+				if !ok || pc != nil {
+					t.Errorf("got (%p, %v); want (nil, true); pc=%v", pc, ok, logger.AsJSON(pc))
+				}
+				return
+			}
+			if (pc != nil) != ok {
+				t.Fatalf("inconsistent ok=%v, pc=%p", ok, pc)
+			}
+			if !reflect.DeepEqual(pc, tt.want) {
+				t.Errorf("mismatch\n got: %v\nwant: %v\n", logger.AsJSON(pc), logger.AsJSON(tt.want))
+			}
+		})
+	}
+}
+
+func TestPeerChangeDiffAllocs(t *testing.T) {
+	a := &tailcfg.Node{ID: 1}
+	b := &tailcfg.Node{ID: 1}
+	n := testing.AllocsPerRun(10000, func() {
+		diff, ok := peerChangeDiff(a.View(), b)
+		if !ok || diff != nil {
+			t.Fatalf("unexpected result: (%s, %v)", logger.AsJSON(diff), ok)
+		}
+	})
+	if n != 0 {
+		t.Errorf("allocs = %v; want 0", int(n))
+	}
+}
+
+type countingNetmapUpdater struct {
+	full atomic.Int64
+}
+
+func (nu *countingNetmapUpdater) UpdateFullNetmap(nm *netmap.NetworkMap) {
+	nu.full.Add(1)
+}
+
+// tests (*mapSession).patchifyPeersChanged; smaller tests are in TestPeerChangeDiff
+func TestPatchifyPeersChanged(t *testing.T) {
+	hi := (&tailcfg.Hostinfo{}).View()
+	tests := []struct {
+		name string
+		mr0  *tailcfg.MapResponse // initial
+		mr1  *tailcfg.MapResponse // incremental
+		want *tailcfg.MapResponse // what the incremental one should've been mutated to
+	}{
+		{
+			name: "change_one_endpoint",
+			mr0: &tailcfg.MapResponse{
+				Node: &tailcfg.Node{Name: "foo.bar.ts.net."},
+				Peers: []*tailcfg.Node{
+					{ID: 1, Hostinfo: hi},
+				},
+			},
+			mr1: &tailcfg.MapResponse{
+				PeersChanged: []*tailcfg.Node{
+					{ID: 1, Endpoints: []string{"10.0.0.1:1111"}, Hostinfo: hi},
+				},
+			},
+			want: &tailcfg.MapResponse{
+				PeersChanged: nil,
+				PeersChangedPatch: []*tailcfg.PeerChange{
+					{NodeID: 1, Endpoints: []string{"10.0.0.1:1111"}},
+				},
+			},
+		},
+		{
+			name: "change_some",
+			mr0: &tailcfg.MapResponse{
+				Node: &tailcfg.Node{Name: "foo.bar.ts.net."},
+				Peers: []*tailcfg.Node{
+					{ID: 1, DERP: "127.3.3.40:1", Hostinfo: hi},
+					{ID: 2, DERP: "127.3.3.40:2", Hostinfo: hi},
+					{ID: 3, DERP: "127.3.3.40:3", Hostinfo: hi},
+				},
+			},
+			mr1: &tailcfg.MapResponse{
+				PeersChanged: []*tailcfg.Node{
+					{ID: 1, DERP: "127.3.3.40:11", Hostinfo: hi},
+					{ID: 2, StableID: "other-change", Hostinfo: hi},
+					{ID: 3, DERP: "127.3.3.40:33", Hostinfo: hi},
+					{ID: 4, DERP: "127.3.3.40:4", Hostinfo: hi},
+				},
+			},
+			want: &tailcfg.MapResponse{
+				PeersChanged: []*tailcfg.Node{
+					{ID: 2, StableID: "other-change", Hostinfo: hi},
+					{ID: 4, DERP: "127.3.3.40:4", Hostinfo: hi},
+				},
+				PeersChangedPatch: []*tailcfg.PeerChange{
+					{NodeID: 1, DERPRegion: 11},
+					{NodeID: 3, DERPRegion: 33},
+				},
+			},
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			nu := &countingNetmapUpdater{}
+			ms := newTestMapSession(t, nu)
+			ms.updateStateFromResponse(tt.mr0)
+			mr1 := new(tailcfg.MapResponse)
+			must.Do(json.Unmarshal(must.Get(json.Marshal(tt.mr1)), mr1))
+			ms.patchifyPeersChanged(mr1)
+			if diff := cmp.Diff(tt.want, mr1); diff != "" {
+				t.Errorf("wrong result (-want +got):\n%s", diff)
+			}
+		})
+	}
+}
+
+func BenchmarkMapSessionDelta(b *testing.B) {
+	for _, size := range []int{10, 100, 1_000, 10_000} {
+		b.Run(fmt.Sprintf("size_%d", size), func(b *testing.B) {
+			ctx := context.Background()
+			nu := &countingNetmapUpdater{}
+			ms := newTestMapSession(b, nu)
+			res := &tailcfg.MapResponse{
+				Node: &tailcfg.Node{
+					ID:   1,
+					Name: "foo.bar.ts.net.",
+				},
+			}
+			for i := 0; i < size; i++ {
+				res.Peers = append(res.Peers, &tailcfg.Node{
+					ID:         tailcfg.NodeID(i + 2),
+					Name:       fmt.Sprintf("peer%d.bar.ts.net.", i),
+					DERP:       "127.3.3.40:10",
+					Addresses:  []netip.Prefix{netip.MustParsePrefix("100.100.2.3/32"), netip.MustParsePrefix("fd7a:115c:a1e0::123/128")},
+					AllowedIPs: []netip.Prefix{netip.MustParsePrefix("100.100.2.3/32"), netip.MustParsePrefix("fd7a:115c:a1e0::123/128")},
+					Endpoints:  []string{"192.168.1.2:345", "192.168.1.3:678"},
+					Hostinfo: (&tailcfg.Hostinfo{
+						OS:       "fooOS",
+						Hostname: "MyHostname",
+						Services: []tailcfg.Service{
+							{Proto: "peerapi4", Port: 1234},
+							{Proto: "peerapi6", Port: 1234},
+							{Proto: "peerapi-dns-proxy", Port: 1},
+						},
+					}).View(),
+					LastSeen: ptr.To(time.Unix(int64(i), 0)),
+				})
+			}
+			ms.HandleNonKeepAliveMapResponse(ctx, res)
+
+			b.ResetTimer()
+			b.ReportAllocs()
+
+			// Now for the core of the benchmark loop, just toggle
+			// a single node's online status.
+			for i := 0; i < b.N; i++ {
+				if err := ms.HandleNonKeepAliveMapResponse(ctx, &tailcfg.MapResponse{
+					OnlineChange: map[tailcfg.NodeID]bool{
+						2: i%2 == 0,
+					},
+				}); err != nil {
+					b.Fatal(err)
 				}
 			}
 		})

control/controlclient/status.go

@@ -8,7 +8,6 @@ import (
 	"fmt"
 	"reflect"
 
-	"tailscale.com/types/empty"
 	"tailscale.com/types/netmap"
 	"tailscale.com/types/persist"
 	"tailscale.com/types/structs"
@@ -62,34 +61,55 @@ func (s State) String() string {
 }
 
 type Status struct {
-	_              structs.Incomparable
-	LoginFinished  *empty.Message // nonempty when login finishes
-	LogoutFinished *empty.Message // nonempty when logout finishes
-	Err            error
-	URL            string             // interactive URL to visit to finish logging in
-	NetMap         *netmap.NetworkMap // server-pushed configuration
+	_ structs.Incomparable
 
-	// The internal state should not be exposed outside this
+	// Err, if non-nil, is an error that occurred while logging in.
+	//
+	// If it's of type UserVisibleError then it's meant to be shown to users in
+	// their Tailscale client. Otherwise it's just logged to tailscaled's logs.
+	Err error
+
+	// URL, if non-empty, is the interactive URL to visit to finish logging in.
+	URL string
+
+	// NetMap is the latest server-pushed state of the tailnet network.
+	NetMap *netmap.NetworkMap
+
+	// Persist, when Valid, is the locally persisted configuration.
+	//
+	// TODO(bradfitz,maisem): clarify this.
+	Persist persist.PersistView
+
+	// state is the internal state. It should not be exposed outside this
 	// package, but we have some automated tests elsewhere that need to
-	// use them. Please don't use these fields.
+	// use it via the StateForTest accessor.
 	// TODO(apenwarr): Unexport or remove these.
-	State   State
-	Persist *persist.PersistView // locally persisted configuration
+	state State
 }
 
+// LoginFinished reports whether the controlclient is in its "StateAuthenticated"
+// state where it's in a happy register state but not yet in a map poll.
+//
+// TODO(bradfitz): delete this and everything around Status.state.
+func (s *Status) LoginFinished() bool { return s.state == StateAuthenticated }
+
+// StateForTest returns the internal state of s for tests only.
+func (s *Status) StateForTest() State { return s.state }
+
+// SetStateForTest sets the internal state of s for tests only.
+func (s *Status) SetStateForTest(state State) { s.state = state }
+
 // Equal reports whether s and s2 are equal.
 func (s *Status) Equal(s2 *Status) bool {
 	if s == nil && s2 == nil {
 		return true
 	}
 	return s != nil && s2 != nil &&
-		(s.LoginFinished == nil) == (s2.LoginFinished == nil) &&
-		(s.LogoutFinished == nil) == (s2.LogoutFinished == nil) &&
 		s.Err == s2.Err &&
 		s.URL == s2.URL &&
+		s.state == s2.state &&
 		reflect.DeepEqual(s.Persist, s2.Persist) &&
-		reflect.DeepEqual(s.NetMap, s2.NetMap) &&
-		s.State == s2.State
+		reflect.DeepEqual(s.NetMap, s2.NetMap)
 }
 
 func (s Status) String() string {
@@ -97,5 +117,5 @@ func (s Status) String() string {
 	if err != nil {
 		panic(err)
 	}
-	return s.State.String() + " " + string(b)
+	return s.state.String() + " " + string(b)
 }