tka: implement filesystem chonk garbage collection

Signed-off-by: Tom DNetto <tom@tailscale.com>

.github/workflows/go-licenses.yml

@@ -17,7 +17,7 @@ concurrency:
   cancel-in-progress: true
 
 jobs:
-  tailscale:
+  update-licenses:
     runs-on: ubuntu-latest
 
     steps:
@@ -25,7 +25,7 @@ jobs:
         uses: actions/checkout@v3
 
       - name: Set up Go
-        uses: actions/setup-go@v3
+        uses: actions/setup-go@v4
         with:
           go-version-file: go.mod
 
@@ -50,7 +50,7 @@ jobs:
           private_key: ${{ secrets.LICENSING_APP_PRIVATE_KEY }}
 
       - name: Send pull request
-        uses: peter-evans/create-pull-request@ad43dccb4d726ca8514126628bec209b8354b6dd #v4.1.4
+        uses: peter-evans/create-pull-request@38e0b6e68b4c852a5500a94740f0e535e0d7ba54 #v4.2.4
         with:
           token: ${{ steps.generate-token.outputs.token }}
           author: License Updater <noreply@tailscale.com>

.github/workflows/update-flake.yml

@@ -16,7 +16,7 @@ concurrency:
   cancel-in-progress: true
 
 jobs:
-  tailscale:
+  update-flake:
     runs-on: ubuntu-latest
 
     steps:
@@ -35,7 +35,7 @@ jobs:
           private_key: ${{ secrets.LICENSING_APP_PRIVATE_KEY }}
 
       - name: Send pull request
-        uses: peter-evans/create-pull-request@ad43dccb4d726ca8514126628bec209b8354b6dd #v4.1.4
+        uses: peter-evans/create-pull-request@38e0b6e68b4c852a5500a94740f0e535e0d7ba54 #v4.2.4
         with:
           token: ${{ steps.generate-token.outputs.token }}
           author: Flakes Updater <noreply@tailscale.com>

VERSION.txt

@@ -1 +1 @@
-1.37.0
+1.39.0

build_dist.sh

@@ -16,7 +16,7 @@ if [ -n "${TS_USE_TOOLCHAIN:-}" ]; then
 	go="./tool/go"
 fi
 
-eval `$go run ./cmd/mkversion`
+eval `GOOS=$($go env GOHOSTOS) GOARCH=$($go env GOHOSTARCH) $go run ./cmd/mkversion`
 
 if [ "$1" = "shellvars" ]; then
 	cat <<EOF

client/tailscale/acl.go

@@ -103,7 +103,7 @@ func (c *Client) ACL(ctx context.Context) (acl *ACL, err error) {
 // it as a string.
 // HuJSON is JSON with a few modifications to make it more human-friendly. The primary
 // changes are allowing comments and trailing comments. See the following links for more info:
-// https://tailscale.com/kb/1018/acls?q=acl#tailscale-acl-policy-format
+// https://tailscale.com/s/acl-format
 // https://github.com/tailscale/hujson
 func (c *Client) ACLHuJSON(ctx context.Context) (acl *ACLHuJSON, err error) {
 	// Format return errors to be descriptive.

cmd/derpprobe/derpprobe.go

@@ -23,13 +23,14 @@ var (
 	derpMapURL = flag.String("derp-map", "https://login.tailscale.com/derpmap/default", "URL to DERP map (https:// or file://)")
 	listen     = flag.String("listen", ":8030", "HTTP listen address")
 	probeOnce  = flag.Bool("once", false, "probe once and print results, then exit; ignores the listen flag")
+	spread     = flag.Bool("spread", true, "whether to spread probing over time")
 	interval   = flag.Duration("interval", 15*time.Second, "probe interval")
 )
 
 func main() {
 	flag.Parse()
 
-	p := prober.New().WithSpread(true).WithOnce(*probeOnce)
+	p := prober.New().WithSpread(*spread).WithOnce(*probeOnce)
 	dp, err := prober.DERP(p, *derpMapURL, *interval, *interval, *interval)
 	if err != nil {
 		log.Fatal(err)

cmd/get-authkey/main.go

@@ -2,7 +2,7 @@
 // SPDX-License-Identifier: BSD-3-Clause
 
 // get-authkey allocates an authkey using an OAuth API client
-// https://tailscale.com/kb/1215/oauth-clients/ and prints it
+// https://tailscale.com/s/oauth-clients and prints it
 // to stdout for scripts to capture and use.
 package main
 

cmd/k8s-operator/manifests/authproxy-rbac.yaml

@@ -7,7 +7,7 @@ metadata:
   name: tailscale-auth-proxy
 rules:
 - apiGroups: [""]
-  resources: ["users"]
+  resources: ["users", "groups"]
   verbs: ["impersonate"]
 ---
 apiVersion: rbac.authorization.k8s.io/v1

cmd/k8s-operator/operator.go

@@ -7,8 +7,10 @@ package main
 
 import (
 	"context"
+	"crypto/tls"
 	_ "embed"
 	"fmt"
+	"net/http"
 	"os"
 	"strings"
 	"time"
@@ -25,7 +27,7 @@ import (
 	"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
 	"k8s.io/apimachinery/pkg/fields"
 	"k8s.io/apimachinery/pkg/types"
-	"k8s.io/client-go/rest"
+	"k8s.io/client-go/transport"
 	"sigs.k8s.io/controller-runtime/pkg/builder"
 	"sigs.k8s.io/controller-runtime/pkg/cache"
 	"sigs.k8s.io/controller-runtime/pkg/client"
@@ -235,15 +237,25 @@ waitOnline:
 
 	startlog.Infof("Startup complete, operator running")
 	if shouldRunAuthProxy {
-		rc, err := rest.TransportFor(restConfig)
+		cfg, err := restConfig.TransportConfig()
 		if err != nil {
-			startlog.Fatalf("could not get rest transport: %v", err)
+			startlog.Fatalf("could not get rest.TransportConfig(): %v", err)
 		}
-		authProxyListener, err := s.Listen("tcp", ":443")
+
+		// Kubernetes uses SPDY for exec and port-forward, however SPDY is
+		// incompatible with HTTP/2; so disable HTTP/2 in the proxy.
+		tr := http.DefaultTransport.(*http.Transport).Clone()
+		tr.TLSClientConfig, err = transport.TLSConfigFor(cfg)
 		if err != nil {
-			startlog.Fatalf("could not listen on :443: %v", err)
+			startlog.Fatalf("could not get transport.TLSConfigFor(): %v", err)
 		}
-		go runAuthProxy(lc, authProxyListener, rc, zlog.Named("auth-proxy").Infof)
+		tr.TLSNextProto = make(map[string]func(authority string, c *tls.Conn) http.RoundTripper)
+
+		rt, err := transport.HTTPWrappersForConfig(cfg, tr)
+		if err != nil {
+			startlog.Fatalf("could not get rest.TransportConfig(): %v", err)
+		}
+		go runAuthProxy(s, rt, zlog.Named("auth-proxy").Infof)
 	}
 	if err := mgr.Start(signals.SetupSignalHandler()); err != nil {
 		startlog.Fatalf("could not start manager: %v", err)

cmd/k8s-operator/proxy.go

@@ -8,7 +8,6 @@ import (
 	"crypto/tls"
 	"fmt"
 	"log"
-	"net"
 	"net/http"
 	"net/http/httputil"
 	"net/url"
@@ -17,6 +16,7 @@ import (
 
 	"tailscale.com/client/tailscale"
 	"tailscale.com/client/tailscale/apitype"
+	"tailscale.com/tsnet"
 	"tailscale.com/types/logger"
 )
 
@@ -41,23 +41,42 @@ func (h *authProxy) ServeHTTP(w http.ResponseWriter, r *http.Request) {
 	h.rp.ServeHTTP(w, r)
 }
 
-func runAuthProxy(lc *tailscale.LocalClient, ls net.Listener, rt http.RoundTripper, logf logger.Logf) {
+// runAuthProxy runs an HTTP server that authenticates requests using the
+// Tailscale LocalAPI and then proxies them to the Kubernetes API.
+// It listens on :443 and uses the Tailscale HTTPS certificate.
+// s will be started if it is not already running.
+// rt is used to proxy requests to the Kubernetes API.
+//
+// It never returns.
+func runAuthProxy(s *tsnet.Server, rt http.RoundTripper, logf logger.Logf) {
+	ln, err := s.Listen("tcp", ":443")
+	if err != nil {
+		log.Fatalf("could not listen on :443: %v", err)
+	}
 	u, err := url.Parse(fmt.Sprintf("https://%s:%s", os.Getenv("KUBERNETES_SERVICE_HOST"), os.Getenv("KUBERNETES_SERVICE_PORT_HTTPS")))
 	if err != nil {
 		log.Fatalf("runAuthProxy: failed to parse URL %v", err)
 	}
+
+	lc, err := s.LocalClient()
+	if err != nil {
+		log.Fatalf("could not get local client: %v", err)
+	}
 	ap := &authProxy{
 		logf: logf,
 		lc:   lc,
 		rp: &httputil.ReverseProxy{
 			Director: func(r *http.Request) {
-				// Replace the request with the user's identity.
-				who := r.Context().Value(whoIsKey{}).(*apitype.WhoIsResponse)
-				r.Header.Set("Impersonate-User", who.UserProfile.LoginName)
+				// We want to proxy to the Kubernetes API, but we want to use
+				// the caller's identity to do so. We do this by impersonating
+				// the caller using the Kubernetes User Impersonation feature:
+				// https://kubernetes.io/docs/reference/access-authn-authz/authentication/#user-impersonation
 
-				// Remove all authentication headers.
+				// Out of paranoia, remove all authentication headers that might
+				// have been set by the client.
 				r.Header.Del("Authorization")
 				r.Header.Del("Impersonate-Group")
+				r.Header.Del("Impersonate-User")
 				r.Header.Del("Impersonate-Uid")
 				for k := range r.Header {
 					if strings.HasPrefix(k, "Impersonate-Extra-") {
@@ -65,6 +84,19 @@ func runAuthProxy(lc *tailscale.LocalClient, ls net.Listener, rt http.RoundTripp
 					}
 				}
 
+				// Now add the impersonation headers that we want.
+				who := r.Context().Value(whoIsKey{}).(*apitype.WhoIsResponse)
+				if who.Node.IsTagged() {
+					// Use the nodes FQDN as the username, and the nodes tags as the groups.
+					// "Impersonate-Group" requires "Impersonate-User" to be set.
+					r.Header.Set("Impersonate-User", strings.TrimSuffix(who.Node.Name, "."))
+					for _, tag := range who.Node.Tags {
+						r.Header.Add("Impersonate-Group", tag)
+					}
+				} else {
+					r.Header.Set("Impersonate-User", who.UserProfile.LoginName)
+				}
+
 				// Replace the URL with the Kubernetes APIServer.
 				r.URL.Scheme = u.Scheme
 				r.URL.Host = u.Host
@@ -72,9 +104,17 @@ func runAuthProxy(lc *tailscale.LocalClient, ls net.Listener, rt http.RoundTripp
 			Transport: rt,
 		},
 	}
-	if err := http.Serve(tls.NewListener(ls, &tls.Config{
-		GetCertificate: lc.GetCertificate,
-	}), ap); err != nil {
+	hs := &http.Server{
+		// Kubernetes uses SPDY for exec and port-forward, however SPDY is
+		// incompatible with HTTP/2; so disable HTTP/2 in the proxy.
+		TLSConfig: &tls.Config{
+			GetCertificate: lc.GetCertificate,
+			NextProtos:     []string{"http/1.1"},
+		},
+		TLSNextProto: make(map[string]func(*http.Server, *tls.Conn, http.Handler)),
+		Handler:      ap,
+	}
+	if err := hs.ServeTLS(ln, "", ""); err != nil {
 		log.Fatalf("runAuthProxy: failed to serve %v", err)
 	}
 }

cmd/tailscale/cli/cli.go

@@ -113,6 +113,7 @@ change in the future.
 			loginCmd,
 			logoutCmd,
 			switchCmd,
+			configureCmd,
 			netcheckCmd,
 			ipCmd,
 			statusCmd,
@@ -146,12 +147,12 @@ change in the future.
 	switch {
 	case slices.Contains(args, "debug"):
 		rootCmd.Subcommands = append(rootCmd.Subcommands, debugCmd)
+	case slices.Contains(args, "funnel"):
+		rootCmd.Subcommands = append(rootCmd.Subcommands, funnelCmd)
 	case slices.Contains(args, "serve"):
 		rootCmd.Subcommands = append(rootCmd.Subcommands, serveCmd)
 	case slices.Contains(args, "update"):
 		rootCmd.Subcommands = append(rootCmd.Subcommands, updateCmd)
-	case slices.Contains(args, "configure"):
-		rootCmd.Subcommands = append(rootCmd.Subcommands, configureCmd)
 	}
 	if runtime.GOOS == "linux" && distro.Get() == distro.Synology {
 		rootCmd.Subcommands = append(rootCmd.Subcommands, configureHostCmd)

cmd/tailscale/cli/cli_test.go

@@ -1071,6 +1071,32 @@ func TestUpdatePrefs(t *testing.T) {
 			},
 			env: upCheckEnv{backendState: "Running"},
 		},
+		{
+			name:             "force_reauth_over_ssh_no_risk",
+			flags:            []string{"--force-reauth"},
+			sshOverTailscale: true,
+			curPrefs: &ipn.Prefs{
+				ControlURL:       "https://login.tailscale.com",
+				AllowSingleHosts: true,
+				CorpDNS:          true,
+				NetfilterMode:    preftype.NetfilterOn,
+			},
+			env:          upCheckEnv{backendState: "Running"},
+			wantErrSubtr: "aborted, no changes made",
+		},
+		{
+			name:             "force_reauth_over_ssh",
+			flags:            []string{"--force-reauth", "--accept-risk=lose-ssh"},
+			sshOverTailscale: true,
+			curPrefs: &ipn.Prefs{
+				ControlURL:       "https://login.tailscale.com",
+				AllowSingleHosts: true,
+				CorpDNS:          true,
+				NetfilterMode:    preftype.NetfilterOn,
+			},
+			wantJustEditMP: nil,
+			env:            upCheckEnv{backendState: "Running"},
+		},
 	}
 	for _, tt := range tests {
 		t.Run(tt.name, func(t *testing.T) {

cmd/tailscale/cli/configure-kube.go

@@ -26,12 +26,14 @@ func init() {
 
 var configureKubeconfigCmd = &ffcli.Command{
 	Name:       "kubeconfig",
-	ShortHelp:  "Configure kubeconfig to use Tailscale",
+	ShortHelp:  "[ALPHA] Connect to a Kubernetes cluster using a Tailscale Auth Proxy",
 	ShortUsage: "kubeconfig <hostname-or-fqdn>",
 	LongHelp: strings.TrimSpace(`
-Run this command to configure your kubeconfig to use Tailscale for authentication to a Kubernetes cluster.
+Run this command to configure kubectl to connect to a Kubernetes cluster over Tailscale.
 
 The hostname argument should be set to the Tailscale hostname of the peer running as an auth proxy in the cluster.
+
+See: https://tailscale.com/s/k8s-auth-proxy
 `),
 	FlagSet: (func() *flag.FlagSet {
 		fs := newFlagSet("kubeconfig")

cmd/tailscale/cli/configure-synology.go

@@ -35,13 +35,13 @@ var configureHostCmd = &ffcli.Command{
 var synologyConfigureCmd = &ffcli.Command{
 	Name:      "synology",
 	Exec:      runConfigureSynology,
-	ShortHelp: "Configure Synology to enable more Tailscale features",
+	ShortHelp: "Configure Synology to enable outbound connections",
 	LongHelp: strings.TrimSpace(`
-The 'configure-host' command is intended to run at boot as root
-to create the /dev/net/tun device and give the tailscaled binary
-permission to use it.
+This command is intended to run at boot as root on a Synology device to
+create the /dev/net/tun device and give the tailscaled binary permission
+to use it.
 
-See: https://tailscale.com/kb/1152/synology-outbound/
+See: https://tailscale.com/s/synology-outbound
 `),
 	FlagSet: (func() *flag.FlagSet {
 		fs := newFlagSet("synology")

cmd/tailscale/cli/configure.go

@@ -15,10 +15,10 @@ import (
 
 var configureCmd = &ffcli.Command{
 	Name:      "configure",
-	ShortHelp: "Configure the host to enable more Tailscale features",
+	ShortHelp: "[ALPHA] Configure the host to enable more Tailscale features",
 	LongHelp: strings.TrimSpace(`
-The 'configure' command is intended to provide a way to configure different
-services on the host to enable more Tailscale features.
+The 'configure' set of commands are intended to provide a way to enable different
+services on the host to use Tailscale in more ways.
 `),
 	FlagSet: (func() *flag.FlagSet {
 		fs := newFlagSet("configure")

cmd/tailscale/cli/funnel.go

@@ -0,0 +1,138 @@
+// Copyright (c) Tailscale Inc & AUTHORS
+// SPDX-License-Identifier: BSD-3-Clause
+
+package cli
+
+import (
+	"context"
+	"flag"
+	"fmt"
+	"net"
+	"os"
+	"strconv"
+	"strings"
+
+	"github.com/peterbourgon/ff/v3/ffcli"
+	"tailscale.com/ipn"
+	"tailscale.com/util/mak"
+)
+
+var funnelCmd = newFunnelCommand(&serveEnv{lc: &localClient})
+
+// newFunnelCommand returns a new "funnel" subcommand using e as its environment.
+// The funnel subcommand is used to turn on/off the Funnel service.
+// Funnel is off by default.
+// Funnel allows you to publish a 'tailscale serve' server publicly, open to the
+// entire internet.
+// newFunnelCommand shares the same serveEnv as the "serve" subcommand. See
+// newServeCommand and serve.go for more details.
+func newFunnelCommand(e *serveEnv) *ffcli.Command {
+	return &ffcli.Command{
+		Name:      "funnel",
+		ShortHelp: "[ALPHA] turn Tailscale Funnel on or off",
+		ShortUsage: strings.TrimSpace(`
+funnel <serve-port> {on|off}
+  funnel status [--json]
+`),
+		LongHelp: strings.Join([]string{
+			"Funnel allows you to publish a 'tailscale serve'",
+			"server publicly, open to the entire internet.",
+			"",
+			"Turning off Funnel only turns off serving to the internet.",
+			"It does not affect serving to your tailnet.",
+		}, "\n"),
+		Exec:      e.runFunnel,
+		UsageFunc: usageFunc,
+		Subcommands: []*ffcli.Command{
+			{
+				Name:      "status",
+				Exec:      e.runServeStatus,
+				ShortHelp: "show current serve/funnel status",
+				FlagSet: e.newFlags("funnel-status", func(fs *flag.FlagSet) {
+					fs.BoolVar(&e.json, "json", false, "output JSON")
+				}),
+				UsageFunc: usageFunc,
+			},
+		},
+	}
+}
+
+// runFunnel is the entry point for the "tailscale funnel" subcommand and
+// manages turning on/off funnel. Funnel is off by default.
+//
+// Note: funnel is only supported on single DNS name for now. (2022-11-15)
+func (e *serveEnv) runFunnel(ctx context.Context, args []string) error {
+	if len(args) != 2 {
+		return flag.ErrHelp
+	}
+
+	var on bool
+	switch args[1] {
+	case "on", "off":
+		on = args[1] == "on"
+	default:
+		return flag.ErrHelp
+	}
+	sc, err := e.lc.GetServeConfig(ctx)
+	if err != nil {
+		return err
+	}
+	if sc == nil {
+		sc = new(ipn.ServeConfig)
+	}
+	st, err := e.getLocalClientStatus(ctx)
+	if err != nil {
+		return fmt.Errorf("getting client status: %w", err)
+	}
+
+	port64, err := strconv.ParseUint(args[0], 10, 16)
+	if err != nil {
+		return err
+	}
+	port := uint16(port64)
+
+	if err := ipn.CheckFunnelAccess(port, st.Self.Capabilities); err != nil {
+		return err
+	}
+	dnsName := strings.TrimSuffix(st.Self.DNSName, ".")
+	hp := ipn.HostPort(dnsName + ":" + strconv.Itoa(int(port)))
+	if on == sc.AllowFunnel[hp] {
+		printFunnelWarning(sc)
+		// Nothing to do.
+		return nil
+	}
+	if on {
+		mak.Set(&sc.AllowFunnel, hp, true)
+	} else {
+		delete(sc.AllowFunnel, hp)
+		// clear map mostly for testing
+		if len(sc.AllowFunnel) == 0 {
+			sc.AllowFunnel = nil
+		}
+	}
+	if err := e.lc.SetServeConfig(ctx, sc); err != nil {
+		return err
+	}
+	printFunnelWarning(sc)
+	return nil
+}
+
+// printFunnelWarning prints a warning if the Funnel is on but there is no serve
+// config for its host:port.
+func printFunnelWarning(sc *ipn.ServeConfig) {
+	var warn bool
+	for hp, a := range sc.AllowFunnel {
+		if !a {
+			continue
+		}
+		_, portStr, _ := net.SplitHostPort(string(hp))
+		p, _ := strconv.ParseUint(portStr, 10, 16)
+		if _, ok := sc.TCP[uint16(p)]; !ok {
+			warn = true
+			fmt.Fprintf(os.Stderr, "Warning: funnel=on for %s, but no serve config\n", hp)
+		}
+	}
+	if warn {
+		fmt.Fprintf(os.Stderr, "         run: `tailscale serve --help` to see how to configure handlers\n")
+	}
+}

cmd/tailscale/cli/network-lock.go

@@ -40,9 +40,17 @@ var netlockCmd = &ffcli.Command{
 		nlDisablementKDFCmd,
 		nlLogCmd,
 		nlLocalDisableCmd,
-		nlTskeyWrapCmd,
 	},
-	Exec: runNetworkLockStatus,
+	Exec: runNetworkLockNoSubcommand,
+}
+
+func runNetworkLockNoSubcommand(ctx context.Context, args []string) error {
+	// Detect & handle the deprecated command 'lock tskey-wrap'.
+	if len(args) >= 2 && args[0] == "tskey-wrap" {
+		return runTskeyWrapCmd(ctx, args[1:])
+	}
+
+	return runNetworkLockStatus(ctx, args)
 }
 
 var nlInitArgs struct {
@@ -427,13 +435,19 @@ func runNetworkLockModify(ctx context.Context, addArgs, removeArgs []string) err
 
 var nlSignCmd = &ffcli.Command{
 	Name:       "sign",
-	ShortUsage: "sign <node-key> [<rotation-key>]",
-	ShortHelp:  "Signs a node key and transmits the signature to the coordination server",
-	LongHelp:   "Signs a node key and transmits the signature to the coordination server",
-	Exec:       runNetworkLockSign,
+	ShortUsage: "sign <node-key> [<rotation-key>] or sign <auth-key>",
+	ShortHelp:  "Signs a node or pre-approved auth key",
+	LongHelp: `Either:
+  - signs a node key and transmits the signature to the coordination server, or
+  - signs a pre-approved auth key, printing it in a form that can be used to bring up nodes under tailnet lock`,
+	Exec: runNetworkLockSign,
 }
 
 func runNetworkLockSign(ctx context.Context, args []string) error {
+	if len(args) > 0 && strings.HasPrefix(args[0], "tskey-auth-") {
+		return runTskeyWrapCmd(ctx, args)
+	}
+
 	var (
 		nodeKey     key.NodePublic
 		rotationKey key.NLPublic
@@ -636,14 +650,6 @@ func runNetworkLockLog(ctx context.Context, args []string) error {
 	return nil
 }
 
-var nlTskeyWrapCmd = &ffcli.Command{
-	Name:       "tskey-wrap",
-	ShortUsage: "tskey-wrap <tailscale pre-auth key>",
-	ShortHelp:  "Modifies a pre-auth key from the admin panel to work with tailnet lock",
-	LongHelp:   "Modifies a pre-auth key from the admin panel to work with tailnet lock",
-	Exec:       runTskeyWrapCmd,
-}
-
 func runTskeyWrapCmd(ctx context.Context, args []string) error {
 	if len(args) != 1 {
 		return errors.New("usage: lock tskey-wrap <tailscale pre-auth key>")
@@ -657,21 +663,25 @@ func runTskeyWrapCmd(ctx context.Context, args []string) error {
 		return fixTailscaledConnectError(err)
 	}
 
+	return wrapAuthKey(ctx, args[0], st)
+}
+
+func wrapAuthKey(ctx context.Context, keyStr string, status *ipnstate.Status) error {
 	// Generate a separate tailnet-lock key just for the credential signature.
 	// We use the free-form meta strings to mark a little bit of metadata about this
 	// key.
 	priv := key.NewNLPrivate()
 	m := map[string]string{
 		"purpose":            "pre-auth key",
-		"wrapper_stableid":   string(st.Self.ID),
+		"wrapper_stableid":   string(status.Self.ID),
 		"wrapper_createtime": fmt.Sprint(time.Now().Unix()),
 	}
-	if strings.HasPrefix(args[0], "tskey-auth-") && strings.Index(args[0][len("tskey-auth-"):], "-") > 0 {
+	if strings.HasPrefix(keyStr, "tskey-auth-") && strings.Index(keyStr[len("tskey-auth-"):], "-") > 0 {
 		// We don't want to accidentally embed the nonce part of the authkey in
 		// the event the format changes. As such, we make sure its in the format we
 		// expect (tskey-auth-<stableID, inc CNTRL suffix>-nonce) before we parse
 		// out and embed the stableID.
-		s := strings.TrimPrefix(args[0], "tskey-auth-")
+		s := strings.TrimPrefix(keyStr, "tskey-auth-")
 		m["authkey_stableid"] = s[:strings.Index(s, "-")]
 	}
 	k := tka.Key{
@@ -681,7 +691,7 @@ func runTskeyWrapCmd(ctx context.Context, args []string) error {
 		Meta:   m,
 	}
 
-	wrapped, err := localClient.NetworkLockWrapPreauthKey(ctx, args[0], priv)
+	wrapped, err := localClient.NetworkLockWrapPreauthKey(ctx, keyStr, priv)
 	if err != nil {
 		return fmt.Errorf("wrapping failed: %w", err)
 	}

cmd/tailscale/cli/serve.go

@@ -35,78 +35,57 @@ func newServeCommand(e *serveEnv) *ffcli.Command {
 		Name:      "serve",
 		ShortHelp: "[ALPHA] Serve from your Tailscale node",
 		ShortUsage: strings.TrimSpace(`
-  serve [flags] <mount-point> {proxy|path|text} <arg>
-  serve [flags] <sub-command> [sub-flags] <args>`),
+serve https:<port> <mount-point> <source> [off]
+  serve tcp:<port> tcp://localhost:<local-port> [off]
+  serve tls-terminated-tcp:<port> tcp://localhost:<local-port> [off]
+  serve status [--json]
+`),
 		LongHelp: strings.TrimSpace(`
 *** ALPHA; all of this is subject to change ***
 
 The 'tailscale serve' set of commands allows you to serve
 content and local servers from your Tailscale node to
-your tailnet. 
+your tailnet.
 
 You can also choose to enable the Tailscale Funnel with:
-'tailscale serve funnel on'. Funnel allows you to publish
+'tailscale funnel on'. Funnel allows you to publish
 a 'tailscale serve' server publicly, open to the entire
 internet. See https://tailscale.com/funnel.
 
 EXAMPLES
   - To proxy requests to a web server at 127.0.0.1:3000:
-    $ tailscale serve / proxy 3000
+    $ tailscale serve https:443 / http://127.0.0.1:3000
+
+	Or, using the default port:
+	$ tailscale serve https / http://127.0.0.1:3000
 
   - To serve a single file or a directory of files:
-    $ tailscale serve / path /home/alice/blog/index.html
-    $ tailscale serve /images/ path /home/alice/blog/images
+    $ tailscale serve https / /home/alice/blog/index.html
+    $ tailscale serve https /images/ /home/alice/blog/images
 
   - To serve simple static text:
-    $ tailscale serve / text "Hello, world!"
+    $ tailscale serve https:8080 / text:"Hello, world!"
+
+  - To forward incoming TCP connections on port 2222 to a local TCP server on
+    port 22 (e.g. to run OpenSSH in parallel with Tailscale SSH):
+    $ tailscale serve tcp:2222 tcp://localhost:22
+
+  - To accept TCP TLS connections (terminated within tailscaled) proxied to a
+    local plaintext server on port 80:
+    $ tailscale serve tls-terminated-tcp:443 tcp://localhost:80
 `),
-		Exec: e.runServe,
-		FlagSet: e.newFlags("serve", func(fs *flag.FlagSet) {
-			fs.BoolVar(&e.remove, "remove", false, "remove an existing serve config")
-			fs.UintVar(&e.servePort, "serve-port", 443, "port to serve on (443, 8443 or 10000)")
-		}),
+		Exec:      e.runServe,
 		UsageFunc: usageFunc,
 		Subcommands: []*ffcli.Command{
 			{
 				Name:      "status",
 				Exec:      e.runServeStatus,
-				ShortHelp: "show current serve status",
+				ShortHelp: "show current serve/funnel status",
 				FlagSet: e.newFlags("serve-status", func(fs *flag.FlagSet) {
 					fs.BoolVar(&e.json, "json", false, "output JSON")
 				}),
 				UsageFunc: usageFunc,
 			},
-			{
-				Name:      "tcp",
-				Exec:      e.runServeTCP,
-				ShortHelp: "add or remove a TCP port forward",
-				LongHelp: strings.Join([]string{
-					"EXAMPLES",
-					"  - Forward TLS over TCP to a local TCP server on port 5432:",
-					"    $ tailscale serve tcp 5432",
-					"",
-					"  - Forward raw, TLS-terminated TCP packets to a local TCP server on port 5432:",
-					"    $ tailscale serve tcp --terminate-tls 5432",
-				}, "\n"),
-				FlagSet: e.newFlags("serve-tcp", func(fs *flag.FlagSet) {
-					fs.BoolVar(&e.terminateTLS, "terminate-tls", false, "terminate TLS before forwarding TCP connection")
-				}),
-				UsageFunc: usageFunc,
-			},
-			{
-				Name:       "funnel",
-				Exec:       e.runServeFunnel,
-				ShortUsage: "funnel [flags] {on|off}",
-				ShortHelp:  "turn Tailscale Funnel on or off",
-				LongHelp: strings.Join([]string{
-					"Funnel allows you to publish a 'tailscale serve'",
-					"server publicly, open to the entire internet.",
-					"",
-					"Turning off Funnel only turns off serving to the internet.",
-					"It does not affect serving to your tailnet.",
-				}, "\n"),
-				UsageFunc: usageFunc,
-			},
 		},
 	}
 }
@@ -143,10 +122,7 @@ type localServeClient interface {
 // It also contains the flags, as registered with newServeCommand.
 type serveEnv struct {
 	// flags
-	servePort    uint // Port to serve on. Defaults to 443.
-	terminateTLS bool
-	remove       bool // remove a serve config
-	json         bool // output JSON (status only for now)
+	json bool // output JSON (status only for now)
 
 	lc localServeClient // localClient interface, specific to serve
 
@@ -186,24 +162,15 @@ func (e *serveEnv) getLocalClientStatus(ctx context.Context) (*ipnstate.Status,
 	return st, nil
 }
 
-// validateServePort returns --serve-port flag value,
-// or an error if the port is not a valid port to serve on.
-func (e *serveEnv) validateServePort() (port uint16, err error) {
-	// Make sure e.servePort is uint16.
-	port = uint16(e.servePort)
-	if uint(port) != e.servePort {
-		return 0, fmt.Errorf("serve-port %d is out of range", e.servePort)
-	}
-	return port, nil
-}
-
 // runServe is the entry point for the "serve" subcommand, managing Web
 // serve config types like proxy, path, and text.
 //
 // Examples:
-// - tailscale serve / proxy 3000
-// - tailscale serve /images/ path /var/www/images/
-// - tailscale --serve-port=10000 serve /motd.txt text "Hello, world!"
+// - tailscale serve https / http://localhost:3000
+// - tailscale serve https /images/ /var/www/images/
+// - tailscale serve https:10000 /motd.txt text:"Hello, world!"
+// - tailscale serve tcp:2222 tcp://localhost:22
+// - tailscale serve tls-terminated-tcp:443 tcp://localhost:80
 func (e *serveEnv) runServe(ctx context.Context, args []string) error {
 	if len(args) == 0 {
 		return flag.ErrHelp
@@ -223,39 +190,94 @@ func (e *serveEnv) runServe(ctx context.Context, args []string) error {
 		return e.lc.SetServeConfig(ctx, sc)
 	}
 
-	if !(len(args) == 3 || (e.remove && len(args) >= 1)) {
+	parsePort := func(portStr string) (uint16, error) {
+		port64, err := strconv.ParseUint(portStr, 10, 16)
+		if err != nil {
+			return 0, err
+		}
+		return uint16(port64), nil
+	}
+
+	srcType, srcPortStr, found := strings.Cut(args[0], ":")
+	if !found {
+		if srcType == "https" && srcPortStr == "" {
+			// Default https port to 443.
+			srcPortStr = "443"
+		} else {
+			return flag.ErrHelp
+		}
+	}
+
+	turnOff := "off" == args[len(args)-1]
+
+	if len(args) < 2 || (srcType == "https" && !turnOff && len(args) < 3) {
 		fmt.Fprintf(os.Stderr, "error: invalid number of arguments\n\n")
 		return flag.ErrHelp
 	}
 
-	srvPort, err := e.validateServePort()
-	if err != nil {
-		return err
-	}
-	srvPortStr := strconv.Itoa(int(srvPort))
-
-	mount, err := cleanMountPoint(args[0])
+	srcPort, err := parsePort(srcPortStr)
 	if err != nil {
 		return err
 	}
 
-	if e.remove {
-		return e.handleWebServeRemove(ctx, mount)
+	switch srcType {
+	case "https":
+		mount, err := cleanMountPoint(args[1])
+		if err != nil {
+			return err
+		}
+		if turnOff {
+			return e.handleWebServeRemove(ctx, srcPort, mount)
+		}
+		return e.handleWebServe(ctx, srcPort, mount, args[2])
+	case "tcp", "tls-terminated-tcp":
+		if turnOff {
+			return e.handleTCPServeRemove(ctx, srcPort)
+		}
+		return e.handleTCPServe(ctx, srcType, srcPort, args[1])
+	default:
+		fmt.Fprintf(os.Stderr, "error: invalid serve type %q\n", srcType)
+		fmt.Fprint(os.Stderr, "must be one of: https:<port>, tcp:<port> or tls-terminated-tcp:<port>\n\n", srcType)
+		return flag.ErrHelp
 	}
+}
 
+// handleWebServe handles the "tailscale serve https:..." subcommand.
+// It configures the serve config to forward HTTPS connections to the
+// given source.
+//
+// Examples:
+//   - tailscale serve https / http://localhost:3000
+//   - tailscale serve https:8443 /files/ /home/alice/shared-files/
+//   - tailscale serve https:10000 /motd.txt text:"Hello, world!"
+func (e *serveEnv) handleWebServe(ctx context.Context, srvPort uint16, mount, source string) error {
 	h := new(ipn.HTTPHandler)
 
-	switch args[1] {
-	case "path":
+	ts, _, _ := strings.Cut(source, ":")
+	switch {
+	case ts == "text":
+		text := strings.TrimPrefix(source, "text:")
+		if text == "" {
+			return errors.New("unable to serve; text cannot be an empty string")
+		}
+		h.Text = text
+	case isProxyTarget(source):
+		t, err := expandProxyTarget(source)
+		if err != nil {
+			return err
+		}
+		h.Proxy = t
+	default: // assume path
 		if version.IsSandboxedMacOS() {
 			// don't allow path serving for now on macOS (2022-11-15)
 			return fmt.Errorf("path serving is not supported if sandboxed on macOS")
 		}
-		if !filepath.IsAbs(args[2]) {
+		if !filepath.IsAbs(source) {
 			fmt.Fprintf(os.Stderr, "error: path must be absolute\n\n")
 			return flag.ErrHelp
 		}
-		fi, err := os.Stat(args[2])
+		source = filepath.Clean(source)
+		fi, err := os.Stat(source)
 		if err != nil {
 			fmt.Fprintf(os.Stderr, "error: invalid path: %v\n\n", err)
 			return flag.ErrHelp
@@ -265,21 +287,7 @@ func (e *serveEnv) runServe(ctx context.Context, args []string) error {
 			// for relative file links to work
 			mount += "/"
 		}
-		h.Path = args[2]
-	case "proxy":
-		t, err := expandProxyTarget(args[2])
-		if err != nil {
-			return err
-		}
-		h.Proxy = t
-	case "text":
-		if args[2] == "" {
-			return errors.New("unable to serve; text cannot be an empty string")
-		}
-		h.Text = args[2]
-	default:
-		fmt.Fprintf(os.Stderr, "error: unknown serve type %q\n\n", args[1])
-		return flag.ErrHelp
+		h.Path = source
 	}
 
 	cursc, err := e.lc.GetServeConfig(ctx)
@@ -294,7 +302,7 @@ func (e *serveEnv) runServe(ctx context.Context, args []string) error {
 	if err != nil {
 		return err
 	}
-	hp := ipn.HostPort(net.JoinHostPort(dnsName, srvPortStr))
+	hp := ipn.HostPort(net.JoinHostPort(dnsName, strconv.Itoa(int(srvPort))))
 
 	if sc.IsTCPForwardingOnPort(srvPort) {
 		fmt.Fprintf(os.Stderr, "error: cannot serve web; already serving TCP\n")
@@ -333,12 +341,36 @@ func (e *serveEnv) runServe(ctx context.Context, args []string) error {
 	return nil
 }
 
-func (e *serveEnv) handleWebServeRemove(ctx context.Context, mount string) error {
-	srvPort, err := e.validateServePort()
-	if err != nil {
-		return err
+// isProxyTarget reports whether source is a valid proxy target.
+func isProxyTarget(source string) bool {
+	if strings.HasPrefix(source, "http://") ||
+		strings.HasPrefix(source, "https://") ||
+		strings.HasPrefix(source, "https+insecure://") {
+		return true
 	}
-	srvPortStr := strconv.Itoa(int(srvPort))
+	// support "localhost:3000", for example
+	_, portStr, ok := strings.Cut(source, ":")
+	if ok && allNumeric(portStr) {
+		return true
+	}
+	return false
+}
+
+// allNumeric reports whether s only comprises of digits
+// and has at least one digit.
+func allNumeric(s string) bool {
+	for i := 0; i < len(s); i++ {
+		if s[i] < '0' || s[i] > '9' {
+			return false
+		}
+	}
+	return s != ""
+}
+
+// handleWebServeRemove removes a web handler from the serve config.
+// The srvPort argument is the serving port and the mount argument is
+// the mount point or registered path to remove.
+func (e *serveEnv) handleWebServeRemove(ctx context.Context, srvPort uint16, mount string) error {
 	sc, err := e.lc.GetServeConfig(ctx)
 	if err != nil {
 		return err
@@ -353,9 +385,9 @@ func (e *serveEnv) handleWebServeRemove(ctx context.Context, mount string) error
 	if sc.IsTCPForwardingOnPort(srvPort) {
 		return errors.New("cannot remove web handler; currently serving TCP")
 	}
-	hp := ipn.HostPort(net.JoinHostPort(dnsName, srvPortStr))
+	hp := ipn.HostPort(net.JoinHostPort(dnsName, strconv.Itoa(int(srvPort))))
 	if !sc.WebHandlerExists(hp, mount) {
-		return errors.New("error: serve config does not exist")
+		return errors.New("error: handler does not exist")
 	}
 	// delete existing handler, then cascade delete if empty
 	delete(sc.Web[hp].Handlers, mount)
@@ -390,18 +422,11 @@ func cleanMountPoint(mount string) (string, error) {
 	return "", fmt.Errorf("invalid mount point %q", mount)
 }
 
-func expandProxyTarget(target string) (string, error) {
-	if allNumeric(target) {
-		p, err := strconv.ParseUint(target, 10, 16)
-		if p == 0 || err != nil {
-			return "", fmt.Errorf("invalid port %q", target)
-		}
-		return "http://127.0.0.1:" + target, nil
+func expandProxyTarget(source string) (string, error) {
+	if !strings.Contains(source, "://") {
+		source = "http://" + source
 	}
-	if !strings.Contains(target, "://") {
-		target = "http://" + target
-	}
-	u, err := url.ParseRequestURI(target)
+	u, err := url.ParseRequestURI(source)
 	if err != nil {
 		return "", fmt.Errorf("parsing url: %w", err)
 	}
@@ -411,9 +436,14 @@ func expandProxyTarget(target string) (string, error) {
 	default:
 		return "", fmt.Errorf("must be a URL starting with http://, https://, or https+insecure://")
 	}
+
+	port, err := strconv.ParseUint(u.Port(), 10, 16)
+	if port == 0 || err != nil {
+		return "", fmt.Errorf("invalid port %q: %w", u.Port(), err)
+	}
+
 	host := u.Hostname()
 	switch host {
-	// TODO(shayne,bradfitz): do we want to do this?
 	case "localhost", "127.0.0.1":
 		host = "127.0.0.1"
 	default:
@@ -426,16 +456,111 @@ func expandProxyTarget(target string) (string, error) {
 	return url, nil
 }
 
-func allNumeric(s string) bool {
-	for i := 0; i < len(s); i++ {
-		if s[i] < '0' || s[i] > '9' {
-			return false
+// handleTCPServe handles the "tailscale serve tls-terminated-tcp:..." subcommand.
+// It configures the serve config to forward TCP connections to the
+// given source.
+//
+// Examples:
+//   - tailscale serve tcp:2222 tcp://localhost:22
+//   - tailscale serve tls-terminated-tcp:8443 tcp://localhost:8080
+func (e *serveEnv) handleTCPServe(ctx context.Context, srcType string, srcPort uint16, dest string) error {
+	var terminateTLS bool
+	switch srcType {
+	case "tcp":
+		terminateTLS = false
+	case "tls-terminated-tcp":
+		terminateTLS = true
+	default:
+		fmt.Fprintf(os.Stderr, "error: invalid TCP source %q\n\n", dest)
+		return flag.ErrHelp
+	}
+
+	dstURL, err := url.Parse(dest)
+	if err != nil {
+		fmt.Fprintf(os.Stderr, "error: invalid TCP source %q: %v\n\n", dest, err)
+		return flag.ErrHelp
+	}
+	host, dstPortStr, err := net.SplitHostPort(dstURL.Host)
+	if err != nil {
+		fmt.Fprintf(os.Stderr, "error: invalid TCP source %q: %v\n\n", dest, err)
+		return flag.ErrHelp
+	}
+
+	switch host {
+	case "localhost", "127.0.0.1":
+		// ok
+	default:
+		fmt.Fprintf(os.Stderr, "error: invalid TCP source %q\n", dest)
+		fmt.Fprint(os.Stderr, "must be one of: localhost or 127.0.0.1\n\n", dest)
+		return flag.ErrHelp
+	}
+
+	if p, err := strconv.ParseUint(dstPortStr, 10, 16); p == 0 || err != nil {
+		fmt.Fprintf(os.Stderr, "error: invalid port %q\n\n", dstPortStr)
+		return flag.ErrHelp
+	}
+
+	cursc, err := e.lc.GetServeConfig(ctx)
+	if err != nil {
+		return err
+	}
+	sc := cursc.Clone() // nil if no config
+	if sc == nil {
+		sc = new(ipn.ServeConfig)
+	}
+
+	fwdAddr := "127.0.0.1:" + dstPortStr
+
+	if sc.IsServingWeb(srcPort) {
+		return fmt.Errorf("cannot serve TCP; already serving web on %d", srcPort)
+	}
+
+	mak.Set(&sc.TCP, srcPort, &ipn.TCPPortHandler{TCPForward: fwdAddr})
+
+	dnsName, err := e.getSelfDNSName(ctx)
+	if err != nil {
+		return err
+	}
+	if terminateTLS {
+		sc.TCP[srcPort].TerminateTLS = dnsName
+	}
+
+	if !reflect.DeepEqual(cursc, sc) {
+		if err := e.lc.SetServeConfig(ctx, sc); err != nil {
+			return err
 		}
 	}
-	return s != ""
+
+	return nil
 }
 
-// runServeStatus prints the current serve config.
+// handleTCPServeRemove removes the TCP forwarding configuration for the
+// given srvPort, or serving port.
+func (e *serveEnv) handleTCPServeRemove(ctx context.Context, src uint16) error {
+	cursc, err := e.lc.GetServeConfig(ctx)
+	if err != nil {
+		return err
+	}
+	sc := cursc.Clone() // nil if no config
+	if sc == nil {
+		sc = new(ipn.ServeConfig)
+	}
+	if sc.IsServingWeb(src) {
+		return fmt.Errorf("unable to remove; serving web, not TCP forwarding on serve port %d", src)
+	}
+	if ph := sc.GetTCPPortHandler(src); ph != nil {
+		delete(sc.TCP, src)
+		// clear map mostly for testing
+		if len(sc.TCP) == 0 {
+			sc.TCP = nil
+		}
+		return e.lc.SetServeConfig(ctx, sc)
+	}
+	return errors.New("error: serve config does not exist")
+}
+
+// runServeStatus is the entry point for the "serve status"
+// subcommand and prints the current serve config.
 //
 // Examples:
 //   - tailscale status
@@ -454,6 +579,7 @@ func (e *serveEnv) runServeStatus(ctx context.Context, args []string) error {
 		e.stdout().Write(j)
 		return nil
 	}
+	printFunnelStatus(ctx)
 	if sc == nil || (len(sc.TCP) == 0 && len(sc.Web) == 0 && len(sc.AllowFunnel) == 0) {
 		printf("No serve config\n")
 		return nil
@@ -472,17 +598,7 @@ func (e *serveEnv) runServeStatus(ctx context.Context, args []string) error {
 		printWebStatusTree(sc, hp)
 		printf("\n")
 	}
-	// warn when funnel on without handlers
-	for hp, a := range sc.AllowFunnel {
-		if !a {
-			continue
-		}
-		_, portStr, _ := net.SplitHostPort(string(hp))
-		p, _ := strconv.ParseUint(portStr, 10, 16)
-		if _, ok := sc.TCP[uint16(p)]; !ok {
-			printf("WARNING: funnel=on for %s, but no serve config\n", hp)
-		}
-	}
+	printFunnelWarning(sc)
 	return nil
 }
 
@@ -566,133 +682,3 @@ func elipticallyTruncate(s string, max int) string {
 	}
 	return s[:max-3] + "..."
 }
-
-// runServeTCP is the entry point for the "serve tcp" subcommand and
-// manages the serve config for TCP forwarding.
-//
-// Examples:
-//   - tailscale serve tcp 5432
-//   - tailscale serve --serve-port=8443 tcp 4430
-//   - tailscale serve --serve-port=10000 tcp --terminate-tls 8080
-func (e *serveEnv) runServeTCP(ctx context.Context, args []string) error {
-	if len(args) != 1 {
-		fmt.Fprintf(os.Stderr, "error: invalid number of arguments\n\n")
-		return flag.ErrHelp
-	}
-
-	srvPort, err := e.validateServePort()
-	if err != nil {
-		return err
-	}
-
-	portStr := args[0]
-	p, err := strconv.ParseUint(portStr, 10, 16)
-	if p == 0 || err != nil {
-		fmt.Fprintf(os.Stderr, "error: invalid port %q\n\n", portStr)
-	}
-
-	cursc, err := e.lc.GetServeConfig(ctx)
-	if err != nil {
-		return err
-	}
-	sc := cursc.Clone() // nil if no config
-	if sc == nil {
-		sc = new(ipn.ServeConfig)
-	}
-
-	fwdAddr := "127.0.0.1:" + portStr
-
-	if sc.IsServingWeb(srvPort) {
-		if e.remove {
-			return fmt.Errorf("unable to remove; serving web, not TCP forwarding on serve port %d", srvPort)
-		}
-		return fmt.Errorf("cannot serve TCP; already serving web on %d", srvPort)
-	}
-
-	if e.remove {
-		if ph := sc.GetTCPPortHandler(srvPort); ph != nil && ph.TCPForward == fwdAddr {
-			delete(sc.TCP, srvPort)
-			// clear map mostly for testing
-			if len(sc.TCP) == 0 {
-				sc.TCP = nil
-			}
-			return e.lc.SetServeConfig(ctx, sc)
-		}
-		return errors.New("error: serve config does not exist")
-	}
-
-	mak.Set(&sc.TCP, srvPort, &ipn.TCPPortHandler{TCPForward: fwdAddr})
-
-	dnsName, err := e.getSelfDNSName(ctx)
-	if err != nil {
-		return err
-	}
-	if e.terminateTLS {
-		sc.TCP[srvPort].TerminateTLS = dnsName
-	}
-
-	if !reflect.DeepEqual(cursc, sc) {
-		if err := e.lc.SetServeConfig(ctx, sc); err != nil {
-			return err
-		}
-	}
-
-	return nil
-}
-
-// runServeFunnel is the entry point for the "serve funnel" subcommand and
-// manages turning on/off funnel. Funnel is off by default.
-//
-// Note: funnel is only supported on single DNS name for now. (2022-11-15)
-func (e *serveEnv) runServeFunnel(ctx context.Context, args []string) error {
-	if len(args) != 1 {
-		return flag.ErrHelp
-	}
-
-	srvPort, err := e.validateServePort()
-	if err != nil {
-		return err
-	}
-	srvPortStr := strconv.Itoa(int(srvPort))
-
-	var on bool
-	switch args[0] {
-	case "on", "off":
-		on = args[0] == "on"
-	default:
-		return flag.ErrHelp
-	}
-	sc, err := e.lc.GetServeConfig(ctx)
-	if err != nil {
-		return err
-	}
-	if sc == nil {
-		sc = new(ipn.ServeConfig)
-	}
-	st, err := e.getLocalClientStatus(ctx)
-	if err != nil {
-		return fmt.Errorf("getting client status: %w", err)
-	}
-	if err := ipn.CheckFunnelAccess(srvPort, st.Self.Capabilities); err != nil {
-		return err
-	}
-	dnsName := strings.TrimSuffix(st.Self.DNSName, ".")
-	hp := ipn.HostPort(dnsName + ":" + srvPortStr)
-	if on == sc.AllowFunnel[hp] {
-		// Nothing to do.
-		return nil
-	}
-	if on {
-		mak.Set(&sc.AllowFunnel, hp, true)
-	} else {
-		delete(sc.AllowFunnel, hp)
-		// clear map mostly for testing
-		if len(sc.AllowFunnel) == 0 {
-			sc.AllowFunnel = nil
-		}
-	}
-	if err := e.lc.SetServeConfig(ctx, sc); err != nil {
-		return err
-	}
-	return nil
-}

cmd/tailscale/cli/serve_test.go

@@ -15,6 +15,7 @@ import (
 	"strings"
 	"testing"
 
+	"github.com/peterbourgon/ff/v3/ffcli"
 	"tailscale.com/ipn"
 	"tailscale.com/ipn/ipnstate"
 	"tailscale.com/tailcfg"
@@ -56,6 +57,8 @@ func TestServeConfigMutations(t *testing.T) {
 		want    *ipn.ServeConfig               // non-nil means we want a save of this value
 		wantErr func(error) (badErrMsg string) // nil means no error is wanted
 		line    int                            // line number of addStep call, for error messages
+
+		debugBreak func()
 	}
 	var steps []step
 	add := func(s step) {
@@ -66,19 +69,19 @@ func TestServeConfigMutations(t *testing.T) {
 	// funnel
 	add(step{reset: true})
 	add(step{
-		command: cmd("funnel on"),
+		command: cmd("funnel 443 on"),
 		want:    &ipn.ServeConfig{AllowFunnel: map[ipn.HostPort]bool{"foo.test.ts.net:443": true}},
 	})
 	add(step{
-		command: cmd("funnel on"),
+		command: cmd("funnel 443 on"),
 		want:    nil, // nothing to save
 	})
 	add(step{
-		command: cmd("funnel off"),
+		command: cmd("funnel 443 off"),
 		want:    &ipn.ServeConfig{},
 	})
 	add(step{
-		command: cmd("funnel off"),
+		command: cmd("funnel 443 off"),
 		want:    nil, // nothing to save
 	})
 	add(step{
@@ -89,27 +92,48 @@ func TestServeConfigMutations(t *testing.T) {
 	// https
 	add(step{reset: true})
 	add(step{
-		command: cmd("/ proxy 0"), // invalid port, too low
+		command: cmd("https:443 / http://localhost:0"), // invalid port, too low
 		wantErr: anyErr(),
 	})
 	add(step{
-		command: cmd("/ proxy 65536"), // invalid port, too high
+		command: cmd("https:443 / http://localhost:65536"), // invalid port, too high
 		wantErr: anyErr(),
 	})
 	add(step{
-		command: cmd("/ proxy somehost"), // invalid host
+		command: cmd("https:443 / http://somehost:3000"), // invalid host
 		wantErr: anyErr(),
 	})
 	add(step{
-		command: cmd("/ proxy http://otherhost"), // invalid host
+		command: cmd("https:443 / httpz://127.0.0.1"), // invalid scheme
 		wantErr: anyErr(),
 	})
-	add(step{
-		command: cmd("/ proxy httpz://127.0.0.1"), // invalid scheme
-		wantErr: anyErr(),
+	add(step{ // allow omitting port (default to 443)
+		command: cmd("https / http://localhost:3000"),
+		want: &ipn.ServeConfig{
+			TCP: map[uint16]*ipn.TCPPortHandler{443: {HTTPS: true}},
+			Web: map[ipn.HostPort]*ipn.WebServerConfig{
+				"foo.test.ts.net:443": {Handlers: map[string]*ipn.HTTPHandler{
+					"/": {Proxy: "http://127.0.0.1:3000"},
+				}},
+			},
+		},
+	})
+	add(step{ // support non Funnel port
+		command: cmd("https:9999 /abc http://localhost:3001"),
+		want: &ipn.ServeConfig{
+			TCP: map[uint16]*ipn.TCPPortHandler{443: {HTTPS: true}, 9999: {HTTPS: true}},
+			Web: map[ipn.HostPort]*ipn.WebServerConfig{
+				"foo.test.ts.net:443": {Handlers: map[string]*ipn.HTTPHandler{
+					"/": {Proxy: "http://127.0.0.1:3000"},
+				}},
+				"foo.test.ts.net:9999": {Handlers: map[string]*ipn.HTTPHandler{
+					"/abc": {Proxy: "http://127.0.0.1:3001"},
+				}},
+			},
+		},
 	})
 	add(step{
-		command: cmd("/ proxy 3000"),
+		command: cmd("https:9999 /abc off"),
 		want: &ipn.ServeConfig{
 			TCP: map[uint16]*ipn.TCPPortHandler{443: {HTTPS: true}},
 			Web: map[ipn.HostPort]*ipn.WebServerConfig{
@@ -120,7 +144,7 @@ func TestServeConfigMutations(t *testing.T) {
 		},
 	})
 	add(step{
-		command: cmd("--serve-port=8443 /abc proxy 3001"),
+		command: cmd("https:8443 /abc http://127.0.0.1:3001"),
 		want: &ipn.ServeConfig{
 			TCP: map[uint16]*ipn.TCPPortHandler{443: {HTTPS: true}, 8443: {HTTPS: true}},
 			Web: map[ipn.HostPort]*ipn.WebServerConfig{
@@ -134,7 +158,7 @@ func TestServeConfigMutations(t *testing.T) {
 		},
 	})
 	add(step{
-		command: cmd("--serve-port=10000 / text hi"),
+		command: cmd("https:10000 / text:hi"),
 		want: &ipn.ServeConfig{
 			TCP: map[uint16]*ipn.TCPPortHandler{
 				443: {HTTPS: true}, 8443: {HTTPS: true}, 10000: {HTTPS: true}},
@@ -152,12 +176,12 @@ func TestServeConfigMutations(t *testing.T) {
 		},
 	})
 	add(step{
-		command: cmd("--remove /foo"),
+		command: cmd("https:443 /foo off"),
 		want:    nil, // nothing to save
 		wantErr: anyErr(),
 	}) // handler doesn't exist, so we get an error
 	add(step{
-		command: cmd("--remove --serve-port=10000 /"),
+		command: cmd("https:10000 / off"),
 		want: &ipn.ServeConfig{
 			TCP: map[uint16]*ipn.TCPPortHandler{443: {HTTPS: true}, 8443: {HTTPS: true}},
 			Web: map[ipn.HostPort]*ipn.WebServerConfig{
@@ -171,7 +195,7 @@ func TestServeConfigMutations(t *testing.T) {
 		},
 	})
 	add(step{
-		command: cmd("--remove /"),
+		command: cmd("https:443 / off"),
 		want: &ipn.ServeConfig{
 			TCP: map[uint16]*ipn.TCPPortHandler{8443: {HTTPS: true}},
 			Web: map[ipn.HostPort]*ipn.WebServerConfig{
@@ -182,11 +206,11 @@ func TestServeConfigMutations(t *testing.T) {
 		},
 	})
 	add(step{
-		command: cmd("--remove --serve-port=8443 /abc"),
+		command: cmd("https:8443 /abc off"),
 		want:    &ipn.ServeConfig{},
 	})
-	add(step{
-		command: cmd("bar proxy https://127.0.0.1:8443"),
+	add(step{ // clean mount: "bar" becomes "/bar"
+		command: cmd("https:443 bar https://127.0.0.1:8443"),
 		want: &ipn.ServeConfig{
 			TCP: map[uint16]*ipn.TCPPortHandler{443: {HTTPS: true}},
 			Web: map[ipn.HostPort]*ipn.WebServerConfig{
@@ -197,12 +221,12 @@ func TestServeConfigMutations(t *testing.T) {
 		},
 	})
 	add(step{
-		command: cmd("bar proxy https://127.0.0.1:8443"),
+		command: cmd("https:443 bar https://127.0.0.1:8443"),
 		want:    nil, // nothing to save
 	})
 	add(step{reset: true})
 	add(step{
-		command: cmd("/ proxy https+insecure://127.0.0.1:3001"),
+		command: cmd("https:443 / https+insecure://127.0.0.1:3001"),
 		want: &ipn.ServeConfig{
 			TCP: map[uint16]*ipn.TCPPortHandler{443: {HTTPS: true}},
 			Web: map[ipn.HostPort]*ipn.WebServerConfig{
@@ -214,7 +238,7 @@ func TestServeConfigMutations(t *testing.T) {
 	})
 	add(step{reset: true})
 	add(step{
-		command: cmd("/foo proxy localhost:3000"),
+		command: cmd("https:443 /foo localhost:3000"),
 		want: &ipn.ServeConfig{
 			TCP: map[uint16]*ipn.TCPPortHandler{443: {HTTPS: true}},
 			Web: map[ipn.HostPort]*ipn.WebServerConfig{
@@ -225,7 +249,7 @@ func TestServeConfigMutations(t *testing.T) {
 		},
 	})
 	add(step{ // test a second handler on the same port
-		command: cmd("--serve-port=8443 /foo proxy localhost:3000"),
+		command: cmd("https:8443 /foo localhost:3000"),
 		want: &ipn.ServeConfig{
 			TCP: map[uint16]*ipn.TCPPortHandler{443: {HTTPS: true}, 8443: {HTTPS: true}},
 			Web: map[ipn.HostPort]*ipn.WebServerConfig{
@@ -241,16 +265,35 @@ func TestServeConfigMutations(t *testing.T) {
 
 	// tcp
 	add(step{reset: true})
+	add(step{ // must include scheme for tcp
+		command: cmd("tls-terminated-tcp:443 localhost:5432"),
+		wantErr: exactErr(flag.ErrHelp, "flag.ErrHelp"),
+	})
+	add(step{ // !somehost, must be localhost or 127.0.0.1
+		command: cmd("tls-terminated-tcp:443 tcp://somehost:5432"),
+		wantErr: exactErr(flag.ErrHelp, "flag.ErrHelp"),
+	})
+	add(step{ // bad target port, too low
+		command: cmd("tls-terminated-tcp:443 tcp://somehost:0"),
+		wantErr: exactErr(flag.ErrHelp, "flag.ErrHelp"),
+	})
+	add(step{ // bad target port, too high
+		command: cmd("tls-terminated-tcp:443 tcp://somehost:65536"),
+		wantErr: exactErr(flag.ErrHelp, "flag.ErrHelp"),
+	})
 	add(step{
-		command: cmd("tcp 5432"),
+		command: cmd("tls-terminated-tcp:443 tcp://localhost:5432"),
 		want: &ipn.ServeConfig{
 			TCP: map[uint16]*ipn.TCPPortHandler{
-				443: {TCPForward: "127.0.0.1:5432"},
+				443: {
+					TCPForward:   "127.0.0.1:5432",
+					TerminateTLS: "foo.test.ts.net",
+				},
 			},
 		},
 	})
 	add(step{
-		command: cmd("tcp -terminate-tls 8443"),
+		command: cmd("tls-terminated-tcp:443 tcp://127.0.0.1:8443"),
 		want: &ipn.ServeConfig{
 			TCP: map[uint16]*ipn.TCPPortHandler{
 				443: {
@@ -261,11 +304,11 @@ func TestServeConfigMutations(t *testing.T) {
 		},
 	})
 	add(step{
-		command: cmd("tcp -terminate-tls 8443"),
+		command: cmd("tls-terminated-tcp:443 tcp://127.0.0.1:8443"),
 		want:    nil, // nothing to save
 	})
 	add(step{
-		command: cmd("tcp --terminate-tls 8444"),
+		command: cmd("tls-terminated-tcp:443 tcp://localhost:8444"),
 		want: &ipn.ServeConfig{
 			TCP: map[uint16]*ipn.TCPPortHandler{
 				443: {
@@ -276,35 +319,41 @@ func TestServeConfigMutations(t *testing.T) {
 		},
 	})
 	add(step{
-		command: cmd("tcp -terminate-tls=false 8445"),
+		command: cmd("tls-terminated-tcp:443 tcp://127.0.0.1:8445"),
 		want: &ipn.ServeConfig{
 			TCP: map[uint16]*ipn.TCPPortHandler{
-				443: {TCPForward: "127.0.0.1:8445"},
+				443: {
+					TCPForward:   "127.0.0.1:8445",
+					TerminateTLS: "foo.test.ts.net",
+				},
 			},
 		},
 	})
 	add(step{reset: true})
 	add(step{
-		command: cmd("tcp 123"),
+		command: cmd("tls-terminated-tcp:443 tcp://localhost:123"),
 		want: &ipn.ServeConfig{
 			TCP: map[uint16]*ipn.TCPPortHandler{
-				443: {TCPForward: "127.0.0.1:123"},
+				443: {
+					TCPForward:   "127.0.0.1:123",
+					TerminateTLS: "foo.test.ts.net",
+				},
 			},
 		},
 	})
-	add(step{
-		command: cmd("--remove tcp 321"),
+	add(step{ // handler doesn't exist, so we get an error
+		command: cmd("tls-terminated-tcp:8443 off"),
 		wantErr: anyErr(),
-	}) // handler doesn't exist, so we get an error
+	})
 	add(step{
-		command: cmd("--remove tcp 123"),
+		command: cmd("tls-terminated-tcp:443 off"),
 		want:    &ipn.ServeConfig{},
 	})
 
 	// text
 	add(step{reset: true})
 	add(step{
-		command: cmd("/ text hello"),
+		command: cmd("https:443 / text:hello"),
 		want: &ipn.ServeConfig{
 			TCP: map[uint16]*ipn.TCPPortHandler{443: {HTTPS: true}},
 			Web: map[ipn.HostPort]*ipn.WebServerConfig{
@@ -325,7 +374,7 @@ func TestServeConfigMutations(t *testing.T) {
 	add(step{reset: true})
 	writeFile("foo", "this is foo")
 	add(step{
-		command: cmd("/ path " + filepath.Join(td, "foo")),
+		command: cmd("https:443 / " + filepath.Join(td, "foo")),
 		want: &ipn.ServeConfig{
 			TCP: map[uint16]*ipn.TCPPortHandler{443: {HTTPS: true}},
 			Web: map[ipn.HostPort]*ipn.WebServerConfig{
@@ -338,7 +387,7 @@ func TestServeConfigMutations(t *testing.T) {
 	os.MkdirAll(filepath.Join(td, "subdir"), 0700)
 	writeFile("subdir/file-a", "this is A")
 	add(step{
-		command: cmd("/some/where path " + filepath.Join(td, "subdir/file-a")),
+		command: cmd("https:443 /some/where " + filepath.Join(td, "subdir/file-a")),
 		want: &ipn.ServeConfig{
 			TCP: map[uint16]*ipn.TCPPortHandler{443: {HTTPS: true}},
 			Web: map[ipn.HostPort]*ipn.WebServerConfig{
@@ -349,13 +398,13 @@ func TestServeConfigMutations(t *testing.T) {
 			},
 		},
 	})
-	add(step{
-		command: cmd("/ path missing"),
+	add(step{ // bad path
+		command: cmd("https:443 / bad/path"),
 		wantErr: exactErr(flag.ErrHelp, "flag.ErrHelp"),
 	})
 	add(step{reset: true})
 	add(step{
-		command: cmd("/ path " + filepath.Join(td, "subdir")),
+		command: cmd("https:443 / " + filepath.Join(td, "subdir")),
 		want: &ipn.ServeConfig{
 			TCP: map[uint16]*ipn.TCPPortHandler{443: {HTTPS: true}},
 			Web: map[ipn.HostPort]*ipn.WebServerConfig{
@@ -366,14 +415,14 @@ func TestServeConfigMutations(t *testing.T) {
 		},
 	})
 	add(step{
-		command: cmd("--remove /"),
+		command: cmd("https:443 / off"),
 		want:    &ipn.ServeConfig{},
 	})
 
 	// combos
 	add(step{reset: true})
 	add(step{
-		command: cmd("/ proxy 3000"),
+		command: cmd("https:443 / localhost:3000"),
 		want: &ipn.ServeConfig{
 			TCP: map[uint16]*ipn.TCPPortHandler{443: {HTTPS: true}},
 			Web: map[ipn.HostPort]*ipn.WebServerConfig{
@@ -384,7 +433,7 @@ func TestServeConfigMutations(t *testing.T) {
 		},
 	})
 	add(step{
-		command: cmd("funnel on"),
+		command: cmd("funnel 443 on"),
 		want: &ipn.ServeConfig{
 			AllowFunnel: map[ipn.HostPort]bool{"foo.test.ts.net:443": true},
 			TCP:         map[uint16]*ipn.TCPPortHandler{443: {HTTPS: true}},
@@ -396,7 +445,7 @@ func TestServeConfigMutations(t *testing.T) {
 		},
 	})
 	add(step{ // serving on secondary port doesn't change funnel
-		command: cmd("--serve-port=8443 /bar proxy 3001"),
+		command: cmd("https:8443 /bar localhost:3001"),
 		want: &ipn.ServeConfig{
 			AllowFunnel: map[ipn.HostPort]bool{"foo.test.ts.net:443": true},
 			TCP:         map[uint16]*ipn.TCPPortHandler{443: {HTTPS: true}, 8443: {HTTPS: true}},
@@ -411,7 +460,7 @@ func TestServeConfigMutations(t *testing.T) {
 		},
 	})
 	add(step{ // turn funnel on for secondary port
-		command: cmd("--serve-port=8443 funnel on"),
+		command: cmd("funnel 8443 on"),
 		want: &ipn.ServeConfig{
 			AllowFunnel: map[ipn.HostPort]bool{"foo.test.ts.net:443": true, "foo.test.ts.net:8443": true},
 			TCP:         map[uint16]*ipn.TCPPortHandler{443: {HTTPS: true}, 8443: {HTTPS: true}},
@@ -426,7 +475,7 @@ func TestServeConfigMutations(t *testing.T) {
 		},
 	})
 	add(step{ // turn funnel off for primary port 443
-		command: cmd("funnel off"),
+		command: cmd("funnel 443 off"),
 		want: &ipn.ServeConfig{
 			AllowFunnel: map[ipn.HostPort]bool{"foo.test.ts.net:8443": true},
 			TCP:         map[uint16]*ipn.TCPPortHandler{443: {HTTPS: true}, 8443: {HTTPS: true}},
@@ -441,7 +490,7 @@ func TestServeConfigMutations(t *testing.T) {
 		},
 	})
 	add(step{ // remove secondary port
-		command: cmd("--serve-port=8443 --remove /bar"),
+		command: cmd("https:8443 /bar off"),
 		want: &ipn.ServeConfig{
 			AllowFunnel: map[ipn.HostPort]bool{"foo.test.ts.net:8443": true},
 			TCP:         map[uint16]*ipn.TCPPortHandler{443: {HTTPS: true}},
@@ -453,7 +502,7 @@ func TestServeConfigMutations(t *testing.T) {
 		},
 	})
 	add(step{ // start a tcp forwarder on 8443
-		command: cmd("--serve-port=8443 tcp 5432"),
+		command: cmd("tcp:8443 tcp://localhost:5432"),
 		want: &ipn.ServeConfig{
 			AllowFunnel: map[ipn.HostPort]bool{"foo.test.ts.net:8443": true},
 			TCP:         map[uint16]*ipn.TCPPortHandler{443: {HTTPS: true}, 8443: {TCPForward: "127.0.0.1:5432"}},
@@ -465,27 +514,27 @@ func TestServeConfigMutations(t *testing.T) {
 		},
 	})
 	add(step{ // remove primary port http handler
-		command: cmd("--remove /"),
+		command: cmd("https:443 / off"),
 		want: &ipn.ServeConfig{
 			AllowFunnel: map[ipn.HostPort]bool{"foo.test.ts.net:8443": true},
 			TCP:         map[uint16]*ipn.TCPPortHandler{8443: {TCPForward: "127.0.0.1:5432"}},
 		},
 	})
 	add(step{ // remove tcp forwarder
-		command: cmd("--serve-port=8443 --remove tcp 5432"),
+		command: cmd("tls-terminated-tcp:8443 off"),
 		want: &ipn.ServeConfig{
 			AllowFunnel: map[ipn.HostPort]bool{"foo.test.ts.net:8443": true},
 		},
 	})
 	add(step{ // turn off funnel
-		command: cmd("--serve-port=8443 funnel off"),
+		command: cmd("funnel 8443 off"),
 		want:    &ipn.ServeConfig{},
 	})
 
 	// tricky steps
 	add(step{reset: true})
 	add(step{ // a directory with a trailing slash mount point
-		command: cmd("/dir path " + filepath.Join(td, "subdir")),
+		command: cmd("https:443 /dir " + filepath.Join(td, "subdir")),
 		want: &ipn.ServeConfig{
 			TCP: map[uint16]*ipn.TCPPortHandler{443: {HTTPS: true}},
 			Web: map[ipn.HostPort]*ipn.WebServerConfig{
@@ -496,7 +545,7 @@ func TestServeConfigMutations(t *testing.T) {
 		},
 	})
 	add(step{ // this should overwrite the previous one
-		command: cmd("/dir path " + filepath.Join(td, "foo")),
+		command: cmd("https:443 /dir " + filepath.Join(td, "foo")),
 		want: &ipn.ServeConfig{
 			TCP: map[uint16]*ipn.TCPPortHandler{443: {HTTPS: true}},
 			Web: map[ipn.HostPort]*ipn.WebServerConfig{
@@ -508,7 +557,7 @@ func TestServeConfigMutations(t *testing.T) {
 	})
 	add(step{reset: true}) // reset and do the opposite
 	add(step{              // a file without a trailing slash mount point
-		command: cmd("/dir path " + filepath.Join(td, "foo")),
+		command: cmd("https:443 /dir " + filepath.Join(td, "foo")),
 		want: &ipn.ServeConfig{
 			TCP: map[uint16]*ipn.TCPPortHandler{443: {HTTPS: true}},
 			Web: map[ipn.HostPort]*ipn.WebServerConfig{
@@ -519,7 +568,7 @@ func TestServeConfigMutations(t *testing.T) {
 		},
 	})
 	add(step{ // this should overwrite the previous one
-		command: cmd("/dir path " + filepath.Join(td, "subdir")),
+		command: cmd("https:443 /dir " + filepath.Join(td, "subdir")),
 		want: &ipn.ServeConfig{
 			TCP: map[uint16]*ipn.TCPPortHandler{443: {HTTPS: true}},
 			Web: map[ipn.HostPort]*ipn.WebServerConfig{
@@ -532,37 +581,24 @@ func TestServeConfigMutations(t *testing.T) {
 
 	// error states
 	add(step{reset: true})
-	add(step{ // make sure we can't add "tcp" as if it was a mount
-		command: cmd("tcp text foo"),
-		wantErr: exactErr(flag.ErrHelp, "flag.ErrHelp"),
-	})
-	add(step{ // "/tcp" is fine though as a mount
-		command: cmd("/tcp text foo"),
-		want: &ipn.ServeConfig{
-			TCP: map[uint16]*ipn.TCPPortHandler{443: {HTTPS: true}},
-			Web: map[ipn.HostPort]*ipn.WebServerConfig{
-				"foo.test.ts.net:443": {Handlers: map[string]*ipn.HTTPHandler{
-					"/tcp": {Text: "foo"},
-				}},
-			},
-		},
-	})
-	add(step{reset: true})
 	add(step{ // tcp forward 5432 on serve port 443
-		command: cmd("tcp 5432"),
+		command: cmd("tls-terminated-tcp:443 tcp://localhost:5432"),
 		want: &ipn.ServeConfig{
 			TCP: map[uint16]*ipn.TCPPortHandler{
-				443: {TCPForward: "127.0.0.1:5432"},
+				443: {
+					TCPForward:   "127.0.0.1:5432",
+					TerminateTLS: "foo.test.ts.net",
+				},
 			},
 		},
 	})
 	add(step{ // try to start a web handler on the same port
-		command: cmd("/ proxy 3000"),
+		command: cmd("https:443 / localhost:3000"),
 		wantErr: exactErr(flag.ErrHelp, "flag.ErrHelp"),
 	})
 	add(step{reset: true})
 	add(step{ // start a web handler on port 443
-		command: cmd("/ proxy 3000"),
+		command: cmd("https:443 / localhost:3000"),
 		want: &ipn.ServeConfig{
 			TCP: map[uint16]*ipn.TCPPortHandler{443: {HTTPS: true}},
 			Web: map[ipn.HostPort]*ipn.WebServerConfig{
@@ -572,14 +608,17 @@ func TestServeConfigMutations(t *testing.T) {
 			},
 		},
 	})
-	add(step{ // try to start a tcp forwarder on the same serve port (443 default)
-		command: cmd("tcp 5432"),
+	add(step{ // try to start a tcp forwarder on the same serve port
+		command: cmd("tls-terminated-tcp:443 tcp://localhost:5432"),
 		wantErr: anyErr(),
 	})
 
 	lc := &fakeLocalServeClient{}
 	// And now run the steps above.
 	for i, st := range steps {
+		if st.debugBreak != nil {
+			st.debugBreak()
+		}
 		if st.reset {
 			t.Logf("Executing step #%d, line %v: [reset]", i, st.line)
 			lc.config = nil
@@ -597,8 +636,16 @@ func TestServeConfigMutations(t *testing.T) {
 			testStdout:  &stdout,
 		}
 		lastCount := lc.setCount
-		cmd := newServeCommand(e)
-		err := cmd.ParseAndRun(context.Background(), st.command)
+		var cmd *ffcli.Command
+		var args []string
+		if st.command[0] == "funnel" {
+			cmd = newFunnelCommand(e)
+			args = st.command[1:]
+		} else {
+			cmd = newServeCommand(e)
+			args = st.command
+		}
+		err := cmd.ParseAndRun(context.Background(), args)
 		if flagOut.Len() > 0 {
 			t.Logf("flag package output: %q", flagOut.Bytes())
 		}
@@ -689,7 +736,5 @@ func anyErr() func(error) string {
 }
 
 func cmd(s string) []string {
-	cmds := strings.Fields(s)
-	fmt.Printf("cmd: %v", cmds)
-	return cmds
+	return strings.Fields(s)
 }

cmd/tailscale/cli/status.go

@@ -258,6 +258,7 @@ func printFunnelStatus(ctx context.Context) {
 		}
 		printf("#     - %s\n", url)
 	}
+	outln()
 }
 
 // isRunningOrStarting reports whether st is in state Running or Starting.

cmd/tailscale/cli/up.go

@@ -409,6 +409,12 @@ func updatePrefs(prefs, curPrefs *ipn.Prefs, env upCheckEnv) (simpleUp bool, jus
 		return false, nil, err
 	}
 
+	if env.upArgs.forceReauth && isSSHOverTailscale() {
+		if err := presentRiskToUser(riskLoseSSH, `You are connected over Tailscale; this action will result in your SSH session disconnecting.`, env.upArgs.acceptedRisks); err != nil {
+			return false, nil, err
+		}
+	}
+
 	tagsChanged := !reflect.DeepEqual(curPrefs.AdvertiseTags, prefs.AdvertiseTags)
 
 	simpleUp = env.flagSet.NFlag() == 0 &&

cmd/tailscale/cli/update.go

@@ -145,11 +145,11 @@ func newUpdater() (*updater, error) {
 		case strings.HasSuffix(os.Getenv("HOME"), "/io.tailscale.ipn.macsys/Data"):
 			up.update = up.updateMacSys
 		default:
-			return nil, errors.New("This is the macOS App Store version of Tailscale; update in the App Store, or see https://tailscale.com/kb/1083/install-unstable/ to use TestFlight or to install the non-App Store version")
+			return nil, errors.New("This is the macOS App Store version of Tailscale; update in the App Store, or see https://tailscale.com/s/unstable-clients to use TestFlight or to install the non-App Store version")
 		}
 	}
 	if up.update == nil {
-		return nil, errors.New("The 'update' command is not supported on this platform; see https://tailscale.com/kb/1067/update/")
+		return nil, errors.New("The 'update' command is not supported on this platform; see https://tailscale.com/s/client-updates")
 	}
 	return up, nil
 }

cmd/tailscaled/depaware.txt

@@ -212,17 +212,18 @@ tailscale.com/cmd/tailscaled dependencies: (generated by github.com/tailscale/de
         tailscale.com/ipn/ipnstate                                   from tailscale.com/control/controlclient+
         tailscale.com/ipn/localapi                                   from tailscale.com/ipn/ipnserver
         tailscale.com/ipn/policy                                     from tailscale.com/ipn/ipnlocal
-        tailscale.com/ipn/store                                      from tailscale.com/cmd/tailscaled
+        tailscale.com/ipn/store                                      from tailscale.com/cmd/tailscaled+
    L    tailscale.com/ipn/store/awsstore                             from tailscale.com/ipn/store
    L    tailscale.com/ipn/store/kubestore                            from tailscale.com/ipn/store
         tailscale.com/ipn/store/mem                                  from tailscale.com/ipn/store+
    L    tailscale.com/kube                                           from tailscale.com/ipn/store/kubestore
         tailscale.com/log/filelogger                                 from tailscale.com/logpolicy
         tailscale.com/log/logheap                                    from tailscale.com/control/controlclient
+        tailscale.com/log/sockstatlog                                from tailscale.com/ipn/ipnlocal
         tailscale.com/logpolicy                                      from tailscale.com/cmd/tailscaled+
         tailscale.com/logtail                                        from tailscale.com/control/controlclient+
         tailscale.com/logtail/backoff                                from tailscale.com/control/controlclient+
-        tailscale.com/logtail/filch                                  from tailscale.com/logpolicy
+        tailscale.com/logtail/filch                                  from tailscale.com/logpolicy+
         tailscale.com/metrics                                        from tailscale.com/derp+
         tailscale.com/net/connstats                                  from tailscale.com/net/tstun+
         tailscale.com/net/dns                                        from tailscale.com/ipn/ipnlocal+

cmd/tailscaled/tailscaled.go

@@ -51,6 +51,7 @@ import (
 	"tailscale.com/tsweb"
 	"tailscale.com/types/flagtype"
 	"tailscale.com/types/logger"
+	"tailscale.com/types/logid"
 	"tailscale.com/util/clientmetric"
 	"tailscale.com/util/multierr"
 	"tailscale.com/util/osshare"
@@ -377,11 +378,10 @@ func run() error {
 		debugMux = newDebugMux()
 	}
 
-	logid := pol.PublicID.String()
-	return startIPNServer(context.Background(), logf, logid)
+	return startIPNServer(context.Background(), logf, pol.PublicID)
 }
 
-func startIPNServer(ctx context.Context, logf logger.Logf, logid string) error {
+func startIPNServer(ctx context.Context, logf logger.Logf, logID logid.PublicID) error {
 	ln, err := safesocket.Listen(args.socketpath)
 	if err != nil {
 		return fmt.Errorf("safesocket.Listen: %v", err)
@@ -407,7 +407,7 @@ func startIPNServer(ctx context.Context, logf logger.Logf, logid string) error {
 		}
 	}()
 
-	srv := ipnserver.New(logf, logid)
+	srv := ipnserver.New(logf, logID)
 	if debugMux != nil {
 		debugMux.HandleFunc("/debug/ipn", srv.ServeHTMLStatus)
 	}
@@ -425,7 +425,7 @@ func startIPNServer(ctx context.Context, logf logger.Logf, logid string) error {
 				return
 			}
 		}
-		lb, err := getLocalBackend(ctx, logf, logid)
+		lb, err := getLocalBackend(ctx, logf, logID)
 		if err == nil {
 			logf("got LocalBackend in %v", time.Since(t0).Round(time.Millisecond))
 			srv.SetLocalBackend(lb)
@@ -449,7 +449,7 @@ func startIPNServer(ctx context.Context, logf logger.Logf, logid string) error {
 	return nil
 }
 
-func getLocalBackend(ctx context.Context, logf logger.Logf, logid string) (_ *ipnlocal.LocalBackend, retErr error) {
+func getLocalBackend(ctx context.Context, logf logger.Logf, logID logid.PublicID) (_ *ipnlocal.LocalBackend, retErr error) {
 	linkMon, err := monitor.New(logf)
 	if err != nil {
 		return nil, fmt.Errorf("monitor.New: %w", err)
@@ -520,7 +520,7 @@ func getLocalBackend(ctx context.Context, logf logger.Logf, logid string) (_ *ip
 		return nil, fmt.Errorf("store.New: %w", err)
 	}
 
-	lb, err := ipnlocal.NewLocalBackend(logf, logid, store, dialer, e, opts.LoginFlags)
+	lb, err := ipnlocal.NewLocalBackend(logf, logID, store, dialer, e, opts.LoginFlags)
 	if err != nil {
 		return nil, fmt.Errorf("ipnlocal.NewLocalBackend: %w", err)
 	}

cmd/tailscaled/tailscaled_windows.go

@@ -47,6 +47,7 @@ import (
 	"tailscale.com/net/dns"
 	"tailscale.com/net/tstun"
 	"tailscale.com/types/logger"
+	"tailscale.com/types/logid"
 	"tailscale.com/util/winutil"
 	"tailscale.com/version"
 	"tailscale.com/wf"
@@ -262,13 +263,13 @@ func beWindowsSubprocess() bool {
 	if len(os.Args) != 3 || os.Args[1] != "/subproc" {
 		return false
 	}
-	logid := os.Args[2]
+	logID := os.Args[2]
 
 	// Remove the date/time prefix; the logtail + file loggers add it.
 	log.SetFlags(0)
 
 	log.Printf("Program starting: v%v: %#v", version.Long(), os.Args)
-	log.Printf("subproc mode: logid=%v", logid)
+	log.Printf("subproc mode: logid=%v", logID)
 	if err := envknob.ApplyDiskConfigError(); err != nil {
 		log.Printf("Error reading environment config: %v", err)
 	}
@@ -290,7 +291,8 @@ func beWindowsSubprocess() bool {
 		}
 	}()
 
-	err := startIPNServer(ctx, log.Printf, logid)
+	publicLogID, _ := logid.ParsePublicID(logID)
+	err := startIPNServer(ctx, log.Printf, publicLogID)
 	if err != nil {
 		log.Fatalf("ipnserver: %v", err)
 	}

cmd/tsconnect/wasm/wasm_js.go

@@ -122,7 +122,7 @@ func newIPN(jsConfig js.Value) map[string]any {
 		return ns.DialContextTCP(ctx, dst)
 	}
 
-	logid := lpc.PublicID.String()
+	logid := lpc.PublicID
 	srv := ipnserver.New(logf, logid)
 	lb, err := ipnlocal.NewLocalBackend(logf, logid, store, dialer, eng, controlclient.LoginEphemeral)
 	if err != nil {

control/controlclient/auto.go

@@ -407,7 +407,13 @@ func (c *Auto) authRoutine() {
 				c.mu.Unlock()
 
 				c.sendStatus("authRoutine-url", err, url, nil)
-				bo.BackOff(ctx, err)
+				if goal.url == url {
+					// The server sent us the same URL we already tried,
+					// backoff to avoid a busy loop.
+					bo.BackOff(ctx, errors.New("login URL not changing"))
+				} else {
+					bo.BackOff(ctx, nil)
+				}
 				continue
 			}
 

derp/derp_server.go

@@ -279,6 +279,7 @@ func (s *dupClientSet) removeClient(c *sclient) bool {
 // public key gets more than one PacketForwarder registered for it.
 type PacketForwarder interface {
 	ForwardPacket(src, dst key.NodePublic, payload []byte) error
+	String() string
 }
 
 // Conn is the subset of the underlying net.Conn the DERP Server needs.
@@ -495,6 +496,7 @@ func (s *Server) registerClient(c *sclient) {
 	switch set := set.(type) {
 	case nil:
 		s.clients[c.key] = singleClient{c}
+		c.debug("register single client")
 	case singleClient:
 		s.dupClientKeys.Add(1)
 		s.dupClientConns.Add(2) // both old and new count
@@ -510,6 +512,7 @@ func (s *Server) registerClient(c *sclient) {
 			},
 			sendHistory: []*sclient{old},
 		}
+		c.debug("register duplicate client")
 	case *dupClientSet:
 		s.dupClientConns.Add(1)     // the gauge
 		s.dupClientConnTotal.Add(1) // the counter
@@ -517,6 +520,7 @@ func (s *Server) registerClient(c *sclient) {
 		set.set[c] = true
 		set.last = c
 		set.sendHistory = append(set.sendHistory, c)
+		c.debug("register another duplicate client")
 	}
 
 	if _, ok := s.clientsMesh[c.key]; !ok {
@@ -549,7 +553,7 @@ func (s *Server) unregisterClient(c *sclient) {
 	case nil:
 		c.logf("[unexpected]; clients map is empty")
 	case singleClient:
-		c.logf("removing connection")
+		c.logf("removed connection")
 		delete(s.clients, c.key)
 		if v, ok := s.clientsMesh[c.key]; ok && v == nil {
 			delete(s.clientsMesh, c.key)
@@ -557,6 +561,7 @@ func (s *Server) unregisterClient(c *sclient) {
 		}
 		s.broadcastPeerStateChangeLocked(c.key, false)
 	case *dupClientSet:
+		c.debug("removed duplicate client")
 		if set.removeClient(c) {
 			s.dupClientConns.Add(-1)
 		} else {
@@ -673,7 +678,7 @@ func (s *Server) accept(ctx context.Context, nc Conn, brw *bufio.ReadWriter, rem
 		nc:             nc,
 		br:             br,
 		bw:             bw,
-		logf:           logger.WithPrefix(s.logf, fmt.Sprintf("derp client %v/%x: ", remoteAddr, clientKey)),
+		logf:           logger.WithPrefix(s.logf, fmt.Sprintf("derp client %v%s: ", remoteAddr, clientKey.ShortString())),
 		done:           ctx.Done(),
 		remoteAddr:     remoteAddr,
 		remoteIPPort:   remoteIPPort,
@@ -690,6 +695,9 @@ func (s *Server) accept(ctx context.Context, nc Conn, brw *bufio.ReadWriter, rem
 	}
 	if clientInfo != nil {
 		c.info = *clientInfo
+		if envknob.Bool("DERP_PROBER_DEBUG_LOGS") && clientInfo.IsProber {
+			c.debugLogging = true
+		}
 	}
 
 	s.registerClient(c)
@@ -726,6 +734,7 @@ func (c *sclient) run(ctx context.Context) error {
 
 	for {
 		ft, fl, err := readFrameHeader(c.br)
+		c.debug("read frame type %d len %d err %v", ft, fl, err)
 		if err != nil {
 			if errors.Is(err, io.EOF) {
 				c.logf("read EOF")
@@ -735,7 +744,7 @@ func (c *sclient) run(ctx context.Context) error {
 				c.logf("closing; server closed")
 				return nil
 			}
-			return fmt.Errorf("client %x: readFrameHeader: %w", c.key, err)
+			return fmt.Errorf("client %s: readFrameHeader: %w", c.key.ShortString(), err)
 		}
 		c.s.noteClientActivity(c)
 		switch ft {
@@ -883,6 +892,8 @@ func (c *sclient) handleFrameForwardPacket(ft frameType, fl uint32) error {
 		return nil
 	}
 
+	dst.debug("received forwarded packet from %s via %s", srcKey.ShortString(), c.key.ShortString())
+
 	return c.sendPkt(dst, pkt{
 		bs:         contents,
 		enqueuedAt: time.Now(),
@@ -930,7 +941,9 @@ func (c *sclient) handleFrameSendPacket(ft frameType, fl uint32) error {
 	if dst == nil {
 		if fwd != nil {
 			s.packetsForwardedOut.Add(1)
-			if err := fwd.ForwardPacket(c.key, dstKey, contents); err != nil {
+			err := fwd.ForwardPacket(c.key, dstKey, contents)
+			c.debug("SendPacket for %s, forwarding via %s: %v", dstKey.ShortString(), fwd, err)
+			if err != nil {
 				// TODO:
 				return nil
 			}
@@ -941,8 +954,10 @@ func (c *sclient) handleFrameSendPacket(ft frameType, fl uint32) error {
 			reason = dropReasonDupClient
 		}
 		s.recordDrop(contents, c.key, dstKey, reason)
+		c.debug("SendPacket for %s, dropping with reason=%s", dstKey.ShortString(), reason)
 		return nil
 	}
+	c.debug("SendPacket for %s, sending directly", dstKey.ShortString())
 
 	p := pkt{
 		bs:         contents,
@@ -952,6 +967,12 @@ func (c *sclient) handleFrameSendPacket(ft frameType, fl uint32) error {
 	return c.sendPkt(dst, p)
 }
 
+func (c *sclient) debug(format string, v ...any) {
+	if c.debugLogging {
+		c.logf(format, v...)
+	}
+}
+
 // dropReason is why we dropped a DERP frame.
 type dropReason int
 
@@ -1003,11 +1024,13 @@ func (c *sclient) sendPkt(dst *sclient, p pkt) error {
 		select {
 		case <-dst.done:
 			s.recordDrop(p.bs, c.key, dstKey, dropReasonGone)
+			dst.debug("sendPkt attempt %d dropped, dst gone", attempt)
 			return nil
 		default:
 		}
 		select {
 		case sendQueue <- p:
+			dst.debug("sendPkt attempt %d enqueued", attempt)
 			return nil
 		default:
 		}
@@ -1023,6 +1046,7 @@ func (c *sclient) sendPkt(dst *sclient, p pkt) error {
 	// contended queue with racing writers. Give up and tail-drop in
 	// this case to keep reader unblocked.
 	s.recordDrop(p.bs, c.key, dstKey, dropReasonQueueTail)
+	dst.debug("sendPkt attempt %d dropped, queue full")
 
 	return nil
 }
@@ -1258,6 +1282,8 @@ type sclient struct {
 	isDup          atomic.Bool         // whether more than 1 sclient for key is connected
 	isDisabled     atomic.Bool         // whether sends to this peer are disabled due to active/active dups
 
+	debugLogging bool
+
 	// replaceLimiter controls how quickly two connections with
 	// the same client key can kick each other off the server by
 	// taking over ownership of a key.
@@ -1529,6 +1555,7 @@ func (c *sclient) sendPacket(srcKey key.NodePublic, contents []byte) (err error)
 			c.s.packetsSent.Add(1)
 			c.s.bytesSent.Add(int64(len(contents)))
 		}
+		c.debug("sendPacket from %s: %v", srcKey.ShortString(), err)
 	}()
 
 	c.setWriteDeadline()
@@ -1689,6 +1716,10 @@ func (f *multiForwarder) ForwardPacket(src, dst key.NodePublic, payload []byte)
 	return f.fwd.Load().ForwardPacket(src, dst, payload)
 }
 
+func (f *multiForwarder) String() string {
+	return fmt.Sprintf("<MultiForwarder fwd=%s total=%d>", f.fwd.Load(), len(f.all))
+}
+
 func (s *Server) expVarFunc(f func() any) expvar.Func {
 	return expvar.Func(func() any {
 		s.mu.Lock()

derp/derp_test.go

@@ -660,6 +660,9 @@ type testFwd int
 func (testFwd) ForwardPacket(key.NodePublic, key.NodePublic, []byte) error {
 	panic("not called in tests")
 }
+func (testFwd) String() string {
+	panic("not called in tests")
+}
 
 func pubAll(b byte) (ret key.NodePublic) {
 	var bs [32]byte
@@ -787,6 +790,7 @@ type channelFwd struct {
 	c  chan []byte
 }
 
+func (f channelFwd) String() string { return "" }
 func (f channelFwd) ForwardPacket(_ key.NodePublic, _ key.NodePublic, packet []byte) error {
 	f.c <- packet
 	return nil

derp/derphttp/derphttp_client.go

@@ -82,6 +82,10 @@ type Client struct {
 	pingOut      map[derp.PingMessage]chan<- bool // chan to send to on pong
 }
 
+func (c *Client) String() string {
+	return fmt.Sprintf("<derphttp_client.Client %s url=%s>", c.serverPubKey.ShortString(), c.url)
+}
+
 // NewRegionClient returns a new DERP-over-HTTP client. It connects lazily.
 // To trigger a connection, use Connect.
 func NewRegionClient(privateKey key.NodePrivate, logf logger.Logf, getRegion func() *tailcfg.DERPRegion) *Client {

go.toolchain.rev

@@ -1 +1 @@
-db4dc9046c93dde2c0e534ca7d529bd690ad09c9
+568add9f5d780e86f8b3e7002fd7b4a7479005fa

ipn/ipnlocal/c2n.go

@@ -83,6 +83,14 @@ func (b *LocalBackend) handleC2N(w http.ResponseWriter, r *http.Request) {
 			return
 		}
 		writeJSON(res)
+	case "/sockstats":
+		if r.Method != "POST" {
+			http.Error(w, "bad method", http.StatusMethodNotAllowed)
+			return
+		}
+		w.Header().Set("Content-Type", "text/plain")
+		b.sockstatLogger.Flush()
+		fmt.Fprintln(w, b.sockstatLogger.LogID())
 	default:
 		http.Error(w, "unknown c2n path", http.StatusBadRequest)
 	}

ipn/ipnlocal/cert.go

@@ -31,10 +31,13 @@ import (
 	"time"
 
 	"golang.org/x/crypto/acme"
+	"tailscale.com/atomicfile"
 	"tailscale.com/envknob"
 	"tailscale.com/hostinfo"
 	"tailscale.com/ipn"
 	"tailscale.com/ipn/ipnstate"
+	"tailscale.com/ipn/store"
+	"tailscale.com/ipn/store/mem"
 	"tailscale.com/types/logger"
 	"tailscale.com/version"
 	"tailscale.com/version/distro"
@@ -82,11 +85,6 @@ func (b *LocalBackend) GetCertPEM(ctx context.Context, domain string) (*TLSCertK
 		return nil, errors.New("invalid domain")
 	}
 	logf := logger.WithPrefix(b.logf, fmt.Sprintf("cert(%q): ", domain))
-	dir, err := b.certDir()
-	if err != nil {
-		logf("failed to get certDir: %v", err)
-		return nil, err
-	}
 	now := time.Now()
 	traceACME := func(v any) {
 		if !acmeDebug() {
@@ -96,17 +94,22 @@ func (b *LocalBackend) GetCertPEM(ctx context.Context, domain string) (*TLSCertK
 		log.Printf("acme %T: %s", v, j)
 	}
 
-	if pair, err := b.getCertPEMCached(dir, domain, now); err == nil {
+	cs, err := b.getCertStore()
+	if err != nil {
+		return nil, err
+	}
+
+	if pair, err := getCertPEMCached(cs, domain, now); err == nil {
 		future := now.AddDate(0, 0, 14)
-		if b.shouldStartDomainRenewal(dir, domain, future) {
+		if b.shouldStartDomainRenewal(cs, domain, future) {
 			logf("starting async renewal")
 			// Start renewal in the background.
-			go b.getCertPEM(context.Background(), logf, traceACME, dir, domain, future)
+			go b.getCertPEM(context.Background(), cs, logf, traceACME, domain, future)
 		}
 		return pair, nil
 	}
 
-	pair, err := b.getCertPEM(ctx, logf, traceACME, dir, domain, now)
+	pair, err := b.getCertPEM(ctx, cs, logf, traceACME, domain, now)
 	if err != nil {
 		logf("getCertPEM: %v", err)
 		return nil, err
@@ -114,7 +117,7 @@ func (b *LocalBackend) GetCertPEM(ctx context.Context, domain string) (*TLSCertK
 	return pair, nil
 }
 
-func (b *LocalBackend) shouldStartDomainRenewal(dir, domain string, future time.Time) bool {
+func (b *LocalBackend) shouldStartDomainRenewal(cs certStore, domain string, future time.Time) bool {
 	renewMu.Lock()
 	defer renewMu.Unlock()
 	now := time.Now()
@@ -124,7 +127,7 @@ func (b *LocalBackend) shouldStartDomainRenewal(dir, domain string, future time.
 		return false
 	}
 	lastRenewCheck[domain] = now
-	_, err := b.getCertPEMCached(dir, domain, future)
+	_, err := getCertPEMCached(cs, domain, future)
 	return errors.Is(err, errCertExpired)
 }
 
@@ -140,15 +143,32 @@ type certStore interface {
 	WriteCert(domain string, cert []byte) error
 	// WriteKey writes the key for domain.
 	WriteKey(domain string, key []byte) error
+	// ACMEKey returns the value previously stored via WriteACMEKey.
+	// It is a PEM encoded ECDSA key.
+	ACMEKey() ([]byte, error)
+	// WriteACMEKey stores the provided PEM encoded ECDSA key.
+	WriteACMEKey([]byte) error
 }
 
 var errCertExpired = errors.New("cert expired")
 
-func (b *LocalBackend) getCertStore(dir string) certStore {
-	if hostinfo.GetEnvType() == hostinfo.Kubernetes && dir == "/tmp" {
-		return certStateStore{StateStore: b.store}
+func (b *LocalBackend) getCertStore() (certStore, error) {
+	switch b.store.(type) {
+	case *store.FileStore:
+	case *mem.Store:
+	default:
+		if hostinfo.GetEnvType() == hostinfo.Kubernetes {
+			// We're running in Kubernetes with a custom StateStore,
+			// use that instead of the cert directory.
+			// TODO(maisem): expand this to other environments?
+			return certStateStore{StateStore: b.store}, nil
+		}
 	}
-	return certFileStore{dir: dir}
+	dir, err := b.certDir()
+	if err != nil {
+		return nil, err
+	}
+	return certFileStore{dir: dir}, nil
 }
 
 // certFileStore implements certStore by storing the cert & key files in the named directory.
@@ -160,6 +180,25 @@ type certFileStore struct {
 	testRoots *x509.CertPool
 }
 
+const acmePEMName = "acme-account.key.pem"
+
+func (f certFileStore) ACMEKey() ([]byte, error) {
+	pemName := filepath.Join(f.dir, acmePEMName)
+	v, err := os.ReadFile(pemName)
+	if err != nil {
+		if os.IsNotExist(err) {
+			return nil, ipn.ErrStateNotExist
+		}
+		return nil, err
+	}
+	return v, nil
+}
+
+func (f certFileStore) WriteACMEKey(b []byte) error {
+	pemName := filepath.Join(f.dir, acmePEMName)
+	return atomicfile.WriteFile(pemName, b, 0600)
+}
+
 func (f certFileStore) Read(domain string, now time.Time) (*TLSCertKeyPair, error) {
 	certPEM, err := os.ReadFile(certFile(f.dir, domain))
 	if err != nil {
@@ -182,11 +221,11 @@ func (f certFileStore) Read(domain string, now time.Time) (*TLSCertKeyPair, erro
 }
 
 func (f certFileStore) WriteCert(domain string, cert []byte) error {
-	return os.WriteFile(certFile(f.dir, domain), cert, 0644)
+	return atomicfile.WriteFile(certFile(f.dir, domain), cert, 0644)
 }
 
 func (f certFileStore) WriteKey(domain string, key []byte) error {
-	return os.WriteFile(keyFile(f.dir, domain), key, 0600)
+	return atomicfile.WriteFile(keyFile(f.dir, domain), key, 0600)
 }
 
 // certStateStore implements certStore by storing the cert & key files in an ipn.StateStore.
@@ -221,6 +260,14 @@ func (s certStateStore) WriteKey(domain string, key []byte) error {
 	return s.WriteState(ipn.StateKey(domain+".key"), key)
 }
 
+func (s certStateStore) ACMEKey() ([]byte, error) {
+	return s.ReadState(ipn.StateKey(acmePEMName))
+}
+
+func (s certStateStore) WriteACMEKey(key []byte) error {
+	return s.WriteState(ipn.StateKey(acmePEMName), key)
+}
+
 // TLSCertKeyPair is a TLS public and private key, and whether they were obtained
 // from cache or freshly obtained.
 type TLSCertKeyPair struct {
@@ -236,26 +283,26 @@ func certFile(dir, domain string) string { return filepath.Join(dir, domain+".cr
 // domain exists on disk in dir that is valid at the provided now time.
 // If the keypair is expired, it returns errCertExpired.
 // If the keypair doesn't exist, it returns ipn.ErrStateNotExist.
-func (b *LocalBackend) getCertPEMCached(dir, domain string, now time.Time) (p *TLSCertKeyPair, err error) {
+func getCertPEMCached(cs certStore, domain string, now time.Time) (p *TLSCertKeyPair, err error) {
 	if !validLookingCertDomain(domain) {
 		// Before we read files from disk using it, validate it's halfway
 		// reasonable looking.
 		return nil, fmt.Errorf("invalid domain %q", domain)
 	}
-	return b.getCertStore(dir).Read(domain, now)
+	return cs.Read(domain, now)
 }
 
-func (b *LocalBackend) getCertPEM(ctx context.Context, logf logger.Logf, traceACME func(any), dir, domain string, now time.Time) (*TLSCertKeyPair, error) {
+func (b *LocalBackend) getCertPEM(ctx context.Context, cs certStore, logf logger.Logf, traceACME func(any), domain string, now time.Time) (*TLSCertKeyPair, error) {
 	acmeMu.Lock()
 	defer acmeMu.Unlock()
 
-	if p, err := b.getCertPEMCached(dir, domain, now); err == nil {
+	if p, err := getCertPEMCached(cs, domain, now); err == nil {
 		return p, nil
 	} else if !errors.Is(err, ipn.ErrStateNotExist) && !errors.Is(err, errCertExpired) {
 		return nil, err
 	}
 
-	key, err := acmeKey(dir)
+	key, err := acmeKey(cs)
 	if err != nil {
 		return nil, fmt.Errorf("acmeKey: %w", err)
 	}
@@ -366,8 +413,7 @@ func (b *LocalBackend) getCertPEM(ctx context.Context, logf logger.Logf, traceAC
 	if err := encodeECDSAKey(&privPEM, certPrivKey); err != nil {
 		return nil, err
 	}
-	certStore := b.getCertStore(dir)
-	if err := certStore.WriteKey(domain, privPEM.Bytes()); err != nil {
+	if err := cs.WriteKey(domain, privPEM.Bytes()); err != nil {
 		return nil, err
 	}
 
@@ -390,7 +436,7 @@ func (b *LocalBackend) getCertPEM(ctx context.Context, logf logger.Logf, traceAC
 			return nil, err
 		}
 	}
-	if err := certStore.WriteCert(domain, certPEM.Bytes()); err != nil {
+	if err := cs.WriteCert(domain, certPEM.Bytes()); err != nil {
 		return nil, err
 	}
 
@@ -444,14 +490,15 @@ func parsePrivateKey(der []byte) (crypto.Signer, error) {
 	return nil, errors.New("acme/autocert: failed to parse private key")
 }
 
-func acmeKey(dir string) (crypto.Signer, error) {
-	pemName := filepath.Join(dir, "acme-account.key.pem")
-	if v, err := os.ReadFile(pemName); err == nil {
+func acmeKey(cs certStore) (crypto.Signer, error) {
+	if v, err := cs.ACMEKey(); err == nil {
 		priv, _ := pem.Decode(v)
 		if priv == nil || !strings.Contains(priv.Type, "PRIVATE") {
 			return nil, errors.New("acme/autocert: invalid account key found in cache")
 		}
 		return parsePrivateKey(priv.Bytes)
+	} else if err != nil && !errors.Is(err, ipn.ErrStateNotExist) {
+		return nil, err
 	}
 
 	privKey, err := ecdsa.GenerateKey(elliptic.P256(), rand.Reader)
@@ -462,7 +509,7 @@ func acmeKey(dir string) (crypto.Signer, error) {
 	if err := encodeECDSAKey(&pemBuf, privKey); err != nil {
 		return nil, err
 	}
-	if err := os.WriteFile(pemName, pemBuf.Bytes(), 0600); err != nil {
+	if err := cs.WriteACMEKey(pemBuf.Bytes()); err != nil {
 		return nil, err
 	}
 	return privKey, nil

ipn/ipnlocal/local.go

@@ -11,6 +11,7 @@ import (
 	"errors"
 	"fmt"
 	"io"
+	"log"
 	"net"
 	"net/http"
 	"net/http/httputil"
@@ -43,6 +44,8 @@ import (
 	"tailscale.com/ipn/ipnauth"
 	"tailscale.com/ipn/ipnstate"
 	"tailscale.com/ipn/policy"
+	"tailscale.com/log/sockstatlog"
+	"tailscale.com/logpolicy"
 	"tailscale.com/net/dns"
 	"tailscale.com/net/dnscache"
 	"tailscale.com/net/dnsfallback"
@@ -60,6 +63,7 @@ import (
 	"tailscale.com/types/empty"
 	"tailscale.com/types/key"
 	"tailscale.com/types/logger"
+	"tailscale.com/types/logid"
 	"tailscale.com/types/netmap"
 	"tailscale.com/types/persist"
 	"tailscale.com/types/preftype"
@@ -136,7 +140,7 @@ type LocalBackend struct {
 	pm                    *profileManager
 	store                 ipn.StateStore
 	dialer                *tsdial.Dialer // non-nil
-	backendLogID          string
+	backendLogID          logid.PublicID
 	unregisterLinkMon     func()
 	unregisterHealthWatch func()
 	portpoll              *portlist.Poller // may be nil
@@ -149,6 +153,7 @@ type LocalBackend struct {
 	sshAtomicBool         atomic.Bool
 	shutdownCalled        bool // if Shutdown has been called
 	debugSink             *capture.Sink
+	sockstatLogger        *sockstatlog.Logger
 
 	// getTCPHandlerForFunnelFlow returns a handler for an incoming TCP flow for
 	// the provided srcAddr and dstPort if one exists.
@@ -261,7 +266,7 @@ type clientGen func(controlclient.Options) (controlclient.Client, error)
 // but is not actually running.
 //
 // If dialer is nil, a new one is made.
-func NewLocalBackend(logf logger.Logf, logid string, store ipn.StateStore, dialer *tsdial.Dialer, e wgengine.Engine, loginFlags controlclient.LoginFlags) (*LocalBackend, error) {
+func NewLocalBackend(logf logger.Logf, logID logid.PublicID, store ipn.StateStore, dialer *tsdial.Dialer, e wgengine.Engine, loginFlags controlclient.LoginFlags) (*LocalBackend, error) {
 	if e == nil {
 		panic("ipn.NewLocalBackend: engine must not be nil")
 	}
@@ -294,9 +299,9 @@ func NewLocalBackend(logf logger.Logf, logid string, store ipn.StateStore, diale
 		statsLogf:      logger.LogOnChange(logf, 5*time.Minute, time.Now),
 		e:              e,
 		pm:             pm,
-		store:          pm.Store(),
+		store:          store,
 		dialer:         dialer,
-		backendLogID:   logid,
+		backendLogID:   logID,
 		state:          ipn.NoState,
 		portpoll:       portpoll,
 		em:             newExpiryManager(logf),
@@ -304,6 +309,14 @@ func NewLocalBackend(logf logger.Logf, logid string, store ipn.StateStore, diale
 		loginFlags:     loginFlags,
 	}
 
+	// for now, only log sockstats on unstable builds
+	if version.IsUnstableBuild() {
+		b.sockstatLogger, err = sockstatlog.NewLogger(logpolicy.LogsDir(logf), logf, logID)
+		if err != nil {
+			log.Printf("error setting up sockstat logger: %v", err)
+		}
+	}
+
 	// Default filter blocks everything and logs nothing, until Start() is called.
 	b.setFilter(filter.NewAllowNone(logf, &netipx.IPSet{}))
 
@@ -541,6 +554,10 @@ func (b *LocalBackend) Shutdown() {
 	}
 	b.mu.Unlock()
 
+	if b.sockstatLogger != nil {
+		b.sockstatLogger.Shutdown()
+	}
+
 	b.unregisterLinkMon()
 	b.unregisterHealthWatch()
 	if cc != nil {
@@ -1278,7 +1295,7 @@ func (b *LocalBackend) Start(opts ipn.Options) error {
 	}
 
 	hostinfo := hostinfo.New()
-	hostinfo.BackendLogID = b.backendLogID
+	hostinfo.BackendLogID = b.backendLogID.String()
 	hostinfo.FrontendLogID = opts.FrontendLogID
 	hostinfo.Userspace.Set(wgengine.IsNetstack(b.e))
 	hostinfo.UserspaceRouter.Set(wgengine.IsNetstackRouter(b.e))
@@ -1432,7 +1449,7 @@ func (b *LocalBackend) Start(opts ipn.Options) error {
 
 	b.e.SetNetInfoCallback(b.setNetInfo)
 
-	blid := b.backendLogID
+	blid := b.backendLogID.String()
 	b.logf("Backend: logs: be:%v fe:%v", blid, opts.FrontendLogID)
 	b.send(ipn.Notify{BackendLogID: &blid})
 	b.send(ipn.Notify{Prefs: &prefs})
@@ -4783,6 +4800,10 @@ func (b *LocalBackend) initTKALocked() error {
 		if err != nil {
 			return fmt.Errorf("opening tailchonk: %v", err)
 		}
+		// Actually delete data which has been purged for 7 days:
+		if err := storage.CollectGarbage(7 * 24 * time.Hour); err != nil {
+			b.logf("tka garbage collection failed: %v", err)
+		}
 		authority, err := tka.Open(storage)
 		if err != nil {
 			return fmt.Errorf("initializing tka: %v", err)

ipn/ipnlocal/local_test.go

@@ -23,6 +23,7 @@ import (
 	"tailscale.com/tstest"
 	"tailscale.com/types/key"
 	"tailscale.com/types/logger"
+	"tailscale.com/types/logid"
 	"tailscale.com/types/netmap"
 	"tailscale.com/wgengine"
 	"tailscale.com/wgengine/filter"
@@ -506,7 +507,7 @@ func TestLazyMachineKeyGeneration(t *testing.T) {
 		t.Fatalf("NewFakeUserspaceEngine: %v", err)
 	}
 	t.Cleanup(eng.Close)
-	lb, err := NewLocalBackend(logf, "logid", store, nil, eng, 0)
+	lb, err := NewLocalBackend(logf, logid.PublicID{}, store, nil, eng, 0)
 	if err != nil {
 		t.Fatalf("NewLocalBackend: %v", err)
 	}
@@ -770,7 +771,7 @@ func TestStatusWithoutPeers(t *testing.T) {
 	}
 	t.Cleanup(e.Close)
 
-	b, err := NewLocalBackend(logf, "logid", store, nil, e, 0)
+	b, err := NewLocalBackend(logf, logid.PublicID{}, store, nil, e, 0)
 	if err != nil {
 		t.Fatalf("NewLocalBackend: %v", err)
 	}

ipn/ipnlocal/loglines_test.go

@@ -54,7 +54,7 @@ func TestLocalLogLines(t *testing.T) {
 	}
 	t.Cleanup(e.Close)
 
-	lb, err := NewLocalBackend(logf, idA.String(), store, nil, e, 0)
+	lb, err := NewLocalBackend(logf, idA, store, nil, e, 0)
 	if err != nil {
 		t.Fatal(err)
 	}

ipn/ipnlocal/peerapi.go

@@ -865,7 +865,7 @@ func (h *peerAPIHandler) handleServeSockStats(w http.ResponseWriter, r *http.Req
 	w.Header().Set("Content-Type", "text/html; charset=utf-8")
 	fmt.Fprintln(w, "<!DOCTYPE html><h1>Socket Stats</h1>")
 
-	stats, validation := sockstats.GetWithValidation()
+	stats, interfaceStats, validation := sockstats.Get(), sockstats.GetInterfaces(), sockstats.GetValidation()
 	if stats == nil {
 		fmt.Fprintln(w, "No socket stats available")
 		return
@@ -876,7 +876,7 @@ func (h *peerAPIHandler) handleServeSockStats(w http.ResponseWriter, r *http.Req
 	fmt.Fprintln(w, "<th>Label</th>")
 	fmt.Fprintln(w, "<th>Tx</th>")
 	fmt.Fprintln(w, "<th>Rx</th>")
-	for _, iface := range stats.Interfaces {
+	for _, iface := range interfaceStats.Interfaces {
 		fmt.Fprintf(w, "<th>Tx (%s)</th>", html.EscapeString(iface))
 		fmt.Fprintf(w, "<th>Rx (%s)</th>", html.EscapeString(iface))
 	}
@@ -907,11 +907,13 @@ func (h *peerAPIHandler) handleServeSockStats(w http.ResponseWriter, r *http.Req
 		txTotal += stat.TxBytes
 		rxTotal += stat.RxBytes
 
-		for _, iface := range stats.Interfaces {
-			fmt.Fprintf(w, "<td align=right>%d</td>", stat.TxBytesByInterface[iface])
-			fmt.Fprintf(w, "<td align=right>%d</td>", stat.RxBytesByInterface[iface])
-			txTotalByInterface[iface] += stat.TxBytesByInterface[iface]
-			rxTotalByInterface[iface] += stat.RxBytesByInterface[iface]
+		if interfaceStat, ok := interfaceStats.Stats[label]; ok {
+			for _, iface := range interfaceStats.Interfaces {
+				fmt.Fprintf(w, "<td align=right>%d</td>", interfaceStat.TxBytesByInterface[iface])
+				fmt.Fprintf(w, "<td align=right>%d</td>", interfaceStat.RxBytesByInterface[iface])
+				txTotalByInterface[iface] += interfaceStat.TxBytesByInterface[iface]
+				rxTotalByInterface[iface] += interfaceStat.RxBytesByInterface[iface]
+			}
 		}
 
 		if validationStat, ok := validation.Stats[label]; ok && (validationStat.RxBytes > 0 || validationStat.TxBytes > 0) {
@@ -932,7 +934,7 @@ func (h *peerAPIHandler) handleServeSockStats(w http.ResponseWriter, r *http.Req
 	fmt.Fprintln(w, "<th>Total</th>")
 	fmt.Fprintf(w, "<th>%d</th>", txTotal)
 	fmt.Fprintf(w, "<th>%d</th>", rxTotal)
-	for _, iface := range stats.Interfaces {
+	for _, iface := range interfaceStats.Interfaces {
 		fmt.Fprintf(w, "<th>%d</th>", txTotalByInterface[iface])
 		fmt.Fprintf(w, "<th>%d</th>", rxTotalByInterface[iface])
 	}

ipn/ipnlocal/state_test.go

@@ -21,6 +21,7 @@ import (
 	"tailscale.com/types/empty"
 	"tailscale.com/types/key"
 	"tailscale.com/types/logger"
+	"tailscale.com/types/logid"
 	"tailscale.com/types/netmap"
 	"tailscale.com/types/persist"
 	"tailscale.com/wgengine"
@@ -303,7 +304,7 @@ func TestStateMachine(t *testing.T) {
 	}
 	t.Cleanup(e.Close)
 
-	b, err := NewLocalBackend(logf, "logid", store, nil, e, 0)
+	b, err := NewLocalBackend(logf, logid.PublicID{}, store, nil, e, 0)
 	if err != nil {
 		t.Fatalf("NewLocalBackend: %v", err)
 	}
@@ -946,7 +947,7 @@ func TestEditPrefsHasNoKeys(t *testing.T) {
 	}
 	t.Cleanup(e.Close)
 
-	b, err := NewLocalBackend(logf, "logid", new(mem.Store), nil, e, 0)
+	b, err := NewLocalBackend(logf, logid.PublicID{}, new(mem.Store), nil, e, 0)
 	if err != nil {
 		t.Fatalf("NewLocalBackend: %v", err)
 	}
@@ -1025,7 +1026,7 @@ func TestWGEngineStatusRace(t *testing.T) {
 	eng, err := wgengine.NewFakeUserspaceEngine(logf, 0)
 	c.Assert(err, qt.IsNil)
 	t.Cleanup(eng.Close)
-	b, err := NewLocalBackend(logf, "logid", new(mem.Store), nil, eng, 0)
+	b, err := NewLocalBackend(logf, logid.PublicID{}, new(mem.Store), nil, eng, 0)
 	c.Assert(err, qt.IsNil)
 
 	var cc *mockControl

ipn/ipnserver/server.go

@@ -25,6 +25,7 @@ import (
 	"tailscale.com/ipn/ipnlocal"
 	"tailscale.com/ipn/localapi"
 	"tailscale.com/types/logger"
+	"tailscale.com/types/logid"
 	"tailscale.com/util/mak"
 	"tailscale.com/util/set"
 	"tailscale.com/util/systemd"
@@ -35,7 +36,7 @@ import (
 type Server struct {
 	lb           atomic.Pointer[ipnlocal.LocalBackend]
 	logf         logger.Logf
-	backendLogID string
+	backendLogID logid.PublicID
 	// resetOnZero is whether to call bs.Reset on transition from
 	// 1->0 active HTTP requests. That is, this is whether the backend is
 	// being run in "client mode" that requires an active GUI
@@ -412,9 +413,9 @@ func (s *Server) addActiveHTTPRequest(req *http.Request, ci *ipnauth.ConnIdentit
 //
 // At some point, either before or after Run, the Server's SetLocalBackend
 // method must also be called before Server can do anything useful.
-func New(logf logger.Logf, logid string) *Server {
+func New(logf logger.Logf, logID logid.PublicID) *Server {
 	return &Server{
-		backendLogID: logid,
+		backendLogID: logID,
 		logf:         logf,
 		resetOnZero:  envknob.GOOS() == "windows",
 	}

ipn/localapi/localapi.go

@@ -40,6 +40,7 @@ import (
 	"tailscale.com/tka"
 	"tailscale.com/types/key"
 	"tailscale.com/types/logger"
+	"tailscale.com/types/logid"
 	"tailscale.com/types/ptr"
 	"tailscale.com/util/clientmetric"
 	"tailscale.com/util/httpm"
@@ -73,6 +74,7 @@ var handler = map[string]localAPIHandler{
 	"debug-portmap":               (*Handler).serveDebugPortmap,
 	"debug-peer-endpoint-changes": (*Handler).serveDebugPeerEndpointChanges,
 	"debug-capture":               (*Handler).serveDebugCapture,
+	"debug-log":                   (*Handler).serveDebugLog,
 	"derpmap":                     (*Handler).serveDERPMap,
 	"dev-set-state-store":         (*Handler).serveDevSetStateStore,
 	"set-push-device-token":       (*Handler).serveSetPushDeviceToken,
@@ -123,7 +125,7 @@ var (
 	metrics   = map[string]*clientmetric.Metric{}
 )
 
-func NewHandler(b *ipnlocal.LocalBackend, logf logger.Logf, logID string) *Handler {
+func NewHandler(b *ipnlocal.LocalBackend, logf logger.Logf, logID logid.PublicID) *Handler {
 	return &Handler{b: b, logf: logf, backendLogID: logID}
 }
 
@@ -148,7 +150,7 @@ type Handler struct {
 
 	b            *ipnlocal.LocalBackend
 	logf         logger.Logf
-	backendLogID string
+	backendLogID logid.PublicID
 }
 
 func (h *Handler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
@@ -1820,6 +1822,47 @@ func (h *Handler) serveDebugCapture(w http.ResponseWriter, r *http.Request) {
 	h.b.StreamDebugCapture(r.Context(), w)
 }
 
+func (h *Handler) serveDebugLog(w http.ResponseWriter, r *http.Request) {
+	if !h.PermitRead {
+		http.Error(w, "debug-log access denied", http.StatusForbidden)
+		return
+	}
+	if r.Method != httpm.POST {
+		http.Error(w, "only POST allowed", http.StatusMethodNotAllowed)
+		return
+	}
+	defer h.b.TryFlushLogs() // kick off upload after we're done logging
+
+	type logRequestJSON struct {
+		Lines  []string
+		Prefix string
+	}
+
+	var logRequest logRequestJSON
+	if err := json.NewDecoder(r.Body).Decode(&logRequest); err != nil {
+		http.Error(w, "invalid JSON body", 400)
+		return
+	}
+
+	prefix := logRequest.Prefix
+	if prefix == "" {
+		prefix = "debug-log"
+	}
+	logf := logger.WithPrefix(h.logf, prefix+": ")
+
+	// We can write logs too fast for logtail to handle, even when
+	// opting-out of rate limits. Limit ourselves to at most one message
+	// per 20ms and a burst of 60 log lines, which should be fast enough to
+	// not block for too long but slow enough that we can upload all lines.
+	logf = logger.SlowLoggerWithClock(r.Context(), logf, 20*time.Millisecond, 60, time.Now)
+
+	for _, line := range logRequest.Lines {
+		logf("%s", line)
+	}
+
+	w.WriteHeader(http.StatusNoContent)
+}
+
 var (
 	metricInvalidRequests = clientmetric.NewCounter("localapi_invalid_requests")
 

ipn/serve.go

@@ -190,13 +190,13 @@ func (sc *ServeConfig) IsFunnelOn() bool {
 // Funnel.
 func CheckFunnelAccess(port uint16, nodeAttrs []string) error {
 	if slices.Contains(nodeAttrs, tailcfg.CapabilityWarnFunnelNoInvite) {
-		return errors.New("Funnel not available; an invite is required to join the alpha. See https://tailscale.com/kb/1223/tailscale-funnel/.")
+		return errors.New("Funnel not available; an invite is required to join the alpha. See https://tailscale.com/s/no-funnel.")
 	}
 	if slices.Contains(nodeAttrs, tailcfg.CapabilityWarnFunnelNoHTTPS) {
-		return errors.New("Funnel not available; HTTPS must be enabled. See https://tailscale.com/kb/1153/enabling-https/.")
+		return errors.New("Funnel not available; HTTPS must be enabled. See https://tailscale.com/s/https.")
 	}
 	if !slices.Contains(nodeAttrs, tailcfg.NodeAttrFunnel) {
-		return errors.New("Funnel not available; \"funnel\" node attribute not set. See https://tailscale.com/kb/1223/tailscale-funnel/.")
+		return errors.New("Funnel not available; \"funnel\" node attribute not set. See https://tailscale.com/s/no-funnel.")
 	}
 	return checkFunnelPort(port, nodeAttrs)
 }

ipn/store/kubestore/store_kube.go

@@ -7,6 +7,7 @@ package kubestore
 
 import (
 	"context"
+	"strings"
 	"time"
 
 	"tailscale.com/ipn"
@@ -46,13 +47,24 @@ func (s *Store) ReadState(id ipn.StateKey) ([]byte, error) {
 		}
 		return nil, err
 	}
-	b, ok := secret.Data[string(id)]
+	b, ok := secret.Data[sanitizeKey(id)]
 	if !ok {
 		return nil, ipn.ErrStateNotExist
 	}
 	return b, nil
 }
 
+func sanitizeKey(k ipn.StateKey) string {
+	// The only valid characters in a Kubernetes secret key are alphanumeric, -,
+	// _, and .
+	return strings.Map(func(r rune) rune {
+		if r >= 'a' && r <= 'z' || r >= 'A' && r <= 'Z' || r >= '0' && r <= '9' || r == '-' || r == '_' || r == '.' {
+			return r
+		}
+		return '_'
+	}, string(k))
+}
+
 // WriteState implements the StateStore interface.
 func (s *Store) WriteState(id ipn.StateKey, bs []byte) error {
 	ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
@@ -70,13 +82,13 @@ func (s *Store) WriteState(id ipn.StateKey, bs []byte) error {
 					Name: s.secretName,
 				},
 				Data: map[string][]byte{
-					string(id): bs,
+					sanitizeKey(id): bs,
 				},
 			})
 		}
 		return err
 	}
-	secret.Data[string(id)] = bs
+	secret.Data[sanitizeKey(id)] = bs
 	if err := s.client.UpdateSecret(ctx, secret); err != nil {
 		return err
 	}

licenses/apple.md

@@ -15,7 +15,7 @@ and [iOS][].  See also the dependencies in the [Tailscale CLI][].
  - [github.com/fxamacker/cbor/v2](https://pkg.go.dev/github.com/fxamacker/cbor/v2) ([MIT](https://github.com/fxamacker/cbor/blob/v2.4.0/LICENSE))
  - [github.com/godbus/dbus/v5](https://pkg.go.dev/github.com/godbus/dbus/v5) ([BSD-2-Clause](https://github.com/godbus/dbus/blob/v5.0.6/LICENSE))
  - [github.com/golang/groupcache/lru](https://pkg.go.dev/github.com/golang/groupcache/lru) ([Apache-2.0](https://github.com/golang/groupcache/blob/41bb18bfe9da/LICENSE))
- - [github.com/google/btree](https://pkg.go.dev/github.com/google/btree) ([Apache-2.0](https://github.com/google/btree/blob/v1.0.1/LICENSE))
+ - [github.com/google/btree](https://pkg.go.dev/github.com/google/btree) ([Apache-2.0](https://github.com/google/btree/blob/v1.1.2/LICENSE))
  - [github.com/hdevalence/ed25519consensus](https://pkg.go.dev/github.com/hdevalence/ed25519consensus) ([BSD-3-Clause](https://github.com/hdevalence/ed25519consensus/blob/c00d1f31bab3/LICENSE))
  - [github.com/illarion/gonotify](https://pkg.go.dev/github.com/illarion/gonotify) ([MIT](https://github.com/illarion/gonotify/blob/v1.0.1/LICENSE))
  - [github.com/insomniacslk/dhcp](https://pkg.go.dev/github.com/insomniacslk/dhcp) ([BSD-3-Clause](https://github.com/insomniacslk/dhcp/blob/de60144f33f8/LICENSE))

log/sockstatlog/logger.go

@@ -0,0 +1,195 @@
+// Copyright (c) Tailscale Inc & AUTHORS
+// SPDX-License-Identifier: BSD-3-Clause
+
+// Package sockstatlog provides a logger for capturing and storing network socket stats.
+package sockstatlog
+
+import (
+	"context"
+	"crypto/sha256"
+	"encoding/json"
+	"io"
+	"net/http"
+	"os"
+	"path/filepath"
+	"time"
+
+	"tailscale.com/logpolicy"
+	"tailscale.com/logtail"
+	"tailscale.com/logtail/filch"
+	"tailscale.com/net/sockstats"
+	"tailscale.com/smallzstd"
+	"tailscale.com/types/logger"
+	"tailscale.com/types/logid"
+	"tailscale.com/util/mak"
+)
+
+// pollPeriod specifies how often to poll for socket stats.
+const pollPeriod = time.Second / 10
+
+// Logger logs statistics about network sockets.
+type Logger struct {
+	ctx      context.Context
+	cancelFn context.CancelFunc
+
+	ticker *time.Ticker
+	logf   logger.Logf
+
+	logger *logtail.Logger
+	filch  *filch.Filch
+	tr     *http.Transport
+}
+
+// deltaStat represents the bytes transferred during a time period.
+// The first element is transmitted bytes, the second element is received bytes.
+type deltaStat [2]uint64
+
+// event represents the socket stats on a specific interface during a time period.
+type event struct {
+	// Time is when the event started as a Unix timestamp in milliseconds.
+	Time int64 `json:"t"`
+
+	// Duration is the duration of this event in milliseconds.
+	Duration int64 `json:"d"`
+
+	// IsCellularInterface is set to 1 if the traffic was sent over a cellular interface.
+	IsCellularInterface int `json:"c,omitempty"`
+
+	// Stats records the stats for each Label during the time period.
+	Stats map[sockstats.Label]deltaStat `json:"s"`
+}
+
+// SockstatLogID reproducibly derives a new logid.PrivateID for sockstat logging from a node's public backend log ID.
+// The returned PrivateID is the sha256 sum of logID + "sockstat".
+// If a node's public log ID becomes known, it is trivial to spoof sockstat logs for that node.
+// Given the this is just for debugging, we're not too concerned about that.
+func SockstatLogID(logID logid.PublicID) logid.PrivateID {
+	return logid.PrivateID(sha256.Sum256([]byte(logID.String() + "sockstat")))
+}
+
+// NewLogger returns a new Logger that will store stats in logdir.
+// On platforms that do not support sockstat logging, a nil Logger will be returned.
+// The returned Logger must be shut down with Shutdown when it is no longer needed.
+func NewLogger(logdir string, logf logger.Logf, logID logid.PublicID) (*Logger, error) {
+	if !sockstats.IsAvailable {
+		return nil, nil
+	}
+
+	if err := os.MkdirAll(logdir, 0755); err != nil && !os.IsExist(err) {
+		return nil, err
+	}
+	filchPrefix := filepath.Join(logdir, "sockstats")
+	filch, err := filch.New(filchPrefix, filch.Options{ReplaceStderr: false})
+	if err != nil {
+		return nil, err
+	}
+
+	ctx, cancel := context.WithCancel(context.Background())
+	logger := &Logger{
+		ctx:      ctx,
+		cancelFn: cancel,
+		ticker:   time.NewTicker(pollPeriod),
+		logf:     logf,
+		filch:    filch,
+		tr:       logpolicy.NewLogtailTransport(logtail.DefaultHost),
+	}
+	logger.logger = logtail.NewLogger(logtail.Config{
+		BaseURL:    logpolicy.LogURL(),
+		PrivateID:  SockstatLogID(logID),
+		Collection: "sockstats.log.tailscale.io",
+		Buffer:     filch,
+		NewZstdEncoder: func() logtail.Encoder {
+			w, err := smallzstd.NewEncoder(nil)
+			if err != nil {
+				panic(err)
+			}
+			return w
+		},
+		FlushDelayFn: func() time.Duration {
+			// set flush delay to 100 years so it never flushes automatically
+			return 100 * 365 * 24 * time.Hour
+		},
+		Stderr: io.Discard, // don't log to stderr
+
+		HTTPC: &http.Client{Transport: logger.tr},
+	}, logf)
+
+	go logger.poll()
+
+	return logger, nil
+}
+
+// poll fetches the current socket stats at the configured time interval,
+// calculates the delta since the last poll, and logs any non-zero values.
+// This method does not return.
+func (l *Logger) poll() {
+	// last is the last set of socket stats we saw.
+	var lastStats *sockstats.SockStats
+	var lastTime time.Time
+
+	enc := json.NewEncoder(l.logger)
+	for {
+		select {
+		case <-l.ctx.Done():
+			return
+		case t := <-l.ticker.C:
+			stats := sockstats.Get()
+			if lastStats != nil {
+				diffstats := delta(lastStats, stats)
+				if len(diffstats) > 0 {
+					e := event{
+						Time:     lastTime.UnixMilli(),
+						Duration: t.Sub(lastTime).Milliseconds(),
+						Stats:    diffstats,
+					}
+					if stats.CurrentInterfaceCellular {
+						e.IsCellularInterface = 1
+					}
+					if err := enc.Encode(e); err != nil {
+						l.logf("sockstatlog: error encoding log: %v", err)
+					}
+				}
+			}
+			lastTime = t
+			lastStats = stats
+		}
+	}
+}
+
+func (l *Logger) LogID() string {
+	if l.logger == nil {
+		return ""
+	}
+	return l.logger.PrivateID().Public().String()
+}
+
+// Flush sends pending logs to the log server and flushes them from the local buffer.
+func (l *Logger) Flush() {
+	l.logger.StartFlush()
+}
+
+func (l *Logger) Shutdown() {
+	l.ticker.Stop()
+	l.logger.Shutdown(l.ctx)
+	l.cancelFn()
+	l.filch.Close()
+	l.tr.CloseIdleConnections()
+}
+
+// delta calculates the delta stats between two SockStats snapshots.
+// b is assumed to have occurred after a.
+// Zero values are omitted from the returned map, and an empty map is returned if no bytes were transferred.
+func delta(a, b *sockstats.SockStats) (stats map[sockstats.Label]deltaStat) {
+	if a == nil || b == nil {
+		return nil
+	}
+	for label, bs := range b.Stats {
+		as := a.Stats[label]
+		if as.TxBytes == bs.TxBytes && as.RxBytes == bs.RxBytes {
+			// fast path for unchanged stats
+			continue
+		}
+		mak.Set(&stats, label, deltaStat{bs.TxBytes - as.TxBytes, bs.RxBytes - as.RxBytes})
+	}
+	return stats
+}

log/sockstatlog/logger_test.go

@@ -0,0 +1,95 @@
+// Copyright (c) Tailscale Inc & AUTHORS
+// SPDX-License-Identifier: BSD-3-Clause
+
+package sockstatlog
+
+import (
+	"testing"
+
+	"github.com/google/go-cmp/cmp"
+	"tailscale.com/net/sockstats"
+)
+
+func TestDelta(t *testing.T) {
+	tests := []struct {
+		name      string
+		a, b      *sockstats.SockStats
+		wantStats map[sockstats.Label]deltaStat
+	}{
+		{
+			name:      "nil a stat",
+			a:         nil,
+			b:         &sockstats.SockStats{},
+			wantStats: nil,
+		},
+		{
+			name:      "nil b stat",
+			a:         &sockstats.SockStats{},
+			b:         nil,
+			wantStats: nil,
+		},
+		{
+			name: "no change",
+			a: &sockstats.SockStats{
+				Stats: map[sockstats.Label]sockstats.SockStat{
+					sockstats.LabelDERPHTTPClient: {
+						TxBytes: 10,
+					},
+				},
+			},
+			b: &sockstats.SockStats{
+				Stats: map[sockstats.Label]sockstats.SockStat{
+					sockstats.LabelDERPHTTPClient: {
+						TxBytes: 10,
+					},
+				},
+			},
+			wantStats: nil,
+		},
+		{
+			name: "tx after empty stat",
+			a:    &sockstats.SockStats{},
+			b: &sockstats.SockStats{
+				Stats: map[sockstats.Label]sockstats.SockStat{
+					sockstats.LabelDERPHTTPClient: {
+						TxBytes: 10,
+					},
+				},
+			},
+			wantStats: map[sockstats.Label]deltaStat{
+				sockstats.LabelDERPHTTPClient: {10, 0},
+			},
+		},
+		{
+			name: "rx after non-empty stat",
+			a: &sockstats.SockStats{
+				Stats: map[sockstats.Label]sockstats.SockStat{
+					sockstats.LabelDERPHTTPClient: {
+						TxBytes: 10,
+						RxBytes: 10,
+					},
+				},
+			},
+			b: &sockstats.SockStats{
+				Stats: map[sockstats.Label]sockstats.SockStat{
+					sockstats.LabelDERPHTTPClient: {
+						TxBytes: 10,
+						RxBytes: 30,
+					},
+				},
+			},
+			wantStats: map[sockstats.Label]deltaStat{
+				sockstats.LabelDERPHTTPClient: {0, 20},
+			},
+		},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			gotStats := delta(tt.a, tt.b)
+			if !cmp.Equal(gotStats, tt.wantStats) {
+				t.Errorf("gotStats = %v, want %v", gotStats, tt.wantStats)
+			}
+		})
+	}
+}

logpolicy/logpolicy.go

@@ -192,9 +192,9 @@ func (l logWriter) Write(buf []byte) (int, error) {
 	return len(buf), nil
 }
 
-// logsDir returns the directory to use for log configuration and
+// LogsDir returns the directory to use for log configuration and
 // buffer storage.
-func logsDir(logf logger.Logf) string {
+func LogsDir(logf logger.Logf) string {
 	if d := os.Getenv("TS_LOGS_DIR"); d != "" {
 		fi, err := os.Stat(d)
 		if err == nil && fi.IsDir() {
@@ -478,7 +478,7 @@ func NewWithConfigPath(collection, dir, cmdName string) *Policy {
 	}
 
 	if dir == "" {
-		dir = logsDir(earlyLogf)
+		dir = LogsDir(earlyLogf)
 	}
 	if cmdName == "" {
 		cmdName = version.CmdName()

net/interfaces/interfaces.go

@@ -153,11 +153,9 @@ func LocalAddresses() (regular, loopback []netip.Addr, err error) {
 	if len(regular4) == 0 && len(regular6) == 0 {
 		// if we have no usable IP addresses then be willing to accept
 		// addresses we otherwise wouldn't, like:
-		//   + 169.254.x.x (AWS Lambda uses NAT with these)
+		//   + 169.254.x.x (AWS Lambda and Azure App Services use NAT with these)
 		//   + IPv6 ULA (Google Cloud Run uses these with address translation)
-		if hostinfo.GetEnvType() == hostinfo.AWSLambda {
-			regular4 = linklocal4
-		}
+		regular4 = linklocal4
 		regular6 = ula6
 	}
 	regular = append(regular4, regular6...)
@@ -645,7 +643,14 @@ func isUsableV4(ip netip.Addr) bool {
 		return false
 	}
 	if ip.IsLinkLocalUnicast() {
-		return hostinfo.GetEnvType() == hostinfo.AWSLambda
+		switch hostinfo.GetEnvType() {
+		case hostinfo.AWSLambda:
+			return true
+		case hostinfo.AzureAppService:
+			return true
+		default:
+			return false
+		}
 	}
 	return true
 }

net/netcheck/netcheck.go

@@ -638,20 +638,23 @@ func (rs *reportState) waitHairCheck(ctx context.Context) {
 		return
 	}
 
+	// First, check whether we have a value before we check for timeouts.
+	select {
+	case <-rs.gotHairSTUN:
+		ret.HairPinning.Set(true)
+		return
+	default:
+	}
+
+	// Now, wait for a response or a timeout.
 	select {
 	case <-rs.gotHairSTUN:
 		ret.HairPinning.Set(true)
 	case <-rs.hairTimeout:
 		rs.c.vlogf("hairCheck timeout")
 		ret.HairPinning.Set(false)
-	default:
-		select {
-		case <-rs.gotHairSTUN:
-			ret.HairPinning.Set(true)
-		case <-rs.hairTimeout:
-			ret.HairPinning.Set(false)
-		case <-ctx.Done():
-		}
+	case <-ctx.Done():
+		rs.c.vlogf("hairCheck context timeout")
 	}
 }
 

net/netcheck/netcheck_test.go

@@ -47,6 +47,113 @@ func TestHairpinSTUN(t *testing.T) {
 	}
 }
 
+func TestHairpinWait(t *testing.T) {
+	makeClient := func(t *testing.T) (*Client, *reportState) {
+		tx := stun.NewTxID()
+		c := &Client{}
+		req := stun.Request(tx)
+		if !stun.Is(req) {
+			t.Fatal("expected STUN message")
+		}
+
+		var err error
+		rs := &reportState{
+			c:           c,
+			hairTX:      tx,
+			gotHairSTUN: make(chan netip.AddrPort, 1),
+			hairTimeout: make(chan struct{}),
+			report:      newReport(),
+		}
+		rs.pc4Hair, err = net.ListenUDP("udp4", &net.UDPAddr{
+			IP:   net.ParseIP("127.0.0.1"),
+			Port: 0,
+		})
+		if err != nil {
+			t.Fatal(err)
+		}
+
+		c.curState = rs
+		return c, rs
+	}
+
+	ll, err := net.ListenPacket("udp", "localhost:0")
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer ll.Close()
+	dstAddr := netip.MustParseAddrPort(ll.LocalAddr().String())
+
+	t.Run("Success", func(t *testing.T) {
+		c, rs := makeClient(t)
+		req := stun.Request(rs.hairTX)
+
+		// Start a hairpin check to ourselves.
+		rs.startHairCheckLocked(dstAddr)
+
+		// Fake receiving the stun check from ourselves after some period of time.
+		src := netip.MustParseAddrPort(rs.pc4Hair.LocalAddr().String())
+		c.handleHairSTUNLocked(req, src)
+
+		rs.waitHairCheck(context.Background())
+
+		// Verify that we set HairPinning
+		if got := rs.report.HairPinning; !got.EqualBool(true) {
+			t.Errorf("wanted HairPinning=true, got %v", got)
+		}
+	})
+
+	t.Run("LateReply", func(t *testing.T) {
+		c, rs := makeClient(t)
+		req := stun.Request(rs.hairTX)
+
+		// Start a hairpin check to ourselves.
+		rs.startHairCheckLocked(dstAddr)
+
+		// Wait until we've timed out, to mimic the race in #1795.
+		<-rs.hairTimeout
+
+		// Fake receiving the stun check from ourselves after some period of time.
+		src := netip.MustParseAddrPort(rs.pc4Hair.LocalAddr().String())
+		c.handleHairSTUNLocked(req, src)
+
+		// Wait for a hairpin response
+		rs.waitHairCheck(context.Background())
+
+		// Verify that we set HairPinning
+		if got := rs.report.HairPinning; !got.EqualBool(true) {
+			t.Errorf("wanted HairPinning=true, got %v", got)
+		}
+	})
+
+	t.Run("Timeout", func(t *testing.T) {
+		_, rs := makeClient(t)
+
+		// Start a hairpin check to ourselves.
+		rs.startHairCheckLocked(dstAddr)
+
+		ctx, cancel := context.WithTimeout(context.Background(), hairpinCheckTimeout*50)
+		defer cancel()
+
+		// Wait in the background
+		waitDone := make(chan struct{})
+		go func() {
+			rs.waitHairCheck(ctx)
+			close(waitDone)
+		}()
+
+		// If we do nothing, then we time out; confirm that we set
+		// HairPinning to false in this case.
+		select {
+		case <-waitDone:
+			if got := rs.report.HairPinning; !got.EqualBool(false) {
+				t.Errorf("wanted HairPinning=false, got %v", got)
+			}
+		case <-ctx.Done():
+			t.Fatalf("timed out waiting for hairpin channel")
+		}
+	})
+}
+
 func TestBasic(t *testing.T) {
 	stunAddr, cleanup := stuntest.Serve(t)
 	defer cleanup()

net/netutil/ip_forward.go

@@ -65,7 +65,7 @@ func CheckIPForwarding(routes []netip.Prefix, state *interfaces.State) (warn, er
 		}
 		return nil, nil
 	}
-	const kbLink = "\nSee https://tailscale.com/kb/1104/enable-ip-forwarding/"
+	const kbLink = "\nSee https://tailscale.com/s/ip-forwarding"
 	if state == nil {
 		var err error
 		state, err = interfaces.GetState()

net/packet/packet.go

@@ -440,6 +440,40 @@ func (q *Parsed) IsEchoResponse() bool {
 	}
 }
 
+// UpdateSrcAddr updates the source address in the packet buffer (e.g. during
+// SNAT). It also updates the checksum. Currently (2022-12-10) only TCP/UDP/ICMP
+// over IPv4 is supported. It panics if called with IPv6 addr.
+func (q *Parsed) UpdateSrcAddr(src netip.Addr) {
+	if q.IPVersion != 4 || src.Is6() {
+		panic("UpdateSrcAddr: only IPv4 is supported")
+	}
+
+	old := q.Src.Addr()
+	q.Src = netip.AddrPortFrom(src, q.Src.Port())
+
+	b := q.Buffer()
+	v4 := src.As4()
+	copy(b[12:16], v4[:])
+	updateV4PacketChecksums(q, old, src)
+}
+
+// UpdateDstAddr updates the source address in the packet buffer (e.g. during
+// DNAT). It also updates the checksum. Currently (2022-12-10) only TCP/UDP/ICMP
+// over IPv4 is supported. It panics if called with IPv6 addr.
+func (q *Parsed) UpdateDstAddr(dst netip.Addr) {
+	if q.IPVersion != 4 || dst.Is6() {
+		panic("UpdateDstAddr: only IPv4 is supported")
+	}
+
+	old := q.Dst.Addr()
+	q.Dst = netip.AddrPortFrom(dst, q.Dst.Port())
+
+	b := q.Buffer()
+	v4 := dst.As4()
+	copy(b[16:20], v4[:])
+	updateV4PacketChecksums(q, old, dst)
+}
+
 // EchoIDSeq extracts the identifier/sequence bytes from an ICMP Echo response,
 // and returns them as a uint32, used to lookup internally routed ICMP echo
 // responses. This function is intentionally lightweight as it is called on
@@ -502,3 +536,73 @@ func withIP(ap netip.AddrPort, ip netip.Addr) netip.AddrPort {
 func withPort(ap netip.AddrPort, port uint16) netip.AddrPort {
 	return netip.AddrPortFrom(ap.Addr(), port)
 }
+
+// updateV4PacketChecksums updates the checksums in the packet buffer.
+// Currently (2023-03-01) only TCP/UDP/ICMP over IPv4 is supported.
+// p is modified in place.
+// If p.IPProto is unknown, only the IP header checksum is updated.
+// TODO(maisem): more protocols (sctp, gre, dccp)
+func updateV4PacketChecksums(p *Parsed, old, new netip.Addr) {
+	o4, n4 := old.As4(), new.As4()
+	updateV4Checksum(p.Buffer()[10:12], o4[:], n4[:]) // header
+	switch p.IPProto {
+	case ipproto.UDP:
+		updateV4Checksum(p.Transport()[6:8], o4[:], n4[:])
+	case ipproto.TCP:
+		updateV4Checksum(p.Transport()[16:18], o4[:], n4[:])
+	case ipproto.ICMPv4:
+		// Nothing to do.
+	}
+	// TODO(maisem): more protocols (sctp, gre, dccp)
+}
+
+// updateV4Checksum calculates and updates the checksum in the packet buffer for
+// a change between old and new. The oldSum must point to the 16-bit checksum
+// field in the packet buffer that holds the old checksum value, it will be
+// updated in place.
+//
+// The old and new must be the same length, and must be an even number of bytes.
+func updateV4Checksum(oldSum, old, new []byte) {
+	if len(old) != len(new) {
+		panic("old and new must be the same length")
+	}
+	if len(old)%2 != 0 {
+		panic("old and new must be of even length")
+	}
+	/*
+		RFC 1624
+		Given the following notation:
+
+		    HC  - old checksum in header
+		    C   - one's complement sum of old header
+		    HC' - new checksum in header
+		    C'  - one's complement sum of new header
+		    m   - old value of a 16-bit field
+		    m'  - new value of a 16-bit field
+
+		    HC' = ~(C + (-m) + m')  --    [Eqn. 3]
+		    HC' = ~(~HC + ~m + m')
+
+		This can be simplified to:
+		    HC' = ~(C + ~m + m')    --    [Eqn. 3]
+		    HC' = ~C'
+		    C'  = C + ~m + m'
+	*/
+
+	c := uint32(^binary.BigEndian.Uint16(oldSum))
+
+	cPrime := c
+	for len(new) > 0 {
+		mNot := uint32(^binary.BigEndian.Uint16(old[:2]))
+		mPrime := uint32(binary.BigEndian.Uint16(new[:2]))
+		cPrime += mPrime + mNot
+		new, old = new[2:], old[2:]
+	}
+
+	// Account for overflows by adding the carry bits back into the sum.
+	for (cPrime >> 16) > 0 {
+		cPrime = cPrime&0xFFFF + cPrime>>16
+	}
+	hcPrime := ^uint16(cPrime)
+	binary.BigEndian.PutUint16(oldSum, hcPrime)
+}

net/packet/packet_test.go

@@ -5,6 +5,7 @@ package packet
 
 import (
 	"bytes"
+	"encoding/binary"
 	"encoding/hex"
 	"net/netip"
 	"reflect"
@@ -29,6 +30,72 @@ const (
 	Fragment = ipproto.Fragment
 )
 
+func fullHeaderChecksumV4(b []byte) uint16 {
+	s := uint32(0)
+	for i := 0; i < len(b); i += 2 {
+		if i == 10 {
+			// Skip checksum field.
+			continue
+		}
+		s += uint32(binary.BigEndian.Uint16(b[i : i+2]))
+	}
+	for s>>16 > 0 {
+		s = s&0xFFFF + s>>16
+	}
+	return ^uint16(s)
+}
+
+func TestHeaderChecksums(t *testing.T) {
+	// This is not a good enough test, because it doesn't
+	// check the various packet types or the many edge cases
+	// of the checksum algorithm. But it's a start.
+
+	tests := []struct {
+		name   string
+		packet []byte
+	}{
+		{
+			name: "ICMPv4",
+			packet: []byte{
+				0x45, 0x00, 0x00, 0x54, 0xb7, 0x96, 0x40, 0x00, 0x40, 0x01, 0x7a, 0x06, 0x64, 0x7f, 0x3f, 0x4c, 0x64, 0x40, 0x01, 0x01, 0x08, 0x00, 0x47, 0x1a, 0x00, 0x11, 0x01, 0xac, 0xcc, 0xf5, 0x95, 0x63, 0x00, 0x00, 0x00, 0x00, 0x8d, 0xfc, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x10, 0x11, 0x12, 0x13, 0x14, 0x15, 0x16, 0x17, 0x18, 0x19, 0x1a, 0x1b, 0x1c, 0x1d, 0x1e, 0x1f, 0x20, 0x21, 0x22, 0x23, 0x24, 0x25, 0x26, 0x27, 0x28, 0x29, 0x2a, 0x2b, 0x2c, 0x2d, 0x2e, 0x2f, 0x30, 0x31, 0x32, 0x33, 0x34, 0x35, 0x36, 0x37,
+			},
+		},
+		{
+			name: "TLS",
+			packet: []byte{
+				0x45, 0x00, 0x00, 0x3c, 0x54, 0x29, 0x40, 0x00, 0x40, 0x06, 0xb1, 0xac, 0x64, 0x42, 0xd4, 0x33, 0x64, 0x61, 0x98, 0x0f, 0xb1, 0x94, 0x01, 0xbb, 0x0a, 0x51, 0xce, 0x7c, 0x00, 0x00, 0x00, 0x00, 0xa0, 0x02, 0xfb, 0xe0, 0x38, 0xf6, 0x00, 0x00, 0x02, 0x04, 0x04, 0xd8, 0x04, 0x02, 0x08, 0x0a, 0x86, 0x2b, 0xcc, 0xd5, 0x00, 0x00, 0x00, 0x00, 0x01, 0x03, 0x03, 0x07,
+			},
+		},
+		{
+			name: "DNS",
+			packet: []byte{
+				0x45, 0x00, 0x00, 0x74, 0xe2, 0x85, 0x00, 0x00, 0x40, 0x11, 0x96, 0xb5, 0x64, 0x64, 0x64, 0x64, 0x64, 0x42, 0xd4, 0x33, 0x00, 0x35, 0xec, 0x55, 0x00, 0x60, 0xd9, 0x19, 0xed, 0xfd, 0x81, 0x80, 0x00, 0x01, 0x00, 0x02, 0x00, 0x00, 0x00, 0x01, 0x08, 0x63, 0x6c, 0x69, 0x65, 0x6e, 0x74, 0x73, 0x34, 0x06, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x03, 0x63, 0x6f, 0x6d, 0x00, 0x00, 0x01, 0x00, 0x01, 0xc0, 0x0c, 0x00, 0x05, 0x00, 0x01, 0x00, 0x00, 0x01, 0x1e, 0x00, 0x0c, 0x07, 0x63, 0x6c, 0x69, 0x65, 0x6e, 0x74, 0x73, 0x01, 0x6c, 0xc0, 0x15, 0xc0, 0x31, 0x00, 0x01, 0x00, 0x01, 0x00, 0x00, 0x01, 0x1e, 0x00, 0x04, 0x8e, 0xfa, 0xbd, 0xce, 0x00, 0x00, 0x29, 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+			},
+		},
+	}
+	var p Parsed
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			p.Decode(tt.packet)
+			t.Log(p.String())
+			p.UpdateSrcAddr(netip.MustParseAddr("100.64.0.1"))
+
+			got := binary.BigEndian.Uint16(tt.packet[10:12])
+			want := fullHeaderChecksumV4(tt.packet[:20])
+			if got != want {
+				t.Fatalf("got %x want %x", got, want)
+			}
+
+			p.UpdateDstAddr(netip.MustParseAddr("100.64.0.2"))
+			got = binary.BigEndian.Uint16(tt.packet[10:12])
+			want = fullHeaderChecksumV4(tt.packet[:20])
+			if got != want {
+				t.Fatalf("got %x want %x", got, want)
+			}
+		})
+	}
+}
+
 func mustIPPort(s string) netip.AddrPort {
 	ipp, err := netip.ParseAddrPort(s)
 	if err != nil {

net/sockstats/sockstats.go

@@ -15,22 +15,17 @@ import (
 )
 
 // SockStats contains statistics for sockets instrumented with the
-// WithSockStats() function, along with the interfaces that we have
-// per-interface statistics for.
+// WithSockStats() function
 type SockStats struct {
-	Stats      map[Label]SockStat
-	Interfaces []string
+	Stats                    map[Label]SockStat
+	CurrentInterfaceCellular bool
 }
 
 // SockStat contains the sent and received bytes for a socket instrumented with
-// the WithSockStats() function. The bytes are also broken down by interface,
-// though this may be a subset of the total if interfaces were added after the
-// instrumented socket was created.
+// the WithSockStats() function.
 type SockStat struct {
-	TxBytes            uint64
-	RxBytes            uint64
-	TxBytesByInterface map[string]uint64
-	RxBytesByInterface map[string]uint64
+	TxBytes uint64
+	RxBytes uint64
 }
 
 // Label is an identifier for a socket that stats are collected for. A finite
@@ -66,6 +61,28 @@ func Get() *SockStats {
 	return get()
 }
 
+// InterfaceSockStats contains statistics for sockets instrumented with the
+// WithSockStats() function, broken down by interface. The statistics may be a
+// subset of the total if interfaces were added after the instrumented socket
+// was created.
+type InterfaceSockStats struct {
+	Stats      map[Label]InterfaceSockStat
+	Interfaces []string
+}
+
+// InterfaceSockStat contains the per-interface sent and received bytes for a
+// socket instrumented with the WithSockStats() function.
+type InterfaceSockStat struct {
+	TxBytesByInterface map[string]uint64
+	RxBytesByInterface map[string]uint64
+}
+
+// GetWithInterfaces is a variant of Get that returns the current socket
+// statistics broken down by interface. It is slightly more expensive than Get.
+func GetInterfaces() *InterfaceSockStats {
+	return getInterfaces()
+}
+
 // ValidationSockStats contains external validation numbers for sockets
 // instrumented with WithSockStats. It may be a subset of the all sockets,
 // depending on what externa measurement mechanisms the platform supports.
@@ -80,11 +97,11 @@ type ValidationSockStat struct {
 	RxBytes uint64
 }
 
-// GetWithValidation is a variant of GetWith that returns both the current stats
-// and external validation numbers for the stats. It is more expensive than
-// Get and should be used in debug interfaces only.
-func GetWithValidation() (*SockStats, *ValidationSockStats) {
-	return get(), getValidation()
+// GetValidation is a variant of Get that returns external validation numbers
+// for stats. It is more expensive than Get and should be used in debug
+// interfaces only.
+func GetValidation() *ValidationSockStats {
+	return getValidation()
 }
 
 // LinkMonitor is the interface for the parts of wgengine/mointor's Mon that we

net/sockstats/sockstats_noop.go

@@ -9,6 +9,8 @@ import (
 	"context"
 )
 
+const IsAvailable = false
+
 func withSockStats(ctx context.Context, label Label) context.Context {
 	return ctx
 }
@@ -17,6 +19,10 @@ func get() *SockStats {
 	return nil
 }
 
+func getInterfaces() *InterfaceSockStats {
+	return nil
+}
+
 func getValidation() *ValidationSockStats {
 	return nil
 }

net/sockstats/sockstats_tsgo.go

@@ -19,11 +19,13 @@ import (
 	"tailscale.com/util/clientmetric"
 )
 
+const IsAvailable = true
+
 type sockStatCounters struct {
 	txBytes, rxBytes                       atomic.Uint64
 	rxBytesByInterface, txBytesByInterface map[int]*atomic.Uint64
 
-	txBytesMetric, rxBytesMetric *clientmetric.Metric
+	txBytesMetric, rxBytesMetric, txBytesCellularMetric, rxBytesCellularMetric *clientmetric.Metric
 
 	// Validate counts for TCP sockets by using the TCP_CONNECTION_INFO
 	// getsockopt. We get current counts, as well as save final values when
@@ -63,10 +65,12 @@ func withSockStats(ctx context.Context, label Label) context.Context {
 	counters, ok := sockStats.countersByLabel[label]
 	if !ok {
 		counters = &sockStatCounters{
-			rxBytesByInterface: make(map[int]*atomic.Uint64),
-			txBytesByInterface: make(map[int]*atomic.Uint64),
-			txBytesMetric:      clientmetric.NewCounter(fmt.Sprintf("sockstats_tx_bytes_%s", label)),
-			rxBytesMetric:      clientmetric.NewCounter(fmt.Sprintf("sockstats_rx_bytes_%s", label)),
+			rxBytesByInterface:    make(map[int]*atomic.Uint64),
+			txBytesByInterface:    make(map[int]*atomic.Uint64),
+			txBytesMetric:         clientmetric.NewCounter(fmt.Sprintf("sockstats_tx_bytes_%s", label)),
+			rxBytesMetric:         clientmetric.NewCounter(fmt.Sprintf("sockstats_rx_bytes_%s", label)),
+			txBytesCellularMetric: clientmetric.NewCounter(fmt.Sprintf("sockstats_tx_bytes_cellular_%s", label)),
+			rxBytesCellularMetric: clientmetric.NewCounter(fmt.Sprintf("sockstats_rx_bytes_cellular_%s", label)),
 		}
 
 		// We might be called before setLinkMonitor has been called (and we've
@@ -117,6 +121,7 @@ func withSockStats(ctx context.Context, label Label) context.Context {
 		}
 		if sockStats.currentInterfaceCellular.Load() {
 			sockStats.rxBytesCellularMetric.Add(int64(n))
+			counters.rxBytesCellularMetric.Add(int64(n))
 		}
 	}
 	didWrite := func(n int) {
@@ -130,6 +135,7 @@ func withSockStats(ctx context.Context, label Label) context.Context {
 		}
 		if sockStats.currentInterfaceCellular.Load() {
 			sockStats.txBytesCellularMetric.Add(int64(n))
+			counters.txBytesCellularMetric.Add(int64(n))
 		}
 	}
 	willOverwrite := func(trace *net.SockTrace) {
@@ -155,19 +161,37 @@ func get() *SockStats {
 	defer sockStats.mu.Unlock()
 
 	r := &SockStats{
-		Stats:      make(map[Label]SockStat),
-		Interfaces: make([]string, 0, len(sockStats.usedInterfaces)),
+		Stats:                    make(map[Label]SockStat, len(sockStats.countersByLabel)),
+		CurrentInterfaceCellular: sockStats.currentInterfaceCellular.Load(),
+	}
+
+	for label, counters := range sockStats.countersByLabel {
+		r.Stats[label] = SockStat{
+			TxBytes: counters.txBytes.Load(),
+			RxBytes: counters.rxBytes.Load(),
+		}
+	}
+
+	return r
+}
+
+func getInterfaces() *InterfaceSockStats {
+	sockStats.mu.Lock()
+	defer sockStats.mu.Unlock()
+
+	interfaceCount := len(sockStats.usedInterfaces)
+	r := &InterfaceSockStats{
+		Stats:      make(map[Label]InterfaceSockStat, len(sockStats.countersByLabel)),
+		Interfaces: make([]string, 0, interfaceCount),
 	}
 	for iface := range sockStats.usedInterfaces {
 		r.Interfaces = append(r.Interfaces, sockStats.knownInterfaces[iface])
 	}
 
 	for label, counters := range sockStats.countersByLabel {
-		s := SockStat{
-			TxBytes:            counters.txBytes.Load(),
-			RxBytes:            counters.rxBytes.Load(),
-			TxBytesByInterface: make(map[string]uint64),
-			RxBytesByInterface: make(map[string]uint64),
+		s := InterfaceSockStat{
+			TxBytesByInterface: make(map[string]uint64, interfaceCount),
+			RxBytesByInterface: make(map[string]uint64, interfaceCount),
 		}
 		for iface, a := range counters.rxBytesByInterface {
 			ifName := sockStats.knownInterfaces[iface]

net/tsaddr/tsaddr.go

@@ -26,7 +26,7 @@ var chromeOSRange oncePrefix
 
 // CGNATRange returns the Carrier Grade NAT address range that
 // is the superset range that Tailscale assigns out of.
-// See https://tailscale.com/kb/1015/100.x-addresses.
+// See https://tailscale.com/s/cgnat
 // Note that Tailscale does not assign out of the ChromeOSVMRange.
 func CGNATRange() netip.Prefix {
 	cgnatRange.Do(func() { mustPrefix(&cgnatRange.v, "100.64.0.0/10") })

net/tstun/wrap.go

@@ -29,7 +29,10 @@ import (
 	"tailscale.com/types/ipproto"
 	"tailscale.com/types/key"
 	"tailscale.com/types/logger"
+	"tailscale.com/types/netmap"
+	"tailscale.com/types/views"
 	"tailscale.com/util/clientmetric"
+	"tailscale.com/util/mak"
 	"tailscale.com/wgengine/capture"
 	"tailscale.com/wgengine/filter"
 )
@@ -88,6 +91,9 @@ type Wrapper struct {
 	destMACAtomic  syncs.AtomicValue[[6]byte]
 	discoKey       syncs.AtomicValue[key.DiscoPublic]
 
+	// natV4Config stores the current NAT configuration.
+	natV4Config atomic.Pointer[natV4Config]
+
 	// vectorBuffer stores the oldest unconsumed packet vector from tdev. It is
 	// allocated in wrap() and the underlying arrays should never grow.
 	vectorBuffer [][]byte
@@ -136,23 +142,23 @@ type Wrapper struct {
 	// filterFlags control the verbosity of logging packet drops/accepts.
 	filterFlags filter.RunFlags
 
-	// PreFilterIn is the inbound filter function that runs before the main filter
+	// PreFilterPacketInboundFromWireGuard is the inbound filter function that runs before the main filter
 	// and therefore sees the packets that may be later dropped by it.
-	PreFilterIn FilterFunc
-	// PostFilterIn is the inbound filter function that runs after the main filter.
-	PostFilterIn FilterFunc
-	// PreFilterFromTunToNetstack is a filter function that runs before the main filter
+	PreFilterPacketInboundFromWireGuard FilterFunc
+	// PostFilterPacketInboundFromWireGaurd is the inbound filter function that runs after the main filter.
+	PostFilterPacketInboundFromWireGaurd FilterFunc
+	// PreFilterPacketOutboundToWireGuardNetstackIntercept is a filter function that runs before the main filter
 	// for packets from the local system. This filter is populated by netstack to hook
 	// packets that should be handled by netstack. If set, this filter runs before
 	// PreFilterFromTunToEngine.
-	PreFilterFromTunToNetstack FilterFunc
-	// PreFilterFromTunToEngine is a filter function that runs before the main filter
+	PreFilterPacketOutboundToWireGuardNetstackIntercept FilterFunc
+	// PreFilterPacketOutboundToWireGuardEngineIntercept is a filter function that runs before the main filter
 	// for packets from the local system. This filter is populated by wgengine to hook
 	// packets which it handles internally. If both this and PreFilterFromTunToNetstack
 	// filter functions are non-nil, this filter runs second.
-	PreFilterFromTunToEngine FilterFunc
-	// PostFilterOut is the outbound filter function that runs after the main filter.
-	PostFilterOut FilterFunc
+	PreFilterPacketOutboundToWireGuardEngineIntercept FilterFunc
+	// PostFilterPacketOutboundToWireGuard is the outbound filter function that runs after the main filter.
+	PostFilterPacketOutboundToWireGuard FilterFunc
 
 	// OnTSMPPongReceived, if non-nil, is called whenever a TSMP pong arrives.
 	OnTSMPPongReceived func(packet.TSMPPongReply)
@@ -459,12 +465,145 @@ func (t *Wrapper) sendVectorOutbound(r tunVectorReadResult) {
 	t.vectorOutbound <- r
 }
 
+// snatV4 does SNAT on p if it's an IPv4 packet and the destination
+// address requires a different source address.
+func (t *Wrapper) snatV4(p *packet.Parsed) {
+	if p.IPVersion != 4 {
+		return
+	}
+
+	nc := t.natV4Config.Load()
+	oldSrc := p.Src.Addr()
+	newSrc := nc.selectSrcIP(oldSrc, p.Dst.Addr())
+	if oldSrc != newSrc {
+		p.UpdateSrcAddr(newSrc)
+	}
+}
+
+// dnatV4 does destination NAT on p if it's an IPv4 packet.
+func (t *Wrapper) dnatV4(p *packet.Parsed) {
+	if p.IPVersion != 4 {
+		return
+	}
+
+	nc := t.natV4Config.Load()
+	oldDst := p.Dst.Addr()
+	newDst := nc.mapDstIP(oldDst)
+	if newDst != oldDst {
+		p.UpdateDstAddr(newDst)
+	}
+}
+
+// findV4 returns the first Tailscale IPv4 address in addrs.
+func findV4(addrs []netip.Prefix) netip.Addr {
+	for _, ap := range addrs {
+		a := ap.Addr()
+		if a.Is4() && tsaddr.IsTailscaleIP(a) {
+			return a
+		}
+	}
+	return netip.Addr{}
+}
+
+// natV4Config is the configuration for IPv4 NAT.
+// It should be treated as immutable.
+//
+// The nil value is a valid configuration.
+type natV4Config struct {
+	// nativeAddr is the IPv4 Tailscale Address of the current node.
+	nativeAddr netip.Addr
+
+	// listenAddrs is the set of IPv4 addresses that should be
+	// mapped to the native address. These are the addresses that
+	// peers will use to connect to this node.
+	listenAddrs views.Map[netip.Addr, struct{}] // masqAddr -> struct{}
+
+	// dstMasqAddrs is map of dst addresses to their respective MasqueradeAsIP
+	// addresses. The MasqueradeAsIP address is the address that should be used
+	// as the source address for packets to dst.
+	dstMasqAddrs views.Map[netip.Addr, netip.Addr] // dst -> masqAddr
+
+	// TODO(maisem/nyghtowl): add support for subnets and exit nodes and test them.
+	// Determine IP routing table algorithm to use - e.g. ART?
+}
+
+// mapDstIP returns the destination IP to use for a packet to dst.
+// If dst is not one of the listen addresses, it is returned as-is,
+// otherwise the native address is returned.
+func (c *natV4Config) mapDstIP(oldDst netip.Addr) netip.Addr {
+	if c == nil {
+		return oldDst
+	}
+	if _, ok := c.listenAddrs.GetOk(oldDst); ok {
+		return c.nativeAddr
+	}
+	return oldDst
+}
+
+// selectSrcIP returns the source IP to use for a packet to dst.
+// If the packet is not from the native address, it is returned as-is.
+func (c *natV4Config) selectSrcIP(oldSrc, dst netip.Addr) netip.Addr {
+	if c == nil {
+		return oldSrc
+	}
+	if oldSrc != c.nativeAddr {
+		return oldSrc
+	}
+	if eip, ok := c.dstMasqAddrs.GetOk(dst); ok {
+		return eip
+	}
+	return oldSrc
+}
+
+// natConfigFromNetMap generates a natV4Config from nm.
+// If v4 NAT is not required, it returns nil.
+func natConfigFromNetMap(nm *netmap.NetworkMap) *natV4Config {
+	if nm == nil || nm.SelfNode == nil {
+		return nil
+	}
+	nativeAddr := findV4(nm.SelfNode.Addresses)
+	if !nativeAddr.IsValid() {
+		return nil
+	}
+	var (
+		dstMasqAddrs map[netip.Addr]netip.Addr
+		listenAddrs  map[netip.Addr]struct{}
+	)
+	for _, p := range nm.Peers {
+		if !p.SelfNodeV4MasqAddrForThisPeer.IsValid() {
+			continue
+		}
+		peerV4 := findV4(p.Addresses)
+		if !peerV4.IsValid() {
+			continue
+		}
+		mak.Set(&dstMasqAddrs, peerV4, p.SelfNodeV4MasqAddrForThisPeer)
+		mak.Set(&listenAddrs, p.SelfNodeV4MasqAddrForThisPeer, struct{}{})
+	}
+	if len(listenAddrs) == 0 || len(dstMasqAddrs) == 0 {
+		return nil
+	}
+	return &natV4Config{
+		nativeAddr:   nativeAddr,
+		listenAddrs:  views.MapOf(listenAddrs),
+		dstMasqAddrs: views.MapOf(dstMasqAddrs),
+	}
+}
+
+// SetNetMap is called when a new NetworkMap is received.
+// It currently (2023-03-01) only updates the IPv4 NAT configuration.
+func (t *Wrapper) SetNetMap(nm *netmap.NetworkMap) {
+	cfg := natConfigFromNetMap(nm)
+	t.natV4Config.Store(cfg)
+	t.logf("nat config: %+v", cfg)
+}
+
 var (
 	magicDNSIPPort   = netip.AddrPortFrom(tsaddr.TailscaleServiceIP(), 0) // 100.100.100.100:0
 	magicDNSIPPortv6 = netip.AddrPortFrom(tsaddr.TailscaleServiceIPv6(), 0)
 )
 
-func (t *Wrapper) filterOut(p *packet.Parsed) filter.Response {
+func (t *Wrapper) filterPacketOutboundToWireGuard(p *packet.Parsed) filter.Response {
 	// Fake ICMP echo responses to MagicDNS (100.100.100.100).
 	if p.IsEchoRequest() {
 		switch p.Dst {
@@ -494,14 +633,14 @@ func (t *Wrapper) filterOut(p *packet.Parsed) filter.Response {
 		return filter.DropSilently
 	}
 
-	if t.PreFilterFromTunToNetstack != nil {
-		if res := t.PreFilterFromTunToNetstack(p, t); res.IsDrop() {
+	if t.PreFilterPacketOutboundToWireGuardNetstackIntercept != nil {
+		if res := t.PreFilterPacketOutboundToWireGuardNetstackIntercept(p, t); res.IsDrop() {
 			// Handled by netstack.Impl.handleLocalPackets (quad-100 DNS primarily)
 			return res
 		}
 	}
-	if t.PreFilterFromTunToEngine != nil {
-		if res := t.PreFilterFromTunToEngine(p, t); res.IsDrop() {
+	if t.PreFilterPacketOutboundToWireGuardEngineIntercept != nil {
+		if res := t.PreFilterPacketOutboundToWireGuardEngineIntercept(p, t); res.IsDrop() {
 			// Handled by userspaceEngine.handleLocalPackets (primarily handles
 			// quad-100 if netstack is not installed).
 			return res
@@ -518,8 +657,8 @@ func (t *Wrapper) filterOut(p *packet.Parsed) filter.Response {
 		return filter.Drop
 	}
 
-	if t.PostFilterOut != nil {
-		if res := t.PostFilterOut(p, t); res.IsDrop() {
+	if t.PostFilterPacketOutboundToWireGuard != nil {
+		if res := t.PostFilterPacketOutboundToWireGuard(p, t); res.IsDrop() {
 			return res
 		}
 	}
@@ -541,8 +680,8 @@ func (t *Wrapper) IdleDuration() time.Duration {
 }
 
 func (t *Wrapper) Read(buffs [][]byte, sizes []int, offset int) (int, error) {
+	// packet from OS read and sent to WG
 	res, ok := <-t.vectorOutbound
-
 	if !ok {
 		return 0, io.EOF
 	}
@@ -566,28 +705,30 @@ func (t *Wrapper) Read(buffs [][]byte, sizes []int, offset int) (int, error) {
 	defer parsedPacketPool.Put(p)
 	for _, data := range res.data {
 		p.Decode(data[res.dataOffset:])
+
+		t.snatV4(p)
 		if m := t.destIPActivity.Load(); m != nil {
 			if fn := m[p.Dst.Addr()]; fn != nil {
 				fn()
 			}
 		}
 		if capt := t.captureHook.Load(); capt != nil {
-			capt(capture.FromLocal, time.Now(), data[res.dataOffset:])
+			capt(capture.FromLocal, time.Now(), p.Buffer())
 		}
 		if !t.disableFilter {
-			response := t.filterOut(p)
+			response := t.filterPacketOutboundToWireGuard(p)
 			if response != filter.Accept {
 				metricPacketOutDrop.Add(1)
 				continue
 			}
 		}
-		n := copy(buffs[buffsPos][offset:], data[res.dataOffset:])
+		n := copy(buffs[buffsPos][offset:], p.Buffer())
 		if n != len(data)-res.dataOffset {
 			panic(fmt.Sprintf("short copy: %d != %d", n, len(data)-res.dataOffset))
 		}
 		sizes[buffsPos] = n
 		if stats := t.stats.Load(); stats != nil {
-			stats.UpdateTxVirtual(data[res.dataOffset:])
+			stats.UpdateTxVirtual(p.Buffer())
 		}
 		buffsPos++
 	}
@@ -622,6 +763,7 @@ func (t *Wrapper) injectedRead(res tunInjectedRead, buf []byte, offset int) (int
 	p := parsedPacketPool.Get().(*packet.Parsed)
 	defer parsedPacketPool.Put(p)
 	p.Decode(buf[offset : offset+n])
+	t.snatV4(p)
 
 	if m := t.destIPActivity.Load(); m != nil {
 		if fn := m[p.Dst.Addr()]; fn != nil {
@@ -636,7 +778,7 @@ func (t *Wrapper) injectedRead(res tunInjectedRead, buf []byte, offset int) (int
 	return n, nil
 }
 
-func (t *Wrapper) filterIn(p *packet.Parsed) filter.Response {
+func (t *Wrapper) filterPacketInboundFromWireGuard(p *packet.Parsed) filter.Response {
 	if capt := t.captureHook.Load(); capt != nil {
 		capt(capture.FromPeer, time.Now(), p.Buffer())
 	}
@@ -672,8 +814,8 @@ func (t *Wrapper) filterIn(p *packet.Parsed) filter.Response {
 		return filter.DropSilently
 	}
 
-	if t.PreFilterIn != nil {
-		if res := t.PreFilterIn(p, t); res.IsDrop() {
+	if t.PreFilterPacketInboundFromWireGuard != nil {
+		if res := t.PreFilterPacketInboundFromWireGuard(p, t); res.IsDrop() {
 			return res
 		}
 	}
@@ -724,8 +866,8 @@ func (t *Wrapper) filterIn(p *packet.Parsed) filter.Response {
 		return filter.Drop
 	}
 
-	if t.PostFilterIn != nil {
-		if res := t.PostFilterIn(p, t); res.IsDrop() {
+	if t.PostFilterPacketInboundFromWireGaurd != nil {
+		if res := t.PostFilterPacketInboundFromWireGaurd(p, t); res.IsDrop() {
 			return res
 		}
 	}
@@ -738,19 +880,21 @@ func (t *Wrapper) filterIn(p *packet.Parsed) filter.Response {
 func (t *Wrapper) Write(buffs [][]byte, offset int) (int, error) {
 	metricPacketIn.Add(int64(len(buffs)))
 	i := 0
-	if !t.disableFilter {
-		p := parsedPacketPool.Get().(*packet.Parsed)
-		defer parsedPacketPool.Put(p)
-		for _, buff := range buffs {
-			p.Decode(buff[offset:])
-			if t.filterIn(p) != filter.Accept {
+	p := parsedPacketPool.Get().(*packet.Parsed)
+	defer parsedPacketPool.Put(p)
+	for _, buff := range buffs {
+		p.Decode(buff[offset:])
+		t.dnatV4(p)
+		if !t.disableFilter {
+			if t.filterPacketInboundFromWireGuard(p) != filter.Accept {
 				metricPacketInDrop.Add(1)
 			} else {
 				buffs[i] = buff
 				i++
 			}
 		}
-	} else {
+	}
+	if t.disableFilter {
 		i = len(buffs)
 	}
 	buffs = buffs[:i]
@@ -798,9 +942,13 @@ func (t *Wrapper) InjectInboundPacketBuffer(pkt stack.PacketBufferPtr) error {
 	}
 	pkt.DecRef()
 
+	p := parsedPacketPool.Get().(*packet.Parsed)
+	defer parsedPacketPool.Put(p)
+	p.Decode(buf[PacketStartOffset:])
 	if capt := t.captureHook.Load(); capt != nil {
-		capt(capture.SynthesizedToLocal, time.Now(), buf[PacketStartOffset:])
+		capt(capture.SynthesizedToLocal, time.Now(), p.Buffer())
 	}
+	t.dnatV4(p)
 
 	return t.InjectInboundDirect(buf, PacketStartOffset)
 }

net/tstun/wrap_test.go

@@ -25,12 +25,14 @@ import (
 	"tailscale.com/net/connstats"
 	"tailscale.com/net/netaddr"
 	"tailscale.com/net/packet"
+	"tailscale.com/tailcfg"
 	"tailscale.com/tstest"
 	"tailscale.com/tstime/mono"
 	"tailscale.com/types/ipproto"
 	"tailscale.com/types/key"
 	"tailscale.com/types/logger"
 	"tailscale.com/types/netlogtype"
+	"tailscale.com/types/netmap"
 	"tailscale.com/util/must"
 	"tailscale.com/wgengine/filter"
 )
@@ -544,7 +546,7 @@ func TestPeerAPIBypass(t *testing.T) {
 			tt.w.SetFilter(tt.filter)
 			tt.w.disableTSMPRejected = true
 			tt.w.logf = t.Logf
-			if got := tt.w.filterIn(p); got != tt.want {
+			if got := tt.w.filterPacketInboundFromWireGuard(p); got != tt.want {
 				t.Errorf("got = %v; want %v", got, tt.want)
 			}
 		})
@@ -574,7 +576,7 @@ func TestFilterDiscoLoop(t *testing.T) {
 
 	p := new(packet.Parsed)
 	p.Decode(pkt)
-	got := tw.filterIn(p)
+	got := tw.filterPacketInboundFromWireGuard(p)
 	if got != filter.DropSilently {
 		t.Errorf("got %v; want DropSilently", got)
 	}
@@ -585,7 +587,7 @@ func TestFilterDiscoLoop(t *testing.T) {
 	memLog.Reset()
 	pp := new(packet.Parsed)
 	pp.Decode(pkt)
-	got = tw.filterOut(pp)
+	got = tw.filterPacketOutboundToWireGuard(pp)
 	if got != filter.DropSilently {
 		t.Errorf("got %v; want DropSilently", got)
 	}
@@ -593,3 +595,129 @@ func TestFilterDiscoLoop(t *testing.T) {
 		t.Errorf("log output mismatch\n got: %q\nwant: %q\n", got, want)
 	}
 }
+
+func TestNATCfg(t *testing.T) {
+	node := func(ip, eip netip.Addr) *tailcfg.Node {
+		return &tailcfg.Node{
+			Addresses: []netip.Prefix{
+				netip.PrefixFrom(ip, ip.BitLen()),
+			},
+			SelfNodeV4MasqAddrForThisPeer: eip,
+		}
+	}
+	var (
+		noIP netip.Addr
+
+		selfNativeIP = netip.MustParseAddr("100.64.0.1")
+		selfEIP1     = netip.MustParseAddr("100.64.1.1")
+		selfEIP2     = netip.MustParseAddr("100.64.1.2")
+
+		peer1IP = netip.MustParseAddr("100.64.0.2")
+		peer2IP = netip.MustParseAddr("100.64.0.3")
+
+		// subnets should not be impacted.
+		// TODO(maisem/nyghtowl): add support for subnets and exit nodes and test them.
+		subnet = netip.MustParseAddr("192.168.0.1")
+	)
+
+	tests := []struct {
+		name    string
+		nm      *netmap.NetworkMap
+		snatMap map[netip.Addr]netip.Addr // dst -> src
+		dnatMap map[netip.Addr]netip.Addr
+	}{
+		{
+			name: "no-netmap",
+			nm:   nil,
+			snatMap: map[netip.Addr]netip.Addr{
+				peer1IP: selfNativeIP,
+				peer2IP: selfNativeIP,
+				subnet:  selfNativeIP,
+			},
+			dnatMap: map[netip.Addr]netip.Addr{
+				selfNativeIP: selfNativeIP,
+				selfEIP1:     selfEIP1,
+				selfEIP2:     selfEIP2,
+			},
+		},
+		{
+			name: "single-peer-requires-nat",
+			nm: &netmap.NetworkMap{
+				SelfNode: node(selfNativeIP, noIP),
+				Peers: []*tailcfg.Node{
+					node(peer1IP, noIP),
+					node(peer2IP, selfEIP1),
+				},
+			},
+			snatMap: map[netip.Addr]netip.Addr{
+				peer1IP: selfNativeIP,
+				peer2IP: selfEIP1,
+				subnet:  selfNativeIP,
+			},
+			dnatMap: map[netip.Addr]netip.Addr{
+				selfNativeIP: selfNativeIP,
+				selfEIP1:     selfNativeIP,
+				selfEIP2:     selfEIP2,
+				subnet:       subnet,
+			},
+		},
+		{
+			name: "multiple-peers-require-nat",
+			nm: &netmap.NetworkMap{
+				SelfNode: node(selfNativeIP, noIP),
+				Peers: []*tailcfg.Node{
+					node(peer1IP, selfEIP1),
+					node(peer2IP, selfEIP2),
+				},
+			},
+			snatMap: map[netip.Addr]netip.Addr{
+				peer1IP: selfEIP1,
+				peer2IP: selfEIP2,
+				subnet:  selfNativeIP,
+			},
+			dnatMap: map[netip.Addr]netip.Addr{
+				selfNativeIP: selfNativeIP,
+				selfEIP1:     selfNativeIP,
+				selfEIP2:     selfNativeIP,
+				subnet:       subnet,
+			},
+		},
+		{
+			name: "no-nat",
+			nm: &netmap.NetworkMap{
+				SelfNode: node(selfNativeIP, noIP),
+				Peers: []*tailcfg.Node{
+					node(peer1IP, noIP),
+					node(peer2IP, noIP),
+				},
+			},
+			snatMap: map[netip.Addr]netip.Addr{
+				peer1IP: selfNativeIP,
+				peer2IP: selfNativeIP,
+				subnet:  selfNativeIP,
+			},
+			dnatMap: map[netip.Addr]netip.Addr{
+				selfNativeIP: selfNativeIP,
+				selfEIP1:     selfEIP1,
+				selfEIP2:     selfEIP2,
+				subnet:       subnet,
+			},
+		},
+	}
+
+	for _, tc := range tests {
+		t.Run(tc.name, func(t *testing.T) {
+			ncfg := natConfigFromNetMap(tc.nm)
+			for peer, want := range tc.snatMap {
+				if got := ncfg.selectSrcIP(selfNativeIP, peer); got != want {
+					t.Errorf("selectSrcIP: got %v; want %v", got, want)
+				}
+			}
+			for dstIP, want := range tc.dnatMap {
+				if got := ncfg.mapDstIP(dstIP); got != want {
+					t.Errorf("mapDstIP: got %v; want %v", got, want)
+				}
+			}
+		})
+	}
+}

prober/derp.go

@@ -273,6 +273,17 @@ func derpProbeNodePair(ctx context.Context, dm *tailcfg.DERPMap, from, to *tailc
 		time.Sleep(100 * time.Millisecond) // pretty arbitrary
 	}
 
+	latency, err = runDerpProbeNodePair(ctx, from, to, fromc, toc)
+	if err != nil {
+		// Record pubkeys on failed probes to aid investigation.
+		err = fmt.Errorf("%s -> %s: %w",
+			fromc.SelfPublicKey().ShortString(),
+			toc.SelfPublicKey().ShortString(), err)
+	}
+	return latency, err
+}
+
+func runDerpProbeNodePair(ctx context.Context, from, to *tailcfg.DERPNode, fromc, toc *derphttp.Client) (latency time.Duration, err error) {
 	// Make a random packet
 	pkt := make([]byte, 8)
 	crand.Read(pkt)

prober/prober.go

@@ -161,11 +161,12 @@ type Probe struct {
 	tick         ticker
 	labels       map[string]string
 
-	mu      sync.Mutex
-	start   time.Time // last time doProbe started
-	end     time.Time // last time doProbe returned
-	result  bool      // whether the last doProbe call succeeded
-	lastErr error
+	mu        sync.Mutex
+	start     time.Time     // last time doProbe started
+	end       time.Time     // last time doProbe returned
+	latency   time.Duration // last successful probe latency
+	succeeded bool          // whether the last doProbe call succeeded
+	lastErr   error
 }
 
 // Close shuts down the Probe and unregisters it from its Prober.
@@ -254,8 +255,13 @@ func (p *Probe) recordEnd(start time.Time, err error) {
 	p.mu.Lock()
 	defer p.mu.Unlock()
 	p.end = end
-	p.result = err == nil
+	p.succeeded = err == nil
 	p.lastErr = err
+	if p.succeeded {
+		p.latency = end.Sub(p.start)
+	} else {
+		p.latency = 0
+	}
 }
 
 type varExporter struct {
@@ -289,13 +295,13 @@ func (v varExporter) probeInfo() map[string]ProbeInfo {
 			Labels: probe.labels,
 			Start:  probe.start,
 			End:    probe.end,
-			Result: probe.result,
+			Result: probe.succeeded,
 		}
 		if probe.lastErr != nil {
 			inf.Error = probe.lastErr.Error()
 		}
-		if probe.end.After(probe.start) {
-			inf.Latency = probe.end.Sub(probe.start).String()
+		if probe.latency > 0 {
+			inf.Latency = probe.latency.String()
 		}
 		out[probe.name] = inf
 		probe.mu.Unlock()
@@ -358,9 +364,10 @@ func (v varExporter) WritePrometheus(w io.Writer, prefix string) {
 		}
 		if !probe.end.IsZero() {
 			fmt.Fprintf(w, "%s_end_secs{%s} %d\n", prefix, labels, probe.end.Unix())
-			// Start is always present if end is.
-			fmt.Fprintf(w, "%s_latency_millis{%s} %d\n", prefix, labels, probe.end.Sub(probe.start).Milliseconds())
-			if probe.result {
+			if probe.latency > 0 {
+				fmt.Fprintf(w, "%s_latency_millis{%s} %d\n", prefix, labels, probe.latency.Milliseconds())
+			}
+			if probe.succeeded {
 				fmt.Fprintf(w, "%s_result{%s} 1\n", prefix, labels)
 			} else {
 				fmt.Fprintf(w, "%s_result{%s} 0\n", prefix, labels)

prober/prober_test.go

@@ -237,12 +237,11 @@ func TestExpvar(t *testing.T) {
 	}
 
 	check("probe", ProbeInfo{
-		Labels:  map[string]string{"label": "value"},
-		Start:   epoch,
-		End:     epoch.Add(aFewMillis),
-		Latency: aFewMillis.String(),
-		Result:  false,
-		Error:   "failing, as instructed by test",
+		Labels: map[string]string{"label": "value"},
+		Start:  epoch,
+		End:    epoch.Add(aFewMillis),
+		Result: false,
+		Error:  "failing, as instructed by test",
 	})
 
 	succeed.Store(true)
@@ -280,9 +279,8 @@ func TestPrometheus(t *testing.T) {
 probe_interval_secs{name="testprobe",label="value"} %f
 probe_start_secs{name="testprobe",label="value"} %d
 probe_end_secs{name="testprobe",label="value"} %d
-probe_latency_millis{name="testprobe",label="value"} %d
 probe_result{name="testprobe",label="value"} 0
-`, probeInterval.Seconds(), epoch.Unix(), epoch.Add(aFewMillis).Unix(), aFewMillis.Milliseconds()))
+`, probeInterval.Seconds(), epoch.Unix(), epoch.Add(aFewMillis).Unix()))
 		if diff := cmp.Diff(strings.TrimSpace(b.String()), want); diff != "" {
 			return fmt.Errorf("wrong probe stats (-got+want):\n%s", diff)
 		}

scripts/installer.sh

@@ -293,82 +293,28 @@ main() {
 		fi
 	fi
 
+	# Ideally we want to use curl, but on some installs we
+	# only have wget. Detect and use what's available.
+	CURL=
+	if type curl >/dev/null; then
+		CURL="curl -fsSL"
+	elif type wget >/dev/null; then
+		CURL="wget -q -O-"
+	fi
+	if [ -z "$CURL" ]; then
+		echo "The installer needs either curl or wget to download files."
+		echo "Please install either curl or wget to proceed."
+		exit 1
+	fi
+
 	# Step 2: having detected an OS we support, is it one of the
 	# versions we support?
 	OS_UNSUPPORTED=
 	case "$OS" in
-		ubuntu)
-			if [ "$VERSION" != "xenial" ] && \
-			   [ "$VERSION" != "bionic" ] && \
-			   [ "$VERSION" != "eoan" ] && \
-			   [ "$VERSION" != "focal" ] && \
-			   [ "$VERSION" != "groovy" ] && \
-			   [ "$VERSION" != "hirsute" ] && \
-			   [ "$VERSION" != "impish" ] && \
-			   [ "$VERSION" != "jammy" ] && \
-			   [ "$VERSION" != "kinetic" ]
-			then
-				OS_UNSUPPORTED=1
-			fi
-		;;
-		debian)
-			if [ "$VERSION" != "stretch" ] && \
-			   [ "$VERSION" != "buster" ] && \
-			   [ "$VERSION" != "bullseye" ] && \
-			   [ "$VERSION" != "bookworm" ] && \
-			   [ "$VERSION" != "sid" ]
-			then
-				OS_UNSUPPORTED=1
-			fi
-		;;
-		raspbian)
-			if [ "$VERSION" != "stretch" ] && \
-			   [ "$VERSION" != "buster" ] && \
-			   [ "$VERSION" != "bullseye" ]
-			then
-				OS_UNSUPPORTED=1
-			fi
-		;;
-		centos)
-			if [ "$VERSION" != "7" ] && \
-			   [ "$VERSION" != "8" ] && \
-			   [ "$VERSION" != "9" ]
-			then
-				OS_UNSUPPORTED=1
-			fi
-		;;
-		oracle)
-			if [ "$VERSION" != "7" ] && \
-			   [ "$VERSION" != "8" ]
-			then
-				OS_UNSUPPORTED=1
-			fi
-		;;
-		rhel)
-			if [ "$VERSION" != "7" ] && \
-			   [ "$VERSION" != "8" ] && \
-			   [ "$VERSION" != "9" ]
-			then
-				OS_UNSUPPORTED=1
-			fi
-		;;
-		amazon-linux)
-			if [ "$VERSION" != "2" ] && \
-			   [ "$VERSION" != "2022" ] && \
-			   [ "$VERSION" != "2023" ]
-			then
-				OS_UNSUPPORTED=1
-			fi
-		;;
-		opensuse)
-			if [ "$VERSION" != "leap/15.1" ] && \
-			   [ "$VERSION" != "leap/15.2" ] && \
-			   [ "$VERSION" != "leap/15.3" ] && \
-			   [ "$VERSION" != "leap/15.4" ] && \
-			   [ "$VERSION" != "tumbleweed" ]
-			then
-				OS_UNSUPPORTED=1
-			fi
+		ubuntu|debian|raspbian|centos|oracle|rhel|amazon-linux|opensuse)
+			# Check with the package server whether a given version is supported.
+			URL="https://pkgs.tailscale.com/$TRACK/$OS/$VERSION/installer-supported"
+			$CURL "$URL" 2> /dev/null | grep -q OK || OS_UNSUPPORTED=1
 			;;
 		fedora)
 			# All versions supported, no version checking required.
@@ -473,19 +419,6 @@ main() {
 	echo "Installing Tailscale for $OS $VERSION, using method $PACKAGETYPE"
 	case "$PACKAGETYPE" in
 		apt)
-			# Ideally we want to use curl, but on some installs we
-			# only have wget. Detect and use what's available.
-			CURL=
-			if type curl >/dev/null; then
-				CURL="curl -fsSL"
-			elif type wget >/dev/null; then
-				CURL="wget -q -O-"
-			fi
-			if [ -z "$CURL" ]; then
-				echo "The installer needs either curl or wget to download files."
-				echo "Please install either curl or wget to proceed."
-				exit 1
-			fi
 			export DEBIAN_FRONTEND=noninteractive
 			if [ "$APT_KEY_TYPE" = "legacy" ] && ! type gpg >/dev/null; then
 				$SUDO apt-get update

ssh/tailssh/incubator.go

@@ -160,7 +160,7 @@ func (stdRWC) Close() error {
 }
 
 type incubatorArgs struct {
-	uid          uint64
+	uid          int
 	gid          int
 	groups       string
 	localUser    string
@@ -177,7 +177,7 @@ type incubatorArgs struct {
 
 func parseIncubatorArgs(args []string) (a incubatorArgs) {
 	flags := flag.NewFlagSet("", flag.ExitOnError)
-	flags.Uint64Var(&a.uid, "uid", 0, "the uid of local-user")
+	flags.IntVar(&a.uid, "uid", 0, "the uid of local-user")
 	flags.IntVar(&a.gid, "gid", 0, "the gid of local-user")
 	flags.StringVar(&a.groups, "groups", "", "comma-separated list of gids of local-user")
 	flags.StringVar(&a.localUser, "local-user", "", "the user to run as")
@@ -204,6 +204,16 @@ func parseIncubatorArgs(args []string) (a incubatorArgs) {
 // OS, sets its UID and groups to the specified `--uid`, `--gid` and
 // `--groups` and then launches the requested `--cmd`.
 func beIncubator(args []string) error {
+	// To defend against issues like https://golang.org/issue/1435,
+	// defensively lock our current goroutine's thread to the current
+	// system thread before we start making any UID/GID/group changes.
+	//
+	// This shouldn't matter on Linux because syscall.AllThreadsSyscall is
+	// used to invoke syscalls on all OS threads, but (as of 2023-03-23)
+	// that function is not implemented on all platforms.
+	runtime.LockOSThread()
+	defer runtime.UnlockOSThread()
+
 	ia := parseIncubatorArgs(args)
 	if ia.isSFTP && ia.isShell {
 		return fmt.Errorf("--sftp and --shell are mutually exclusive")
@@ -217,7 +227,7 @@ func beIncubator(args []string) error {
 		}
 	}
 
-	euid := uint64(os.Geteuid())
+	euid := os.Geteuid()
 	runningAsRoot := euid == 0
 	if runningAsRoot && ia.loginCmdPath != "" {
 		// Check if we can exec into the login command instead of trying to
@@ -235,6 +245,7 @@ func beIncubator(args []string) error {
 	if err == nil && sessionCloser != nil {
 		defer sessionCloser()
 	}
+
 	var groupIDs []int
 	for _, g := range strings.Split(ia.groups, ",") {
 		gid, err := strconv.ParseInt(g, 10, 32)
@@ -244,22 +255,10 @@ func beIncubator(args []string) error {
 		groupIDs = append(groupIDs, int(gid))
 	}
 
-	if err := setGroups(groupIDs); err != nil {
+	if err := dropPrivileges(logf, ia.uid, ia.gid, groupIDs); err != nil {
 		return err
 	}
-	if egid := os.Getegid(); egid != ia.gid {
-		if err := syscall.Setgid(int(ia.gid)); err != nil {
-			logf(err.Error())
-			os.Exit(1)
-		}
-	}
-	if euid != ia.uid {
-		// Switch users if required before starting the desired process.
-		if err := syscall.Setuid(int(ia.uid)); err != nil {
-			logf(err.Error())
-			os.Exit(1)
-		}
-	}
+
 	if ia.isSFTP {
 		logf("handling sftp")
 
@@ -304,6 +303,118 @@ func beIncubator(args []string) error {
 	return err
 }
 
+const (
+	// This controls whether we assert that our privileges were dropped
+	// using geteuid/getegid; it's a const and not an envknob because the
+	// incubator doesn't see the parent's environment.
+	//
+	// TODO(andrew): remove this const and always do this after sufficient
+	// testing, e.g. the 1.40 release
+	assertPrivilegesWereDropped = true
+
+	// TODO(andrew-d): verify that this works in more configurations before
+	// enabling by default.
+	assertPrivilegesWereDroppedByAttemptingToUnDrop = false
+)
+
+// dropPrivileges contains all the logic for dropping privileges to a different
+// UID, GID, and set of supplementary groups. This function is
+// security-sensitive and ordering-dependent; please be very cautious if/when
+// refactoring.
+//
+// WARNING: if you change this function, you *MUST* run the TestDropPrivileges
+// test in this package as root on at least Linux, FreeBSD and Darwin. This can
+// be done by running:
+//
+//	go test -c ./ssh/tailssh/ && sudo ./tailssh.test -test.v -test.run TestDropPrivileges
+func dropPrivileges(logf logger.Logf, wantUid, wantGid int, supplementaryGroups []int) error {
+	fatalf := func(format string, args ...any) {
+		logf("[unexpected] error dropping privileges: "+format, args...)
+		os.Exit(1)
+	}
+
+	euid := os.Geteuid()
+	egid := os.Getegid()
+
+	if runtime.GOOS == "darwin" || runtime.GOOS == "freebsd" {
+		// On FreeBSD and Darwin, the first entry returned from the
+		// getgroups(2) syscall is the egid, and changing it with
+		// setgroups(2) changes the egid of the process. This is
+		// technically a violation of the POSIX standard; see the
+		// following article for more detail:
+		//    https://www.usenix.org/system/files/login/articles/325-tsafrir.pdf
+		//
+		// In this case, we add an entry at the beginning of the
+		// groupIDs list containing the expected gid if it's not
+		// already there, which modifies the egid and additional groups
+		// as one unit.
+		if len(supplementaryGroups) == 0 || supplementaryGroups[0] != wantGid {
+			supplementaryGroups = append([]int{wantGid}, supplementaryGroups...)
+		}
+	}
+
+	if err := setGroups(supplementaryGroups); err != nil {
+		return err
+	}
+	if egid != wantGid {
+		// On FreeBSD and Darwin, we may have already called the
+		// equivalent of setegid(wantGid) via the call to setGroups,
+		// above. However, per the manpage, setgid(getegid()) is an
+		// allowed operation regardless of privilege level.
+		//
+		// FreeBSD:
+		//	The setgid() system call is permitted if the specified ID
+		//	is equal to the real group ID or the effective group ID
+		//	of the process, or if the effective user ID is that of
+		//	the super user.
+		//
+		// Darwin:
+		//	The setgid() function is permitted if the effective
+		//	user ID is that of the super user, or if the specified
+		//	group ID is the same as the effective group ID.  If
+		//	not, but the specified group ID is the same as the real
+		//	group ID, setgid() will set the effective group ID to
+		//	the real group ID.
+		if err := syscall.Setgid(wantGid); err != nil {
+			fatalf("Setgid(%d): %v", wantGid, err)
+		}
+	}
+	if euid != wantUid {
+		// Switch users if required before starting the desired process.
+		if err := syscall.Setuid(wantUid); err != nil {
+			fatalf("Setuid(%d): %v", wantUid, err)
+		}
+	}
+
+	// If we changed either the UID or GID, defensively assert that we
+	// cannot reset the it back to our original values, and that the
+	// current egid/euid are the expected values after we change
+	// everything; if not, we exit the process.
+	if assertPrivilegesWereDroppedByAttemptingToUnDrop {
+		if egid != wantGid {
+			if err := syscall.Setegid(egid); err == nil {
+				fatalf("able to set egid back to %d", egid)
+			}
+		}
+		if euid != wantUid {
+			if err := syscall.Seteuid(euid); err == nil {
+				fatalf("able to set euid back to %d", euid)
+			}
+		}
+	}
+	if assertPrivilegesWereDropped {
+		if got := os.Getegid(); got != wantGid {
+			fatalf("got egid=%d, want %d", got, wantGid)
+		}
+		if got := os.Geteuid(); got != wantUid {
+			fatalf("got euid=%d, want %d", got, wantUid)
+		}
+		// TODO(andrew-d): assert that our supplementary groups are correct
+	}
+
+	return nil
+}
+
 // launchProcess launches an incubator process for the provided session.
 // It is responsible for configuring the process execution environment.
 // The caller can wait for the process to exit by calling cmd.Wait().

ssh/tailssh/privs_test.go

@@ -0,0 +1,295 @@
+// Copyright (c) Tailscale Inc & AUTHORS
+// SPDX-License-Identifier: BSD-3-Clause
+
+//go:build linux || darwin || freebsd || openbsd || netbsd || dragonfly
+
+package tailssh
+
+import (
+	"encoding/json"
+	"errors"
+	"os"
+	"os/exec"
+	"os/user"
+	"path/filepath"
+	"reflect"
+	"regexp"
+	"runtime"
+	"strconv"
+	"syscall"
+	"testing"
+
+	"golang.org/x/exp/slices"
+	"tailscale.com/types/logger"
+)
+
+func TestDropPrivileges(t *testing.T) {
+	type SubprocInput struct {
+		UID              int
+		GID              int
+		AdditionalGroups []int
+	}
+	type SubprocOutput struct {
+		UID              int
+		GID              int
+		EUID             int
+		EGID             int
+		AdditionalGroups []int
+	}
+
+	if v := os.Getenv("TS_TEST_DROP_PRIVILEGES_CHILD"); v != "" {
+		t.Logf("in child process")
+
+		var input SubprocInput
+		if err := json.Unmarshal([]byte(v), &input); err != nil {
+			t.Fatal(err)
+		}
+
+		// Get a handle to our provided JSON file before dropping privs.
+		f := os.NewFile(3, "out.json")
+
+		// We're in our subprocess; actually drop privileges now.
+		dropPrivileges(t.Logf, input.UID, input.GID, input.AdditionalGroups)
+
+		additional, _ := syscall.Getgroups()
+
+		// Print our IDs
+		json.NewEncoder(f).Encode(SubprocOutput{
+			UID:              os.Getuid(),
+			GID:              os.Getgid(),
+			EUID:             os.Geteuid(),
+			EGID:             os.Getegid(),
+			AdditionalGroups: additional,
+		})
+
+		// Close output file to ensure that it's flushed to disk before we exit
+		f.Close()
+
+		// Always exit the process now that we have a different
+		// UID/GID/etc.; we don't want the Go test framework to try and
+		// clean anything up, since it might no longer have access.
+		os.Exit(0)
+	}
+
+	if os.Getuid() != 0 {
+		t.Skip("test only works when run as root")
+	}
+
+	rerunSelf := func(t *testing.T, input SubprocInput) []byte {
+		fpath := filepath.Join(t.TempDir(), "out.json")
+		outf, err := os.Create(fpath)
+		if err != nil {
+			t.Fatal(err)
+		}
+
+		inputb, err := json.Marshal(input)
+		if err != nil {
+			t.Fatal(err)
+		}
+
+		cmd := exec.Command(os.Args[0], "-test.v", "-test.run", "^"+regexp.QuoteMeta(t.Name())+"$")
+		cmd.Env = append(os.Environ(), "TS_TEST_DROP_PRIVILEGES_CHILD="+string(inputb))
+		cmd.ExtraFiles = []*os.File{outf}
+		cmd.Stdout = logger.FuncWriter(logger.WithPrefix(t.Logf, "child: "))
+		cmd.Stderr = logger.FuncWriter(logger.WithPrefix(t.Logf, "child: "))
+		if err := cmd.Run(); err != nil {
+			t.Fatal(err)
+		}
+		outf.Close()
+
+		jj, err := os.ReadFile(fpath)
+		if err != nil {
+			t.Fatal(err)
+		}
+		return jj
+	}
+
+	// We want to ensure we're not colliding with existing users; find some
+	// unused UIDs and GIDs for the tests we run.
+	uid1 := findUnusedUID(t)
+	gid1 := findUnusedGID(t)
+	gid2 := findUnusedGID(t, gid1)
+	gid3 := findUnusedGID(t, gid1, gid2)
+
+	// For some tests, we want a UID/GID pair with the same numerical
+	// value; this finds one.
+	uidgid1 := findUnusedUIDGID(t, uid1, gid1, gid2, gid3)
+
+	t.Logf("uid1=%d gid1=%d gid2=%d gid3=%d uidgid1=%d",
+		uid1, gid1, gid2, gid3, uidgid1)
+
+	testCases := []struct {
+		name             string
+		uid              int
+		gid              int
+		additionalGroups []int
+	}{
+		{
+			name:             "all_different_values",
+			uid:              uid1,
+			gid:              gid1,
+			additionalGroups: []int{gid2, gid3},
+		},
+		{
+			name:             "no_additional_groups",
+			uid:              uid1,
+			gid:              gid1,
+			additionalGroups: []int{},
+		},
+		// This is a regression test for the following bug, triggered
+		// on Darwin & FreeBSD:
+		//    https://github.com/tailscale/tailscale/issues/7616
+		{
+			name:             "same_values",
+			uid:              uidgid1,
+			gid:              uidgid1,
+			additionalGroups: []int{uidgid1},
+		},
+	}
+
+	for _, tt := range testCases {
+		t.Run(tt.name, func(t *testing.T) {
+			subprocOut := rerunSelf(t, SubprocInput{
+				UID:              tt.uid,
+				GID:              tt.gid,
+				AdditionalGroups: tt.additionalGroups,
+			})
+
+			var out SubprocOutput
+			if err := json.Unmarshal(subprocOut, &out); err != nil {
+				t.Logf("%s", subprocOut)
+				t.Fatal(err)
+			}
+			t.Logf("output: %+v", out)
+
+			if out.UID != tt.uid {
+				t.Errorf("got uid %d; want %d", out.UID, tt.uid)
+			}
+			if out.GID != tt.gid {
+				t.Errorf("got gid %d; want %d", out.GID, tt.gid)
+			}
+			if out.EUID != tt.uid {
+				t.Errorf("got euid %d; want %d", out.EUID, tt.uid)
+			}
+			if out.EGID != tt.gid {
+				t.Errorf("got egid %d; want %d", out.EGID, tt.gid)
+			}
+
+			// On FreeBSD and Darwin, the set of additional groups
+			// is prefixed with the egid; handle that case by
+			// modifying our expected set.
+			wantGroups := make(map[int]bool)
+			for _, id := range tt.additionalGroups {
+				wantGroups[id] = true
+			}
+			if runtime.GOOS == "darwin" || runtime.GOOS == "freebsd" {
+				wantGroups[tt.gid] = true
+			}
+
+			gotGroups := make(map[int]bool)
+			for _, id := range out.AdditionalGroups {
+				gotGroups[id] = true
+			}
+
+			if !reflect.DeepEqual(gotGroups, wantGroups) {
+				t.Errorf("got additional groups %+v; want %+v", gotGroups, wantGroups)
+			}
+		})
+	}
+}
+
+func findUnusedUID(t *testing.T, not ...int) int {
+	for i := 1000; i < 65535; i++ {
+		// Skip UIDs that might be valid
+		if maybeValidUID(i) {
+			continue
+		}
+
+		// Skip UIDs that we're avoiding
+		if slices.Contains(not, i) {
+			continue
+		}
+
+		// Not a valid UID, not one we're avoiding... all good!
+		return i
+	}
+
+	t.Fatalf("unable to find an unused UID")
+	return -1
+}
+
+func findUnusedGID(t *testing.T, not ...int) int {
+	for i := 1000; i < 65535; i++ {
+		if maybeValidGID(i) {
+			continue
+		}
+
+		// Skip GIDs that we're avoiding
+		if slices.Contains(not, i) {
+			continue
+		}
+
+		// Not a valid GID, not one we're avoiding... all good!
+		return i
+	}
+
+	t.Fatalf("unable to find an unused GID")
+	return -1
+}
+
+func findUnusedUIDGID(t *testing.T, not ...int) int {
+	for i := 1000; i < 65535; i++ {
+		if maybeValidUID(i) || maybeValidGID(i) {
+			continue
+		}
+
+		// Skip IDs that we're avoiding
+		if slices.Contains(not, i) {
+			continue
+		}
+
+		// Not a valid ID, not one we're avoiding... all good!
+		return i
+	}
+
+	t.Fatalf("unable to find an unused UID/GID pair")
+	return -1
+}
+
+func maybeValidUID(id int) bool {
+	_, err := user.LookupId(strconv.Itoa(id))
+	if err == nil {
+		return true
+	}
+
+	var u1 user.UnknownUserIdError
+	if errors.As(err, &u1) {
+		return false
+	}
+	var u2 user.UnknownUserError
+	if errors.As(err, &u2) {
+		return false
+	}
+
+	// Some other error; might be valid
+	return true
+}
+
+func maybeValidGID(id int) bool {
+	_, err := user.LookupGroupId(strconv.Itoa(id))
+	if err == nil {
+		return true
+	}
+
+	var u1 user.UnknownGroupIdError
+	if errors.As(err, &u1) {
+		return false
+	}
+	var u2 user.UnknownGroupError
+	if errors.As(err, &u2) {
+		return false
+	}
+
+	// Some other error; might be valid
+	return true
+}

ssh/tailssh/tailssh.go

@@ -35,6 +35,7 @@ import (
 	"tailscale.com/ipn/ipnlocal"
 	"tailscale.com/logtail/backoff"
 	"tailscale.com/net/tsaddr"
+	"tailscale.com/net/tsdial"
 	"tailscale.com/tailcfg"
 	"tailscale.com/tempfork/gliderlabs/ssh"
 	"tailscale.com/types/logger"
@@ -62,7 +63,7 @@ type ipnLocalBackend interface {
 	NetMap() *netmap.NetworkMap
 	WhoIs(ipp netip.AddrPort) (n *tailcfg.Node, u tailcfg.UserProfile, ok bool)
 	DoNoiseRequest(req *http.Request) (*http.Response, error)
-	TailscaleVarRoot() string
+	Dialer() *tsdial.Dialer
 }
 
 type server struct {
@@ -77,11 +78,33 @@ type server struct {
 
 	// mu protects the following
 	mu                   sync.Mutex
+	httpc                *http.Client                // for calling out to peers.
 	activeConns          map[*conn]bool              // set; value is always true
 	fetchPublicKeysCache map[string]pubKeyCacheEntry // by https URL
 	shutdownCalled       bool
 }
 
+// sessionRecordingClient returns an http.Client that uses srv.lb.Dialer() to
+// dial connections. This is used to make requests to the session recording
+// server to upload session recordings.
+func (srv *server) sessionRecordingClient() *http.Client {
+	srv.mu.Lock()
+	defer srv.mu.Unlock()
+	if srv.httpc != nil {
+		return srv.httpc
+	}
+	tr := http.DefaultTransport.(*http.Transport).Clone()
+	tr.DialContext = func(ctx context.Context, network, addr string) (net.Conn, error) {
+		ctx, cancel := context.WithTimeout(ctx, 10*time.Second)
+		defer cancel()
+		return srv.lb.Dialer().UserDial(ctx, network, addr)
+	}
+	srv.httpc = &http.Client{
+		Transport: tr,
+	}
+	return srv.httpc
+}
+
 func (srv *server) now() time.Time {
 	if srv != nil && srv.timeNow != nil {
 		return srv.timeNow()
@@ -987,12 +1010,6 @@ func (ss *sshSession) handleSSHAgentForwarding(s ssh.Session, lu *user.User) err
 	return nil
 }
 
-// recordSSH is a temporary dev knob to test the SSH recording
-// functionality and support off-node streaming.
-//
-// TODO(bradfitz,maisem): move this to SSHPolicy.
-var recordSSH = envknob.RegisterBool("TS_DEBUG_LOG_SSH")
-
 // run is the entrypoint for a newly accepted SSH session.
 //
 // It handles ss once it's been accepted and determined
@@ -1125,12 +1142,19 @@ func (ss *sshSession) run() {
 	return
 }
 
+// recorders returns the list of recorders to use for this session.
+// If the final action has a non-empty list of recorders, that list is
+// returned. Otherwise, the list of recorders from the initial action
+// is returned.
+func (ss *sshSession) recorders() []netip.AddrPort {
+	if len(ss.conn.finalAction.Recorders) > 0 {
+		return ss.conn.finalAction.Recorders
+	}
+	return ss.conn.action0.Recorders
+}
+
 func (ss *sshSession) shouldRecord() bool {
-	// for now only record pty sessions
-	// TODO(bradfitz,maisem): make configurable on SSHPolicy and
-	// support recording non-pty stuff too.
-	_, _, isPtyReq := ss.Pty()
-	return recordSSH() && isPtyReq
+	return len(ss.recorders()) > 0
 }
 
 type sshConnInfo struct {
@@ -1312,11 +1336,67 @@ func randBytes(n int) []byte {
 	return b
 }
 
+// CastHeader is the header of an asciinema file.
+type CastHeader struct {
+	// Version is the asciinema file format version.
+	Version int `json:"version"`
+
+	// Width is the terminal width in characters.
+	// It is non-zero for Pty sessions.
+	Width int `json:"width"`
+
+	// Height is the terminal height in characters.
+	// It is non-zero for Pty sessions.
+	Height int `json:"height"`
+
+	// Timestamp is the unix timestamp of when the recording started.
+	Timestamp int64 `json:"timestamp"`
+
+	// Env is the environment variables of the session.
+	// Only "TERM" is set (2023-03-22).
+	Env map[string]string `json:"env"`
+
+	// Command is the command that was executed.
+	// Typically empty for shell sessions.
+	Command string `json:"command,omitempty"`
+
+	// Tailscale-specific fields:
+	// SrcNode is the FQDN of the node originating the connection.
+	// It is also the MagicDNS name for the node.
+	// It does not have a trailing dot.
+	// e.g. "host.tail-scale.ts.net"
+	SrcNode string `json:"srcNode"`
+
+	// SrcNodeID is the node ID of the node originating the connection.
+	SrcNodeID tailcfg.StableNodeID `json:"srcNodeID"`
+
+	// SrcNodeTags is the list of tags on the node originating the connection (if any).
+	SrcNodeTags []string `json:"srcNodeTags,omitempty"`
+
+	// SrcNodeUserID is the user ID of the node originating the connection (if not tagged).
+	SrcNodeUserID tailcfg.UserID `json:"srcNodeUserID,omitempty"` // if not tagged
+
+	// SrcNodeUser is the LoginName of the node originating the connection (if not tagged).
+	SrcNodeUser string `json:"srcNodeUser,omitempty"`
+
+	// SSHUser is the username as presented by the client.
+	SSHUser string `json:"sshUser"` // as presented by the client
+
+	// LocalUser is the effective username on the server.
+	LocalUser string `json:"localUser"`
+}
+
 // startNewRecording starts a new SSH session recording.
-//
-// It writes an asciinema file to
-// $TAILSCALE_VAR_ROOT/ssh-sessions/ssh-session-<unixtime>-*.cast.
 func (ss *sshSession) startNewRecording() (_ *recording, err error) {
+	recorders := ss.recorders()
+	if len(recorders) == 0 {
+		return nil, errors.New("no recorders configured")
+	}
+	recorder := recorders[0]
+	if len(recorders) > 1 {
+		ss.logf("warning: multiple recorders configured, using first one: %v", recorder)
+	}
+
 	var w ssh.Window
 	if ptyReq, _, isPtyReq := ss.Pty(); isPtyReq {
 		w = ptyReq.Window
@@ -1332,39 +1412,44 @@ func (ss *sshSession) startNewRecording() (_ *recording, err error) {
 		ss:    ss,
 		start: now,
 	}
-	varRoot := ss.conn.srv.lb.TailscaleVarRoot()
-	if varRoot == "" {
-		return nil, errors.New("no var root for recording storage")
-	}
-	dir := filepath.Join(varRoot, "ssh-sessions")
-	if err := os.MkdirAll(dir, 0700); err != nil {
+
+	pr, pw := io.Pipe()
+
+	// We want to use a background context for uploading and not ss.ctx.
+	// ss.ctx is closed when the session closes, but we don't want to break the upload at that time.
+	// Instead we want to wait for the session to close the writer when it finishes.
+	ctx := context.Background()
+	req, err := http.NewRequestWithContext(ctx, "POST", fmt.Sprintf("http://%s:%d/record", recorder.Addr(), recorder.Port()), pr)
+	if err != nil {
+		pr.Close()
+		pw.Close()
 		return nil, err
 	}
-	defer func() {
+	go func() {
+		defer pw.Close()
+		ss.logf("starting asciinema recording to %s", recorder)
+		hc := ss.conn.srv.sessionRecordingClient()
+		resp, err := hc.Do(req)
 		if err != nil {
-			rec.Close()
+			ss.cancelCtx(err)
+			ss.logf("recording: error sending recording to %s: %v", recorder, err)
+			return
+		}
+		defer resp.Body.Close()
+		defer ss.cancelCtx(errors.New("recording: done"))
+		if resp.StatusCode != http.StatusOK {
+			ss.logf("recording: error sending recording to %s: %v", recorder, resp.Status)
 		}
 	}()
 
-	f, err := os.CreateTemp(dir, fmt.Sprintf("ssh-session-%v-*.cast", now.UnixNano()))
-	if err != nil {
-		return nil, err
-	}
-	rec.out = f
+	rec.out = pw
 
-	// {"version": 2, "width": 221, "height": 84, "timestamp": 1647146075, "env": {"SHELL": "/bin/bash", "TERM": "screen"}}
-	type CastHeader struct {
-		Version   int               `json:"version"`
-		Width     int               `json:"width"`
-		Height    int               `json:"height"`
-		Timestamp int64             `json:"timestamp"`
-		Env       map[string]string `json:"env"`
-	}
-	j, err := json.Marshal(CastHeader{
+	ch := CastHeader{
 		Version:   2,
 		Width:     w.Width,
 		Height:    w.Height,
 		Timestamp: now.Unix(),
+		Command:   strings.Join(ss.Command(), " "),
 		Env: map[string]string{
 			"TERM": term,
 			// TODO(bradfitz): anything else important?
@@ -1376,15 +1461,23 @@ func (ss *sshSession) startNewRecording() (_ *recording, err error) {
 			// it. Then we can (1) make the cmd, (2) start the
 			// recording, (3) start the process.
 		},
-	})
+		SSHUser:   ss.conn.info.sshUser,
+		LocalUser: ss.conn.localUser.Username,
+		SrcNode:   strings.TrimSuffix(ss.conn.info.node.Name, "."),
+		SrcNodeID: ss.conn.info.node.StableID,
+	}
+	if !ss.conn.info.node.IsTagged() {
+		ch.SrcNodeUser = ss.conn.info.uprof.LoginName
+		ch.SrcNodeUserID = ss.conn.info.node.User
+	} else {
+		ch.SrcNodeTags = ss.conn.info.node.Tags
+	}
+	j, err := json.Marshal(ch)
 	if err != nil {
-		f.Close()
 		return nil, err
 	}
-	ss.logf("starting asciinema recording to %s", f.Name())
 	j = append(j, '\n')
-	if _, err := f.Write(j); err != nil {
-		f.Close()
+	if _, err := pw.Write(j); err != nil {
 		return nil, err
 	}
 	return rec, nil
@@ -1396,7 +1489,7 @@ type recording struct {
 	start time.Time
 
 	mu  sync.Mutex // guards writes to, close of out
-	out *os.File   // nil if closed
+	out io.WriteCloser
 }
 
 func (r *recording) Close() error {
@@ -1415,10 +1508,17 @@ func (r *recording) Close() error {
 // The dir should be "i" for input or "o" for output.
 //
 // If r is nil, it returns w unchanged.
+//
+// Currently (2023-03-21) we only record output, not input.
 func (r *recording) writer(dir string, w io.Writer) io.Writer {
 	if r == nil {
 		return w
 	}
+	if dir == "i" {
+		// TODO: record input? Maybe not, since it might contain
+		// passwords.
+		return w
+	}
 	return &loggingWriter{r, dir, w}
 }
 

ssh/tailssh/tailssh_test.go

@@ -38,6 +38,7 @@ import (
 	"tailscale.com/tempfork/gliderlabs/ssh"
 	"tailscale.com/tstest"
 	"tailscale.com/types/logger"
+	"tailscale.com/types/logid"
 	"tailscale.com/types/netmap"
 	"tailscale.com/util/cibuild"
 	"tailscale.com/util/lineread"
@@ -236,6 +237,10 @@ var (
 	testSignerOnce sync.Once
 )
 
+func (ts *localState) Dialer() *tsdial.Dialer {
+	return nil
+}
+
 func (ts *localState) GetSSH_HostKeys() ([]gossh.Signer, error) {
 	testSignerOnce.Do(func() {
 		_, priv, err := ed25519.GenerateKey(rand.Reader)
@@ -505,7 +510,7 @@ func TestSSH(t *testing.T) {
 	if err != nil {
 		t.Fatal(err)
 	}
-	lb, err := ipnlocal.NewLocalBackend(logf, "",
+	lb, err := ipnlocal.NewLocalBackend(logf, logid.PublicID{},
 		new(mem.Store),
 		new(tsdial.Dialer),
 		eng, 0)
@@ -539,7 +544,8 @@ func TestSSH(t *testing.T) {
 		node:    &tailcfg.Node{},
 		uprof:   tailcfg.UserProfile{},
 	}
-	sc.finalAction = &tailcfg.SSHAction{Accept: true}
+	sc.action0 = &tailcfg.SSHAction{Accept: true}
+	sc.finalAction = sc.action0
 
 	sc.Handler = func(s ssh.Session) {
 		sc.newSSHSession(s).run()

tailcfg/tailcfg.go

@@ -3,7 +3,7 @@
 
 package tailcfg
 
-//go:generate go run tailscale.com/cmd/viewer --type=User,Node,Hostinfo,NetInfo,Login,DNSConfig,RegisterResponse,DERPRegion,DERPMap,DERPNode,SSHRule,SSHPrincipal,ControlDialPlan --clonefunc
+//go:generate go run tailscale.com/cmd/viewer --type=User,Node,Hostinfo,NetInfo,Login,DNSConfig,RegisterResponse,DERPRegion,DERPMap,DERPNode,SSHRule,SSHAction,SSHPrincipal,ControlDialPlan --clonefunc
 
 import (
 	"bytes"
@@ -94,7 +94,9 @@ type CapabilityVersion int
 //   - 55: 2023-01-23: start of c2n GET+POST /update handler
 //   - 56: 2023-01-24: Client understands CapabilityDebugTSDNSResolution
 //   - 57: 2023-01-25: Client understands CapabilityBindToInterfaceByRoute
-const CurrentCapabilityVersion CapabilityVersion = 57
+//   - 58: 2023-03-10: Client retries lite map updates before restarting map poll.
+//   - 59: 2023-03-16: Client understands Peers[].SelfNodeV4MasqAddrForThisPeer
+const CurrentCapabilityVersion CapabilityVersion = 59
 
 type StableID string
 
@@ -182,7 +184,12 @@ func (emptyStructJSONSlice) UnmarshalJSON([]byte) error { return nil }
 type Node struct {
 	ID       NodeID
 	StableID StableNodeID
-	Name     string // DNS
+
+	// Name is the FQDN of the node.
+	// It is also the MagicDNS name for the node.
+	// It has a trailing dot.
+	// e.g. "host.tail-scale.ts.net."
+	Name string
 
 	// User is the user who created the node. If ACL tags are in
 	// use for the node then it doesn't reflect the ACL identity
@@ -267,6 +274,21 @@ type Node struct {
 	// the client, this is calculated client-side based on a timestamp sent
 	// from control, to avoid clock skew issues.
 	Expired bool `json:",omitempty"`
+
+	// SelfNodeV4MasqAddrForThisPeer is the IPv4 that this peer knows the current node as.
+	// It may be empty if the peer knows the current node by its native
+	// IPv4 address.
+	// This field is only populated in a MapResponse for peers and not
+	// for the current node.
+	//
+	// If set, it should be used to masquerade traffic originating from the
+	// current node to this peer. The masquerade address is only relevant
+	// for this peer and not for other peers.
+	//
+	// This only applies to traffic originating from the current node to the
+	// peer or any of its subnets. Traffic originating from subnet routes will
+	// not be masqueraded (e.g. in case of --snat-subnet-routes).
+	SelfNodeV4MasqAddrForThisPeer netip.Addr `json:",omitempty"`
 }
 
 // DisplayName returns the user-facing name for a node which should
@@ -1692,7 +1714,8 @@ func (n *Node) Equal(n2 *Node) bool {
 		n.computedHostIfDifferent == n2.computedHostIfDifferent &&
 		n.ComputedNameWithHost == n2.ComputedNameWithHost &&
 		eqStrings(n.Tags, n2.Tags) &&
-		n.Expired == n2.Expired
+		n.Expired == n2.Expired &&
+		n.SelfNodeV4MasqAddrForThisPeer == n2.SelfNodeV4MasqAddrForThisPeer
 }
 
 func eqBoolPtr(a, b *bool) bool {
@@ -2015,9 +2038,9 @@ type SSHAction struct {
 	// to use local port forwarding if requested.
 	AllowLocalPortForwarding bool `json:"allowLocalPortForwarding,omitempty"`
 
-	// SessionHaulTargetNode, if non-empty, is the Stable ID of a peer to
-	// stream this SSH session's logs to.
-	SessionHaulTargetNode StableNodeID `json:"sessionHaulTargetNode,omitempty"`
+	// Recorders defines the destinations of the SSH session recorders.
+	// The recording will be uploaded to http://addr:port/record.
+	Recorders []netip.AddrPort `json:"recorders"`
 }
 
 // OverTLSPublicKeyResponse is the JSON response to /key?v=<n>

tailcfg/tailcfg_clone.go

@@ -68,36 +68,37 @@ func (src *Node) Clone() *Node {
 
 // A compilation failure here means this code must be regenerated, with the command at the top of this file.
 var _NodeCloneNeedsRegeneration = Node(struct {
-	ID                      NodeID
-	StableID                StableNodeID
-	Name                    string
-	User                    UserID
-	Sharer                  UserID
-	Key                     key.NodePublic
-	KeyExpiry               time.Time
-	KeySignature            tkatype.MarshaledSignature
-	Machine                 key.MachinePublic
-	DiscoKey                key.DiscoPublic
-	Addresses               []netip.Prefix
-	AllowedIPs              []netip.Prefix
-	Endpoints               []string
-	DERP                    string
-	Hostinfo                HostinfoView
-	Created                 time.Time
-	Cap                     CapabilityVersion
-	Tags                    []string
-	PrimaryRoutes           []netip.Prefix
-	LastSeen                *time.Time
-	Online                  *bool
-	KeepAlive               bool
-	MachineAuthorized       bool
-	Capabilities            []string
-	UnsignedPeerAPIOnly     bool
-	ComputedName            string
-	computedHostIfDifferent string
-	ComputedNameWithHost    string
-	DataPlaneAuditLogID     string
-	Expired                 bool
+	ID                            NodeID
+	StableID                      StableNodeID
+	Name                          string
+	User                          UserID
+	Sharer                        UserID
+	Key                           key.NodePublic
+	KeyExpiry                     time.Time
+	KeySignature                  tkatype.MarshaledSignature
+	Machine                       key.MachinePublic
+	DiscoKey                      key.DiscoPublic
+	Addresses                     []netip.Prefix
+	AllowedIPs                    []netip.Prefix
+	Endpoints                     []string
+	DERP                          string
+	Hostinfo                      HostinfoView
+	Created                       time.Time
+	Cap                           CapabilityVersion
+	Tags                          []string
+	PrimaryRoutes                 []netip.Prefix
+	LastSeen                      *time.Time
+	Online                        *bool
+	KeepAlive                     bool
+	MachineAuthorized             bool
+	Capabilities                  []string
+	UnsignedPeerAPIOnly           bool
+	ComputedName                  string
+	computedHostIfDifferent       string
+	ComputedNameWithHost          string
+	DataPlaneAuditLogID           string
+	Expired                       bool
+	SelfNodeV4MasqAddrForThisPeer netip.Addr
 }{})
 
 // Clone makes a deep copy of Hostinfo.
@@ -371,10 +372,7 @@ func (src *SSHRule) Clone() *SSHRule {
 			dst.SSHUsers[k] = v
 		}
 	}
-	if dst.Action != nil {
-		dst.Action = new(SSHAction)
-		*dst.Action = *src.Action
-	}
+	dst.Action = src.Action.Clone()
 	return dst
 }
 
@@ -386,6 +384,30 @@ var _SSHRuleCloneNeedsRegeneration = SSHRule(struct {
 	Action      *SSHAction
 }{})
 
+// Clone makes a deep copy of SSHAction.
+// The result aliases no memory with the original.
+func (src *SSHAction) Clone() *SSHAction {
+	if src == nil {
+		return nil
+	}
+	dst := new(SSHAction)
+	*dst = *src
+	dst.Recorders = append(src.Recorders[:0:0], src.Recorders...)
+	return dst
+}
+
+// A compilation failure here means this code must be regenerated, with the command at the top of this file.
+var _SSHActionCloneNeedsRegeneration = SSHAction(struct {
+	Message                  string
+	Reject                   bool
+	Accept                   bool
+	SessionDuration          time.Duration
+	AllowAgentForwarding     bool
+	HoldAndDelegate          string
+	AllowLocalPortForwarding bool
+	Recorders                []netip.AddrPort
+}{})
+
 // Clone makes a deep copy of SSHPrincipal.
 // The result aliases no memory with the original.
 func (src *SSHPrincipal) Clone() *SSHPrincipal {
@@ -426,7 +448,7 @@ var _ControlDialPlanCloneNeedsRegeneration = ControlDialPlan(struct {
 
 // Clone duplicates src into dst and reports whether it succeeded.
 // To succeed, <src, dst> must be of types <*T, *T> or <*T, **T>,
-// where T is one of User,Node,Hostinfo,NetInfo,Login,DNSConfig,RegisterResponse,DERPRegion,DERPMap,DERPNode,SSHRule,SSHPrincipal,ControlDialPlan.
+// where T is one of User,Node,Hostinfo,NetInfo,Login,DNSConfig,RegisterResponse,DERPRegion,DERPMap,DERPNode,SSHRule,SSHAction,SSHPrincipal,ControlDialPlan.
 func Clone(dst, src any) bool {
 	switch src := src.(type) {
 	case *User:
@@ -528,6 +550,15 @@ func Clone(dst, src any) bool {
 			*dst = src.Clone()
 			return true
 		}
+	case *SSHAction:
+		switch dst := dst.(type) {
+		case *SSHAction:
+			*dst = *src.Clone()
+			return true
+		case **SSHAction:
+			*dst = src.Clone()
+			return true
+		}
 	case *SSHPrincipal:
 		switch dst := dst.(type) {
 		case *SSHPrincipal:

tailcfg/tailcfg_test.go

@@ -349,7 +349,7 @@ func TestNodeEqual(t *testing.T) {
 		"Capabilities",
 		"UnsignedPeerAPIOnly",
 		"ComputedName", "computedHostIfDifferent", "ComputedNameWithHost",
-		"DataPlaneAuditLogID", "Expired",
+		"DataPlaneAuditLogID", "Expired", "SelfNodeV4MasqAddrForThisPeer",
 	}
 	if have := fieldsOf(reflect.TypeOf(Node{})); !reflect.DeepEqual(have, nodeHandles) {
 		t.Errorf("Node.Equal check might be out of sync\nfields: %q\nhandled: %q\n",
@@ -534,6 +534,16 @@ func TestNodeEqual(t *testing.T) {
 			&Node{},
 			false,
 		},
+		{
+			&Node{},
+			&Node{SelfNodeV4MasqAddrForThisPeer: netip.MustParseAddr("100.64.0.1")},
+			false,
+		},
+		{
+			&Node{SelfNodeV4MasqAddrForThisPeer: netip.MustParseAddr("100.64.0.1")},
+			&Node{SelfNodeV4MasqAddrForThisPeer: netip.MustParseAddr("100.64.0.1")},
+			true,
+		},
 	}
 	for i, tt := range tests {
 		got := tt.a.Equal(tt.b)

tailcfg/tailcfg_view.go

@@ -20,7 +20,7 @@ import (
 	"tailscale.com/types/views"
 )
 
-//go:generate go run tailscale.com/cmd/cloner  -clonefunc=true -type=User,Node,Hostinfo,NetInfo,Login,DNSConfig,RegisterResponse,DERPRegion,DERPMap,DERPNode,SSHRule,SSHPrincipal,ControlDialPlan
+//go:generate go run tailscale.com/cmd/cloner  -clonefunc=true -type=User,Node,Hostinfo,NetInfo,Login,DNSConfig,RegisterResponse,DERPRegion,DERPMap,DERPNode,SSHRule,SSHAction,SSHPrincipal,ControlDialPlan
 
 // View returns a readonly view of User.
 func (p *User) View() UserView {
@@ -176,40 +176,44 @@ func (v NodeView) ComputedName() string              { return v.ж.ComputedName
 func (v NodeView) ComputedNameWithHost() string      { return v.ж.ComputedNameWithHost }
 func (v NodeView) DataPlaneAuditLogID() string       { return v.ж.DataPlaneAuditLogID }
 func (v NodeView) Expired() bool                     { return v.ж.Expired }
-func (v NodeView) Equal(v2 NodeView) bool            { return v.ж.Equal(v2.ж) }
+func (v NodeView) SelfNodeV4MasqAddrForThisPeer() netip.Addr {
+	return v.ж.SelfNodeV4MasqAddrForThisPeer
+}
+func (v NodeView) Equal(v2 NodeView) bool { return v.ж.Equal(v2.ж) }
 
 // A compilation failure here means this code must be regenerated, with the command at the top of this file.
 var _NodeViewNeedsRegeneration = Node(struct {
-	ID                      NodeID
-	StableID                StableNodeID
-	Name                    string
-	User                    UserID
-	Sharer                  UserID
-	Key                     key.NodePublic
-	KeyExpiry               time.Time
-	KeySignature            tkatype.MarshaledSignature
-	Machine                 key.MachinePublic
-	DiscoKey                key.DiscoPublic
-	Addresses               []netip.Prefix
-	AllowedIPs              []netip.Prefix
-	Endpoints               []string
-	DERP                    string
-	Hostinfo                HostinfoView
-	Created                 time.Time
-	Cap                     CapabilityVersion
-	Tags                    []string
-	PrimaryRoutes           []netip.Prefix
-	LastSeen                *time.Time
-	Online                  *bool
-	KeepAlive               bool
-	MachineAuthorized       bool
-	Capabilities            []string
-	UnsignedPeerAPIOnly     bool
-	ComputedName            string
-	computedHostIfDifferent string
-	ComputedNameWithHost    string
-	DataPlaneAuditLogID     string
-	Expired                 bool
+	ID                            NodeID
+	StableID                      StableNodeID
+	Name                          string
+	User                          UserID
+	Sharer                        UserID
+	Key                           key.NodePublic
+	KeyExpiry                     time.Time
+	KeySignature                  tkatype.MarshaledSignature
+	Machine                       key.MachinePublic
+	DiscoKey                      key.DiscoPublic
+	Addresses                     []netip.Prefix
+	AllowedIPs                    []netip.Prefix
+	Endpoints                     []string
+	DERP                          string
+	Hostinfo                      HostinfoView
+	Created                       time.Time
+	Cap                           CapabilityVersion
+	Tags                          []string
+	PrimaryRoutes                 []netip.Prefix
+	LastSeen                      *time.Time
+	Online                        *bool
+	KeepAlive                     bool
+	MachineAuthorized             bool
+	Capabilities                  []string
+	UnsignedPeerAPIOnly           bool
+	ComputedName                  string
+	computedHostIfDifferent       string
+	ComputedNameWithHost          string
+	DataPlaneAuditLogID           string
+	Expired                       bool
+	SelfNodeV4MasqAddrForThisPeer netip.Addr
 }{})
 
 // View returns a readonly view of Hostinfo.
@@ -865,13 +869,7 @@ func (v SSHRuleView) Principals() views.SliceView[*SSHPrincipal, SSHPrincipalVie
 }
 
 func (v SSHRuleView) SSHUsers() views.Map[string, string] { return views.MapOf(v.ж.SSHUsers) }
-func (v SSHRuleView) Action() *SSHAction {
-	if v.ж.Action == nil {
-		return nil
-	}
-	x := *v.ж.Action
-	return &x
-}
+func (v SSHRuleView) Action() SSHActionView               { return v.ж.Action.View() }
 
 // A compilation failure here means this code must be regenerated, with the command at the top of this file.
 var _SSHRuleViewNeedsRegeneration = SSHRule(struct {
@@ -881,6 +879,72 @@ var _SSHRuleViewNeedsRegeneration = SSHRule(struct {
 	Action      *SSHAction
 }{})
 
+// View returns a readonly view of SSHAction.
+func (p *SSHAction) View() SSHActionView {
+	return SSHActionView{ж: p}
+}
+
+// SSHActionView provides a read-only view over SSHAction.
+//
+// Its methods should only be called if `Valid()` returns true.
+type SSHActionView struct {
+	// ж is the underlying mutable value, named with a hard-to-type
+	// character that looks pointy like a pointer.
+	// It is named distinctively to make you think of how dangerous it is to escape
+	// to callers. You must not let callers be able to mutate it.
+	ж *SSHAction
+}
+
+// Valid reports whether underlying value is non-nil.
+func (v SSHActionView) Valid() bool { return v.ж != nil }
+
+// AsStruct returns a clone of the underlying value which aliases no memory with
+// the original.
+func (v SSHActionView) AsStruct() *SSHAction {
+	if v.ж == nil {
+		return nil
+	}
+	return v.ж.Clone()
+}
+
+func (v SSHActionView) MarshalJSON() ([]byte, error) { return json.Marshal(v.ж) }
+
+func (v *SSHActionView) UnmarshalJSON(b []byte) error {
+	if v.ж != nil {
+		return errors.New("already initialized")
+	}
+	if len(b) == 0 {
+		return nil
+	}
+	var x SSHAction
+	if err := json.Unmarshal(b, &x); err != nil {
+		return err
+	}
+	v.ж = &x
+	return nil
+}
+
+func (v SSHActionView) Message() string                        { return v.ж.Message }
+func (v SSHActionView) Reject() bool                           { return v.ж.Reject }
+func (v SSHActionView) Accept() bool                           { return v.ж.Accept }
+func (v SSHActionView) SessionDuration() time.Duration         { return v.ж.SessionDuration }
+func (v SSHActionView) AllowAgentForwarding() bool             { return v.ж.AllowAgentForwarding }
+func (v SSHActionView) HoldAndDelegate() string                { return v.ж.HoldAndDelegate }
+func (v SSHActionView) AllowLocalPortForwarding() bool         { return v.ж.AllowLocalPortForwarding }
+func (v SSHActionView) Recorders() views.Slice[netip.AddrPort] { return views.SliceOf(v.ж.Recorders) }
+
+// A compilation failure here means this code must be regenerated, with the command at the top of this file.
+var _SSHActionViewNeedsRegeneration = SSHAction(struct {
+	Message                  string
+	Reject                   bool
+	Accept                   bool
+	SessionDuration          time.Duration
+	AllowAgentForwarding     bool
+	HoldAndDelegate          string
+	AllowLocalPortForwarding bool
+	Recorders                []netip.AddrPort
+}{})
+
 // View returns a readonly view of SSHPrincipal.
 func (p *SSHPrincipal) View() SSHPrincipalView {
 	return SSHPrincipalView{ж: p}

tka/tailchonk.go

@@ -201,10 +201,6 @@ func ChonkDir(dir string) (*FS, error) {
 		return nil, fmt.Errorf("chonk directory %q is a file", dir)
 	}
 
-	// TODO(tom): *FS marks AUMs as deleted but does not actually
-	// delete them, to avoid data loss in the event of a bug.
-	// Implement deletion after we are fairly sure in the implementation.
-
 	return &FS{base: dir}, nil
 }
 
@@ -218,6 +214,9 @@ func ChonkDir(dir string) (*FS, error) {
 // much smaller than JSON for AUMs. The 'keyasint' thing isn't essential
 // but again it saves a bunch of bytes.
 type fsHashInfo struct {
+	// diskHash specifies the AUMHash this structure describes.
+	diskHash AUMHash
+
 	Children    []AUMHash `cbor:"1,keyasint"`
 	AUM         *AUM      `cbor:"2,keyasint"`
 	CreatedUnix int64     `cbor:"3,keyasint,omitempty"`
@@ -344,6 +343,7 @@ func (c *FS) get(h AUMHash) (*fsHashInfo, error) {
 	if out.AUM != nil && out.AUM.Hash() != h {
 		return nil, fmt.Errorf("%s: AUM does not match file name hash %s", f.Name(), out.AUM.Hash())
 	}
+	out.diskHash = h
 	return &out, nil
 }
 
@@ -380,6 +380,104 @@ func (c *FS) AllAUMs() ([]AUMHash, error) {
 	return out, err
 }
 
+// CollectGarbage frees up disk space by removing purged AUMs
+// and files which contain no data.
+func (c *FS) CollectGarbage(maxAge time.Duration) error {
+	c.mu.Lock()
+	defer c.mu.Unlock()
+
+	// Collect the list of all stored hashes which are marked
+	// for deletion & old enough to delete.
+	var (
+		deletionCandidates = make(map[AUMHash]*fsHashInfo)
+		purgeBefore        = time.Now().Add(-maxAge)
+	)
+	err := c.scanHashes(func(info *fsHashInfo) {
+		// Mark for deletion all hashes which are explicitly purged, or
+		// hashes that store no data.
+		purged := info.PurgedUnix > 0 && time.Unix(info.PurgedUnix, 0).Before(purgeBefore)
+		if purged || (info.AUM == nil && len(info.Children) == 0) {
+			deletionCandidates[info.diskHash] = info
+		}
+	})
+	if err != nil {
+		return err
+	}
+
+	// TODO: consistency check that no deletion candidate is the last active
+	// ancestor nor a parent is the last active ancestor.
+
+	for h, info := range deletionCandidates {
+
+		// First, if we store the parent, remove the reference to this
+		// hash as a child.
+		if info.AUM != nil {
+			if parent, haveParent := info.AUM.Parent(); haveParent {
+				dir, base := c.aumDir(parent)
+				_, err := os.Stat(filepath.Join(dir, base))
+				parentExists := err == nil
+
+				if parentExists {
+					err := c.commit(parent, func(info *fsHashInfo) {
+						newChildren := make([]AUMHash, 0, len(info.Children))
+						for _, c := range info.Children {
+							if c != h {
+								newChildren = append(newChildren, c)
+							}
+						}
+						info.Children = newChildren
+					})
+					if err != nil {
+						return fmt.Errorf("mutating parent %x of %x: %v", parent, h, err)
+					}
+				}
+			}
+		}
+
+		dir, base := c.aumDir(h)
+		path := filepath.Join(dir, base)
+		if len(info.Children) == 0 {
+			// This hash has no dependencies.
+			//
+			// Technically, info.Children could be stale, because if this hash was
+			// someones parent then that someone would have removed their hash from
+			// the list. Because thats only ever a deletion tho, this is still safe,
+			// staleness will result in us not deleting this file but it will be
+			// deleted next time.
+			if err := os.Remove(path); err != nil {
+				return fmt.Errorf("removing dead entry: %w", err)
+			}
+			continue
+		}
+
+		// This hash has children it needs to keep track of, so might not
+		// be able to be deleted outright.
+		var delete bool
+		err := c.commit(h, func(info *fsHashInfo) {
+			info.AUM = nil // in all cases this hash shouldnt store its own AUM info
+			newChildren := make([]AUMHash, 0, len(info.Children))
+			for _, c := range info.Children {
+				if _, deleted := deletionCandidates[c]; !deleted {
+					newChildren = append(newChildren, c)
+				}
+			}
+			info.Children = newChildren
+			delete = len(newChildren) == 0
+		})
+		if err != nil {
+			return fmt.Errorf("mutating entry %x: %v", h, err)
+		}
+
+		if delete {
+			if err := os.Remove(path); err != nil {
+				return fmt.Errorf("removing empty entry: %w", err)
+			}
+		}
+	}
+
+	return nil
+}
+
 func (c *FS) scanHashes(eachHashInfo func(*fsHashInfo)) error {
 	prefixDirs, err := os.ReadDir(c.base)
 	if err != nil {

tka/tailchonk_test.go

@@ -630,6 +630,7 @@ func TestCompact(t *testing.T) {
 	// OLD is deleted because it does not match retention criteria, and
 	// though it is a descendant of the new lastActiveAncestor (C), it is not a
 	// descendant of a retained AUM.
+	// O is deleted because it is orphaned.
 	// G, & H are retained as recent (MinChain=2) ancestors of HEAD.
 	// E & F are retained because they are between retained AUMs (G+) and
 	// their newest checkpoint ancestor.
@@ -648,6 +649,9 @@ func TestCompact(t *testing.T) {
                        |  -> F1 -> F2       | -> G2
                        |  -> OLD
 
+        // Orphaned AUM
+        O
+
         // make {A,B,C,D} compaction candidates
         A.template = checkpoint
         B.template = checkpoint
@@ -658,13 +662,14 @@ func TestCompact(t *testing.T) {
         F1.hashSeed = 1
         OLD.hashSeed = 2
         G2.hashSeed = 3
+        O.hashSeed = 4
     `, optTemplate("checkpoint", AUM{MessageKind: AUMCheckpoint, State: fakeState}))
 
 	storage := &compactingChonkFake{
 		Mem:        (*c.Chonk().(*Mem)),
 		aumAge:     map[AUMHash]time.Time{(c.AUMHashes["F1"]): time.Now()},
 		t:          t,
-		wantDelete: []AUMHash{c.AUMHashes["A"], c.AUMHashes["B"], c.AUMHashes["OLD"]},
+		wantDelete: []AUMHash{c.AUMHashes["A"], c.AUMHashes["B"], c.AUMHashes["O"], c.AUMHashes["OLD"]},
 	}
 
 	lastActiveAncestor, err := Compact(storage, c.AUMHashes["H"], CompactionOptions{MinChain: 2, MinAge: time.Hour})
@@ -681,3 +686,87 @@ func TestCompact(t *testing.T) {
 		}
 	}
 }
+
+func TestCollectGarbage(t *testing.T) {
+	fakeState := &State{
+		Keys:               []Key{{Kind: Key25519, Votes: 1}},
+		DisablementSecrets: [][]byte{bytes.Repeat([]byte{1}, 32)},
+	}
+
+	c := newTestchain(t, `
+        A -> B -> C -> C2 -> D -> E -> F -> G -> H
+                       |  -> OLD            | -> G2
+
+        // make {A,B,C,D} compaction candidates
+        A.template = checkpoint
+        B.template = checkpoint
+        C.template = checkpoint
+        D.template = checkpoint
+
+        // tweak seeds of forks so hashes arent identical
+        OLD.hashSeed = 2
+        G2.hashSeed = 3
+    `, optTemplate("checkpoint", AUM{MessageKind: AUMCheckpoint, State: fakeState}))
+
+	// Populate a *FS chonk.
+	storage, err := ChonkDir(t.TempDir())
+	if err != nil {
+		t.Fatal(err)
+	}
+	for _, update := range c.AUMs {
+		if err := storage.CommitVerifiedAUMs([]AUM{update}); err != nil {
+			t.Fatal(err)
+		}
+	}
+	if err := storage.SetLastActiveAncestor(c.AUMHashes["A"]); err != nil {
+		t.Fatal(err)
+	}
+
+	// Run compaction.
+	lastActiveAncestor, err := Compact(storage, c.AUMHashes["H"], CompactionOptions{MinChain: 2, MinAge: 1})
+	if err != nil {
+		t.Errorf("Compact() failed: %v", err)
+	}
+	if lastActiveAncestor != c.AUMHashes["D"] {
+		t.Errorf("last active ancestor = %v, want %v", lastActiveAncestor, c.AUMHashes["C"])
+	}
+
+	deletedAUMs := []AUMHash{c.AUMHashes["A"], c.AUMHashes["B"], c.AUMHashes["C"], c.AUMHashes["C2"], c.AUMHashes["OLD"]}
+
+	// Make sure deleted AUMs are unreadable.
+	for _, h := range deletedAUMs {
+		if _, err := storage.AUM(h); err != os.ErrNotExist {
+			t.Errorf("storage.AUM(%v).err = %v, want ErrNotExist", h, err)
+		}
+	}
+
+	if err := storage.CollectGarbage(0); err != nil {
+		t.Fatal(err)
+	}
+
+	// Make sure files for deleted AUMs are gone.
+	for _, h := range deletedAUMs {
+		dir, base := storage.aumDir(h)
+		path := filepath.Join(dir, base)
+		// C2 is excluded, because its child D exists and the file
+		// stores the parent->child relationship.
+		if _, err := os.Stat(path); err == nil && h != c.AUMHashes["C2"] {
+			t.Errorf("file for deleted AUM %v exists", h)
+		}
+	}
+
+	if t.Failed() {
+		for name, hash := range c.AUMHashes {
+			t.Logf("AUM[%q] = %v", name, hash)
+		}
+	}
+
+	// Lastly, lets make sure an authority can start from the garbage-collected state.
+	a, err := Open(storage)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if a.Head() != c.AUMHashes["H"] {
+		t.Errorf("head = %v, want %v", a.Head(), c.AUMHashes["H"])
+	}
+}

tool/gocross/autoflags.go

@@ -104,10 +104,15 @@ func autoflagsForTest(argv []string, env *Environment, goroot, nativeGOOS, nativ
 		if env.IsSet("XCODE_VERSION_ACTUAL") {
 			var xcodeFlags []string
 			// Minimum OS version being targeted, results in
-			// e.g. -mmacosx-version-min=11.3
-			minOSKey := env.Get("DEPLOYMENT_TARGET_CLANG_FLAG_NAME", "")
-			minOSVal := env.Get(env.Get("DEPLOYMENT_TARGET_CLANG_ENV_NAME", ""), "")
-			xcodeFlags = append(xcodeFlags, fmt.Sprintf("-%s=%s", minOSKey, minOSVal))
+			// e.g. -mmacosx-version-min=11.3, -miphoneos-version-min=15.0
+			switch {
+			case env.IsSet("IPHONEOS_DEPLOYMENT_TARGET"):
+				xcodeFlags = append(xcodeFlags, "-miphoneos-version-min="+env.Get("IPHONEOS_DEPLOYMENT_TARGET", ""))
+			case env.IsSet("MACOSX_DEPLOYMENT_TARGET"):
+				xcodeFlags = append(xcodeFlags, "-mmacosx-version-min="+env.Get("MACOSX_DEPLOYMENT_TARGET", ""))
+			default:
+				return nil, nil, fmt.Errorf("invoked by Xcode but couldn't figure out deployment target. Did Xcode change its envvars again?")
+			}
 
 			// Target-specific SDK directory. Must be passed as two
 			// words ("-isysroot PATH", not "-isysroot=PATH").

tool/gocross/autoflags_test.go

@@ -277,13 +277,11 @@ TS_LINK_FAIL_REFLECT=1 (was <nil>)`,
 		{
 			name: "darwin_arm64_to_darwin_amd64_xcode",
 			env: map[string]string{
-				"GOOS":                              "darwin",
-				"GOARCH":                            "amd64",
-				"XCODE_VERSION_ACTUAL":              "1300",
-				"DEPLOYMENT_TARGET_CLANG_FLAG_NAME": "mmacosx-version-min",
-				"MACOSX_DEPLOYMENT_TARGET":          "11.3",
-				"DEPLOYMENT_TARGET_CLANG_ENV_NAME":  "MACOSX_DEPLOYMENT_TARGET",
-				"SDKROOT":                           "/my/sdk/root",
+				"GOOS":                     "darwin",
+				"GOARCH":                   "amd64",
+				"XCODE_VERSION_ACTUAL":     "1300",
+				"MACOSX_DEPLOYMENT_TARGET": "11.3",
+				"SDKROOT":                  "/my/sdk/root",
 			},
 			argv:         []string{"gocross", "build", "./cmd/tailcontrol"},
 			goroot:       "/goroot",
@@ -308,6 +306,38 @@ TS_LINK_FAIL_REFLECT=0 (was <nil>)`,
 				"./cmd/tailcontrol",
 			},
 		},
+		{
+			name: "darwin_amd64_to_ios_arm64_xcode",
+			env: map[string]string{
+				"GOOS":                       "ios",
+				"GOARCH":                     "arm64",
+				"XCODE_VERSION_ACTUAL":       "1300",
+				"IPHONEOS_DEPLOYMENT_TARGET": "15.0",
+				"SDKROOT":                    "/my/sdk/root",
+			},
+			argv:         []string{"gocross", "build", "./cmd/tailcontrol"},
+			goroot:       "/goroot",
+			nativeGOOS:   "darwin",
+			nativeGOARCH: "amd64",
+
+			envDiff: `CC=cc (was <nil>)
+CGO_CFLAGS=-O3 -std=gnu11 -miphoneos-version-min=15.0 -isysroot /my/sdk/root -arch arm64 (was <nil>)
+CGO_ENABLED=1 (was <nil>)
+CGO_LDFLAGS=-miphoneos-version-min=15.0 -isysroot /my/sdk/root -arch arm64 (was <nil>)
+GOARCH=arm64 (was arm64)
+GOARM=5 (was <nil>)
+GOMIPS=softfloat (was <nil>)
+GOOS=ios (was ios)
+GOROOT=/goroot (was <nil>)
+TS_LINK_FAIL_REFLECT=1 (was <nil>)`,
+			wantArgv: []string{
+				"gocross", "build",
+				"-trimpath",
+				"-tags=tailscale_go,omitidna,omitpemdecrypt",
+				"-ldflags", "-X tailscale.com/version.longStamp=1.2.3-long -X tailscale.com/version.shortStamp=1.2.3 -X tailscale.com/version.gitCommitStamp=abcd -X tailscale.com/version.extraGitCommitStamp=defg -w",
+				"./cmd/tailcontrol",
+			},
+		},
 		{
 			name:         "linux_amd64_to_linux_amd64_in_goroot",
 			argv:         []string{"go", "build", "./cmd/tailcontrol"},

tool/gocross/gocross-wrapper.sh

@@ -19,8 +19,24 @@ fi
 (
 repo_root="$(dirname $0)/../.."
 
+# Figuring out if gocross needs a rebuild, as well as the rebuild itself, need
+# to happen with CWD inside this repo. Since we're in a subshell entirely
+# dedicated to wrangling gocross and toolchains, cd over now before doing
+# anything further so that the rest of this logic works the same if gocross is
+# being invoked from somewhere else.
+cd "$repo_root"
+
 toolchain="$HOME/.cache/tailscale-go"
 
+if [ -d "$toolchain" ]; then
+    # A toolchain exists, but is it recent enough to compile gocross? If not,
+    # wipe it out so that the next if block fetches a usable one.
+    want_go_minor=$(egrep '^go ' "$repo_root/go.mod" | cut -f2 -d'.')
+    have_go_minor=$(cut -f2 -d'.' <$toolchain/VERSION)
+    if [ -z "$have_go_minor" -o "$have_go_minor" -lt "$want_go_minor" ]; then
+        rm -rf "$toolchain" "$toolchain.extracted"
+    fi
+fi
 if [ ! -d "$toolchain" ]; then
     mkdir -p "$HOME/.cache"
 
@@ -53,6 +69,7 @@ if [ ! -d "$toolchain" ]; then
         mkdir -p "$toolchain"
         (cd "$toolchain" && tar --strip-components=1 -xf "$toolchain.tar.gz")
         echo "$REV" >"$toolchain.extracted"
+        rm -f "$toolchain.tar.gz"
         ;;
     esac
 fi
@@ -65,9 +82,9 @@ fi
 # Anyway, build gocross in a stripped down universe.
 gocross_path="$repo_root/gocross"
 gocross_ok=0
+wantver="$(git rev-parse HEAD)"
 if [ -x "$gocross_path" ]; then
 	gotver="$($gocross_path gocross-version 2>/dev/null || echo '')"
-	wantver="$(git rev-parse HEAD)"
 	if [ "$gotver" = "$wantver" ]; then
 		gocross_ok=1
 	fi
@@ -78,7 +95,7 @@ if [ "$gocross_ok" = "0" ]; then
     unset GO111MODULE
     unset GOROOT
     export CGO_ENABLED=0
-    "$toolchain/bin/go" build -o "$gocross_path" -ldflags='-X tailscale.com/version/gitCommitStamp=$wantver' tailscale.com/tool/gocross
+    "$toolchain/bin/go" build -o "$gocross_path" -ldflags "-X tailscale.com/version.gitCommitStamp=$wantver" tailscale.com/tool/gocross
 fi
 ) # End of the subshell execution.
 

tool/gocross/toolchain.go

@@ -15,11 +15,21 @@ import (
 )
 
 func toolchainRev() (string, error) {
-	cwd, err := os.Getwd()
+	// gocross gets built in the root of the repo that has toolchain
+	// information, so we can use os.Args[0] to locate toolchain info.
+	//
+	// We might be getting invoked via the synthetic goroot that we create, so
+	// walk symlinks to find the true location of gocross.
+	start, err := os.Executable()
 	if err != nil {
-		return "", fmt.Errorf("getting CWD: %v", err)
+		return "", err
 	}
-	d := cwd
+	start, err = filepath.EvalSymlinks(start)
+	if err != nil {
+		return "", fmt.Errorf("evaluating symlinks in %q: %v", os.Args[0], err)
+	}
+	start = filepath.Dir(start)
+	d := start
 findTopLevel:
 	for {
 		if _, err := os.Lstat(filepath.Join(d, ".git")); err == nil {
@@ -29,7 +39,7 @@ findTopLevel:
 		}
 		d = filepath.Dir(d)
 		if d == "/" {
-			return "", fmt.Errorf("couldn't find .git starting from %q, cannot manage toolchain", cwd)
+			return "", fmt.Errorf("couldn't find .git starting from %q, cannot manage toolchain", start)
 		}
 	}
 
@@ -39,9 +49,6 @@ findTopLevel:
 func readRevFile(path string) (string, error) {
 	bs, err := os.ReadFile(path)
 	if err != nil {
-		if os.IsNotExist(err) {
-			return "", nil
-		}
 		return "", err
 	}
 	return string(bytes.TrimSpace(bs)), nil

tsnet/tsnet.go

@@ -12,6 +12,7 @@ import (
 	"crypto/tls"
 	"encoding/hex"
 	"errors"
+	"flag"
 	"fmt"
 	"io"
 	"log"
@@ -47,6 +48,7 @@ import (
 	"tailscale.com/net/tsdial"
 	"tailscale.com/smallzstd"
 	"tailscale.com/types/logger"
+	"tailscale.com/types/logid"
 	"tailscale.com/types/nettype"
 	"tailscale.com/util/mak"
 	"tailscale.com/wgengine"
@@ -54,6 +56,8 @@ import (
 	"tailscale.com/wgengine/netstack"
 )
 
+func inTest() bool { return flag.Lookup("test.v") != nil }
+
 // Server is an embedded Tailscale server.
 //
 // Its exported fields may be changed until the first call to Listen.
@@ -83,7 +87,7 @@ type Server struct {
 	Logf logger.Logf
 
 	// Ephemeral, if true, specifies that the instance should register
-	// as an Ephemeral node (https://tailscale.com/kb/1111/ephemeral-nodes/).
+	// as an Ephemeral node (https://tailscale.com/s/ephemeral-nodes).
 	Ephemeral bool
 
 	// AuthKey, if non-empty, is the auth key to create the node
@@ -97,6 +101,8 @@ type Server struct {
 	// If empty, the Tailscale default is used.
 	ControlURL string
 
+	getCertForTesting func(*tls.ClientHelloInfo) (*tls.Certificate, error)
+
 	initOnce         sync.Once
 	initErr          error
 	lb               *ipnlocal.LocalBackend
@@ -113,11 +119,12 @@ type Server struct {
 	localClient      *tailscale.LocalClient // in-memory
 	logbuffer        *filch.Filch
 	logtail          *logtail.Logger
-	logid            string
+	logid            logid.PublicID
 
 	mu        sync.Mutex
 	listeners map[listenKey]*listener
 	dialer    *tsdial.Dialer
+	closed    bool
 }
 
 // Dial connects to the address on the tailnet.
@@ -303,6 +310,11 @@ func (s *Server) Up(ctx context.Context) (*ipnstate.Status, error) {
 //
 // It must not be called before or concurrently with Start.
 func (s *Server) Close() error {
+	s.mu.Lock()
+	defer s.mu.Unlock()
+	if s.closed {
+		return fmt.Errorf("tsnet: %w", net.ErrClosed)
+	}
 	ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
 	defer cancel()
 	var wg sync.WaitGroup
@@ -350,14 +362,12 @@ func (s *Server) Close() error {
 		s.loopbackListener.Close()
 	}
 
-	s.mu.Lock()
-	defer s.mu.Unlock()
 	for _, ln := range s.listeners {
-		ln.Close()
+		ln.closeLocked()
 	}
-	s.listeners = nil
 
 	wg.Wait()
+	s.closed = true
 	return nil
 }
 
@@ -452,44 +462,9 @@ func (s *Server) start() (reterr error) {
 		return fmt.Errorf("%v is not a directory", s.rootPath)
 	}
 
-	cfgPath := filepath.Join(s.rootPath, "tailscaled.log.conf")
-
-	lpc, err := logpolicy.ConfigFromFile(cfgPath)
-	switch {
-	case os.IsNotExist(err):
-		lpc = logpolicy.NewConfig(logtail.CollectionNode)
-		if err := lpc.Save(cfgPath); err != nil {
-			return fmt.Errorf("logpolicy.Config.Save for %v: %w", cfgPath, err)
-		}
-	case err != nil:
-		return fmt.Errorf("logpolicy.LoadConfig for %v: %w", cfgPath, err)
+	if err := s.startLogger(&closePool); err != nil {
+		return err
 	}
-	if err := lpc.Validate(logtail.CollectionNode); err != nil {
-		return fmt.Errorf("logpolicy.Config.Validate for %v: %w", cfgPath, err)
-	}
-	s.logid = lpc.PublicID.String()
-
-	s.logbuffer, err = filch.New(filepath.Join(s.rootPath, "tailscaled"), filch.Options{ReplaceStderr: false})
-	if err != nil {
-		return fmt.Errorf("error creating filch: %w", err)
-	}
-	closePool.add(s.logbuffer)
-	c := logtail.Config{
-		Collection: lpc.Collection,
-		PrivateID:  lpc.PrivateID,
-		Stderr:     io.Discard, // log everything to Buffer
-		Buffer:     s.logbuffer,
-		NewZstdEncoder: func() logtail.Encoder {
-			w, err := smallzstd.NewEncoder(nil)
-			if err != nil {
-				panic(err)
-			}
-			return w
-		},
-		HTTPC: &http.Client{Transport: logpolicy.NewLogtailTransport(logtail.DefaultHost)},
-	}
-	s.logtail = logtail.NewLogger(c, logf)
-	closePool.addFunc(func() { s.logtail.Shutdown(context.Background()) })
 
 	s.linkMon, err = monitor.New(logf)
 	if err != nil {
@@ -597,6 +572,50 @@ func (s *Server) start() (reterr error) {
 	return nil
 }
 
+func (s *Server) startLogger(closePool *closeOnErrorPool) error {
+	if inTest() {
+		return nil
+	}
+	cfgPath := filepath.Join(s.rootPath, "tailscaled.log.conf")
+	lpc, err := logpolicy.ConfigFromFile(cfgPath)
+	switch {
+	case os.IsNotExist(err):
+		lpc = logpolicy.NewConfig(logtail.CollectionNode)
+		if err := lpc.Save(cfgPath); err != nil {
+			return fmt.Errorf("logpolicy.Config.Save for %v: %w", cfgPath, err)
+		}
+	case err != nil:
+		return fmt.Errorf("logpolicy.LoadConfig for %v: %w", cfgPath, err)
+	}
+	if err := lpc.Validate(logtail.CollectionNode); err != nil {
+		return fmt.Errorf("logpolicy.Config.Validate for %v: %w", cfgPath, err)
+	}
+	s.logid = lpc.PublicID
+
+	s.logbuffer, err = filch.New(filepath.Join(s.rootPath, "tailscaled"), filch.Options{ReplaceStderr: false})
+	if err != nil {
+		return fmt.Errorf("error creating filch: %w", err)
+	}
+	closePool.add(s.logbuffer)
+	c := logtail.Config{
+		Collection: lpc.Collection,
+		PrivateID:  lpc.PrivateID,
+		Stderr:     io.Discard, // log everything to Buffer
+		Buffer:     s.logbuffer,
+		NewZstdEncoder: func() logtail.Encoder {
+			w, err := smallzstd.NewEncoder(nil)
+			if err != nil {
+				panic(err)
+			}
+			return w
+		},
+		HTTPC: &http.Client{Transport: logpolicy.NewLogtailTransport(logtail.DefaultHost)},
+	}
+	s.logtail = logtail.NewLogger(c, s.logf)
+	closePool.addFunc(func() { s.logtail.Shutdown(context.Background()) })
+	return nil
+}
+
 type closeOnErrorPool []func()
 
 func (p *closeOnErrorPool) add(c io.Closer)   { *p = append(*p, func() { c.Close() }) }
@@ -822,12 +841,7 @@ func (s *Server) ListenTLS(network, addr string) (net.Listener, error) {
 		return nil, err
 	}
 	if len(st.CertDomains) == 0 {
-		return nil, errors.New("tsnet: you must enable HTTPS in the admin panel to proceed")
-	}
-
-	lc, err := s.LocalClient() // do local client first before listening.
-	if err != nil {
-		return nil, err
+		return nil, errors.New("tsnet: you must enable HTTPS in the admin panel to proceed. See https://tailscale.com/s/https")
 	}
 
 	ln, err := s.listen(network, addr, listenOnTailnet)
@@ -835,10 +849,25 @@ func (s *Server) ListenTLS(network, addr string) (net.Listener, error) {
 		return nil, err
 	}
 	return tls.NewListener(ln, &tls.Config{
-		GetCertificate: lc.GetCertificate,
+		GetCertificate: s.getCert,
 	}), nil
 }
 
+// getCert is the GetCertificate function used by ListenTLS.
+//
+// It calls GetCertificate on the localClient, passing in the ClientHelloInfo.
+// For testing, if s.getCertForTesting is set, it will call that instead.
+func (s *Server) getCert(hi *tls.ClientHelloInfo) (*tls.Certificate, error) {
+	if s.getCertForTesting != nil {
+		return s.getCertForTesting(hi)
+	}
+	lc, err := s.LocalClient()
+	if err != nil {
+		return nil, err
+	}
+	return lc.GetCertificate(hi)
+}
+
 // FunnelOption is an option passed to ListenFunnel to configure the listener.
 type FunnelOption interface {
 	funnelOption()
@@ -892,10 +921,7 @@ func (s *Server) ListenFunnel(network, addr string, opts ...FunnelOption) (net.L
 		return nil, err
 	}
 
-	lc, err := s.LocalClient()
-	if err != nil {
-		return nil, err
-	}
+	lc := s.localClient
 
 	// May not have funnel enabled. Enable it.
 	srvConfig, err := lc.GetServeConfig(ctx)
@@ -927,7 +953,7 @@ func (s *Server) ListenFunnel(network, addr string, opts ...FunnelOption) (net.L
 		return nil, err
 	}
 	return tls.NewListener(ln, &tls.Config{
-		GetCertificate: lc.GetCertificate,
+		GetCertificate: s.getCert,
 	}), nil
 }
 
@@ -1017,10 +1043,11 @@ type listenKey struct {
 }
 
 type listener struct {
-	s    *Server
-	keys []listenKey
-	addr string
-	conn chan net.Conn
+	s      *Server
+	keys   []listenKey
+	addr   string
+	conn   chan net.Conn
+	closed bool // guarded by s.mu
 }
 
 func (ln *listener) Accept() (net.Conn, error) {
@@ -1032,15 +1059,26 @@ func (ln *listener) Accept() (net.Conn, error) {
 }
 
 func (ln *listener) Addr() net.Addr { return addr{ln} }
+
 func (ln *listener) Close() error {
 	ln.s.mu.Lock()
 	defer ln.s.mu.Unlock()
+	return ln.closeLocked()
+}
+
+// closeLocked closes the listener.
+// It must be called with ln.s.mu held.
+func (ln *listener) closeLocked() error {
+	if ln.closed {
+		return fmt.Errorf("tsnet: %w", net.ErrClosed)
+	}
 	for _, key := range ln.keys {
 		if v, ok := ln.s.listeners[key]; ok && v == ln {
 			delete(ln.s.listeners, key)
 		}
 	}
 	close(ln.conn)
+	ln.closed = true
 	return nil
 }
 

tsnet/tsnet_test.go

@@ -4,26 +4,40 @@
 package tsnet
 
 import (
+	"bufio"
 	"context"
+	"crypto/ecdsa"
+	"crypto/elliptic"
+	"crypto/rand"
+	"crypto/tls"
+	"crypto/x509"
+	"crypto/x509/pkix"
 	"errors"
 	"flag"
 	"fmt"
 	"io"
+	"io/ioutil"
+	"math/big"
+	"net"
 	"net/http"
 	"net/http/httptest"
 	"net/netip"
 	"os"
 	"path/filepath"
+	"strings"
+	"sync"
 	"testing"
 	"time"
 
 	"golang.org/x/net/proxy"
+	"tailscale.com/ipn"
 	"tailscale.com/ipn/store/mem"
 	"tailscale.com/net/netns"
 	"tailscale.com/tailcfg"
 	"tailscale.com/tstest/integration"
 	"tailscale.com/tstest/integration/testcontrol"
 	"tailscale.com/types/logger"
+	"tailscale.com/util/must"
 )
 
 // TestListener_Server ensures that the listener type always keeps the Server
@@ -92,6 +106,10 @@ func startControl(t *testing.T) (controlURL string) {
 	derpMap := integration.RunDERPAndSTUN(t, derpLogf, "127.0.0.1")
 	control := &testcontrol.Server{
 		DERPMap: derpMap,
+		DNSConfig: &tailcfg.DNSConfig{
+			Proxied: true,
+		},
+		MagicDNSDomain: "tail-scale.ts.net",
 	}
 	control.HTTPTestServer = httptest.NewUnstartedServer(control)
 	control.HTTPTestServer.Start()
@@ -101,17 +119,96 @@ func startControl(t *testing.T) (controlURL string) {
 	return controlURL
 }
 
+type testCertIssuer struct {
+	mu    sync.Mutex
+	certs map[string]*tls.Certificate
+
+	root    *x509.Certificate
+	rootKey *ecdsa.PrivateKey
+}
+
+func newCertIssuer() *testCertIssuer {
+	rootKey, err := ecdsa.GenerateKey(elliptic.P256(), rand.Reader)
+	if err != nil {
+		panic(err)
+	}
+	t := &x509.Certificate{
+		SerialNumber: big.NewInt(1),
+		Subject: pkix.Name{
+			CommonName: "root",
+		},
+		NotBefore:             time.Now(),
+		NotAfter:              time.Now().Add(time.Hour),
+		IsCA:                  true,
+		ExtKeyUsage:           []x509.ExtKeyUsage{x509.ExtKeyUsageServerAuth},
+		KeyUsage:              x509.KeyUsageCertSign,
+		BasicConstraintsValid: true,
+	}
+	rootDER, err := x509.CreateCertificate(rand.Reader, t, t, &rootKey.PublicKey, rootKey)
+	if err != nil {
+		panic(err)
+	}
+	rootCA, err := x509.ParseCertificate(rootDER)
+	if err != nil {
+		panic(err)
+	}
+	return &testCertIssuer{
+		certs:   make(map[string]*tls.Certificate),
+		root:    rootCA,
+		rootKey: rootKey,
+	}
+}
+
+func (tci *testCertIssuer) getCert(chi *tls.ClientHelloInfo) (*tls.Certificate, error) {
+	tci.mu.Lock()
+	defer tci.mu.Unlock()
+	cert, ok := tci.certs[chi.ServerName]
+	if ok {
+		return cert, nil
+	}
+
+	certPrivKey, err := ecdsa.GenerateKey(elliptic.P256(), rand.Reader)
+	if err != nil {
+		return nil, err
+	}
+	certTmpl := &x509.Certificate{
+		SerialNumber: big.NewInt(1),
+		DNSNames:     []string{chi.ServerName},
+		NotBefore:    time.Now(),
+		NotAfter:     time.Now().Add(time.Hour),
+	}
+	certDER, err := x509.CreateCertificate(rand.Reader, certTmpl, tci.root, &certPrivKey.PublicKey, tci.rootKey)
+	if err != nil {
+		return nil, err
+	}
+	cert = &tls.Certificate{
+		Certificate: [][]byte{certDER, tci.root.Raw},
+		PrivateKey:  certPrivKey,
+	}
+	tci.certs[chi.ServerName] = cert
+	return cert, nil
+}
+
+func (tci *testCertIssuer) Pool() *x509.CertPool {
+	p := x509.NewCertPool()
+	p.AddCert(tci.root)
+	return p
+}
+
+var testCertRoot = newCertIssuer()
+
 func startServer(t *testing.T, ctx context.Context, controlURL, hostname string) (*Server, netip.Addr) {
 	t.Helper()
 
 	tmp := filepath.Join(t.TempDir(), hostname)
 	os.MkdirAll(tmp, 0755)
 	s := &Server{
-		Dir:        tmp,
-		ControlURL: controlURL,
-		Hostname:   hostname,
-		Store:      new(mem.Store),
-		Ephemeral:  true,
+		Dir:               tmp,
+		ControlURL:        controlURL,
+		Hostname:          hostname,
+		Store:             new(mem.Store),
+		Ephemeral:         true,
+		getCertForTesting: testCertRoot.getCert,
 	}
 	if !*verboseNodes {
 		s.Logf = logger.Discard
@@ -344,3 +441,135 @@ func TestTailscaleIPs(t *testing.T) {
 			sIp4, upIp4, sIp6, upIp6)
 	}
 }
+
+// TestListenerCleanup is a regression test to verify that s.Close doesn't
+// deadlock if a listener is still open.
+func TestListenerCleanup(t *testing.T) {
+	ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
+	defer cancel()
+
+	controlURL := startControl(t)
+	s1, _ := startServer(t, ctx, controlURL, "s1")
+
+	ln, err := s1.Listen("tcp", ":8081")
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	if err := s1.Close(); err != nil {
+		t.Fatal(err)
+	}
+
+	if err := ln.Close(); !errors.Is(err, net.ErrClosed) {
+		t.Fatalf("second ln.Close error: %v, want net.ErrClosed", err)
+	}
+}
+
+func TestFunnel(t *testing.T) {
+	ctx, dialCancel := context.WithTimeout(context.Background(), 30*time.Second)
+	defer dialCancel()
+
+	controlURL := startControl(t)
+	s1, _ := startServer(t, ctx, controlURL, "s1")
+	s2, _ := startServer(t, ctx, controlURL, "s2")
+
+	ln := must.Get(s1.ListenFunnel("tcp", ":443"))
+	defer ln.Close()
+	wantSrcAddrPort := netip.MustParseAddrPort("127.0.0.1:1234")
+	wantTarget := ipn.HostPort("s1.tail-scale.ts.net:443")
+	srv := &http.Server{
+		ConnContext: func(ctx context.Context, c net.Conn) context.Context {
+			tc, ok := c.(*tls.Conn)
+			if !ok {
+				t.Errorf("ConnContext called with non-TLS conn: %T", c)
+			}
+			if fc, ok := tc.NetConn().(*ipn.FunnelConn); !ok {
+				t.Errorf("ConnContext called with non-FunnelConn: %T", c)
+			} else if fc.Src != wantSrcAddrPort {
+				t.Errorf("ConnContext called with wrong SrcAddrPort; got %v, want %v", fc.Src, wantSrcAddrPort)
+			} else if fc.Target != wantTarget {
+				t.Errorf("ConnContext called with wrong Target; got %q, want %q", fc.Target, wantTarget)
+			}
+			return ctx
+		},
+		Handler: http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+			fmt.Fprintf(w, "hello")
+		}),
+	}
+	go srv.Serve(ln)
+
+	c := &http.Client{
+		Transport: &http.Transport{
+			DialContext: func(ctx context.Context, network, addr string) (net.Conn, error) {
+				return dialIngressConn(s2, s1, addr)
+			},
+			TLSClientConfig: &tls.Config{
+				RootCAs: testCertRoot.Pool(),
+			},
+		},
+	}
+	resp, err := c.Get("https://s1.tail-scale.ts.net:443")
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer resp.Body.Close()
+	if resp.StatusCode != 200 {
+		t.Errorf("unexpected status code: %v", resp.StatusCode)
+		return
+	}
+	body, err := ioutil.ReadAll(resp.Body)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if string(body) != "hello" {
+		t.Errorf("unexpected body: %q", body)
+	}
+}
+
+func dialIngressConn(from, to *Server, target string) (net.Conn, error) {
+	toLC := must.Get(to.LocalClient())
+	toStatus := must.Get(toLC.StatusWithoutPeers(context.Background()))
+	peer6 := toStatus.Self.PeerAPIURL[1] // IPv6
+	toPeerAPI, ok := strings.CutPrefix(peer6, "http://")
+	if !ok {
+		return nil, fmt.Errorf("unexpected PeerAPIURL %q", peer6)
+	}
+
+	dialCtx, dialCancel := context.WithTimeout(context.Background(), 30*time.Second)
+	outConn, err := from.Dial(dialCtx, "tcp", toPeerAPI)
+	dialCancel()
+	if err != nil {
+		return nil, err
+	}
+
+	req, err := http.NewRequest("POST", "/v0/ingress", nil)
+	if err != nil {
+		return nil, err
+	}
+	req.Host = toPeerAPI
+	req.Header.Set("Tailscale-Ingress-Src", "127.0.0.1:1234")
+	req.Header.Set("Tailscale-Ingress-Target", target)
+	if err := req.Write(outConn); err != nil {
+		return nil, err
+	}
+
+	br := bufio.NewReader(outConn)
+	res, err := http.ReadResponse(br, req)
+	if err != nil {
+		return nil, err
+	}
+	defer res.Body.Close() // just to appease vet
+	if res.StatusCode != 101 {
+		return nil, fmt.Errorf("unexpected status code: %v", res.StatusCode)
+	}
+	return &bufferedConn{outConn, br}, nil
+}
+
+type bufferedConn struct {
+	net.Conn
+	reader *bufio.Reader
+}
+
+func (c *bufferedConn) Read(b []byte) (int, error) {
+	return c.reader.Read(b)
+}

tstest/integration/integration_test.go

@@ -39,6 +39,7 @@ import (
 	"tailscale.com/tailcfg"
 	"tailscale.com/tstest"
 	"tailscale.com/tstest/integration/testcontrol"
+	"tailscale.com/types/key"
 	"tailscale.com/types/logger"
 )
 
@@ -503,6 +504,110 @@ func TestOneNodeUpWindowsStyle(t *testing.T) {
 	d1.MustCleanShutdown(t)
 }
 
+// TestNATPing creates two nodes, n1 and n2, sets up masquerades for both and
+// tries to do bi-directional pings between them.
+func TestNATPing(t *testing.T) {
+	t.Parallel()
+	env := newTestEnv(t)
+	registerNode := func() (*testNode, key.NodePublic) {
+		n := newTestNode(t, env)
+		n.StartDaemon()
+		n.AwaitListening()
+		n.MustUp()
+		n.AwaitRunning()
+		k := n.MustStatus().Self.PublicKey
+		return n, k
+	}
+	n1, k1 := registerNode()
+	n2, k2 := registerNode()
+
+	n1IP := n1.AwaitIP()
+	n2IP := n2.AwaitIP()
+
+	n1ExternalIP := netip.MustParseAddr("100.64.1.1")
+	n2ExternalIP := netip.MustParseAddr("100.64.2.1")
+
+	tests := []struct {
+		name       string
+		pairs      []testcontrol.MasqueradePair
+		n1SeesN2IP netip.Addr
+		n2SeesN1IP netip.Addr
+	}{
+		{
+			name:       "no_nat",
+			n1SeesN2IP: n2IP,
+			n2SeesN1IP: n1IP,
+		},
+		{
+			name: "n1_has_external_ip",
+			pairs: []testcontrol.MasqueradePair{
+				{
+					Node:              k1,
+					Peer:              k2,
+					NodeMasqueradesAs: n1ExternalIP,
+				},
+			},
+			n1SeesN2IP: n2IP,
+			n2SeesN1IP: n1ExternalIP,
+		},
+		{
+			name: "n2_has_external_ip",
+			pairs: []testcontrol.MasqueradePair{
+				{
+					Node:              k2,
+					Peer:              k1,
+					NodeMasqueradesAs: n2ExternalIP,
+				},
+			},
+			n1SeesN2IP: n2ExternalIP,
+			n2SeesN1IP: n1IP,
+		},
+		{
+			name: "both_have_external_ips",
+			pairs: []testcontrol.MasqueradePair{
+				{
+					Node:              k1,
+					Peer:              k2,
+					NodeMasqueradesAs: n1ExternalIP,
+				},
+				{
+					Node:              k2,
+					Peer:              k1,
+					NodeMasqueradesAs: n2ExternalIP,
+				},
+			},
+			n1SeesN2IP: n2ExternalIP,
+			n2SeesN1IP: n1ExternalIP,
+		},
+	}
+
+	for _, tc := range tests {
+		t.Run(tc.name, func(t *testing.T) {
+			env.Control.SetMasqueradeAddresses(tc.pairs)
+
+			s1 := n1.MustStatus()
+			n2AsN1Peer := s1.Peer[k2]
+			if got := n2AsN1Peer.TailscaleIPs[0]; got != tc.n1SeesN2IP {
+				t.Fatalf("n1 sees n2 as %v; want %v", got, tc.n1SeesN2IP)
+			}
+
+			s2 := n2.MustStatus()
+			n1AsN2Peer := s2.Peer[k1]
+			if got := n1AsN2Peer.TailscaleIPs[0]; got != tc.n2SeesN1IP {
+				t.Fatalf("n2 sees n1 as %v; want %v", got, tc.n2SeesN1IP)
+			}
+
+			if err := n1.Tailscale("ping", tc.n1SeesN2IP.String()).Run(); err != nil {
+				t.Fatal(err)
+			}
+
+			if err := n2.Tailscale("ping", tc.n2SeesN1IP.String()).Run(); err != nil {
+				t.Fatal(err)
+			}
+		})
+	}
+}
+
 func TestLogoutRemovesAllPeers(t *testing.T) {
 	t.Parallel()
 	env := newTestEnv(t)

tstest/integration/tailscaled_deps_test_darwin.go

@@ -40,6 +40,7 @@ import (
 	_ "tailscale.com/types/flagtype"
 	_ "tailscale.com/types/key"
 	_ "tailscale.com/types/logger"
+	_ "tailscale.com/types/logid"
 	_ "tailscale.com/util/clientmetric"
 	_ "tailscale.com/util/multierr"
 	_ "tailscale.com/util/osshare"

tstest/integration/tailscaled_deps_test_freebsd.go

@@ -40,6 +40,7 @@ import (
 	_ "tailscale.com/types/flagtype"
 	_ "tailscale.com/types/key"
 	_ "tailscale.com/types/logger"
+	_ "tailscale.com/types/logid"
 	_ "tailscale.com/util/clientmetric"
 	_ "tailscale.com/util/multierr"
 	_ "tailscale.com/util/osshare"

tstest/integration/tailscaled_deps_test_linux.go

@@ -40,6 +40,7 @@ import (
 	_ "tailscale.com/types/flagtype"
 	_ "tailscale.com/types/key"
 	_ "tailscale.com/types/logger"
+	_ "tailscale.com/types/logid"
 	_ "tailscale.com/util/clientmetric"
 	_ "tailscale.com/util/multierr"
 	_ "tailscale.com/util/osshare"

tstest/integration/tailscaled_deps_test_openbsd.go

@@ -40,6 +40,7 @@ import (
 	_ "tailscale.com/types/flagtype"
 	_ "tailscale.com/types/key"
 	_ "tailscale.com/types/logger"
+	_ "tailscale.com/types/logid"
 	_ "tailscale.com/util/clientmetric"
 	_ "tailscale.com/util/multierr"
 	_ "tailscale.com/util/osshare"

tstest/integration/tailscaled_deps_test_windows.go

@@ -47,6 +47,7 @@ import (
 	_ "tailscale.com/types/flagtype"
 	_ "tailscale.com/types/key"
 	_ "tailscale.com/types/logger"
+	_ "tailscale.com/types/logid"
 	_ "tailscale.com/util/clientmetric"
 	_ "tailscale.com/util/multierr"
 	_ "tailscale.com/util/osshare"

tstest/integration/testcontrol/testcontrol.go

@@ -26,6 +26,7 @@ import (
 
 	"github.com/klauspost/compress/zstd"
 	"go4.org/mem"
+	"golang.org/x/exp/slices"
 	"tailscale.com/net/netaddr"
 	"tailscale.com/net/tsaddr"
 	"tailscale.com/smallzstd"
@@ -39,11 +40,12 @@ const msgLimit = 1 << 20 // encrypted message length limit
 // Server is a control plane server. Its zero value is ready for use.
 // Everything is stored in-memory in one tailnet.
 type Server struct {
-	Logf        logger.Logf      // nil means to use the log package
-	DERPMap     *tailcfg.DERPMap // nil means to use prod DERP map
-	RequireAuth bool
-	Verbose     bool
-	DNSConfig   *tailcfg.DNSConfig // nil means no DNS config
+	Logf           logger.Logf      // nil means to use the log package
+	DERPMap        *tailcfg.DERPMap // nil means to use prod DERP map
+	RequireAuth    bool
+	Verbose        bool
+	DNSConfig      *tailcfg.DNSConfig // nil means no DNS config
+	MagicDNSDomain string
 
 	// ExplicitBaseURL or HTTPTestServer must be set.
 	ExplicitBaseURL string           // e.g. "http://127.0.0.1:1234" with no trailing URL
@@ -58,6 +60,12 @@ type Server struct {
 	pubKey     key.MachinePublic
 	privKey    key.ControlPrivate // not strictly needed vs. MachinePrivate, but handy to test type interactions.
 
+	// masquerades is the set of masquerades that should be applied to
+	// MapResponses sent to clients. It is keyed by the requesting nodes
+	// public key, and then the peer node's public key. The value is the
+	// masquerade address to use for that peer.
+	masquerades map[key.NodePublic]map[key.NodePublic]netip.Addr // node => peer => SelfNodeV4MasqAddrForThisPeer IP
+
 	noisePubKey  key.MachinePublic
 	noisePrivKey key.ControlPrivate // not strictly needed vs. MachinePrivate, but handy to test type interactions.
 
@@ -286,6 +294,48 @@ func (s *Server) serveMachine(w http.ResponseWriter, r *http.Request) {
 	}
 }
 
+// MasqueradePair is a pair of nodes and the IP address that the
+// Node masquerades as for the Peer.
+//
+// Setting this will have future MapResponses for Node to have
+// Peer.SelfNodeV4MasqAddrForThisPeer set to NodeMasqueradesAs.
+// MapResponses for the Peer will now see Node.Addresses as
+// NodeMasqueradesAs.
+type MasqueradePair struct {
+	Node              key.NodePublic
+	Peer              key.NodePublic
+	NodeMasqueradesAs netip.Addr
+}
+
+// SetMasqueradeAddresses sets the masquerade addresses for the server.
+// See MasqueradePair for more details.
+func (s *Server) SetMasqueradeAddresses(pairs []MasqueradePair) {
+	m := make(map[key.NodePublic]map[key.NodePublic]netip.Addr)
+	for _, p := range pairs {
+		if m[p.Node] == nil {
+			m[p.Node] = make(map[key.NodePublic]netip.Addr)
+		}
+		m[p.Node][p.Peer] = p.NodeMasqueradesAs
+	}
+	s.mu.Lock()
+	defer s.mu.Unlock()
+	s.masquerades = m
+	s.updateLocked("SetMasqueradeAddresses", s.nodeIDsLocked(0))
+}
+
+// nodeIDsLocked returns the node IDs of all nodes in the server, except
+// for the node with the given ID.
+func (s *Server) nodeIDsLocked(except tailcfg.NodeID) []tailcfg.NodeID {
+	var ids []tailcfg.NodeID
+	for _, n := range s.nodes {
+		if n.ID == except {
+			continue
+		}
+		ids = append(ids, n.ID)
+	}
+	return ids
+}
+
 // Node returns the node for nodeKey. It's always nil or cloned memory.
 func (s *Server) Node(nodeKey key.NodePublic) *tailcfg.Node {
 	s.mu.Lock()
@@ -328,6 +378,15 @@ func (s *Server) AddFakeNode() {
 	// TODO: send updates to other (non-fake?) nodes
 }
 
+func (s *Server) AllUsers() (users []*tailcfg.User) {
+	s.mu.Lock()
+	defer s.mu.Unlock()
+	for _, u := range s.users {
+		users = append(users, u.Clone())
+	}
+	return users
+}
+
 func (s *Server) AllNodes() (nodes []*tailcfg.Node) {
 	s.mu.Lock()
 	defer s.mu.Unlock()
@@ -494,6 +553,11 @@ func (s *Server) serveRegister(w http.ResponseWriter, r *http.Request, mkey key.
 		Addresses:         allowedIPs,
 		AllowedIPs:        allowedIPs,
 		Hostinfo:          req.Hostinfo.View(),
+		Name:              req.Hostinfo.Hostname,
+		Capabilities: []string{
+			tailcfg.NodeAttrFunnel,
+			tailcfg.CapabilityFunnelPorts + "?ports=8080,443",
+		},
 	}
 	requireAuth := s.RequireAuth
 	if requireAuth && s.nodeKeyAuthed[nk] {
@@ -572,12 +636,7 @@ func (s *Server) UpdateNode(n *tailcfg.Node) (peersToUpdate []tailcfg.NodeID) {
 		panic("zero nodekey")
 	}
 	s.nodes[n.Key] = n.Clone()
-	for _, n2 := range s.nodes {
-		if n.ID != n2.ID {
-			peersToUpdate = append(peersToUpdate, n2.ID)
-		}
-	}
-	return peersToUpdate
+	return s.nodeIDsLocked(n.ID)
 }
 
 func (s *Server) incrInServeMap(delta int) {
@@ -729,6 +788,20 @@ var keepAliveMsg = &struct {
 	KeepAlive: true,
 }
 
+func packetFilterWithIngressCaps() []tailcfg.FilterRule {
+	out := slices.Clone(tailcfg.FilterAllowAll)
+	out = append(out, tailcfg.FilterRule{
+		SrcIPs: []string{"*"},
+		CapGrant: []tailcfg.CapGrant{
+			{
+				Dsts: []netip.Prefix{tsaddr.AllIPv4(), tsaddr.AllIPv6()},
+				Caps: []string{tailcfg.CapabilityIngress},
+			},
+		},
+	})
+	return out
+}
+
 // MapResponse generates a MapResponse for a MapRequest.
 //
 // No updates to s are done here.
@@ -741,26 +814,58 @@ func (s *Server) MapResponse(req *tailcfg.MapRequest) (res *tailcfg.MapResponse,
 	}
 	user, _ := s.getUser(nk)
 	t := time.Date(2020, 8, 3, 0, 0, 0, 1, time.UTC)
+	dns := s.DNSConfig
+	if dns != nil && s.MagicDNSDomain != "" {
+		dns = dns.Clone()
+		dns.CertDomains = []string{
+			fmt.Sprintf(node.Hostinfo.Hostname() + "." + s.MagicDNSDomain),
+		}
+	}
+
 	res = &tailcfg.MapResponse{
 		Node:            node,
 		DERPMap:         s.DERPMap,
 		Domain:          string(user.Domain),
 		CollectServices: "true",
-		PacketFilter:    tailcfg.FilterAllowAll,
+		PacketFilter:    packetFilterWithIngressCaps(),
 		Debug: &tailcfg.Debug{
 			DisableUPnP: "true",
 		},
-		DNSConfig:   s.DNSConfig,
+		DNSConfig:   dns,
 		ControlTime: &t,
 	}
+
+	s.mu.Lock()
+	nodeMasqs := s.masquerades[node.Key]
+	s.mu.Unlock()
 	for _, p := range s.AllNodes() {
-		if p.StableID != node.StableID {
-			res.Peers = append(res.Peers, p)
+		if p.StableID == node.StableID {
+			continue
 		}
+		if masqIP := nodeMasqs[p.Key]; masqIP.IsValid() {
+			p.SelfNodeV4MasqAddrForThisPeer = masqIP
+		}
+
+		s.mu.Lock()
+		peerAddress := s.masquerades[p.Key][node.Key]
+		s.mu.Unlock()
+		if peerAddress.IsValid() {
+			p.Addresses[0] = netip.PrefixFrom(peerAddress, peerAddress.BitLen())
+			p.AllowedIPs[0] = netip.PrefixFrom(peerAddress, peerAddress.BitLen())
+		}
+		res.Peers = append(res.Peers, p)
 	}
+
 	sort.Slice(res.Peers, func(i, j int) bool {
 		return res.Peers[i].ID < res.Peers[j].ID
 	})
+	for _, u := range s.AllUsers() {
+		res.UserProfiles = append(res.UserProfiles, tailcfg.UserProfile{
+			ID:          u.ID,
+			LoginName:   u.LoginName,
+			DisplayName: u.DisplayName,
+		})
+	}
 
 	v4Prefix := netip.PrefixFrom(netaddr.IPv4(100, 64, uint8(tailcfg.NodeID(user.ID)>>8), uint8(tailcfg.NodeID(user.ID))), 32)
 	v6Prefix := netip.PrefixFrom(tsaddr.Tailscale4To6(v4Prefix.Addr()), 128)

util/deephash/deephash_test.go

@@ -575,8 +575,8 @@ func TestGetTypeHasher(t *testing.T) {
 		{
 			name:  "tailcfg.Node",
 			val:   &tailcfg.Node{},
-			out:   "\x01\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\tn\x88\xf1\xff\xff\xff\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\tn\x88\xf1\xff\xff\xff\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00",
-			out32: "\x01\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\tn\x88\xf1\xff\xff\xff\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\tn\x88\xf1\xff\xff\xff\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00",
+			out:   "\x01\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\tn\x88\xf1\xff\xff\xff\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\tn\x88\xf1\xff\xff\xff\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00",
+			out32: "\x01\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\tn\x88\xf1\xff\xff\xff\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\tn\x88\xf1\xff\xff\xff\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00",
 		},
 	}
 	for _, tt := range tests {

version/mkversion/mkversion.go

@@ -150,14 +150,14 @@ func InfoFrom(dir string) (VersionInfo, error) {
 	}
 
 	// Note, this mechanism doesn't correctly support go.mod replacements,
-	// or go workdirs. We only parse out the commit hash from go.mod's
+	// or go workdirs. We only parse out the commit ref from go.mod's
 	// "require" line, nothing else.
-	tailscaleHash, err := tailscaleModuleHash(modBs)
+	tailscaleRef, err := tailscaleModuleRef(modBs)
 	if err != nil {
 		return VersionInfo{}, err
 	}
 
-	v, err := infoFromCache(tailscaleHash, runner)
+	v, err := infoFromCache(tailscaleRef, runner)
 	if err != nil {
 		return VersionInfo{}, err
 	}
@@ -171,9 +171,10 @@ func InfoFrom(dir string) (VersionInfo, error) {
 	return mkOutput(v)
 }
 
-// tailscaleModuleHash returns the git hash of the 'require tailscale.com' line
-// in the given go.mod bytes.
-func tailscaleModuleHash(modBs []byte) (string, error) {
+// tailscaleModuleRef returns the git ref of the 'require tailscale.com' line
+// in the given go.mod bytes. The ref is either a short commit hash, or a git
+// tag.
+func tailscaleModuleRef(modBs []byte) (string, error) {
 	mod, err := modfile.Parse("go.mod", modBs, nil)
 	if err != nil {
 		return "", err
@@ -187,7 +188,8 @@ func tailscaleModuleHash(modBs []byte) (string, error) {
 		if i := strings.LastIndexByte(req.Mod.Version, '-'); i != -1 {
 			return req.Mod.Version[i+1:], nil
 		}
-		return "", fmt.Errorf("couldn't parse git hash from tailscale.com version %q", req.Mod.Version)
+		// If there are no dashes, the version is a tag.
+		return req.Mod.Version, nil
 	}
 	return "", fmt.Errorf("no require tailscale.com line in go.mod")
 }
@@ -310,7 +312,7 @@ type verInfo struct {
 // sentinel patch number.
 const unknownPatchVersion = 9999999
 
-func infoFromCache(shortHash string, runner dirRunner) (verInfo, error) {
+func infoFromCache(ref string, runner dirRunner) (verInfo, error) {
 	cacheDir, err := os.UserCacheDir()
 	if err != nil {
 		return verInfo{}, fmt.Errorf("Getting user cache dir: %w", err)
@@ -324,16 +326,16 @@ func infoFromCache(shortHash string, runner dirRunner) (verInfo, error) {
 		}
 	}
 
-	if !r.ok("git", "cat-file", "-e", shortHash) {
+	if !r.ok("git", "cat-file", "-e", ref) {
 		if !r.ok("git", "fetch", "origin") {
 			return verInfo{}, fmt.Errorf("updating OSS repo failed")
 		}
 	}
-	hash, err := r.output("git", "rev-parse", shortHash)
+	hash, err := r.output("git", "rev-parse", ref)
 	if err != nil {
 		return verInfo{}, err
 	}
-	date, err := r.output("git", "log", "-n1", "--format=%ct", shortHash)
+	date, err := r.output("git", "log", "-n1", "--format=%ct", ref)
 	if err != nil {
 		return verInfo{}, err
 	}

wgengine/netstack/netstack.go

@@ -260,8 +260,8 @@ func (ns *Impl) Start(lb *ipnlocal.LocalBackend) error {
 	ns.ipstack.SetTransportProtocolHandler(tcp.ProtocolNumber, ns.wrapProtoHandler(tcpFwd.HandlePacket))
 	ns.ipstack.SetTransportProtocolHandler(udp.ProtocolNumber, ns.wrapProtoHandler(udpFwd.HandlePacket))
 	go ns.inject()
-	ns.tundev.PostFilterIn = ns.injectInbound
-	ns.tundev.PreFilterFromTunToNetstack = ns.handleLocalPackets
+	ns.tundev.PostFilterPacketInboundFromWireGaurd = ns.injectInbound
+	ns.tundev.PreFilterPacketOutboundToWireGuardNetstackIntercept = ns.handleLocalPackets
 	return nil
 }
 

wgengine/netstack/netstack_test.go

@@ -18,6 +18,7 @@ import (
 	"tailscale.com/net/tstun"
 	"tailscale.com/tstest"
 	"tailscale.com/types/ipproto"
+	"tailscale.com/types/logid"
 	"tailscale.com/wgengine"
 	"tailscale.com/wgengine/filter"
 )
@@ -49,7 +50,7 @@ func TestInjectInboundLeak(t *testing.T) {
 		t.Fatal("failed to get internals")
 	}
 
-	lb, err := ipnlocal.NewLocalBackend(logf, "logid", new(mem.Store), dialer, eng, 0)
+	lb, err := ipnlocal.NewLocalBackend(logf, logid.PublicID{}, new(mem.Store), dialer, eng, 0)
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -113,7 +114,7 @@ func makeNetstack(t *testing.T, config func(*Impl)) *Impl {
 	}
 	t.Cleanup(func() { ns.Close() })
 
-	lb, err := ipnlocal.NewLocalBackend(logf, "logid", new(mem.Store), dialer, eng, 0)
+	lb, err := ipnlocal.NewLocalBackend(logf, logid.PublicID{}, new(mem.Store), dialer, eng, 0)
 	if err != nil {
 		t.Fatalf("NewLocalBackend: %v", err)
 	}

wgengine/userspace.go

@@ -373,19 +373,19 @@ func NewUserspaceEngine(logf logger.Logf, conf Config) (_ Engine, reterr error)
 	tsTUNDev.SetDiscoKey(e.magicConn.DiscoPublicKey())
 
 	if conf.RespondToPing {
-		e.tundev.PostFilterIn = echoRespondToAll
+		e.tundev.PostFilterPacketInboundFromWireGaurd = echoRespondToAll
 	}
-	e.tundev.PreFilterFromTunToEngine = e.handleLocalPackets
+	e.tundev.PreFilterPacketOutboundToWireGuardEngineIntercept = e.handleLocalPackets
 
 	if envknob.BoolDefaultTrue("TS_DEBUG_CONNECT_FAILURES") {
-		if e.tundev.PreFilterIn != nil {
+		if e.tundev.PreFilterPacketInboundFromWireGuard != nil {
 			return nil, errors.New("unexpected PreFilterIn already set")
 		}
-		e.tundev.PreFilterIn = e.trackOpenPreFilterIn
-		if e.tundev.PostFilterOut != nil {
+		e.tundev.PreFilterPacketInboundFromWireGuard = e.trackOpenPreFilterIn
+		if e.tundev.PostFilterPacketOutboundToWireGuard != nil {
 			return nil, errors.New("unexpected PostFilterOut already set")
 		}
-		e.tundev.PostFilterOut = e.trackOpenPostFilterOut
+		e.tundev.PostFilterPacketOutboundToWireGuard = e.trackOpenPostFilterOut
 	}
 
 	e.wgLogger = wglog.NewLogger(logf)
@@ -1205,6 +1205,7 @@ func (e *userspaceEngine) SetNetworkMap(nm *netmap.NetworkMap) {
 	e.magicConn.SetNetworkMap(nm)
 	e.mu.Lock()
 	e.netMap = nm
+	e.tundev.SetNetMap(nm)
 	callbacks := make([]NetworkMapCallback, 0, 4)
 	for _, fn := range e.networkMapCallbacks {
 		callbacks = append(callbacks, fn)

words/tails.txt

@@ -541,3 +541,4 @@ uaru
 vimba
 wahoo
 coelacanth
+llama