WIP: debug logs for app connector debugging

If the total number of differences is less than a small amount, just do
the dumb quadratic thing and compare every single object instead of
allocating a map.

Updates tailscale/corp#25479

Signed-off-by: Andrew Dunham <andrew@du.nham.ca>
Change-Id: I8931b4355a2da4ec0f19739927311cf88711a840

.github/workflows/installer.yml

@@ -6,11 +6,13 @@ on:
       - "main"
     paths:
       - scripts/installer.sh
+      - .github/workflows/installer.yml
   pull_request:
     branches:
       - "*"
     paths:
       - scripts/installer.sh
+      - .github/workflows/installer.yml
 
 jobs:
   test:
@@ -29,10 +31,9 @@ jobs:
           - "debian:stable-slim"
           - "debian:testing-slim"
           - "debian:sid-slim"
-          - "ubuntu:18.04"
           - "ubuntu:20.04"
           - "ubuntu:22.04"
-          - "ubuntu:23.04"
+          - "ubuntu:24.04"
           - "elementary/docker:stable"
           - "elementary/docker:unstable"
           - "parrotsec/core:lts-amd64"
@@ -48,7 +49,7 @@ jobs:
           - "opensuse/leap:latest"
           - "opensuse/tumbleweed:latest"
           - "archlinux:latest"
-          - "alpine:3.14"
+          - "alpine:3.21"
           - "alpine:latest"
           - "alpine:edge"
         deps:
@@ -58,10 +59,6 @@ jobs:
           # Check a few images with wget rather than curl.
           - { image: "debian:oldstable-slim", deps: "wget" }
           - { image: "debian:sid-slim", deps: "wget" }
-          - { image: "ubuntu:23.04", deps: "wget" }
-          # Ubuntu 16.04 also needs apt-transport-https installed.
-          - { image: "ubuntu:16.04", deps: "curl apt-transport-https" }
-          - { image: "ubuntu:16.04", deps: "wget apt-transport-https" }
     runs-on: ubuntu-latest
     container:
       image: ${{ matrix.image }}

Makefile

@@ -116,7 +116,6 @@ sshintegrationtest: ## Run the SSH integration tests in various Docker container
 	GOOS=linux GOARCH=amd64 ./tool/go build -o ssh/tailssh/testcontainers/tailscaled ./cmd/tailscaled && \
 	echo "Testing on ubuntu:focal" && docker build --build-arg="BASE=ubuntu:focal" -t ssh-ubuntu-focal ssh/tailssh/testcontainers && \
 	echo "Testing on ubuntu:jammy" && docker build --build-arg="BASE=ubuntu:jammy" -t ssh-ubuntu-jammy ssh/tailssh/testcontainers && \
-	echo "Testing on ubuntu:mantic" && docker build --build-arg="BASE=ubuntu:mantic" -t ssh-ubuntu-mantic ssh/tailssh/testcontainers && \
 	echo "Testing on ubuntu:noble" && docker build --build-arg="BASE=ubuntu:noble" -t ssh-ubuntu-noble ssh/tailssh/testcontainers && \
 	echo "Testing on alpine:latest" && docker build --build-arg="BASE=alpine:latest" -t ssh-alpine-latest ssh/tailssh/testcontainers
 

README.md

@@ -72,7 +72,7 @@ Origin](https://en.wikipedia.org/wiki/Developer_Certificate_of_Origin)
 `Signed-off-by` lines in commits.
 
 See `git log` for our commit message style. It's basically the same as
-[Go's style](https://github.com/golang/go/wiki/CommitMessage).
+[Go's style](https://go.dev/wiki/CommitMessage).
 
 ## About Us
 

appc/appconnector.go

@@ -18,7 +18,6 @@ import (
 	"sync"
 	"time"
 
-	xmaps "golang.org/x/exp/maps"
 	"golang.org/x/net/dns/dnsmessage"
 	"tailscale.com/types/logger"
 	"tailscale.com/types/views"
@@ -291,11 +290,11 @@ func (e *AppConnector) updateDomains(domains []string) {
 			}
 		}
 		if err := e.routeAdvertiser.UnadvertiseRoute(toRemove...); err != nil {
-			e.logf("failed to unadvertise routes on domain removal: %v: %v: %v", xmaps.Keys(oldDomains), toRemove, err)
+			e.logf("failed to unadvertise routes on domain removal: %v: %v: %v", slicesx.MapKeys(oldDomains), toRemove, err)
 		}
 	}
 
-	e.logf("handling domains: %v and wildcards: %v", xmaps.Keys(e.domains), e.wildcards)
+	e.logf("handling domains: %v and wildcards: %v", slicesx.MapKeys(e.domains), e.wildcards)
 }
 
 // updateRoutes merges the supplied routes into the currently configured routes. The routes supplied
@@ -354,7 +353,7 @@ func (e *AppConnector) Domains() views.Slice[string] {
 	e.mu.Lock()
 	defer e.mu.Unlock()
 
-	return views.SliceOf(xmaps.Keys(e.domains))
+	return views.SliceOf(slicesx.MapKeys(e.domains))
 }
 
 // DomainRoutes returns a map of domains to resolved IP

appc/appconnector_test.go

@@ -11,13 +11,13 @@ import (
 	"testing"
 	"time"
 
-	xmaps "golang.org/x/exp/maps"
 	"golang.org/x/net/dns/dnsmessage"
 	"tailscale.com/appc/appctest"
 	"tailscale.com/tstest"
 	"tailscale.com/util/clientmetric"
 	"tailscale.com/util/mak"
 	"tailscale.com/util/must"
+	"tailscale.com/util/slicesx"
 )
 
 func fakeStoreRoutes(*RouteInfo) error { return nil }
@@ -50,7 +50,7 @@ func TestUpdateDomains(t *testing.T) {
 		// domains are explicitly downcased on set.
 		a.UpdateDomains([]string{"UP.EXAMPLE.COM"})
 		a.Wait(ctx)
-		if got, want := xmaps.Keys(a.domains), []string{"up.example.com"}; !slices.Equal(got, want) {
+		if got, want := slicesx.MapKeys(a.domains), []string{"up.example.com"}; !slices.Equal(got, want) {
 			t.Errorf("got %v; want %v", got, want)
 		}
 	}

client/systray/logo.go

@@ -0,0 +1,319 @@
+// Copyright (c) Tailscale Inc & AUTHORS
+// SPDX-License-Identifier: BSD-3-Clause
+
+//go:build cgo || !darwin
+
+package systray
+
+import (
+	"bytes"
+	"context"
+	"image"
+	"image/color"
+	"image/png"
+	"sync"
+	"time"
+
+	"fyne.io/systray"
+	"github.com/fogleman/gg"
+)
+
+// tsLogo represents the Tailscale logo displayed as the systray icon.
+type tsLogo struct {
+	// dots represents the state of the 3x3 dot grid in the logo.
+	// A 0 represents a gray dot, any other value is a white dot.
+	dots [9]byte
+
+	// dotMask returns an image mask to be used when rendering the logo dots.
+	dotMask func(dc *gg.Context, borderUnits int, radius int) *image.Alpha
+
+	// overlay is called after the dots are rendered to draw an additional overlay.
+	overlay func(dc *gg.Context, borderUnits int, radius int)
+}
+
+var (
+	// disconnected is all gray dots
+	disconnected = tsLogo{dots: [9]byte{
+		0, 0, 0,
+		0, 0, 0,
+		0, 0, 0,
+	}}
+
+	// connected is the normal Tailscale logo
+	connected = tsLogo{dots: [9]byte{
+		0, 0, 0,
+		1, 1, 1,
+		0, 1, 0,
+	}}
+
+	// loading is a special tsLogo value that is not meant to be rendered directly,
+	// but indicates that the loading animation should be shown.
+	loading = tsLogo{dots: [9]byte{'l', 'o', 'a', 'd', 'i', 'n', 'g'}}
+
+	// loadingIcons are shown in sequence as an animated loading icon.
+	loadingLogos = []tsLogo{
+		{dots: [9]byte{
+			0, 1, 1,
+			1, 0, 1,
+			0, 0, 1,
+		}},
+		{dots: [9]byte{
+			0, 1, 1,
+			0, 0, 1,
+			0, 1, 0,
+		}},
+		{dots: [9]byte{
+			0, 1, 1,
+			0, 0, 0,
+			0, 0, 1,
+		}},
+		{dots: [9]byte{
+			0, 0, 1,
+			0, 1, 0,
+			0, 0, 0,
+		}},
+		{dots: [9]byte{
+			0, 1, 0,
+			0, 0, 0,
+			0, 0, 0,
+		}},
+		{dots: [9]byte{
+			0, 0, 0,
+			0, 0, 1,
+			0, 0, 0,
+		}},
+		{dots: [9]byte{
+			0, 0, 0,
+			0, 0, 0,
+			0, 0, 0,
+		}},
+		{dots: [9]byte{
+			0, 0, 1,
+			0, 0, 0,
+			0, 0, 0,
+		}},
+		{dots: [9]byte{
+			0, 0, 0,
+			0, 0, 0,
+			1, 0, 0,
+		}},
+		{dots: [9]byte{
+			0, 0, 0,
+			0, 0, 0,
+			1, 1, 0,
+		}},
+		{dots: [9]byte{
+			0, 0, 0,
+			1, 0, 0,
+			1, 1, 0,
+		}},
+		{dots: [9]byte{
+			0, 0, 0,
+			1, 1, 0,
+			0, 1, 0,
+		}},
+		{dots: [9]byte{
+			0, 0, 0,
+			1, 1, 0,
+			0, 1, 1,
+		}},
+		{dots: [9]byte{
+			0, 0, 0,
+			1, 1, 1,
+			0, 0, 1,
+		}},
+		{dots: [9]byte{
+			0, 1, 0,
+			0, 1, 1,
+			1, 0, 1,
+		}},
+	}
+
+	// exitNodeOnline is the Tailscale logo with an additional arrow overlay in the corner.
+	exitNodeOnline = tsLogo{
+		dots: [9]byte{
+			0, 0, 0,
+			1, 1, 1,
+			0, 1, 0,
+		},
+		// draw an arrow mask in the bottom right corner with a reasonably thick line width.
+		dotMask: func(dc *gg.Context, borderUnits int, radius int) *image.Alpha {
+			bu, r := float64(borderUnits), float64(radius)
+
+			x1 := r * (bu + 3.5)
+			y := r * (bu + 7)
+			x2 := x1 + (r * 5)
+
+			mc := gg.NewContext(dc.Width(), dc.Height())
+			mc.DrawLine(x1, y, x2, y)                 // arrow center line
+			mc.DrawLine(x2-(1.5*r), y-(1.5*r), x2, y) // top of arrow tip
+			mc.DrawLine(x2-(1.5*r), y+(1.5*r), x2, y) // bottom of arrow tip
+			mc.SetLineWidth(r * 3)
+			mc.Stroke()
+			return mc.AsMask()
+		},
+		// draw an arrow in the bottom right corner over the masked area.
+		overlay: func(dc *gg.Context, borderUnits int, radius int) {
+			bu, r := float64(borderUnits), float64(radius)
+
+			x1 := r * (bu + 3.5)
+			y := r * (bu + 7)
+			x2 := x1 + (r * 5)
+
+			dc.DrawLine(x1, y, x2, y)                 // arrow center line
+			dc.DrawLine(x2-(1.5*r), y-(1.5*r), x2, y) // top of arrow tip
+			dc.DrawLine(x2-(1.5*r), y+(1.5*r), x2, y) // bottom of arrow tip
+			dc.SetColor(fg)
+			dc.SetLineWidth(r)
+			dc.Stroke()
+		},
+	}
+
+	// exitNodeOffline is the Tailscale logo with a red "x" in the corner.
+	exitNodeOffline = tsLogo{
+		dots: [9]byte{
+			0, 0, 0,
+			1, 1, 1,
+			0, 1, 0,
+		},
+		// Draw a square that hides the four dots in the bottom right corner,
+		dotMask: func(dc *gg.Context, borderUnits int, radius int) *image.Alpha {
+			bu, r := float64(borderUnits), float64(radius)
+			x := r * (bu + 3)
+
+			mc := gg.NewContext(dc.Width(), dc.Height())
+			mc.DrawRectangle(x, x, r*6, r*6)
+			mc.Fill()
+			return mc.AsMask()
+		},
+		// draw a red "x" over the bottom right corner.
+		overlay: func(dc *gg.Context, borderUnits int, radius int) {
+			bu, r := float64(borderUnits), float64(radius)
+
+			x1 := r * (bu + 4)
+			x2 := x1 + (r * 3.5)
+			dc.DrawLine(x1, x1, x2, x2) // top-left to bottom-right stroke
+			dc.DrawLine(x1, x2, x2, x1) // bottom-left to top-right stroke
+			dc.SetColor(red)
+			dc.SetLineWidth(r)
+			dc.Stroke()
+		},
+	}
+)
+
+var (
+	bg   = color.NRGBA{0, 0, 0, 255}
+	fg   = color.NRGBA{255, 255, 255, 255}
+	gray = color.NRGBA{255, 255, 255, 102}
+	red  = color.NRGBA{229, 111, 74, 255}
+)
+
+// render returns a PNG image of the logo.
+func (logo tsLogo) render() *bytes.Buffer {
+	const borderUnits = 1
+	return logo.renderWithBorder(borderUnits)
+}
+
+// renderWithBorder returns a PNG image of the logo with the specified border width.
+// One border unit is equal to the radius of a tailscale logo dot.
+func (logo tsLogo) renderWithBorder(borderUnits int) *bytes.Buffer {
+	const radius = 25
+	dim := radius * (8 + borderUnits*2)
+
+	dc := gg.NewContext(dim, dim)
+	dc.DrawRectangle(0, 0, float64(dim), float64(dim))
+	dc.SetColor(bg)
+	dc.Fill()
+
+	if logo.dotMask != nil {
+		mask := logo.dotMask(dc, borderUnits, radius)
+		dc.SetMask(mask)
+		dc.InvertMask()
+	}
+
+	for y := 0; y < 3; y++ {
+		for x := 0; x < 3; x++ {
+			px := (borderUnits + 1 + 3*x) * radius
+			py := (borderUnits + 1 + 3*y) * radius
+			col := fg
+			if logo.dots[y*3+x] == 0 {
+				col = gray
+			}
+			dc.DrawCircle(float64(px), float64(py), radius)
+			dc.SetColor(col)
+			dc.Fill()
+		}
+	}
+
+	if logo.overlay != nil {
+		dc.ResetClip()
+		logo.overlay(dc, borderUnits, radius)
+	}
+
+	b := bytes.NewBuffer(nil)
+	png.Encode(b, dc.Image())
+	return b
+}
+
+// setAppIcon renders logo and sets it as the systray icon.
+func setAppIcon(icon tsLogo) {
+	if icon.dots == loading.dots {
+		startLoadingAnimation()
+	} else {
+		stopLoadingAnimation()
+		systray.SetIcon(icon.render().Bytes())
+	}
+}
+
+var (
+	loadingMu sync.Mutex // protects loadingCancel
+
+	// loadingCancel stops the loading animation in the systray icon.
+	// This is nil if the animation is not currently active.
+	loadingCancel func()
+)
+
+// startLoadingAnimation starts the animated loading icon in the system tray.
+// The animation continues until [stopLoadingAnimation] is called.
+// If the loading animation is already active, this func does nothing.
+func startLoadingAnimation() {
+	loadingMu.Lock()
+	defer loadingMu.Unlock()
+
+	if loadingCancel != nil {
+		// loading icon already displayed
+		return
+	}
+
+	ctx := context.Background()
+	ctx, loadingCancel = context.WithCancel(ctx)
+
+	go func() {
+		t := time.NewTicker(500 * time.Millisecond)
+		var i int
+		for {
+			select {
+			case <-ctx.Done():
+				return
+			case <-t.C:
+				systray.SetIcon(loadingLogos[i].render().Bytes())
+				i++
+				if i >= len(loadingLogos) {
+					i = 0
+				}
+			}
+		}
+	}()
+}
+
+// stopLoadingAnimation stops the animated loading icon in the system tray.
+// If the loading animation is not currently active, this func does nothing.
+func stopLoadingAnimation() {
+	loadingMu.Lock()
+	defer loadingMu.Unlock()
+
+	if loadingCancel != nil {
+		loadingCancel()
+		loadingCancel = nil
+	}
+}

client/systray/systray.go

@@ -0,0 +1,712 @@
+// Copyright (c) Tailscale Inc & AUTHORS
+// SPDX-License-Identifier: BSD-3-Clause
+
+//go:build cgo || !darwin
+
+// Package systray provides a minimal Tailscale systray application.
+package systray
+
+import (
+	"context"
+	"errors"
+	"fmt"
+	"io"
+	"log"
+	"net/http"
+	"os"
+	"os/signal"
+	"runtime"
+	"slices"
+	"strings"
+	"sync"
+	"syscall"
+	"time"
+
+	"fyne.io/systray"
+	"github.com/atotto/clipboard"
+	dbus "github.com/godbus/dbus/v5"
+	"github.com/toqueteos/webbrowser"
+	"tailscale.com/client/tailscale"
+	"tailscale.com/ipn"
+	"tailscale.com/ipn/ipnstate"
+	"tailscale.com/tailcfg"
+	"tailscale.com/util/slicesx"
+	"tailscale.com/util/stringsx"
+)
+
+var (
+	// newMenuDelay is the amount of time to sleep after creating a new menu,
+	// but before adding items to it. This works around a bug in some dbus implementations.
+	newMenuDelay time.Duration
+
+	// if true, treat all mullvad exit node countries as single-city.
+	// Instead of rendering a submenu with cities, just select the highest-priority peer.
+	hideMullvadCities bool
+)
+
+// Run starts the systray menu and blocks until the menu exits.
+func (menu *Menu) Run() {
+	menu.updateState()
+
+	// exit cleanly on SIGINT and SIGTERM
+	go func() {
+		interrupt := make(chan os.Signal, 1)
+		signal.Notify(interrupt, syscall.SIGINT, syscall.SIGTERM)
+		select {
+		case <-interrupt:
+			menu.onExit()
+		case <-menu.bgCtx.Done():
+		}
+	}()
+	go menu.lc.IncrementCounter(menu.bgCtx, "systray_start", 1)
+
+	systray.Run(menu.onReady, menu.onExit)
+}
+
+// Menu represents the systray menu, its items, and the current Tailscale state.
+type Menu struct {
+	mu sync.Mutex // protects the entire Menu
+
+	lc          tailscale.LocalClient
+	status      *ipnstate.Status
+	curProfile  ipn.LoginProfile
+	allProfiles []ipn.LoginProfile
+
+	bgCtx    context.Context // ctx for background tasks not involving menu item clicks
+	bgCancel context.CancelFunc
+
+	// Top-level menu items
+	connect    *systray.MenuItem
+	disconnect *systray.MenuItem
+	self       *systray.MenuItem
+	exitNodes  *systray.MenuItem
+	more       *systray.MenuItem
+	quit       *systray.MenuItem
+
+	rebuildCh  chan struct{} // triggers a menu rebuild
+	accountsCh chan ipn.ProfileID
+	exitNodeCh chan tailcfg.StableNodeID // ID of selected exit node
+
+	eventCancel context.CancelFunc // cancel eventLoop
+
+	notificationIcon *os.File // icon used for desktop notifications
+}
+
+func (menu *Menu) init() {
+	if menu.bgCtx != nil {
+		// already initialized
+		return
+	}
+
+	menu.rebuildCh = make(chan struct{}, 1)
+	menu.accountsCh = make(chan ipn.ProfileID)
+	menu.exitNodeCh = make(chan tailcfg.StableNodeID)
+
+	// dbus wants a file path for notification icons, so copy to a temp file.
+	menu.notificationIcon, _ = os.CreateTemp("", "tailscale-systray.png")
+	io.Copy(menu.notificationIcon, connected.renderWithBorder(3))
+
+	menu.bgCtx, menu.bgCancel = context.WithCancel(context.Background())
+	go menu.watchIPNBus()
+}
+
+func init() {
+	if runtime.GOOS != "linux" {
+		// so far, these tweaks are only needed on Linux
+		return
+	}
+
+	desktop := strings.ToLower(os.Getenv("XDG_CURRENT_DESKTOP"))
+	switch desktop {
+	case "gnome":
+		// GNOME expands submenus downward in the main menu, rather than flyouts to the side.
+		// Either as a result of that or another limitation, there seems to be a maximum depth of submenus.
+		// Mullvad countries that have a city submenu are not being rendered, and so can't be selected.
+		// Handle this by simply treating all mullvad countries as single-city and select the best peer.
+		hideMullvadCities = true
+	case "kde":
+		// KDE doesn't need a delay, and actually won't render submenus
+		// if we delay for more than about 400µs.
+		newMenuDelay = 0
+	default:
+		// Add a slight delay to ensure the menu is created before adding items.
+		//
+		// Systray implementations that use libdbusmenu sometimes process messages out of order,
+		// resulting in errors such as:
+		//    (waybar:153009): LIBDBUSMENU-GTK-WARNING **: 18:07:11.551: Children but no menu, someone's been naughty with their 'children-display' property: 'submenu'
+		//
+		// See also: https://github.com/fyne-io/systray/issues/12
+		newMenuDelay = 10 * time.Millisecond
+	}
+}
+
+// onReady is called by the systray package when the menu is ready to be built.
+func (menu *Menu) onReady() {
+	log.Printf("starting")
+	setAppIcon(disconnected)
+	menu.rebuild()
+}
+
+// updateState updates the Menu state from the Tailscale local client.
+func (menu *Menu) updateState() {
+	menu.mu.Lock()
+	defer menu.mu.Unlock()
+	menu.init()
+
+	var err error
+	menu.status, err = menu.lc.Status(menu.bgCtx)
+	if err != nil {
+		log.Print(err)
+	}
+	menu.curProfile, menu.allProfiles, err = menu.lc.ProfileStatus(menu.bgCtx)
+	if err != nil {
+		log.Print(err)
+	}
+}
+
+// rebuild the systray menu based on the current Tailscale state.
+//
+// We currently rebuild the entire menu because it is not easy to update the existing menu.
+// You cannot iterate over the items in a menu, nor can you remove some items like separators.
+// So for now we rebuild the whole thing, and can optimize this later if needed.
+func (menu *Menu) rebuild() {
+	menu.mu.Lock()
+	defer menu.mu.Unlock()
+	menu.init()
+
+	if menu.eventCancel != nil {
+		menu.eventCancel()
+	}
+	ctx := context.Background()
+	ctx, menu.eventCancel = context.WithCancel(ctx)
+
+	systray.ResetMenu()
+
+	menu.connect = systray.AddMenuItem("Connect", "")
+	menu.disconnect = systray.AddMenuItem("Disconnect", "")
+	menu.disconnect.Hide()
+	systray.AddSeparator()
+
+	// delay to prevent race setting icon on first start
+	time.Sleep(newMenuDelay)
+
+	// Set systray menu icon and title.
+	// Also adjust connect/disconnect menu items if needed.
+	var backendState string
+	if menu.status != nil {
+		backendState = menu.status.BackendState
+	}
+	switch backendState {
+	case ipn.Running.String():
+		if menu.status.ExitNodeStatus != nil && !menu.status.ExitNodeStatus.ID.IsZero() {
+			if menu.status.ExitNodeStatus.Online {
+				setTooltip("Using exit node")
+				setAppIcon(exitNodeOnline)
+			} else {
+				setTooltip("Exit node offline")
+				setAppIcon(exitNodeOffline)
+			}
+		} else {
+			setTooltip(fmt.Sprintf("Connected to %s", menu.status.CurrentTailnet.Name))
+			setAppIcon(connected)
+		}
+		menu.connect.SetTitle("Connected")
+		menu.connect.Disable()
+		menu.disconnect.Show()
+		menu.disconnect.Enable()
+	case ipn.Starting.String():
+		setTooltip("Connecting")
+		setAppIcon(loading)
+	default:
+		setTooltip("Disconnected")
+		setAppIcon(disconnected)
+	}
+
+	account := "Account"
+	if pt := profileTitle(menu.curProfile); pt != "" {
+		account = pt
+	}
+	accounts := systray.AddMenuItem(account, "")
+	setRemoteIcon(accounts, menu.curProfile.UserProfile.ProfilePicURL)
+	time.Sleep(newMenuDelay)
+	for _, profile := range menu.allProfiles {
+		title := profileTitle(profile)
+		var item *systray.MenuItem
+		if profile.ID == menu.curProfile.ID {
+			item = accounts.AddSubMenuItemCheckbox(title, "", true)
+		} else {
+			item = accounts.AddSubMenuItem(title, "")
+		}
+		setRemoteIcon(item, profile.UserProfile.ProfilePicURL)
+		onClick(ctx, item, func(ctx context.Context) {
+			select {
+			case <-ctx.Done():
+			case menu.accountsCh <- profile.ID:
+			}
+		})
+	}
+
+	if menu.status != nil && menu.status.Self != nil && len(menu.status.Self.TailscaleIPs) > 0 {
+		title := fmt.Sprintf("This Device: %s (%s)", menu.status.Self.HostName, menu.status.Self.TailscaleIPs[0])
+		menu.self = systray.AddMenuItem(title, "")
+	} else {
+		menu.self = systray.AddMenuItem("This Device: not connected", "")
+		menu.self.Disable()
+	}
+	systray.AddSeparator()
+
+	menu.rebuildExitNodeMenu(ctx)
+
+	if menu.status != nil {
+		menu.more = systray.AddMenuItem("More settings", "")
+		onClick(ctx, menu.more, func(_ context.Context) {
+			webbrowser.Open("http://100.100.100.100/")
+		})
+	}
+
+	menu.quit = systray.AddMenuItem("Quit", "Quit the app")
+	menu.quit.Enable()
+
+	go menu.eventLoop(ctx)
+}
+
+// profileTitle returns the title string for a profile menu item.
+func profileTitle(profile ipn.LoginProfile) string {
+	title := profile.Name
+	if profile.NetworkProfile.DomainName != "" {
+		if runtime.GOOS == "windows" || runtime.GOOS == "darwin" {
+			// windows and mac don't support multi-line menu
+			title += " (" + profile.NetworkProfile.DomainName + ")"
+		} else {
+			title += "\n" + profile.NetworkProfile.DomainName
+		}
+	}
+	return title
+}
+
+var (
+	cacheMu   sync.Mutex
+	httpCache = map[string][]byte{} // URL => response body
+)
+
+// setRemoteIcon sets the icon for menu to the specified remote image.
+// Remote images are fetched as needed and cached.
+func setRemoteIcon(menu *systray.MenuItem, urlStr string) {
+	if menu == nil || urlStr == "" {
+		return
+	}
+
+	cacheMu.Lock()
+	b, ok := httpCache[urlStr]
+	if !ok {
+		resp, err := http.Get(urlStr)
+		if err == nil && resp.StatusCode == http.StatusOK {
+			b, _ = io.ReadAll(resp.Body)
+			httpCache[urlStr] = b
+			resp.Body.Close()
+		}
+	}
+	cacheMu.Unlock()
+
+	if len(b) > 0 {
+		menu.SetIcon(b)
+	}
+}
+
+// setTooltip sets the tooltip text for the systray icon.
+func setTooltip(text string) {
+	if runtime.GOOS == "darwin" || runtime.GOOS == "windows" {
+		systray.SetTooltip(text)
+	} else {
+		// on Linux, SetTitle actually sets the tooltip
+		systray.SetTitle(text)
+	}
+}
+
+// eventLoop is the main event loop for handling click events on menu items
+// and responding to Tailscale state changes.
+// This method does not return until ctx.Done is closed.
+func (menu *Menu) eventLoop(ctx context.Context) {
+	for {
+		select {
+		case <-ctx.Done():
+			return
+		case <-menu.rebuildCh:
+			menu.updateState()
+			menu.rebuild()
+		case <-menu.connect.ClickedCh:
+			_, err := menu.lc.EditPrefs(ctx, &ipn.MaskedPrefs{
+				Prefs: ipn.Prefs{
+					WantRunning: true,
+				},
+				WantRunningSet: true,
+			})
+			if err != nil {
+				log.Printf("error connecting: %v", err)
+			}
+
+		case <-menu.disconnect.ClickedCh:
+			_, err := menu.lc.EditPrefs(ctx, &ipn.MaskedPrefs{
+				Prefs: ipn.Prefs{
+					WantRunning: false,
+				},
+				WantRunningSet: true,
+			})
+			if err != nil {
+				log.Printf("error disconnecting: %v", err)
+			}
+
+		case <-menu.self.ClickedCh:
+			menu.copyTailscaleIP(menu.status.Self)
+
+		case id := <-menu.accountsCh:
+			if err := menu.lc.SwitchProfile(ctx, id); err != nil {
+				log.Printf("error switching to profile ID %v: %v", id, err)
+			}
+
+		case exitNode := <-menu.exitNodeCh:
+			if exitNode.IsZero() {
+				log.Print("disable exit node")
+				if err := menu.lc.SetUseExitNode(ctx, false); err != nil {
+					log.Printf("error disabling exit node: %v", err)
+				}
+			} else {
+				log.Printf("enable exit node: %v", exitNode)
+				mp := &ipn.MaskedPrefs{
+					Prefs: ipn.Prefs{
+						ExitNodeID: exitNode,
+					},
+					ExitNodeIDSet: true,
+				}
+				if _, err := menu.lc.EditPrefs(ctx, mp); err != nil {
+					log.Printf("error setting exit node: %v", err)
+				}
+			}
+
+		case <-menu.quit.ClickedCh:
+			systray.Quit()
+		}
+	}
+}
+
+// onClick registers a click handler for a menu item.
+func onClick(ctx context.Context, item *systray.MenuItem, fn func(ctx context.Context)) {
+	go func() {
+		for {
+			select {
+			case <-ctx.Done():
+				return
+			case <-item.ClickedCh:
+				fn(ctx)
+			}
+		}
+	}()
+}
+
+// watchIPNBus subscribes to the tailscale event bus and sends state updates to chState.
+// This method does not return.
+func (menu *Menu) watchIPNBus() {
+	for {
+		if err := menu.watchIPNBusInner(); err != nil {
+			log.Println(err)
+			if errors.Is(err, context.Canceled) {
+				// If the context got canceled, we will never be able to
+				// reconnect to IPN bus, so exit the process.
+				log.Fatalf("watchIPNBus: %v", err)
+			}
+		}
+		// If our watch connection breaks, wait a bit before reconnecting. No
+		// reason to spam the logs if e.g. tailscaled is restarting or goes
+		// down.
+		time.Sleep(3 * time.Second)
+	}
+}
+
+func (menu *Menu) watchIPNBusInner() error {
+	watcher, err := menu.lc.WatchIPNBus(menu.bgCtx, ipn.NotifyNoPrivateKeys)
+	if err != nil {
+		return fmt.Errorf("watching ipn bus: %w", err)
+	}
+	defer watcher.Close()
+	for {
+		select {
+		case <-menu.bgCtx.Done():
+			return nil
+		default:
+			n, err := watcher.Next()
+			if err != nil {
+				return fmt.Errorf("ipnbus error: %w", err)
+			}
+			var rebuild bool
+			if n.State != nil {
+				log.Printf("new state: %v", n.State)
+				rebuild = true
+			}
+			if n.Prefs != nil {
+				rebuild = true
+			}
+			if rebuild {
+				menu.rebuildCh <- struct{}{}
+			}
+		}
+	}
+}
+
+// copyTailscaleIP copies the first Tailscale IP of the given device to the clipboard
+// and sends a notification with the copied value.
+func (menu *Menu) copyTailscaleIP(device *ipnstate.PeerStatus) {
+	if device == nil || len(device.TailscaleIPs) == 0 {
+		return
+	}
+	name := strings.Split(device.DNSName, ".")[0]
+	ip := device.TailscaleIPs[0].String()
+	err := clipboard.WriteAll(ip)
+	if err != nil {
+		log.Printf("clipboard error: %v", err)
+	}
+
+	menu.sendNotification(fmt.Sprintf("Copied Address for %v", name), ip)
+}
+
+// sendNotification sends a desktop notification with the given title and content.
+func (menu *Menu) sendNotification(title, content string) {
+	conn, err := dbus.SessionBus()
+	if err != nil {
+		log.Printf("dbus: %v", err)
+		return
+	}
+	timeout := 3 * time.Second
+	obj := conn.Object("org.freedesktop.Notifications", "/org/freedesktop/Notifications")
+	call := obj.Call("org.freedesktop.Notifications.Notify", 0, "Tailscale", uint32(0),
+		menu.notificationIcon.Name(), title, content, []string{}, map[string]dbus.Variant{}, int32(timeout.Milliseconds()))
+	if call.Err != nil {
+		log.Printf("dbus: %v", call.Err)
+	}
+}
+
+func (menu *Menu) rebuildExitNodeMenu(ctx context.Context) {
+	if menu.status == nil {
+		return
+	}
+
+	status := menu.status
+	menu.exitNodes = systray.AddMenuItem("Exit Nodes", "")
+	time.Sleep(newMenuDelay)
+
+	// register a click handler for a menu item to set nodeID as the exit node.
+	setExitNodeOnClick := func(item *systray.MenuItem, nodeID tailcfg.StableNodeID) {
+		onClick(ctx, item, func(ctx context.Context) {
+			select {
+			case <-ctx.Done():
+			case menu.exitNodeCh <- nodeID:
+			}
+		})
+	}
+
+	noExitNodeMenu := menu.exitNodes.AddSubMenuItemCheckbox("None", "", status.ExitNodeStatus == nil)
+	setExitNodeOnClick(noExitNodeMenu, "")
+
+	// Show recommended exit node if available.
+	if status.Self.CapMap.Contains(tailcfg.NodeAttrSuggestExitNodeUI) {
+		sugg, err := menu.lc.SuggestExitNode(ctx)
+		if err == nil {
+			title := "Recommended: "
+			if loc := sugg.Location; loc.Valid() && loc.Country() != "" {
+				flag := countryFlag(loc.CountryCode())
+				title += fmt.Sprintf("%s %s: %s", flag, loc.Country(), loc.City())
+			} else {
+				title += strings.Split(sugg.Name, ".")[0]
+			}
+			menu.exitNodes.AddSeparator()
+			rm := menu.exitNodes.AddSubMenuItemCheckbox(title, "", false)
+			setExitNodeOnClick(rm, sugg.ID)
+			if status.ExitNodeStatus != nil && sugg.ID == status.ExitNodeStatus.ID {
+				rm.Check()
+			}
+		}
+	}
+
+	// Add tailnet exit nodes if present.
+	var tailnetExitNodes []*ipnstate.PeerStatus
+	for _, ps := range status.Peer {
+		if ps.ExitNodeOption && ps.Location == nil {
+			tailnetExitNodes = append(tailnetExitNodes, ps)
+		}
+	}
+	if len(tailnetExitNodes) > 0 {
+		menu.exitNodes.AddSeparator()
+		menu.exitNodes.AddSubMenuItem("Tailnet Exit Nodes", "").Disable()
+		for _, ps := range status.Peer {
+			if !ps.ExitNodeOption || ps.Location != nil {
+				continue
+			}
+			name := strings.Split(ps.DNSName, ".")[0]
+			if !ps.Online {
+				name += " (offline)"
+			}
+			sm := menu.exitNodes.AddSubMenuItemCheckbox(name, "", false)
+			if !ps.Online {
+				sm.Disable()
+			}
+			if status.ExitNodeStatus != nil && ps.ID == status.ExitNodeStatus.ID {
+				sm.Check()
+			}
+			setExitNodeOnClick(sm, ps.ID)
+		}
+	}
+
+	// Add mullvad exit nodes if present.
+	var mullvadExitNodes mullvadPeers
+	if status.Self.CapMap.Contains("mullvad") {
+		mullvadExitNodes = newMullvadPeers(status)
+	}
+	if len(mullvadExitNodes.countries) > 0 {
+		menu.exitNodes.AddSeparator()
+		menu.exitNodes.AddSubMenuItem("Location-based Exit Nodes", "").Disable()
+		mullvadMenu := menu.exitNodes.AddSubMenuItemCheckbox("Mullvad VPN", "", false)
+
+		for _, country := range mullvadExitNodes.sortedCountries() {
+			flag := countryFlag(country.code)
+			countryMenu := mullvadMenu.AddSubMenuItemCheckbox(flag+" "+country.name, "", false)
+
+			// single-city country, no submenu
+			if len(country.cities) == 1 || hideMullvadCities {
+				setExitNodeOnClick(countryMenu, country.best.ID)
+				if status.ExitNodeStatus != nil {
+					for _, city := range country.cities {
+						for _, ps := range city.peers {
+							if status.ExitNodeStatus.ID == ps.ID {
+								mullvadMenu.Check()
+								countryMenu.Check()
+							}
+						}
+					}
+				}
+				continue
+			}
+
+			// multi-city country, build submenu with "best available" option and cities.
+			time.Sleep(newMenuDelay)
+			bm := countryMenu.AddSubMenuItemCheckbox("Best Available", "", false)
+			setExitNodeOnClick(bm, country.best.ID)
+			countryMenu.AddSeparator()
+
+			for _, city := range country.sortedCities() {
+				cityMenu := countryMenu.AddSubMenuItemCheckbox(city.name, "", false)
+				setExitNodeOnClick(cityMenu, city.best.ID)
+				if status.ExitNodeStatus != nil {
+					for _, ps := range city.peers {
+						if status.ExitNodeStatus.ID == ps.ID {
+							mullvadMenu.Check()
+							countryMenu.Check()
+							cityMenu.Check()
+						}
+					}
+				}
+			}
+		}
+	}
+
+	// TODO: "Allow Local Network Access" and "Run Exit Node" menu items
+}
+
+// mullvadPeers contains all mullvad peer nodes, sorted by country and city.
+type mullvadPeers struct {
+	countries map[string]*mvCountry // country code (uppercase) => country
+}
+
+// sortedCountries returns countries containing mullvad nodes, sorted by name.
+func (mp mullvadPeers) sortedCountries() []*mvCountry {
+	countries := slicesx.MapValues(mp.countries)
+	slices.SortFunc(countries, func(a, b *mvCountry) int {
+		return stringsx.CompareFold(a.name, b.name)
+	})
+	return countries
+}
+
+type mvCountry struct {
+	code   string
+	name   string
+	best   *ipnstate.PeerStatus // highest priority peer in the country
+	cities map[string]*mvCity   // city code => city
+}
+
+// sortedCities returns cities containing mullvad nodes, sorted by name.
+func (mc *mvCountry) sortedCities() []*mvCity {
+	cities := slicesx.MapValues(mc.cities)
+	slices.SortFunc(cities, func(a, b *mvCity) int {
+		return stringsx.CompareFold(a.name, b.name)
+	})
+	return cities
+}
+
+// countryFlag takes a 2-character ASCII string and returns the corresponding emoji flag.
+// It returns the empty string on error.
+func countryFlag(code string) string {
+	if len(code) != 2 {
+		return ""
+	}
+	runes := make([]rune, 0, 2)
+	for i := range 2 {
+		b := code[i] | 32 // lowercase
+		if b < 'a' || b > 'z' {
+			return ""
+		}
+		// https://en.wikipedia.org/wiki/Regional_indicator_symbol
+		runes = append(runes, 0x1F1E6+rune(b-'a'))
+	}
+	return string(runes)
+}
+
+type mvCity struct {
+	name  string
+	best  *ipnstate.PeerStatus // highest priority peer in the city
+	peers []*ipnstate.PeerStatus
+}
+
+func newMullvadPeers(status *ipnstate.Status) mullvadPeers {
+	countries := make(map[string]*mvCountry)
+	for _, ps := range status.Peer {
+		if !ps.ExitNodeOption || ps.Location == nil {
+			continue
+		}
+		loc := ps.Location
+		country, ok := countries[loc.CountryCode]
+		if !ok {
+			country = &mvCountry{
+				code:   loc.CountryCode,
+				name:   loc.Country,
+				cities: make(map[string]*mvCity),
+			}
+			countries[loc.CountryCode] = country
+		}
+		city, ok := countries[loc.CountryCode].cities[loc.CityCode]
+		if !ok {
+			city = &mvCity{
+				name: loc.City,
+			}
+			countries[loc.CountryCode].cities[loc.CityCode] = city
+		}
+		city.peers = append(city.peers, ps)
+		if city.best == nil || ps.Location.Priority > city.best.Location.Priority {
+			city.best = ps
+		}
+		if country.best == nil || ps.Location.Priority > country.best.Location.Priority {
+			country.best = ps
+		}
+	}
+	return mullvadPeers{countries}
+}
+
+// onExit is called by the systray package when the menu is exiting.
+func (menu *Menu) onExit() {
+	log.Printf("exiting")
+	if menu.bgCancel != nil {
+		menu.bgCancel()
+	}
+	if menu.eventCancel != nil {
+		menu.eventCancel()
+	}
+
+	os.Remove(menu.notificationIcon.Name())
+}

client/tailscale/localclient.go

@@ -1,7 +1,7 @@
 // Copyright (c) Tailscale Inc & AUTHORS
 // SPDX-License-Identifier: BSD-3-Clause
 
-//go:build go1.19
+//go:build go1.22
 
 package tailscale
 

client/web/web.go

@@ -804,8 +804,8 @@ type nodeData struct {
 	DeviceName  string
 	TailnetName string // TLS cert name
 	DomainName  string
-	IPv4        string
-	IPv6        string
+	IPv4        netip.Addr
+	IPv6        netip.Addr
 	OS          string
 	IPNVersion  string
 
@@ -864,10 +864,14 @@ func (s *Server) serveGetNodeData(w http.ResponseWriter, r *http.Request) {
 		return
 	}
 	filterRules, _ := s.lc.DebugPacketFilterRules(r.Context())
+	ipv4, ipv6 := s.selfNodeAddresses(r, st)
+
 	data := &nodeData{
 		ID:               st.Self.ID,
 		Status:           st.BackendState,
 		DeviceName:       strings.Split(st.Self.DNSName, ".")[0],
+		IPv4:             ipv4,
+		IPv6:             ipv6,
 		OS:               st.Self.OS,
 		IPNVersion:       strings.Split(st.Version, "-")[0],
 		Profile:          st.User[st.Self.UserID],
@@ -887,10 +891,6 @@ func (s *Server) serveGetNodeData(w http.ResponseWriter, r *http.Request) {
 		ACLAllowsAnyIncomingTraffic: s.aclsAllowAccess(filterRules),
 	}
 
-	ipv4, ipv6 := s.selfNodeAddresses(r, st)
-	data.IPv4 = ipv4.String()
-	data.IPv6 = ipv6.String()
-
 	if hostinfo.GetEnvType() == hostinfo.HomeAssistantAddOn && data.URLPrefix == "" {
 		// X-Ingress-Path is the path prefix in use for Home Assistant
 		// https://developers.home-assistant.io/docs/add-ons/presentation#ingress

cmd/addlicense/main.go

@@ -18,12 +18,12 @@ var (
 )
 
 func usage() {
-	fmt.Fprintf(os.Stderr, `
+	fmt.Fprint(os.Stderr, `
 usage: addlicense -file FILE <subcommand args...>
 `[1:])
 
 	flag.PrintDefaults()
-	fmt.Fprintf(os.Stderr, `
+	fmt.Fprint(os.Stderr, `
 addlicense adds a Tailscale license to the beginning of file.
 
 It is intended for use with 'go generate', so it also runs a subcommand,

cmd/derper/bootstrap_dns_test.go

@@ -20,10 +20,10 @@ import (
 )
 
 func BenchmarkHandleBootstrapDNS(b *testing.B) {
-	tstest.Replace(b, bootstrapDNS, "log.tailscale.io,login.tailscale.com,controlplane.tailscale.com,login.us.tailscale.com")
+	tstest.Replace(b, bootstrapDNS, "log.tailscale.com,login.tailscale.com,controlplane.tailscale.com,login.us.tailscale.com")
 	refreshBootstrapDNS()
 	w := new(bitbucketResponseWriter)
-	req, _ := http.NewRequest("GET", "https://localhost/bootstrap-dns?q="+url.QueryEscape("log.tailscale.io"), nil)
+	req, _ := http.NewRequest("GET", "https://localhost/bootstrap-dns?q="+url.QueryEscape("log.tailscale.com"), nil)
 	b.ReportAllocs()
 	b.ResetTimer()
 	b.RunParallel(func(b *testing.PB) {
@@ -63,7 +63,7 @@ func TestUnpublishedDNS(t *testing.T) {
 	nettest.SkipIfNoNetwork(t)
 
 	const published = "login.tailscale.com"
-	const unpublished = "log.tailscale.io"
+	const unpublished = "log.tailscale.com"
 
 	prev1, prev2 := *bootstrapDNS, *unpublishedDNS
 	*bootstrapDNS = published
@@ -119,18 +119,18 @@ func TestUnpublishedDNSEmptyList(t *testing.T) {
 
 	unpublishedDNSCache.Store(&dnsEntryMap{
 		IPs: map[string][]net.IP{
-			"log.tailscale.io":           {},
+			"log.tailscale.com":          {},
 			"controlplane.tailscale.com": {net.IPv4(1, 2, 3, 4)},
 		},
 		Percent: map[string]float64{
-			"log.tailscale.io":           1.0,
+			"log.tailscale.com":          1.0,
 			"controlplane.tailscale.com": 1.0,
 		},
 	})
 
 	t.Run("CacheMiss", func(t *testing.T) {
 		// One domain in map but empty, one not in map at all
-		for _, q := range []string{"log.tailscale.io", "login.tailscale.com"} {
+		for _, q := range []string{"log.tailscale.com", "login.tailscale.com"} {
 			resetMetrics()
 			ips := getBootstrapDNS(t, q)
 

cmd/derper/mesh.go

@@ -47,6 +47,7 @@ func startMeshWithHost(s *derp.Server, host string) error {
 	c.SetURLDialer(func(ctx context.Context, network, addr string) (net.Conn, error) {
 		host, port, err := net.SplitHostPort(addr)
 		if err != nil {
+			logf("failed to split %q: %v", addr, err)
 			return nil, err
 		}
 		var d net.Dialer
@@ -55,15 +56,18 @@ func startMeshWithHost(s *derp.Server, host string) error {
 			subCtx, cancel := context.WithTimeout(ctx, 2*time.Second)
 			defer cancel()
 			vpcHost := base + "-vpc.tailscale.com"
-			ips, _ := r.LookupIP(subCtx, "ip", vpcHost)
+			ips, err := r.LookupIP(subCtx, "ip", vpcHost)
+			if err != nil {
+				logf("failed to resolve %v: %v", vpcHost, err)
+			}
 			if len(ips) > 0 {
 				vpcAddr := net.JoinHostPort(ips[0].String(), port)
 				c, err := d.DialContext(subCtx, network, vpcAddr)
 				if err == nil {
-					log.Printf("connected to %v (%v) instead of %v", vpcHost, ips[0], base)
+					logf("connected to %v (%v) instead of %v", vpcHost, ips[0], base)
 					return c, nil
 				}
-				log.Printf("failed to connect to %v (%v): %v; trying non-VPC route", vpcHost, ips[0], err)
+				logf("failed to connect to %v (%v): %v; trying non-VPC route", vpcHost, ips[0], err)
 			}
 		}
 		return d.DialContext(ctx, network, addr)

cmd/derpprobe/derpprobe.go

@@ -18,18 +18,21 @@ import (
 )
 
 var (
-	derpMapURL   = flag.String("derp-map", "https://login.tailscale.com/derpmap/default", "URL to DERP map (https:// or file://) or 'local' to use the local tailscaled's DERP map")
-	versionFlag  = flag.Bool("version", false, "print version and exit")
-	listen       = flag.String("listen", ":8030", "HTTP listen address")
-	probeOnce    = flag.Bool("once", false, "probe once and print results, then exit; ignores the listen flag")
-	spread       = flag.Bool("spread", true, "whether to spread probing over time")
-	interval     = flag.Duration("interval", 15*time.Second, "probe interval")
-	meshInterval = flag.Duration("mesh-interval", 15*time.Second, "mesh probe interval")
-	stunInterval = flag.Duration("stun-interval", 15*time.Second, "STUN probe interval")
-	tlsInterval  = flag.Duration("tls-interval", 15*time.Second, "TLS probe interval")
-	bwInterval   = flag.Duration("bw-interval", 0, "bandwidth probe interval (0 = no bandwidth probing)")
-	bwSize       = flag.Int64("bw-probe-size-bytes", 1_000_000, "bandwidth probe size")
-	regionCode   = flag.String("region-code", "", "probe only this region (e.g. 'lax'); if left blank, all regions will be probed")
+	derpMapURL         = flag.String("derp-map", "https://login.tailscale.com/derpmap/default", "URL to DERP map (https:// or file://) or 'local' to use the local tailscaled's DERP map")
+	versionFlag        = flag.Bool("version", false, "print version and exit")
+	listen             = flag.String("listen", ":8030", "HTTP listen address")
+	probeOnce          = flag.Bool("once", false, "probe once and print results, then exit; ignores the listen flag")
+	spread             = flag.Bool("spread", true, "whether to spread probing over time")
+	interval           = flag.Duration("interval", 15*time.Second, "probe interval")
+	meshInterval       = flag.Duration("mesh-interval", 15*time.Second, "mesh probe interval")
+	stunInterval       = flag.Duration("stun-interval", 15*time.Second, "STUN probe interval")
+	tlsInterval        = flag.Duration("tls-interval", 15*time.Second, "TLS probe interval")
+	bwInterval         = flag.Duration("bw-interval", 0, "bandwidth probe interval (0 = no bandwidth probing)")
+	bwSize             = flag.Int64("bw-probe-size-bytes", 1_000_000, "bandwidth probe size")
+	bwTUNIPv4Address   = flag.String("bw-tun-ipv4-addr", "", "if specified, bandwidth probes will be performed over a TUN device at this address in order to exercise TCP-in-TCP in similar fashion to TCP over Tailscale via DERP; we will use a /30 subnet including this IP address")
+	qdPacketsPerSecond = flag.Int("qd-packets-per-second", 0, "if greater than 0, queuing delay will be measured continuously using 260 byte packets (approximate size of a CallMeMaybe packet) sent at this rate per second")
+	qdPacketTimeout    = flag.Duration("qd-packet-timeout", 5*time.Second, "queuing delay packets arriving after this period of time from being sent are treated like dropped packets and don't count toward queuing delay timings")
+	regionCode         = flag.String("region-code", "", "probe only this region (e.g. 'lax'); if left blank, all regions will be probed")
 )
 
 func main() {
@@ -44,9 +47,10 @@ func main() {
 		prober.WithMeshProbing(*meshInterval),
 		prober.WithSTUNProbing(*stunInterval),
 		prober.WithTLSProbing(*tlsInterval),
+		prober.WithQueuingDelayProbing(*qdPacketsPerSecond, *qdPacketTimeout),
 	}
 	if *bwInterval > 0 {
-		opts = append(opts, prober.WithBandwidthProbing(*bwInterval, *bwSize))
+		opts = append(opts, prober.WithBandwidthProbing(*bwInterval, *bwSize, *bwTUNIPv4Address))
 	}
 	if *regionCode != "" {
 		opts = append(opts, prober.WithRegion(*regionCode))
@@ -106,7 +110,7 @@ func getOverallStatus(p *prober.Prober) (o overallStatus) {
 			// Do not show probes that have not finished yet.
 			continue
 		}
-		if i.Result {
+		if i.Status == prober.ProbeStatusSucceeded {
 			o.addGoodf("%s: %s", p, i.Latency)
 		} else {
 			o.addBadf("%s: %s", p, i.Error)

cmd/k8s-operator/connector_test.go

@@ -203,7 +203,7 @@ func TestConnectorWithProxyClass(t *testing.T) {
 	pc := &tsapi.ProxyClass{
 		ObjectMeta: metav1.ObjectMeta{Name: "custom-metadata"},
 		Spec: tsapi.ProxyClassSpec{StatefulSet: &tsapi.StatefulSet{
-			Labels:      map[string]string{"foo": "bar"},
+			Labels:      tsapi.Labels{"foo": "bar"},
 			Annotations: map[string]string{"bar.io/foo": "some-val"},
 			Pod:         &tsapi.Pod{Annotations: map[string]string{"foo.io/bar": "some-val"}}}},
 	}

cmd/k8s-operator/depaware.txt

@@ -225,7 +225,6 @@ tailscale.com/cmd/k8s-operator dependencies: (generated by github.com/tailscale/
         github.com/tailscale/wireguard-go/rwcancel                   from github.com/tailscale/wireguard-go/device+
         github.com/tailscale/wireguard-go/tai64n                     from github.com/tailscale/wireguard-go/device
      💣 github.com/tailscale/wireguard-go/tun                        from github.com/tailscale/wireguard-go/device+
-        github.com/tcnksm/go-httpstat                                from tailscale.com/net/netcheck
    L    github.com/u-root/uio/rand                                   from github.com/insomniacslk/dhcp/dhcpv4
    L    github.com/u-root/uio/uio                                    from github.com/insomniacslk/dhcp/dhcpv4+
    L    github.com/vishvananda/netns                                 from github.com/tailscale/netlink+

cmd/k8s-operator/deploy/crds/tailscale.com_proxyclasses.yaml

@@ -99,6 +99,16 @@ spec:
                         enable:
                           description: If Enable is set to true, a Prometheus ServiceMonitor will be created. Enable can only be set to true if metrics are enabled.
                           type: boolean
+                        labels:
+                          description: |-
+                            Labels to add to the ServiceMonitor.
+                            Labels must be valid Kubernetes labels.
+                            https://kubernetes.io/docs/concepts/overview/working-with-objects/labels/#syntax-and-character-set
+                          type: object
+                          additionalProperties:
+                            type: string
+                            maxLength: 63
+                            pattern: ^(([a-zA-Z0-9][-._a-zA-Z0-9]*)?[a-zA-Z0-9])?$
                   x-kubernetes-validations:
                     - rule: '!(has(self.serviceMonitor) && self.serviceMonitor.enable  && !self.enable)'
                       message: ServiceMonitor can only be enabled if metrics are enabled
@@ -133,6 +143,8 @@ spec:
                       type: object
                       additionalProperties:
                         type: string
+                        maxLength: 63
+                        pattern: ^(([a-zA-Z0-9][-._a-zA-Z0-9]*)?[a-zA-Z0-9])?$
                     pod:
                       description: Configuration for the proxy Pod.
                       type: object
@@ -1062,6 +1074,8 @@ spec:
                           type: object
                           additionalProperties:
                             type: string
+                            maxLength: 63
+                            pattern: ^(([a-zA-Z0-9][-._a-zA-Z0-9]*)?[a-zA-Z0-9])?$
                         nodeName:
                           description: |-
                             Proxy Pod's node name.

cmd/k8s-operator/deploy/crds/tailscale.com_proxygroups.yaml

@@ -20,9 +20,24 @@ spec:
           jsonPath: .status.conditions[?(@.type == "ProxyGroupReady")].reason
           name: Status
           type: string
+        - description: ProxyGroup type.
+          jsonPath: .spec.type
+          name: Type
+          type: string
       name: v1alpha1
       schema:
         openAPIV3Schema:
+          description: |-
+            ProxyGroup defines a set of Tailscale devices that will act as proxies.
+            Currently only egress ProxyGroups are supported.
+
+            Use the tailscale.com/proxy-group annotation on a Service to specify that
+            the egress proxy should be implemented by a ProxyGroup instead of a single
+            dedicated proxy. In addition to running a highly available set of proxies,
+            ProxyGroup also allows for serving many annotated Services from a single
+            set of proxies to minimise resource consumption.
+
+            More info: https://tailscale.com/kb/1438/kubernetes-operator-cluster-egress
           type: object
           required:
             - spec
@@ -73,6 +88,7 @@ spec:
                     Defaults to 2.
                   type: integer
                   format: int32
+                  minimum: 0
                 tags:
                   description: |-
                     Tags that the Tailscale devices will be tagged with. Defaults to [tag:k8s].
@@ -86,10 +102,16 @@ spec:
                     type: string
                     pattern: ^tag:[a-zA-Z][a-zA-Z0-9-]*$
                 type:
-                  description: Type of the ProxyGroup proxies. Currently the only supported type is egress.
+                  description: |-
+                    Type of the ProxyGroup proxies. Supported types are egress and ingress.
+                    Type is immutable once a ProxyGroup is created.
                   type: string
                   enum:
                     - egress
+                    - ingress
+                  x-kubernetes-validations:
+                    - rule: self == oldSelf
+                      message: ProxyGroup type is immutable
             status:
               description: |-
                 ProxyGroupStatus describes the status of the ProxyGroup resources. This is

cmd/k8s-operator/deploy/crds/tailscale.com_recorders.yaml

@@ -27,6 +27,12 @@ spec:
       name: v1alpha1
       schema:
         openAPIV3Schema:
+          description: |-
+            Recorder defines a tsrecorder device for recording SSH sessions. By default,
+            it will store recordings in a local ephemeral volume. If you want to persist
+            recordings, you can configure an S3-compatible API for storage.
+
+            More info: https://tailscale.com/kb/1484/kubernetes-operator-deploying-tsrecorder
           type: object
           required:
             - spec

cmd/k8s-operator/deploy/manifests/operator.yaml

@@ -563,6 +563,16 @@ spec:
                                             enable:
                                                 description: If Enable is set to true, a Prometheus ServiceMonitor will be created. Enable can only be set to true if metrics are enabled.
                                                 type: boolean
+                                            labels:
+                                                additionalProperties:
+                                                    maxLength: 63
+                                                    pattern: ^(([a-zA-Z0-9][-._a-zA-Z0-9]*)?[a-zA-Z0-9])?$
+                                                    type: string
+                                                description: |-
+                                                    Labels to add to the ServiceMonitor.
+                                                    Labels must be valid Kubernetes labels.
+                                                    https://kubernetes.io/docs/concepts/overview/working-with-objects/labels/#syntax-and-character-set
+                                                type: object
                                         required:
                                             - enable
                                         type: object
@@ -592,6 +602,8 @@ spec:
                                         type: object
                                     labels:
                                         additionalProperties:
+                                            maxLength: 63
+                                            pattern: ^(([a-zA-Z0-9][-._a-zA-Z0-9]*)?[a-zA-Z0-9])?$
                                             type: string
                                         description: |-
                                             Labels that will be added to the StatefulSet created for the proxy.
@@ -1522,6 +1534,8 @@ spec:
                                                 type: array
                                             labels:
                                                 additionalProperties:
+                                                    maxLength: 63
+                                                    pattern: ^(([a-zA-Z0-9][-._a-zA-Z0-9]*)?[a-zA-Z0-9])?$
                                                     type: string
                                                 description: |-
                                                     Labels that will be added to the proxy Pod.
@@ -2721,9 +2735,24 @@ spec:
               jsonPath: .status.conditions[?(@.type == "ProxyGroupReady")].reason
               name: Status
               type: string
+            - description: ProxyGroup type.
+              jsonPath: .spec.type
+              name: Type
+              type: string
           name: v1alpha1
           schema:
             openAPIV3Schema:
+                description: |-
+                    ProxyGroup defines a set of Tailscale devices that will act as proxies.
+                    Currently only egress ProxyGroups are supported.
+
+                    Use the tailscale.com/proxy-group annotation on a Service to specify that
+                    the egress proxy should be implemented by a ProxyGroup instead of a single
+                    dedicated proxy. In addition to running a highly available set of proxies,
+                    ProxyGroup also allows for serving many annotated Services from a single
+                    set of proxies to minimise resource consumption.
+
+                    More info: https://tailscale.com/kb/1438/kubernetes-operator-cluster-egress
                 properties:
                     apiVersion:
                         description: |-
@@ -2767,6 +2796,7 @@ spec:
                                     Replicas specifies how many replicas to create the StatefulSet with.
                                     Defaults to 2.
                                 format: int32
+                                minimum: 0
                                 type: integer
                             tags:
                                 description: |-
@@ -2781,10 +2811,16 @@ spec:
                                     type: string
                                 type: array
                             type:
-                                description: Type of the ProxyGroup proxies. Currently the only supported type is egress.
+                                description: |-
+                                    Type of the ProxyGroup proxies. Supported types are egress and ingress.
+                                    Type is immutable once a ProxyGroup is created.
                                 enum:
                                     - egress
+                                    - ingress
                                 type: string
+                                x-kubernetes-validations:
+                                    - message: ProxyGroup type is immutable
+                                      rule: self == oldSelf
                         required:
                             - type
                         type: object
@@ -2916,6 +2952,12 @@ spec:
           name: v1alpha1
           schema:
             openAPIV3Schema:
+                description: |-
+                    Recorder defines a tsrecorder device for recording SSH sessions. By default,
+                    it will store recordings in a local ephemeral volume. If you want to persist
+                    recordings, you can configure an S3-compatible API for storage.
+
+                    More info: https://tailscale.com/kb/1484/kubernetes-operator-deploying-tsrecorder
                 properties:
                     apiVersion:
                         description: |-

cmd/k8s-operator/egress-services.go

@@ -495,13 +495,6 @@ func (esr *egressSvcsReconciler) validateClusterResources(ctx context.Context, s
 		tsoperator.RemoveServiceCondition(svc, tsapi.EgressSvcConfigured)
 		return false, err
 	}
-	if !tsoperator.ProxyGroupIsReady(pg) {
-		l.Infof("ProxyGroup %s is not ready, waiting...", proxyGroupName)
-		tsoperator.SetServiceCondition(svc, tsapi.EgressSvcValid, metav1.ConditionUnknown, reasonProxyGroupNotReady, reasonProxyGroupNotReady, esr.clock, l)
-		tsoperator.RemoveServiceCondition(svc, tsapi.EgressSvcConfigured)
-		return false, nil
-	}
-
 	if violations := validateEgressService(svc, pg); len(violations) > 0 {
 		msg := fmt.Sprintf("invalid egress Service: %s", strings.Join(violations, ", "))
 		esr.recorder.Event(svc, corev1.EventTypeWarning, "INVALIDSERVICE", msg)
@@ -510,6 +503,13 @@ func (esr *egressSvcsReconciler) validateClusterResources(ctx context.Context, s
 		tsoperator.RemoveServiceCondition(svc, tsapi.EgressSvcConfigured)
 		return false, nil
 	}
+	if !tsoperator.ProxyGroupIsReady(pg) {
+		l.Infof("ProxyGroup %s is not ready, waiting...", proxyGroupName)
+		tsoperator.SetServiceCondition(svc, tsapi.EgressSvcValid, metav1.ConditionUnknown, reasonProxyGroupNotReady, reasonProxyGroupNotReady, esr.clock, l)
+		tsoperator.RemoveServiceCondition(svc, tsapi.EgressSvcConfigured)
+		return false, nil
+	}
+
 	l.Debugf("egress service is valid")
 	tsoperator.SetServiceCondition(svc, tsapi.EgressSvcValid, metav1.ConditionTrue, reasonEgressSvcValid, reasonEgressSvcValid, esr.clock, l)
 	return true, nil

cmd/k8s-operator/ingress_test.go

@@ -295,7 +295,7 @@ func TestTailscaleIngressWithProxyClass(t *testing.T) {
 	pc := &tsapi.ProxyClass{
 		ObjectMeta: metav1.ObjectMeta{Name: "custom-metadata"},
 		Spec: tsapi.ProxyClassSpec{StatefulSet: &tsapi.StatefulSet{
-			Labels:      map[string]string{"foo": "bar"},
+			Labels:      tsapi.Labels{"foo": "bar"},
 			Annotations: map[string]string{"bar.io/foo": "some-val"},
 			Pod:         &tsapi.Pod{Annotations: map[string]string{"foo.io/bar": "some-val"}}}},
 	}
@@ -424,12 +424,7 @@ func TestTailscaleIngressWithProxyClass(t *testing.T) {
 func TestTailscaleIngressWithServiceMonitor(t *testing.T) {
 	pc := &tsapi.ProxyClass{
 		ObjectMeta: metav1.ObjectMeta{Name: "metrics", Generation: 1},
-		Spec: tsapi.ProxyClassSpec{
-			Metrics: &tsapi.Metrics{
-				Enable:         true,
-				ServiceMonitor: &tsapi.ServiceMonitor{Enable: true},
-			},
-		},
+		Spec:       tsapi.ProxyClassSpec{},
 		Status: tsapi.ProxyClassStatus{
 			Conditions: []metav1.Condition{{
 				Status:             metav1.ConditionTrue,
@@ -437,32 +432,6 @@ func TestTailscaleIngressWithServiceMonitor(t *testing.T) {
 				ObservedGeneration: 1,
 			}}},
 	}
-	crd := &apiextensionsv1.CustomResourceDefinition{ObjectMeta: metav1.ObjectMeta{Name: serviceMonitorCRD}}
-	tsIngressClass := &networkingv1.IngressClass{ObjectMeta: metav1.ObjectMeta{Name: "tailscale"}, Spec: networkingv1.IngressClassSpec{Controller: "tailscale.com/ts-ingress"}}
-	fc := fake.NewClientBuilder().
-		WithScheme(tsapi.GlobalScheme).
-		WithObjects(pc, tsIngressClass).
-		WithStatusSubresource(pc).
-		Build()
-	ft := &fakeTSClient{}
-	fakeTsnetServer := &fakeTSNetServer{certDomains: []string{"foo.com"}}
-	zl, err := zap.NewDevelopment()
-	if err != nil {
-		t.Fatal(err)
-	}
-	ingR := &IngressReconciler{
-		Client: fc,
-		ssr: &tailscaleSTSReconciler{
-			Client:            fc,
-			tsClient:          ft,
-			tsnetServer:       fakeTsnetServer,
-			defaultTags:       []string{"tag:k8s"},
-			operatorNamespace: "operator-ns",
-			proxyImage:        "tailscale/tailscale",
-		},
-		logger: zl.Sugar(),
-	}
-	// 1. Enable metrics- expect metrics Service to be created
 	ing := &networkingv1.Ingress{
 		TypeMeta: metav1.TypeMeta{Kind: "Ingress", APIVersion: "networking.k8s.io/v1"},
 		ObjectMeta: metav1.ObjectMeta{
@@ -491,8 +460,7 @@ func TestTailscaleIngressWithServiceMonitor(t *testing.T) {
 			},
 		},
 	}
-	mustCreate(t, fc, ing)
-	mustCreate(t, fc, &corev1.Service{
+	svc := &corev1.Service{
 		ObjectMeta: metav1.ObjectMeta{
 			Name:      "test",
 			Namespace: "default",
@@ -504,11 +472,38 @@ func TestTailscaleIngressWithServiceMonitor(t *testing.T) {
 				Name: "http"},
 			},
 		},
-	})
-
+	}
+	crd := &apiextensionsv1.CustomResourceDefinition{ObjectMeta: metav1.ObjectMeta{Name: serviceMonitorCRD}}
+	tsIngressClass := &networkingv1.IngressClass{ObjectMeta: metav1.ObjectMeta{Name: "tailscale"}, Spec: networkingv1.IngressClassSpec{Controller: "tailscale.com/ts-ingress"}}
+	fc := fake.NewClientBuilder().
+		WithScheme(tsapi.GlobalScheme).
+		WithObjects(pc, tsIngressClass, ing, svc).
+		WithStatusSubresource(pc).
+		Build()
+	ft := &fakeTSClient{}
+	fakeTsnetServer := &fakeTSNetServer{certDomains: []string{"foo.com"}}
+	zl, err := zap.NewDevelopment()
+	if err != nil {
+		t.Fatal(err)
+	}
+	ingR := &IngressReconciler{
+		Client: fc,
+		ssr: &tailscaleSTSReconciler{
+			Client:            fc,
+			tsClient:          ft,
+			tsnetServer:       fakeTsnetServer,
+			defaultTags:       []string{"tag:k8s"},
+			operatorNamespace: "operator-ns",
+			proxyImage:        "tailscale/tailscale",
+		},
+		logger: zl.Sugar(),
+	}
 	expectReconciled(t, ingR, "default", "test")
-
 	fullName, shortName := findGenName(t, fc, "default", "test", "ingress")
+	serveConfig := &ipn.ServeConfig{
+		TCP: map[uint16]*ipn.TCPPortHandler{443: {HTTPS: true}},
+		Web: map[ipn.HostPort]*ipn.WebServerConfig{"${TS_CERT_DOMAIN}:443": {Handlers: map[string]*ipn.HTTPHandler{"/": {Proxy: "http://1.2.3.4:8080/"}}}},
+	}
 	opts := configOpts{
 		stsName:            shortName,
 		secretName:         fullName,
@@ -517,27 +512,51 @@ func TestTailscaleIngressWithServiceMonitor(t *testing.T) {
 		parentType:         "ingress",
 		hostname:           "default-test",
 		app:                kubetypes.AppIngressResource,
-		enableMetrics:      true,
 		namespaced:         true,
 		proxyType:          proxyTypeIngressResource,
+		serveConfig:        serveConfig,
+		resourceVersion:    "1",
 	}
-	serveConfig := &ipn.ServeConfig{
-		TCP: map[uint16]*ipn.TCPPortHandler{443: {HTTPS: true}},
-		Web: map[ipn.HostPort]*ipn.WebServerConfig{"${TS_CERT_DOMAIN}:443": {Handlers: map[string]*ipn.HTTPHandler{"/": {Proxy: "http://1.2.3.4:8080/"}}}},
-	}
-	opts.serveConfig = serveConfig
 
-	expectEqual(t, fc, expectedSecret(t, fc, opts), nil)
-	expectEqual(t, fc, expectedHeadlessService(shortName, "ingress"), nil)
+	// 1. Enable metrics- expect metrics Service to be created
+	mustUpdate(t, fc, "", "metrics", func(proxyClass *tsapi.ProxyClass) {
+		proxyClass.Spec.Metrics = &tsapi.Metrics{Enable: true}
+	})
+	opts.enableMetrics = true
+
+	expectReconciled(t, ingR, "default", "test")
+
 	expectEqual(t, fc, expectedMetricsService(opts), nil)
-	expectEqual(t, fc, expectedSTSUserspace(t, fc, opts), removeHashAnnotation)
+
 	// 2. Enable ServiceMonitor - should not error when there is no ServiceMonitor CRD in cluster
 	mustUpdate(t, fc, "", "metrics", func(pc *tsapi.ProxyClass) {
-		pc.Spec.Metrics.ServiceMonitor = &tsapi.ServiceMonitor{Enable: true}
+		pc.Spec.Metrics.ServiceMonitor = &tsapi.ServiceMonitor{Enable: true, Labels: tsapi.Labels{"foo": "bar"}}
 	})
 	expectReconciled(t, ingR, "default", "test")
+	expectEqual(t, fc, expectedMetricsService(opts), nil)
+
 	// 3. Create ServiceMonitor CRD and reconcile- ServiceMonitor should get created
 	mustCreate(t, fc, crd)
 	expectReconciled(t, ingR, "default", "test")
+	opts.serviceMonitorLabels = tsapi.Labels{"foo": "bar"}
+	expectEqual(t, fc, expectedMetricsService(opts), nil)
 	expectEqualUnstructured(t, fc, expectedServiceMonitor(t, opts))
+
+	// 4. Update ServiceMonitor CRD and reconcile- ServiceMonitor should get updated
+	mustUpdate(t, fc, pc.Namespace, pc.Name, func(proxyClass *tsapi.ProxyClass) {
+		proxyClass.Spec.Metrics.ServiceMonitor.Labels = nil
+	})
+	expectReconciled(t, ingR, "default", "test")
+	opts.serviceMonitorLabels = nil
+	opts.resourceVersion = "2"
+	expectEqual(t, fc, expectedMetricsService(opts), nil)
+	expectEqualUnstructured(t, fc, expectedServiceMonitor(t, opts))
+
+	// 5. Disable metrics - metrics resources should get deleted.
+	mustUpdate(t, fc, pc.Namespace, pc.Name, func(proxyClass *tsapi.ProxyClass) {
+		proxyClass.Spec.Metrics = nil
+	})
+	expectReconciled(t, ingR, "default", "test")
+	expectMissing[corev1.Service](t, fc, "operator-ns", metricsResourceName(shortName))
+	// ServiceMonitor gets garbage collected when the Service is deleted - we cannot test that here.
 }

cmd/k8s-operator/metrics_resources.go

@@ -8,6 +8,7 @@ package main
 import (
 	"context"
 	"fmt"
+	"reflect"
 
 	"go.uber.org/zap"
 	corev1 "k8s.io/api/core/v1"
@@ -115,15 +116,15 @@ func reconcileMetricsResources(ctx context.Context, logger *zap.SugaredLogger, o
 		return maybeCleanupServiceMonitor(ctx, cl, opts.proxyStsName, opts.tsNamespace)
 	}
 
-	logger.Info("ensuring ServiceMonitor for metrics Service %s/%s", metricsSvc.Namespace, metricsSvc.Name)
-	svcMonitor, err := newServiceMonitor(metricsSvc)
+	logger.Infof("ensuring ServiceMonitor for metrics Service %s/%s", metricsSvc.Namespace, metricsSvc.Name)
+	svcMonitor, err := newServiceMonitor(metricsSvc, pc.Spec.Metrics.ServiceMonitor)
 	if err != nil {
 		return fmt.Errorf("error creating ServiceMonitor: %w", err)
 	}
-	// We don't use createOrUpdate here because that does not work with unstructured types. We also do not update
-	// the ServiceMonitor because it is not expected that any of its fields would change. Currently this is good
-	// enough, but in future we might want to add logic to create-or-update unstructured types.
-	err = cl.Get(ctx, client.ObjectKeyFromObject(metricsSvc), svcMonitor.DeepCopy())
+
+	// We don't use createOrUpdate here because that does not work with unstructured types.
+	existing := svcMonitor.DeepCopy()
+	err = cl.Get(ctx, client.ObjectKeyFromObject(metricsSvc), existing)
 	if apierrors.IsNotFound(err) {
 		if err := cl.Create(ctx, svcMonitor); err != nil {
 			return fmt.Errorf("error creating ServiceMonitor: %w", err)
@@ -133,6 +134,13 @@ func reconcileMetricsResources(ctx context.Context, logger *zap.SugaredLogger, o
 	if err != nil {
 		return fmt.Errorf("error getting ServiceMonitor: %w", err)
 	}
+	// Currently, we only update labels on the ServiceMonitor as those are the only values that can change.
+	if !reflect.DeepEqual(existing.GetLabels(), svcMonitor.GetLabels()) {
+		existing.SetLabels(svcMonitor.GetLabels())
+		if err := cl.Update(ctx, existing); err != nil {
+			return fmt.Errorf("error updating ServiceMonitor: %w", err)
+		}
+	}
 	return nil
 }
 
@@ -165,9 +173,13 @@ func maybeCleanupServiceMonitor(ctx context.Context, cl client.Client, stsName,
 // newServiceMonitor takes a metrics Service created for a proxy and constructs and returns a ServiceMonitor for that
 // proxy that can be applied to the kube API server.
 // The ServiceMonitor is returned as Unstructured type - this allows us to avoid importing prometheus-operator API server client/schema.
-func newServiceMonitor(metricsSvc *corev1.Service) (*unstructured.Unstructured, error) {
+func newServiceMonitor(metricsSvc *corev1.Service, spec *tsapi.ServiceMonitor) (*unstructured.Unstructured, error) {
 	sm := serviceMonitorTemplate(metricsSvc.Name, metricsSvc.Namespace)
 	sm.ObjectMeta.Labels = metricsSvc.Labels
+	if spec != nil && len(spec.Labels) > 0 {
+		sm.ObjectMeta.Labels = mergeMapKeys(sm.ObjectMeta.Labels, spec.Labels.Parse())
+	}
+
 	sm.ObjectMeta.OwnerReferences = []metav1.OwnerReference{*metav1.NewControllerRef(metricsSvc, corev1.SchemeGroupVersion.WithKind("Service"))}
 	sm.Spec = ServiceMonitorSpec{
 		Selector: metav1.LabelSelector{MatchLabels: metricsSvc.Labels},
@@ -270,3 +282,14 @@ type metricsOpts struct {
 func isNamespacedProxyType(typ string) bool {
 	return typ == proxyTypeIngressResource || typ == proxyTypeIngressService
 }
+
+func mergeMapKeys(a, b map[string]string) map[string]string {
+	m := make(map[string]string, len(a)+len(b))
+	for key, val := range b {
+		m[key] = val
+	}
+	for key, val := range a {
+		m[key] = val
+	}
+	return m
+}

cmd/k8s-operator/operator.go

@@ -499,7 +499,7 @@ func runReconcilers(opts reconcilerOpts) {
 		startlog.Fatalf("could not create Recorder reconciler: %v", err)
 	}
 
-	// Recorder reconciler.
+	// ProxyGroup reconciler.
 	ownedByProxyGroupFilter := handler.EnqueueRequestForOwner(mgr.GetScheme(), mgr.GetRESTMapper(), &tsapi.ProxyGroup{})
 	proxyClassFilterForProxyGroup := handler.EnqueueRequestsFromMapFunc(proxyClassHandlerForProxyGroup(mgr.GetClient(), startlog))
 	err = builder.ControllerManagedBy(mgr).

cmd/k8s-operator/operator_test.go

@@ -16,6 +16,7 @@ import (
 	appsv1 "k8s.io/api/apps/v1"
 	corev1 "k8s.io/api/core/v1"
 	networkingv1 "k8s.io/api/networking/v1"
+	apiextensionsv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/types"
 	"k8s.io/client-go/tools/record"
@@ -1129,7 +1130,7 @@ func TestProxyClassForService(t *testing.T) {
 				AcceptRoutes: true,
 			},
 			StatefulSet: &tsapi.StatefulSet{
-				Labels:      map[string]string{"foo": "bar"},
+				Labels:      tsapi.Labels{"foo": "bar"},
 				Annotations: map[string]string{"bar.io/foo": "some-val"},
 				Pod:         &tsapi.Pod{Annotations: map[string]string{"foo.io/bar": "some-val"}}}},
 	}
@@ -1766,6 +1767,106 @@ func Test_externalNameService(t *testing.T) {
 	expectEqual(t, fc, expectedSTS(t, fc, opts), removeHashAnnotation)
 }
 
+func Test_metricsResourceCreation(t *testing.T) {
+	pc := &tsapi.ProxyClass{
+		ObjectMeta: metav1.ObjectMeta{Name: "metrics", Generation: 1},
+		Spec:       tsapi.ProxyClassSpec{},
+		Status: tsapi.ProxyClassStatus{
+			Conditions: []metav1.Condition{{
+				Status:             metav1.ConditionTrue,
+				Type:               string(tsapi.ProxyClassReady),
+				ObservedGeneration: 1,
+			}}},
+	}
+	svc := &corev1.Service{
+		ObjectMeta: metav1.ObjectMeta{
+			Name:      "test",
+			Namespace: "default",
+			UID:       types.UID("1234-UID"),
+			Labels:    map[string]string{LabelProxyClass: "metrics"},
+		},
+		Spec: corev1.ServiceSpec{
+			ClusterIP:         "10.20.30.40",
+			Type:              corev1.ServiceTypeLoadBalancer,
+			LoadBalancerClass: ptr.To("tailscale"),
+		},
+	}
+	crd := &apiextensionsv1.CustomResourceDefinition{ObjectMeta: metav1.ObjectMeta{Name: serviceMonitorCRD}}
+	fc := fake.NewClientBuilder().
+		WithScheme(tsapi.GlobalScheme).
+		WithObjects(pc, svc).
+		WithStatusSubresource(pc).
+		Build()
+	ft := &fakeTSClient{}
+	zl, err := zap.NewDevelopment()
+	if err != nil {
+		t.Fatal(err)
+	}
+	clock := tstest.NewClock(tstest.ClockOpts{})
+	sr := &ServiceReconciler{
+		Client: fc,
+		ssr: &tailscaleSTSReconciler{
+			Client:            fc,
+			tsClient:          ft,
+			operatorNamespace: "operator-ns",
+		},
+		logger: zl.Sugar(),
+		clock:  clock,
+	}
+	expectReconciled(t, sr, "default", "test")
+	fullName, shortName := findGenName(t, fc, "default", "test", "svc")
+	opts := configOpts{
+		stsName:            shortName,
+		secretName:         fullName,
+		namespace:          "default",
+		parentType:         "svc",
+		tailscaleNamespace: "operator-ns",
+		hostname:           "default-test",
+		namespaced:         true,
+		proxyType:          proxyTypeIngressService,
+		app:                kubetypes.AppIngressProxy,
+		resourceVersion:    "1",
+	}
+
+	// 1. Enable metrics- expect metrics Service to be created
+	mustUpdate(t, fc, "", "metrics", func(pc *tsapi.ProxyClass) {
+		pc.Spec = tsapi.ProxyClassSpec{Metrics: &tsapi.Metrics{Enable: true}}
+	})
+	expectReconciled(t, sr, "default", "test")
+	opts.enableMetrics = true
+	expectEqual(t, fc, expectedMetricsService(opts), nil)
+
+	// 2. Enable ServiceMonitor - should not error when there is no ServiceMonitor CRD in cluster
+	mustUpdate(t, fc, "", "metrics", func(pc *tsapi.ProxyClass) {
+		pc.Spec.Metrics.ServiceMonitor = &tsapi.ServiceMonitor{Enable: true}
+	})
+	expectReconciled(t, sr, "default", "test")
+
+	// 3. Create ServiceMonitor CRD and reconcile- ServiceMonitor should get created
+	mustCreate(t, fc, crd)
+	expectReconciled(t, sr, "default", "test")
+	expectEqualUnstructured(t, fc, expectedServiceMonitor(t, opts))
+
+	// 4. A change to ServiceMonitor config gets reflected in the ServiceMonitor resource
+	mustUpdate(t, fc, "", "metrics", func(pc *tsapi.ProxyClass) {
+		pc.Spec.Metrics.ServiceMonitor.Labels = tsapi.Labels{"foo": "bar"}
+	})
+	expectReconciled(t, sr, "default", "test")
+	opts.serviceMonitorLabels = tsapi.Labels{"foo": "bar"}
+	opts.resourceVersion = "2"
+	expectEqual(t, fc, expectedMetricsService(opts), nil)
+	expectEqualUnstructured(t, fc, expectedServiceMonitor(t, opts))
+
+	// 5. Disable metrics- expect metrics Service to be deleted
+	mustUpdate(t, fc, "", "metrics", func(pc *tsapi.ProxyClass) {
+		pc.Spec.Metrics = nil
+	})
+	expectReconciled(t, sr, "default", "test")
+	expectMissing[corev1.Service](t, fc, "operator-ns", metricsResourceName(opts.stsName))
+	// ServiceMonitor gets garbage collected when Service gets deleted (it has OwnerReference of the Service
+	// object). We cannot test this using the fake client.
+}
+
 func toFQDN(t *testing.T, s string) dnsname.FQDN {
 	t.Helper()
 	fqdn, err := dnsname.ToFQDN(s)

cmd/k8s-operator/proxy.go

@@ -311,7 +311,7 @@ func (h *apiserverProxy) addImpersonationHeadersAsRequired(r *http.Request) {
 
 	// Now add the impersonation headers that we want.
 	if err := addImpersonationHeaders(r, h.log); err != nil {
-		log.Printf("failed to add impersonation headers: " + err.Error())
+		log.Print("failed to add impersonation headers: ", err.Error())
 	}
 }
 

cmd/k8s-operator/proxyclass.go

@@ -115,7 +115,7 @@ func (pcr *ProxyClassReconciler) Reconcile(ctx context.Context, req reconcile.Re
 func (pcr *ProxyClassReconciler) validate(ctx context.Context, pc *tsapi.ProxyClass) (violations field.ErrorList) {
 	if sts := pc.Spec.StatefulSet; sts != nil {
 		if len(sts.Labels) > 0 {
-			if errs := metavalidation.ValidateLabels(sts.Labels, field.NewPath(".spec.statefulSet.labels")); errs != nil {
+			if errs := metavalidation.ValidateLabels(sts.Labels.Parse(), field.NewPath(".spec.statefulSet.labels")); errs != nil {
 				violations = append(violations, errs...)
 			}
 		}
@@ -126,7 +126,7 @@ func (pcr *ProxyClassReconciler) validate(ctx context.Context, pc *tsapi.ProxyCl
 		}
 		if pod := sts.Pod; pod != nil {
 			if len(pod.Labels) > 0 {
-				if errs := metavalidation.ValidateLabels(pod.Labels, field.NewPath(".spec.statefulSet.pod.labels")); errs != nil {
+				if errs := metavalidation.ValidateLabels(pod.Labels.Parse(), field.NewPath(".spec.statefulSet.pod.labels")); errs != nil {
 					violations = append(violations, errs...)
 				}
 			}
@@ -178,6 +178,11 @@ func (pcr *ProxyClassReconciler) validate(ctx context.Context, pc *tsapi.ProxyCl
 			violations = append(violations, field.TypeInvalid(field.NewPath("spec", "metrics", "serviceMonitor"), "enable", msg))
 		}
 	}
+	if pc.Spec.Metrics != nil && pc.Spec.Metrics.ServiceMonitor != nil && len(pc.Spec.Metrics.ServiceMonitor.Labels) > 0 {
+		if errs := metavalidation.ValidateLabels(pc.Spec.Metrics.ServiceMonitor.Labels.Parse(), field.NewPath(".spec.metrics.serviceMonitor.labels")); errs != nil {
+			violations = append(violations, errs...)
+		}
+	}
 	// We do not validate embedded fields (security context, resource
 	// requirements etc) as we inherit upstream validation for those fields.
 	// Invalid values would get rejected by upstream validations at apply

cmd/k8s-operator/proxyclass_test.go

@@ -36,10 +36,10 @@ func TestProxyClass(t *testing.T) {
 		},
 		Spec: tsapi.ProxyClassSpec{
 			StatefulSet: &tsapi.StatefulSet{
-				Labels:      map[string]string{"foo": "bar", "xyz1234": "abc567"},
+				Labels:      tsapi.Labels{"foo": "bar", "xyz1234": "abc567"},
 				Annotations: map[string]string{"foo.io/bar": "{'key': 'val1232'}"},
 				Pod: &tsapi.Pod{
-					Labels:      map[string]string{"foo": "bar", "xyz1234": "abc567"},
+					Labels:      tsapi.Labels{"foo": "bar", "xyz1234": "abc567"},
 					Annotations: map[string]string{"foo.io/bar": "{'key': 'val1232'}"},
 					TailscaleContainer: &tsapi.Container{
 						Env:             []tsapi.Env{{Name: "FOO", Value: "BAR"}},
@@ -155,6 +155,25 @@ func TestProxyClass(t *testing.T) {
 	expectReconciled(t, pcr, "", "test")
 	tsoperator.SetProxyClassCondition(pc, tsapi.ProxyClassReady, metav1.ConditionTrue, reasonProxyClassValid, reasonProxyClassValid, 0, cl, zl.Sugar())
 	expectEqual(t, fc, pc, nil)
+
+	// 7. A ProxyClass with invalid ServiceMonitor labels gets its status updated to Invalid with an error message.
+	pc.Spec.Metrics.ServiceMonitor.Labels = tsapi.Labels{"foo": "bar!"}
+	mustUpdate(t, fc, "", "test", func(proxyClass *tsapi.ProxyClass) {
+		proxyClass.Spec.Metrics.ServiceMonitor.Labels = pc.Spec.Metrics.ServiceMonitor.Labels
+	})
+	expectReconciled(t, pcr, "", "test")
+	msg = `ProxyClass is not valid: .spec.metrics.serviceMonitor.labels: Invalid value: "bar!": a valid label must be an empty string or consist of alphanumeric characters, '-', '_' or '.', and must start and end with an alphanumeric character (e.g. 'MyValue',  or 'my_value',  or '12345', regex used for validation is '(([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9])?')`
+	tsoperator.SetProxyClassCondition(pc, tsapi.ProxyClassReady, metav1.ConditionFalse, reasonProxyClassInvalid, msg, 0, cl, zl.Sugar())
+	expectEqual(t, fc, pc, nil)
+
+	// 8. A ProxyClass with valid ServiceMonitor labels gets its status updated to Valid.
+	pc.Spec.Metrics.ServiceMonitor.Labels = tsapi.Labels{"foo": "bar", "xyz1234": "abc567", "empty": "", "onechar": "a"}
+	mustUpdate(t, fc, "", "test", func(proxyClass *tsapi.ProxyClass) {
+		proxyClass.Spec.Metrics.ServiceMonitor.Labels = pc.Spec.Metrics.ServiceMonitor.Labels
+	})
+	expectReconciled(t, pcr, "", "test")
+	tsoperator.SetProxyClassCondition(pc, tsapi.ProxyClassReady, metav1.ConditionTrue, reasonProxyClassValid, reasonProxyClassValid, 0, cl, zl.Sugar())
+	expectEqual(t, fc, pc, nil)
 }
 
 func TestValidateProxyClass(t *testing.T) {

cmd/k8s-operator/proxygroup.go

@@ -51,7 +51,10 @@ const (
 	optimisticLockErrorMsg = "the object has been modified; please apply your changes to the latest version and try again"
 )
 
-var gaugeProxyGroupResources = clientmetric.NewGauge(kubetypes.MetricProxyGroupEgressCount)
+var (
+	gaugeEgressProxyGroupResources  = clientmetric.NewGauge(kubetypes.MetricProxyGroupEgressCount)
+	gaugeIngressProxyGroupResources = clientmetric.NewGauge(kubetypes.MetricProxyGroupIngressCount)
+)
 
 // ProxyGroupReconciler ensures cluster resources for a ProxyGroup definition.
 type ProxyGroupReconciler struct {
@@ -68,8 +71,9 @@ type ProxyGroupReconciler struct {
 	tsFirewallMode    string
 	defaultProxyClass string
 
-	mu          sync.Mutex           // protects following
-	proxyGroups set.Slice[types.UID] // for proxygroups gauge
+	mu                 sync.Mutex           // protects following
+	egressProxyGroups  set.Slice[types.UID] // for egress proxygroups gauge
+	ingressProxyGroups set.Slice[types.UID] // for ingress proxygroups gauge
 }
 
 func (r *ProxyGroupReconciler) logger(name string) *zap.SugaredLogger {
@@ -203,8 +207,7 @@ func (r *ProxyGroupReconciler) Reconcile(ctx context.Context, req reconcile.Requ
 func (r *ProxyGroupReconciler) maybeProvision(ctx context.Context, pg *tsapi.ProxyGroup, proxyClass *tsapi.ProxyClass) error {
 	logger := r.logger(pg.Name)
 	r.mu.Lock()
-	r.proxyGroups.Add(pg.UID)
-	gaugeProxyGroupResources.Set(int64(r.proxyGroups.Len()))
+	r.ensureAddedToGaugeForProxyGroup(pg)
 	r.mu.Unlock()
 
 	cfgHash, err := r.ensureConfigSecretsCreated(ctx, pg, proxyClass)
@@ -358,8 +361,7 @@ func (r *ProxyGroupReconciler) maybeCleanup(ctx context.Context, pg *tsapi.Proxy
 
 	logger.Infof("cleaned up ProxyGroup resources")
 	r.mu.Lock()
-	r.proxyGroups.Remove(pg.UID)
-	gaugeProxyGroupResources.Set(int64(r.proxyGroups.Len()))
+	r.ensureRemovedFromGaugeForProxyGroup(pg)
 	r.mu.Unlock()
 	return true, nil
 }
@@ -469,6 +471,32 @@ func (r *ProxyGroupReconciler) ensureConfigSecretsCreated(ctx context.Context, p
 	return configSHA256Sum, nil
 }
 
+// ensureAddedToGaugeForProxyGroup ensures the gauge metric for the ProxyGroup resource is updated when the ProxyGroup
+// is created. r.mu must be held.
+func (r *ProxyGroupReconciler) ensureAddedToGaugeForProxyGroup(pg *tsapi.ProxyGroup) {
+	switch pg.Spec.Type {
+	case tsapi.ProxyGroupTypeEgress:
+		r.egressProxyGroups.Add(pg.UID)
+	case tsapi.ProxyGroupTypeIngress:
+		r.ingressProxyGroups.Add(pg.UID)
+	}
+	gaugeEgressProxyGroupResources.Set(int64(r.egressProxyGroups.Len()))
+	gaugeIngressProxyGroupResources.Set(int64(r.ingressProxyGroups.Len()))
+}
+
+// ensureRemovedFromGaugeForProxyGroup ensures the gauge metric for the ProxyGroup resource type is updated when the
+// ProxyGroup is deleted. r.mu must be held.
+func (r *ProxyGroupReconciler) ensureRemovedFromGaugeForProxyGroup(pg *tsapi.ProxyGroup) {
+	switch pg.Spec.Type {
+	case tsapi.ProxyGroupTypeEgress:
+		r.egressProxyGroups.Remove(pg.UID)
+	case tsapi.ProxyGroupTypeIngress:
+		r.ingressProxyGroups.Remove(pg.UID)
+	}
+	gaugeEgressProxyGroupResources.Set(int64(r.egressProxyGroups.Len()))
+	gaugeIngressProxyGroupResources.Set(int64(r.ingressProxyGroups.Len()))
+}
+
 func pgTailscaledConfig(pg *tsapi.ProxyGroup, class *tsapi.ProxyClass, idx int32, authKey string, oldSecret *corev1.Secret) (tailscaledConfigs, error) {
 	conf := &ipn.ConfigVAlpha{
 		Version:      "alpha0",
@@ -479,7 +507,7 @@ func pgTailscaledConfig(pg *tsapi.ProxyGroup, class *tsapi.ProxyClass, idx int32
 	}
 
 	if pg.Spec.HostnamePrefix != "" {
-		conf.Hostname = ptr.To(fmt.Sprintf("%s%d", pg.Spec.HostnamePrefix, idx))
+		conf.Hostname = ptr.To(fmt.Sprintf("%s-%d", pg.Spec.HostnamePrefix, idx))
 	}
 
 	if shouldAcceptRoutes(class) {

cmd/k8s-operator/proxygroup_specs.go

@@ -138,10 +138,6 @@ func pgStatefulSet(pg *tsapi.ProxyGroup, namespace, image, tsFirewallMode, cfgHa
 				Name:  "TS_EXPERIMENTAL_VERSIONED_CONFIG_DIR",
 				Value: "/etc/tsconfig/$(POD_NAME)",
 			},
-			{
-				Name:  "TS_INTERNAL_APP",
-				Value: kubetypes.AppProxyGroupEgress,
-			},
 		}
 
 		if tsFirewallMode != "" {
@@ -155,9 +151,18 @@ func pgStatefulSet(pg *tsapi.ProxyGroup, namespace, image, tsFirewallMode, cfgHa
 			envs = append(envs, corev1.EnvVar{
 				Name:  "TS_EGRESS_SERVICES_CONFIG_PATH",
 				Value: fmt.Sprintf("/etc/proxies/%s", egressservices.KeyEgressServices),
+			},
+				corev1.EnvVar{
+					Name:  "TS_INTERNAL_APP",
+					Value: kubetypes.AppProxyGroupEgress,
+				},
+			)
+		} else {
+			envs = append(envs, corev1.EnvVar{
+				Name:  "TS_INTERNAL_APP",
+				Value: kubetypes.AppProxyGroupIngress,
 			})
 		}
-
 		return append(c.Env, envs...)
 	}()
 

cmd/k8s-operator/proxygroup_test.go

@@ -25,6 +25,8 @@ import (
 	"tailscale.com/client/tailscale"
 	tsoperator "tailscale.com/k8s-operator"
 	tsapi "tailscale.com/k8s-operator/apis/v1alpha1"
+	"tailscale.com/kube/egressservices"
+	"tailscale.com/kube/kubetypes"
 	"tailscale.com/tstest"
 	"tailscale.com/types/ptr"
 )
@@ -53,6 +55,9 @@ func TestProxyGroup(t *testing.T) {
 			Name:       "test",
 			Finalizers: []string{"tailscale.com/finalizer"},
 		},
+		Spec: tsapi.ProxyGroupSpec{
+			Type: tsapi.ProxyGroupTypeEgress,
+		},
 	}
 
 	fc := fake.NewClientBuilder().
@@ -83,6 +88,7 @@ func TestProxyGroup(t *testing.T) {
 		stsName:            pg.Name,
 		parentType:         "proxygroup",
 		tailscaleNamespace: "tailscale",
+		resourceVersion:    "1",
 	}
 
 	t.Run("proxyclass_not_ready", func(t *testing.T) {
@@ -112,8 +118,8 @@ func TestProxyGroup(t *testing.T) {
 		tsoperator.SetProxyGroupCondition(pg, tsapi.ProxyGroupReady, metav1.ConditionFalse, reasonProxyGroupCreating, "0/2 ProxyGroup pods running", 0, cl, zl.Sugar())
 		expectEqual(t, fc, pg, nil)
 		expectProxyGroupResources(t, fc, pg, true, initialCfgHash)
-		if expected := 1; reconciler.proxyGroups.Len() != expected {
-			t.Fatalf("expected %d recorders, got %d", expected, reconciler.proxyGroups.Len())
+		if expected := 1; reconciler.egressProxyGroups.Len() != expected {
+			t.Fatalf("expected %d egress ProxyGroups, got %d", expected, reconciler.egressProxyGroups.Len())
 		}
 		expectProxyGroupResources(t, fc, pg, true, initialCfgHash)
 		keyReq := tailscale.KeyCapabilities{
@@ -227,8 +233,8 @@ func TestProxyGroup(t *testing.T) {
 		expectReconciled(t, reconciler, "", pg.Name)
 
 		expectMissing[tsapi.ProxyGroup](t, fc, "", pg.Name)
-		if expected := 0; reconciler.proxyGroups.Len() != expected {
-			t.Fatalf("expected %d ProxyGroups, got %d", expected, reconciler.proxyGroups.Len())
+		if expected := 0; reconciler.egressProxyGroups.Len() != expected {
+			t.Fatalf("expected %d ProxyGroups, got %d", expected, reconciler.egressProxyGroups.Len())
 		}
 		// 2 nodes should get deleted as part of the scale down, and then finally
 		// the first node gets deleted with the ProxyGroup cleanup.
@@ -241,6 +247,131 @@ func TestProxyGroup(t *testing.T) {
 	})
 }
 
+func TestProxyGroupTypes(t *testing.T) {
+	fc := fake.NewClientBuilder().
+		WithScheme(tsapi.GlobalScheme).
+		Build()
+
+	zl, _ := zap.NewDevelopment()
+	reconciler := &ProxyGroupReconciler{
+		tsNamespace: tsNamespace,
+		proxyImage:  testProxyImage,
+		Client:      fc,
+		l:           zl.Sugar(),
+		tsClient:    &fakeTSClient{},
+		clock:       tstest.NewClock(tstest.ClockOpts{}),
+	}
+
+	t.Run("egress_type", func(t *testing.T) {
+		pg := &tsapi.ProxyGroup{
+			ObjectMeta: metav1.ObjectMeta{
+				Name: "test-egress",
+				UID:  "test-egress-uid",
+			},
+			Spec: tsapi.ProxyGroupSpec{
+				Type:     tsapi.ProxyGroupTypeEgress,
+				Replicas: ptr.To[int32](0),
+			},
+		}
+		if err := fc.Create(context.Background(), pg); err != nil {
+			t.Fatal(err)
+		}
+
+		expectReconciled(t, reconciler, "", pg.Name)
+		verifyProxyGroupCounts(t, reconciler, 0, 1)
+
+		sts := &appsv1.StatefulSet{}
+		if err := fc.Get(context.Background(), client.ObjectKey{Namespace: tsNamespace, Name: pg.Name}, sts); err != nil {
+			t.Fatalf("failed to get StatefulSet: %v", err)
+		}
+		verifyEnvVar(t, sts, "TS_INTERNAL_APP", kubetypes.AppProxyGroupEgress)
+		verifyEnvVar(t, sts, "TS_EGRESS_SERVICES_CONFIG_PATH", fmt.Sprintf("/etc/proxies/%s", egressservices.KeyEgressServices))
+
+		// Verify that egress configuration has been set up.
+		cm := &corev1.ConfigMap{}
+		cmName := fmt.Sprintf("%s-egress-config", pg.Name)
+		if err := fc.Get(context.Background(), client.ObjectKey{Namespace: tsNamespace, Name: cmName}, cm); err != nil {
+			t.Fatalf("failed to get ConfigMap: %v", err)
+		}
+
+		expectedVolumes := []corev1.Volume{
+			{
+				Name: cmName,
+				VolumeSource: corev1.VolumeSource{
+					ConfigMap: &corev1.ConfigMapVolumeSource{
+						LocalObjectReference: corev1.LocalObjectReference{
+							Name: cmName,
+						},
+					},
+				},
+			},
+		}
+
+		expectedVolumeMounts := []corev1.VolumeMount{
+			{
+				Name:      cmName,
+				MountPath: "/etc/proxies",
+				ReadOnly:  true,
+			},
+		}
+
+		if diff := cmp.Diff(expectedVolumes, sts.Spec.Template.Spec.Volumes); diff != "" {
+			t.Errorf("unexpected volumes (-want +got):\n%s", diff)
+		}
+
+		if diff := cmp.Diff(expectedVolumeMounts, sts.Spec.Template.Spec.Containers[0].VolumeMounts); diff != "" {
+			t.Errorf("unexpected volume mounts (-want +got):\n%s", diff)
+		}
+	})
+
+	t.Run("ingress_type", func(t *testing.T) {
+		pg := &tsapi.ProxyGroup{
+			ObjectMeta: metav1.ObjectMeta{
+				Name: "test-ingress",
+				UID:  "test-ingress-uid",
+			},
+			Spec: tsapi.ProxyGroupSpec{
+				Type: tsapi.ProxyGroupTypeIngress,
+			},
+		}
+		if err := fc.Create(context.Background(), pg); err != nil {
+			t.Fatal(err)
+		}
+
+		expectReconciled(t, reconciler, "", pg.Name)
+		verifyProxyGroupCounts(t, reconciler, 1, 1)
+
+		sts := &appsv1.StatefulSet{}
+		if err := fc.Get(context.Background(), client.ObjectKey{Namespace: tsNamespace, Name: pg.Name}, sts); err != nil {
+			t.Fatalf("failed to get StatefulSet: %v", err)
+		}
+		verifyEnvVar(t, sts, "TS_INTERNAL_APP", kubetypes.AppProxyGroupIngress)
+	})
+}
+
+func verifyProxyGroupCounts(t *testing.T, r *ProxyGroupReconciler, wantIngress, wantEgress int) {
+	t.Helper()
+	if r.ingressProxyGroups.Len() != wantIngress {
+		t.Errorf("expected %d ingress proxy groups, got %d", wantIngress, r.ingressProxyGroups.Len())
+	}
+	if r.egressProxyGroups.Len() != wantEgress {
+		t.Errorf("expected %d egress proxy groups, got %d", wantEgress, r.egressProxyGroups.Len())
+	}
+}
+
+func verifyEnvVar(t *testing.T, sts *appsv1.StatefulSet, name, expectedValue string) {
+	t.Helper()
+	for _, env := range sts.Spec.Template.Spec.Containers[0].Env {
+		if env.Name == name {
+			if env.Value != expectedValue {
+				t.Errorf("expected %s=%s, got %s", name, expectedValue, env.Value)
+			}
+			return
+		}
+	}
+	t.Errorf("%s environment variable not found", name)
+}
+
 func expectProxyGroupResources(t *testing.T, fc client.WithWatch, pg *tsapi.ProxyGroup, shouldExist bool, cfgHash string) {
 	t.Helper()
 

cmd/k8s-operator/sts.go

@@ -761,7 +761,7 @@ func applyProxyClassToStatefulSet(pc *tsapi.ProxyClass, ss *appsv1.StatefulSet,
 	}
 
 	// Update StatefulSet metadata.
-	if wantsSSLabels := pc.Spec.StatefulSet.Labels; len(wantsSSLabels) > 0 {
+	if wantsSSLabels := pc.Spec.StatefulSet.Labels.Parse(); len(wantsSSLabels) > 0 {
 		ss.ObjectMeta.Labels = mergeStatefulSetLabelsOrAnnots(ss.ObjectMeta.Labels, wantsSSLabels, tailscaleManagedLabels)
 	}
 	if wantsSSAnnots := pc.Spec.StatefulSet.Annotations; len(wantsSSAnnots) > 0 {
@@ -773,7 +773,7 @@ func applyProxyClassToStatefulSet(pc *tsapi.ProxyClass, ss *appsv1.StatefulSet,
 		return ss
 	}
 	wantsPod := pc.Spec.StatefulSet.Pod
-	if wantsPodLabels := wantsPod.Labels; len(wantsPodLabels) > 0 {
+	if wantsPodLabels := wantsPod.Labels.Parse(); len(wantsPodLabels) > 0 {
 		ss.Spec.Template.ObjectMeta.Labels = mergeStatefulSetLabelsOrAnnots(ss.Spec.Template.ObjectMeta.Labels, wantsPodLabels, tailscaleManagedLabels)
 	}
 	if wantsPodAnnots := wantsPod.Annotations; len(wantsPodAnnots) > 0 {

cmd/k8s-operator/sts_test.go

@@ -61,10 +61,10 @@ func Test_applyProxyClassToStatefulSet(t *testing.T) {
 	proxyClassAllOpts := &tsapi.ProxyClass{
 		Spec: tsapi.ProxyClassSpec{
 			StatefulSet: &tsapi.StatefulSet{
-				Labels:      map[string]string{"foo": "bar"},
+				Labels:      tsapi.Labels{"foo": "bar"},
 				Annotations: map[string]string{"foo.io/bar": "foo"},
 				Pod: &tsapi.Pod{
-					Labels:      map[string]string{"bar": "foo"},
+					Labels:      tsapi.Labels{"bar": "foo"},
 					Annotations: map[string]string{"bar.io/foo": "foo"},
 					SecurityContext: &corev1.PodSecurityContext{
 						RunAsUser: ptr.To(int64(0)),
@@ -116,10 +116,10 @@ func Test_applyProxyClassToStatefulSet(t *testing.T) {
 	proxyClassJustLabels := &tsapi.ProxyClass{
 		Spec: tsapi.ProxyClassSpec{
 			StatefulSet: &tsapi.StatefulSet{
-				Labels:      map[string]string{"foo": "bar"},
+				Labels:      tsapi.Labels{"foo": "bar"},
 				Annotations: map[string]string{"foo.io/bar": "foo"},
 				Pod: &tsapi.Pod{
-					Labels:      map[string]string{"bar": "foo"},
+					Labels:      tsapi.Labels{"bar": "foo"},
 					Annotations: map[string]string{"bar.io/foo": "foo"},
 				},
 			},
@@ -146,7 +146,6 @@ func Test_applyProxyClassToStatefulSet(t *testing.T) {
 			},
 		}
 	}
-
 	var userspaceProxySS, nonUserspaceProxySS appsv1.StatefulSet
 	if err := yaml.Unmarshal(userspaceProxyYaml, &userspaceProxySS); err != nil {
 		t.Fatalf("unmarshaling userspace proxy template: %v", err)
@@ -176,9 +175,9 @@ func Test_applyProxyClassToStatefulSet(t *testing.T) {
 	// 1. Test that a ProxyClass with all fields set gets correctly applied
 	// to a Statefulset built from non-userspace proxy template.
 	wantSS := nonUserspaceProxySS.DeepCopy()
-	wantSS.ObjectMeta.Labels = mergeMapKeys(wantSS.ObjectMeta.Labels, proxyClassAllOpts.Spec.StatefulSet.Labels)
-	wantSS.ObjectMeta.Annotations = mergeMapKeys(wantSS.ObjectMeta.Annotations, proxyClassAllOpts.Spec.StatefulSet.Annotations)
-	wantSS.Spec.Template.Labels = proxyClassAllOpts.Spec.StatefulSet.Pod.Labels
+	updateMap(wantSS.ObjectMeta.Labels, proxyClassAllOpts.Spec.StatefulSet.Labels.Parse())
+	updateMap(wantSS.ObjectMeta.Annotations, proxyClassAllOpts.Spec.StatefulSet.Annotations)
+	wantSS.Spec.Template.Labels = proxyClassAllOpts.Spec.StatefulSet.Pod.Labels.Parse()
 	wantSS.Spec.Template.Annotations = proxyClassAllOpts.Spec.StatefulSet.Pod.Annotations
 	wantSS.Spec.Template.Spec.SecurityContext = proxyClassAllOpts.Spec.StatefulSet.Pod.SecurityContext
 	wantSS.Spec.Template.Spec.ImagePullSecrets = proxyClassAllOpts.Spec.StatefulSet.Pod.ImagePullSecrets
@@ -207,9 +206,9 @@ func Test_applyProxyClassToStatefulSet(t *testing.T) {
 	// StatefulSet and Pod set gets correctly applied to a Statefulset built
 	// from non-userspace proxy template.
 	wantSS = nonUserspaceProxySS.DeepCopy()
-	wantSS.ObjectMeta.Labels = mergeMapKeys(wantSS.ObjectMeta.Labels, proxyClassJustLabels.Spec.StatefulSet.Labels)
-	wantSS.ObjectMeta.Annotations = mergeMapKeys(wantSS.ObjectMeta.Annotations, proxyClassJustLabels.Spec.StatefulSet.Annotations)
-	wantSS.Spec.Template.Labels = proxyClassJustLabels.Spec.StatefulSet.Pod.Labels
+	updateMap(wantSS.ObjectMeta.Labels, proxyClassJustLabels.Spec.StatefulSet.Labels.Parse())
+	updateMap(wantSS.ObjectMeta.Annotations, proxyClassJustLabels.Spec.StatefulSet.Annotations)
+	wantSS.Spec.Template.Labels = proxyClassJustLabels.Spec.StatefulSet.Pod.Labels.Parse()
 	wantSS.Spec.Template.Annotations = proxyClassJustLabels.Spec.StatefulSet.Pod.Annotations
 	gotSS = applyProxyClassToStatefulSet(proxyClassJustLabels, nonUserspaceProxySS.DeepCopy(), new(tailscaleSTSConfig), zl.Sugar())
 	if diff := cmp.Diff(gotSS, wantSS); diff != "" {
@@ -219,9 +218,9 @@ func Test_applyProxyClassToStatefulSet(t *testing.T) {
 	// 3. Test that a ProxyClass with all fields set gets correctly applied
 	// to a Statefulset built from a userspace proxy template.
 	wantSS = userspaceProxySS.DeepCopy()
-	wantSS.ObjectMeta.Labels = mergeMapKeys(wantSS.ObjectMeta.Labels, proxyClassAllOpts.Spec.StatefulSet.Labels)
-	wantSS.ObjectMeta.Annotations = mergeMapKeys(wantSS.ObjectMeta.Annotations, proxyClassAllOpts.Spec.StatefulSet.Annotations)
-	wantSS.Spec.Template.Labels = proxyClassAllOpts.Spec.StatefulSet.Pod.Labels
+	updateMap(wantSS.ObjectMeta.Labels, proxyClassAllOpts.Spec.StatefulSet.Labels.Parse())
+	updateMap(wantSS.ObjectMeta.Annotations, proxyClassAllOpts.Spec.StatefulSet.Annotations)
+	wantSS.Spec.Template.Labels = proxyClassAllOpts.Spec.StatefulSet.Pod.Labels.Parse()
 	wantSS.Spec.Template.Annotations = proxyClassAllOpts.Spec.StatefulSet.Pod.Annotations
 	wantSS.Spec.Template.Spec.SecurityContext = proxyClassAllOpts.Spec.StatefulSet.Pod.SecurityContext
 	wantSS.Spec.Template.Spec.ImagePullSecrets = proxyClassAllOpts.Spec.StatefulSet.Pod.ImagePullSecrets
@@ -243,9 +242,9 @@ func Test_applyProxyClassToStatefulSet(t *testing.T) {
 	// 4. Test that a ProxyClass with custom labels and annotations gets correctly applied
 	// to a Statefulset built from a userspace proxy template.
 	wantSS = userspaceProxySS.DeepCopy()
-	wantSS.ObjectMeta.Labels = mergeMapKeys(wantSS.ObjectMeta.Labels, proxyClassJustLabels.Spec.StatefulSet.Labels)
-	wantSS.ObjectMeta.Annotations = mergeMapKeys(wantSS.ObjectMeta.Annotations, proxyClassJustLabels.Spec.StatefulSet.Annotations)
-	wantSS.Spec.Template.Labels = proxyClassJustLabels.Spec.StatefulSet.Pod.Labels
+	updateMap(wantSS.ObjectMeta.Labels, proxyClassJustLabels.Spec.StatefulSet.Labels.Parse())
+	updateMap(wantSS.ObjectMeta.Annotations, proxyClassJustLabels.Spec.StatefulSet.Annotations)
+	wantSS.Spec.Template.Labels = proxyClassJustLabels.Spec.StatefulSet.Pod.Labels.Parse()
 	wantSS.Spec.Template.Annotations = proxyClassJustLabels.Spec.StatefulSet.Pod.Annotations
 	gotSS = applyProxyClassToStatefulSet(proxyClassJustLabels, userspaceProxySS.DeepCopy(), new(tailscaleSTSConfig), zl.Sugar())
 	if diff := cmp.Diff(gotSS, wantSS); diff != "" {
@@ -294,13 +293,6 @@ func Test_applyProxyClassToStatefulSet(t *testing.T) {
 	}
 }
 
-func mergeMapKeys(a, b map[string]string) map[string]string {
-	for key, val := range b {
-		a[key] = val
-	}
-	return a
-}
-
 func Test_mergeStatefulSetLabelsOrAnnots(t *testing.T) {
 	tests := []struct {
 		name    string
@@ -392,3 +384,10 @@ func Test_mergeStatefulSetLabelsOrAnnots(t *testing.T) {
 		})
 	}
 }
+
+// updateMap updates map a with the values from map b.
+func updateMap(a, b map[string]string) {
+	for key, val := range b {
+		a[key] = val
+	}
+}

cmd/k8s-operator/testutils_test.go

@@ -61,7 +61,10 @@ type configOpts struct {
 	app                                            string
 	shouldRemoveAuthKey                            bool
 	secretExtraData                                map[string][]byte
-	enableMetrics                                  bool
+	resourceVersion                                string
+
+	enableMetrics        bool
+	serviceMonitorLabels tsapi.Labels
 }
 
 func expectedSTS(t *testing.T, cl client.Client, opts configOpts) *appsv1.StatefulSet {
@@ -431,14 +434,17 @@ func metricsLabels(opts configOpts) map[string]string {
 
 func expectedServiceMonitor(t *testing.T, opts configOpts) *unstructured.Unstructured {
 	t.Helper()
-	labels := metricsLabels(opts)
+	smLabels := metricsLabels(opts)
+	if len(opts.serviceMonitorLabels) != 0 {
+		smLabels = mergeMapKeys(smLabels, opts.serviceMonitorLabels.Parse())
+	}
 	name := metricsResourceName(opts.stsName)
 	sm := &ServiceMonitor{
 		ObjectMeta: metav1.ObjectMeta{
 			Name:            name,
 			Namespace:       opts.tailscaleNamespace,
-			Labels:          labels,
-			ResourceVersion: "1",
+			Labels:          smLabels,
+			ResourceVersion: opts.resourceVersion,
 			OwnerReferences: []metav1.OwnerReference{{APIVersion: "v1", Kind: "Service", Name: name, BlockOwnerDeletion: ptr.To(true), Controller: ptr.To(true)}},
 		},
 		TypeMeta: metav1.TypeMeta{
@@ -446,7 +452,7 @@ func expectedServiceMonitor(t *testing.T, opts configOpts) *unstructured.Unstruc
 			APIVersion: "monitoring.coreos.com/v1",
 		},
 		Spec: ServiceMonitorSpec{
-			Selector: metav1.LabelSelector{MatchLabels: labels},
+			Selector: metav1.LabelSelector{MatchLabels: metricsLabels(opts)},
 			Endpoints: []ServiceMonitorEndpoint{{
 				Port: "metrics",
 			}},
@@ -653,10 +659,11 @@ func expectEqualUnstructured(t *testing.T, client client.Client, want *unstructu
 func expectMissing[T any, O ptrObject[T]](t *testing.T, client client.Client, ns, name string) {
 	t.Helper()
 	obj := O(new(T))
-	if err := client.Get(context.Background(), types.NamespacedName{
+	err := client.Get(context.Background(), types.NamespacedName{
 		Name:      name,
 		Namespace: ns,
-	}, obj); !apierrors.IsNotFound(err) {
+	}, obj)
+	if !apierrors.IsNotFound(err) {
 		t.Fatalf("%s %s/%s unexpectedly present, wanted missing", reflect.TypeOf(obj).Elem().Name(), ns, name)
 	}
 }

cmd/stund/depaware.txt

@@ -55,6 +55,7 @@ tailscale.com/cmd/stund dependencies: (generated by github.com/tailscale/depawar
         tailscale.com/net/stun                                       from tailscale.com/net/stunserver
         tailscale.com/net/stunserver                                 from tailscale.com/cmd/stund
         tailscale.com/net/tsaddr                                     from tailscale.com/tsweb
+        tailscale.com/syncs                                          from tailscale.com/metrics
         tailscale.com/tailcfg                                        from tailscale.com/version
         tailscale.com/tsweb                                          from tailscale.com/cmd/stund
         tailscale.com/tsweb/promvarz                                 from tailscale.com/tsweb
@@ -74,6 +75,7 @@ tailscale.com/cmd/stund dependencies: (generated by github.com/tailscale/depawar
    L 💣 tailscale.com/util/dirwalk                                   from tailscale.com/metrics
         tailscale.com/util/dnsname                                   from tailscale.com/tailcfg
         tailscale.com/util/lineiter                                  from tailscale.com/version/distro
+        tailscale.com/util/mak                                       from tailscale.com/syncs
         tailscale.com/util/nocasemaps                                from tailscale.com/types/ipproto
         tailscale.com/util/rands                                     from tailscale.com/tsweb
         tailscale.com/util/slicesx                                   from tailscale.com/tailcfg

cmd/systray/logo.go

@@ -1,220 +0,0 @@
-// Copyright (c) Tailscale Inc & AUTHORS
-// SPDX-License-Identifier: BSD-3-Clause
-
-//go:build cgo || !darwin
-
-package main
-
-import (
-	"bytes"
-	"context"
-	"image/color"
-	"image/png"
-	"sync"
-	"time"
-
-	"fyne.io/systray"
-	"github.com/fogleman/gg"
-)
-
-// tsLogo represents the state of the 3x3 dot grid in the Tailscale logo.
-// A 0 represents a gray dot, any other value is a white dot.
-type tsLogo [9]byte
-
-var (
-	// disconnected is all gray dots
-	disconnected = tsLogo{
-		0, 0, 0,
-		0, 0, 0,
-		0, 0, 0,
-	}
-
-	// connected is the normal Tailscale logo
-	connected = tsLogo{
-		0, 0, 0,
-		1, 1, 1,
-		0, 1, 0,
-	}
-
-	// loading is a special tsLogo value that is not meant to be rendered directly,
-	// but indicates that the loading animation should be shown.
-	loading = tsLogo{'l', 'o', 'a', 'd', 'i', 'n', 'g'}
-
-	// loadingIcons are shown in sequence as an animated loading icon.
-	loadingLogos = []tsLogo{
-		{
-			0, 1, 1,
-			1, 0, 1,
-			0, 0, 1,
-		},
-		{
-			0, 1, 1,
-			0, 0, 1,
-			0, 1, 0,
-		},
-		{
-			0, 1, 1,
-			0, 0, 0,
-			0, 0, 1,
-		},
-		{
-			0, 0, 1,
-			0, 1, 0,
-			0, 0, 0,
-		},
-		{
-			0, 1, 0,
-			0, 0, 0,
-			0, 0, 0,
-		},
-		{
-			0, 0, 0,
-			0, 0, 1,
-			0, 0, 0,
-		},
-		{
-			0, 0, 0,
-			0, 0, 0,
-			0, 0, 0,
-		},
-		{
-			0, 0, 1,
-			0, 0, 0,
-			0, 0, 0,
-		},
-		{
-			0, 0, 0,
-			0, 0, 0,
-			1, 0, 0,
-		},
-		{
-			0, 0, 0,
-			0, 0, 0,
-			1, 1, 0,
-		},
-		{
-			0, 0, 0,
-			1, 0, 0,
-			1, 1, 0,
-		},
-		{
-			0, 0, 0,
-			1, 1, 0,
-			0, 1, 0,
-		},
-		{
-			0, 0, 0,
-			1, 1, 0,
-			0, 1, 1,
-		},
-		{
-			0, 0, 0,
-			1, 1, 1,
-			0, 0, 1,
-		},
-		{
-			0, 1, 0,
-			0, 1, 1,
-			1, 0, 1,
-		},
-	}
-)
-
-var (
-	black = color.NRGBA{0, 0, 0, 255}
-	white = color.NRGBA{255, 255, 255, 255}
-	gray  = color.NRGBA{255, 255, 255, 102}
-)
-
-// render returns a PNG image of the logo.
-func (logo tsLogo) render() *bytes.Buffer {
-	const radius = 25
-	const borderUnits = 1
-	dim := radius * (8 + borderUnits*2)
-
-	dc := gg.NewContext(dim, dim)
-	dc.DrawRectangle(0, 0, float64(dim), float64(dim))
-	dc.SetColor(black)
-	dc.Fill()
-
-	for y := 0; y < 3; y++ {
-		for x := 0; x < 3; x++ {
-			px := (borderUnits + 1 + 3*x) * radius
-			py := (borderUnits + 1 + 3*y) * radius
-			col := white
-			if logo[y*3+x] == 0 {
-				col = gray
-			}
-			dc.DrawCircle(float64(px), float64(py), radius)
-			dc.SetColor(col)
-			dc.Fill()
-		}
-	}
-
-	b := bytes.NewBuffer(nil)
-	png.Encode(b, dc.Image())
-	return b
-}
-
-// setAppIcon renders logo and sets it as the systray icon.
-func setAppIcon(icon tsLogo) {
-	if icon == loading {
-		startLoadingAnimation()
-	} else {
-		stopLoadingAnimation()
-		systray.SetIcon(icon.render().Bytes())
-	}
-}
-
-var (
-	loadingMu sync.Mutex // protects loadingCancel
-
-	// loadingCancel stops the loading animation in the systray icon.
-	// This is nil if the animation is not currently active.
-	loadingCancel func()
-)
-
-// startLoadingAnimation starts the animated loading icon in the system tray.
-// The animation continues until [stopLoadingAnimation] is called.
-// If the loading animation is already active, this func does nothing.
-func startLoadingAnimation() {
-	loadingMu.Lock()
-	defer loadingMu.Unlock()
-
-	if loadingCancel != nil {
-		// loading icon already displayed
-		return
-	}
-
-	ctx := context.Background()
-	ctx, loadingCancel = context.WithCancel(ctx)
-
-	go func() {
-		t := time.NewTicker(500 * time.Millisecond)
-		var i int
-		for {
-			select {
-			case <-ctx.Done():
-				return
-			case <-t.C:
-				systray.SetIcon(loadingLogos[i].render().Bytes())
-				i++
-				if i >= len(loadingLogos) {
-					i = 0
-				}
-			}
-		}
-	}()
-}
-
-// stopLoadingAnimation stops the animated loading icon in the system tray.
-// If the loading animation is not currently active, this func does nothing.
-func stopLoadingAnimation() {
-	loadingMu.Lock()
-	defer loadingMu.Unlock()
-
-	if loadingCancel != nil {
-		loadingCancel()
-		loadingCancel = nil
-	}
-}

cmd/systray/systray.go

@@ -3,256 +3,13 @@
 
 //go:build cgo || !darwin
 
-// The systray command is a minimal Tailscale systray application for Linux.
+// systray is a minimal Tailscale systray application.
 package main
 
 import (
-	"context"
-	"errors"
-	"fmt"
-	"io"
-	"log"
-	"os"
-	"strings"
-	"sync"
-	"time"
-
-	"fyne.io/systray"
-	"github.com/atotto/clipboard"
-	dbus "github.com/godbus/dbus/v5"
-	"github.com/toqueteos/webbrowser"
-	"tailscale.com/client/tailscale"
-	"tailscale.com/ipn"
-	"tailscale.com/ipn/ipnstate"
-)
-
-var (
-	localClient tailscale.LocalClient
-	chState     chan ipn.State // tailscale state changes
-
-	appIcon *os.File
+	"tailscale.com/client/systray"
 )
 
 func main() {
-	systray.Run(onReady, onExit)
-}
-
-// Menu represents the systray menu, its items, and the current Tailscale state.
-type Menu struct {
-	mu     sync.Mutex // protects the entire Menu
-	status *ipnstate.Status
-
-	connect    *systray.MenuItem
-	disconnect *systray.MenuItem
-
-	self *systray.MenuItem
-	more *systray.MenuItem
-	quit *systray.MenuItem
-
-	eventCancel func() // cancel eventLoop
-}
-
-func onReady() {
-	log.Printf("starting")
-	ctx := context.Background()
-
-	setAppIcon(disconnected)
-
-	// dbus wants a file path for notification icons, so copy to a temp file.
-	appIcon, _ = os.CreateTemp("", "tailscale-systray.png")
-	io.Copy(appIcon, connected.render())
-
-	chState = make(chan ipn.State, 1)
-
-	status, err := localClient.Status(ctx)
-	if err != nil {
-		log.Print(err)
-	}
-
-	menu := new(Menu)
-	menu.rebuild(status)
-
-	go watchIPNBus(ctx)
-}
-
-// rebuild the systray menu based on the current Tailscale state.
-//
-// We currently rebuild the entire menu because it is not easy to update the existing menu.
-// You cannot iterate over the items in a menu, nor can you remove some items like separators.
-// So for now we rebuild the whole thing, and can optimize this later if needed.
-func (menu *Menu) rebuild(status *ipnstate.Status) {
-	menu.mu.Lock()
-	defer menu.mu.Unlock()
-
-	if menu.eventCancel != nil {
-		menu.eventCancel()
-	}
-	menu.status = status
-	systray.ResetMenu()
-
-	menu.connect = systray.AddMenuItem("Connect", "")
-	menu.disconnect = systray.AddMenuItem("Disconnect", "")
-	menu.disconnect.Hide()
-	systray.AddSeparator()
-
-	if status != nil && status.Self != nil {
-		title := fmt.Sprintf("This Device: %s (%s)", status.Self.HostName, status.Self.TailscaleIPs[0])
-		menu.self = systray.AddMenuItem(title, "")
-	}
-	systray.AddSeparator()
-
-	menu.more = systray.AddMenuItem("More settings", "")
-	menu.more.Enable()
-
-	menu.quit = systray.AddMenuItem("Quit", "Quit the app")
-	menu.quit.Enable()
-
-	ctx := context.Background()
-	ctx, menu.eventCancel = context.WithCancel(ctx)
-	go menu.eventLoop(ctx)
-}
-
-// eventLoop is the main event loop for handling click events on menu items
-// and responding to Tailscale state changes.
-// This method does not return until ctx.Done is closed.
-func (menu *Menu) eventLoop(ctx context.Context) {
-	for {
-		select {
-		case <-ctx.Done():
-			return
-		case state := <-chState:
-			switch state {
-			case ipn.Running:
-				setAppIcon(loading)
-				status, err := localClient.Status(ctx)
-				if err != nil {
-					log.Printf("error getting tailscale status: %v", err)
-				}
-				menu.rebuild(status)
-				setAppIcon(connected)
-				menu.connect.SetTitle("Connected")
-				menu.connect.Disable()
-				menu.disconnect.Show()
-				menu.disconnect.Enable()
-			case ipn.NoState, ipn.Stopped:
-				menu.connect.SetTitle("Connect")
-				menu.connect.Enable()
-				menu.disconnect.Hide()
-				setAppIcon(disconnected)
-			case ipn.Starting:
-				setAppIcon(loading)
-			}
-		case <-menu.connect.ClickedCh:
-			_, err := localClient.EditPrefs(ctx, &ipn.MaskedPrefs{
-				Prefs: ipn.Prefs{
-					WantRunning: true,
-				},
-				WantRunningSet: true,
-			})
-			if err != nil {
-				log.Print(err)
-				continue
-			}
-
-		case <-menu.disconnect.ClickedCh:
-			_, err := localClient.EditPrefs(ctx, &ipn.MaskedPrefs{
-				Prefs: ipn.Prefs{
-					WantRunning: false,
-				},
-				WantRunningSet: true,
-			})
-			if err != nil {
-				log.Printf("disconnecting: %v", err)
-				continue
-			}
-
-		case <-menu.self.ClickedCh:
-			copyTailscaleIP(menu.status.Self)
-
-		case <-menu.more.ClickedCh:
-			webbrowser.Open("http://100.100.100.100/")
-
-		case <-menu.quit.ClickedCh:
-			systray.Quit()
-		}
-	}
-}
-
-// watchIPNBus subscribes to the tailscale event bus and sends state updates to chState.
-// This method does not return.
-func watchIPNBus(ctx context.Context) {
-	for {
-		if err := watchIPNBusInner(ctx); err != nil {
-			log.Println(err)
-			if errors.Is(err, context.Canceled) {
-				// If the context got canceled, we will never be able to
-				// reconnect to IPN bus, so exit the process.
-				log.Fatalf("watchIPNBus: %v", err)
-			}
-		}
-		// If our watch connection breaks, wait a bit before reconnecting. No
-		// reason to spam the logs if e.g. tailscaled is restarting or goes
-		// down.
-		time.Sleep(3 * time.Second)
-	}
-}
-
-func watchIPNBusInner(ctx context.Context) error {
-	watcher, err := localClient.WatchIPNBus(ctx, ipn.NotifyInitialState|ipn.NotifyNoPrivateKeys)
-	if err != nil {
-		return fmt.Errorf("watching ipn bus: %w", err)
-	}
-	defer watcher.Close()
-	for {
-		select {
-		case <-ctx.Done():
-			return nil
-		default:
-			n, err := watcher.Next()
-			if err != nil {
-				return fmt.Errorf("ipnbus error: %w", err)
-			}
-			if n.State != nil {
-				chState <- *n.State
-				log.Printf("new state: %v", n.State)
-			}
-		}
-	}
-}
-
-// copyTailscaleIP copies the first Tailscale IP of the given device to the clipboard
-// and sends a notification with the copied value.
-func copyTailscaleIP(device *ipnstate.PeerStatus) {
-	if device == nil || len(device.TailscaleIPs) == 0 {
-		return
-	}
-	name := strings.Split(device.DNSName, ".")[0]
-	ip := device.TailscaleIPs[0].String()
-	err := clipboard.WriteAll(ip)
-	if err != nil {
-		log.Printf("clipboard error: %v", err)
-	}
-
-	sendNotification(fmt.Sprintf("Copied Address for %v", name), ip)
-}
-
-// sendNotification sends a desktop notification with the given title and content.
-func sendNotification(title, content string) {
-	conn, err := dbus.SessionBus()
-	if err != nil {
-		log.Printf("dbus: %v", err)
-		return
-	}
-	timeout := 3 * time.Second
-	obj := conn.Object("org.freedesktop.Notifications", "/org/freedesktop/Notifications")
-	call := obj.Call("org.freedesktop.Notifications.Notify", 0, "Tailscale", uint32(0),
-		appIcon.Name(), title, content, []string{}, map[string]dbus.Variant{}, int32(timeout.Milliseconds()))
-	if call.Err != nil {
-		log.Printf("dbus: %v", call.Err)
-	}
-}
-
-func onExit() {
-	log.Printf("exiting")
-	os.Remove(appIcon.Name())
+	new(systray.Menu).Run()
 }

cmd/tailscale/cli/exitnode.go

@@ -15,10 +15,10 @@ import (
 
 	"github.com/kballard/go-shellquote"
 	"github.com/peterbourgon/ff/v3/ffcli"
-	xmaps "golang.org/x/exp/maps"
 	"tailscale.com/envknob"
 	"tailscale.com/ipn/ipnstate"
 	"tailscale.com/tailcfg"
+	"tailscale.com/util/slicesx"
 )
 
 func exitNodeCmd() *ffcli.Command {
@@ -255,7 +255,7 @@ func filterFormatAndSortExitNodes(peers []*ipnstate.PeerStatus, filterBy string)
 	}
 
 	filteredExitNodes := filteredExitNodes{
-		Countries: xmaps.Values(countries),
+		Countries: slicesx.MapValues(countries),
 	}
 
 	for _, country := range filteredExitNodes.Countries {

cmd/tailscale/cli/risks.go

@@ -77,7 +77,7 @@ func presentRiskToUser(riskType, riskMessage, acceptedRisks string) error {
 	for left := riskAbortTimeSeconds; left > 0; left-- {
 		msg := fmt.Sprintf("\rContinuing in %d seconds...", left)
 		msgLen = len(msg)
-		printf(msg)
+		printf("%s", msg)
 		select {
 		case <-interrupt:
 			printf("\r%s\r", strings.Repeat("x", msgLen+1))

cmd/tailscale/cli/serve_legacy.go

@@ -27,6 +27,7 @@ import (
 	"tailscale.com/ipn"
 	"tailscale.com/ipn/ipnstate"
 	"tailscale.com/tailcfg"
+	"tailscale.com/util/slicesx"
 	"tailscale.com/version"
 )
 
@@ -707,10 +708,7 @@ func (e *serveEnv) printWebStatusTree(sc *ipn.ServeConfig, hp ipn.HostPort) erro
 		return "", ""
 	}
 
-	var mounts []string
-	for k := range sc.Web[hp].Handlers {
-		mounts = append(mounts, k)
-	}
+	mounts := slicesx.MapKeys(sc.Web[hp].Handlers)
 	sort.Slice(mounts, func(i, j int) bool {
 		return len(mounts[i]) < len(mounts[j])
 	})

cmd/tailscale/cli/serve_v2.go

@@ -28,6 +28,7 @@ import (
 	"tailscale.com/ipn/ipnstate"
 	"tailscale.com/tailcfg"
 	"tailscale.com/util/mak"
+	"tailscale.com/util/slicesx"
 	"tailscale.com/version"
 )
 
@@ -439,11 +440,7 @@ func (e *serveEnv) messageForPort(sc *ipn.ServeConfig, st *ipnstate.Status, dnsN
 	}
 
 	if sc.Web[hp] != nil {
-		var mounts []string
-
-		for k := range sc.Web[hp].Handlers {
-			mounts = append(mounts, k)
-		}
+		mounts := slicesx.MapKeys(sc.Web[hp].Handlers)
 		sort.Slice(mounts, func(i, j int) bool {
 			return len(mounts[i]) < len(mounts[j])
 		})

cmd/tailscale/cli/up.go

@@ -379,7 +379,7 @@ func updatePrefs(prefs, curPrefs *ipn.Prefs, env upCheckEnv) (simpleUp bool, jus
 		return false, nil, err
 	}
 
-	if runtime.GOOS == "darwin" && env.upArgs.advertiseConnector {
+	if env.goos == "darwin" && env.upArgs.advertiseConnector {
 		if err := presentRiskToUser(riskMacAppConnector, riskMacAppConnectorMessage, env.upArgs.acceptedRisks); err != nil {
 			return false, nil, err
 		}

cmd/tailscale/depaware.txt

@@ -58,7 +58,6 @@ tailscale.com/cmd/tailscale dependencies: (generated by github.com/tailscale/dep
    L 💣 github.com/tailscale/netlink                                 from tailscale.com/util/linuxfw
    L 💣 github.com/tailscale/netlink/nl                              from github.com/tailscale/netlink
         github.com/tailscale/web-client-prebuilt                     from tailscale.com/client/web
-        github.com/tcnksm/go-httpstat                                from tailscale.com/net/netcheck
         github.com/toqueteos/webbrowser                              from tailscale.com/cmd/tailscale/cli
    L    github.com/vishvananda/netns                                 from github.com/tailscale/netlink+
         github.com/x448/float16                                      from github.com/fxamacker/cbor/v2
@@ -203,7 +202,7 @@ tailscale.com/cmd/tailscale dependencies: (generated by github.com/tailscale/dep
         golang.org/x/crypto/salsa20/salsa                            from golang.org/x/crypto/nacl/box+
         golang.org/x/crypto/sha3                                     from crypto/internal/mlkem768+
    W    golang.org/x/exp/constraints                                 from github.com/dblohm7/wingoes/pe+
-        golang.org/x/exp/maps                                        from tailscale.com/cmd/tailscale/cli+
+        golang.org/x/exp/maps                                        from tailscale.com/util/syspolicy/internal/metrics+
         golang.org/x/net/bpf                                         from github.com/mdlayher/netlink+
         golang.org/x/net/dns/dnsmessage                              from net+
         golang.org/x/net/http/httpguts                               from net/http+
@@ -306,7 +305,7 @@ tailscale.com/cmd/tailscale dependencies: (generated by github.com/tailscale/dep
         net                                                          from crypto/tls+
         net/http                                                     from expvar+
         net/http/cgi                                                 from tailscale.com/cmd/tailscale/cli
-        net/http/httptrace                                           from github.com/tcnksm/go-httpstat+
+        net/http/httptrace                                           from golang.org/x/net/http2+
         net/http/httputil                                            from tailscale.com/client/web+
         net/http/internal                                            from net/http+
         net/netip                                                    from go4.org/netipx+

cmd/tailscaled/depaware.txt

@@ -181,7 +181,6 @@ tailscale.com/cmd/tailscaled dependencies: (generated by github.com/tailscale/de
      💣 github.com/tailscale/wireguard-go/tun                        from github.com/tailscale/wireguard-go/device+
         github.com/tailscale/xnet/webdav                             from tailscale.com/drive/driveimpl+
         github.com/tailscale/xnet/webdav/internal/xml                from github.com/tailscale/xnet/webdav
-        github.com/tcnksm/go-httpstat                                from tailscale.com/net/netcheck
   LD    github.com/u-root/u-root/pkg/termios                         from tailscale.com/ssh/tailssh
    L    github.com/u-root/uio/rand                                   from github.com/insomniacslk/dhcp/dhcpv4
    L    github.com/u-root/uio/uio                                    from github.com/insomniacslk/dhcp/dhcpv4+
@@ -450,7 +449,7 @@ tailscale.com/cmd/tailscaled dependencies: (generated by github.com/tailscale/de
         golang.org/x/crypto/sha3                                     from crypto/internal/mlkem768+
   LD    golang.org/x/crypto/ssh                                      from github.com/pkg/sftp+
         golang.org/x/exp/constraints                                 from github.com/dblohm7/wingoes/pe+
-        golang.org/x/exp/maps                                        from tailscale.com/appc+
+        golang.org/x/exp/maps                                        from tailscale.com/ipn/store/mem+
         golang.org/x/net/bpf                                         from github.com/mdlayher/genetlink+
         golang.org/x/net/dns/dnsmessage                              from net+
         golang.org/x/net/http/httpguts                               from golang.org/x/net/http2+
@@ -553,7 +552,7 @@ tailscale.com/cmd/tailscaled dependencies: (generated by github.com/tailscale/de
         net                                                          from crypto/tls+
         net/http                                                     from expvar+
         net/http/httptest                                            from tailscale.com/control/controlclient
-        net/http/httptrace                                           from github.com/tcnksm/go-httpstat+
+        net/http/httptrace                                           from github.com/prometheus-community/pro-bing+
         net/http/httputil                                            from github.com/aws/smithy-go/transport/http+
         net/http/internal                                            from net/http+
         net/http/pprof                                               from tailscale.com/cmd/tailscaled+

cmd/testwrapper/testwrapper.go

@@ -29,8 +29,8 @@ import (
 	"github.com/dave/courtney/tester"
 	"github.com/dave/patsy"
 	"github.com/dave/patsy/vos"
-	xmaps "golang.org/x/exp/maps"
 	"tailscale.com/cmd/testwrapper/flakytest"
+	"tailscale.com/util/slicesx"
 )
 
 const (
@@ -350,7 +350,7 @@ func main() {
 		if len(toRetry) == 0 {
 			continue
 		}
-		pkgs := xmaps.Keys(toRetry)
+		pkgs := slicesx.MapKeys(toRetry)
 		sort.Strings(pkgs)
 		nextRun := &nextRun{
 			attempt: thisRun.attempt + 1,

cmd/tsconnect/tsconnect.go

@@ -53,12 +53,12 @@ func main() {
 }
 
 func usage() {
-	fmt.Fprintf(os.Stderr, `
+	fmt.Fprint(os.Stderr, `
 usage: tsconnect {dev|build|serve}
 `[1:])
 
 	flag.PrintDefaults()
-	fmt.Fprintf(os.Stderr, `
+	fmt.Fprint(os.Stderr, `
 
 tsconnect implements development/build/serving workflows for Tailscale Connect.
 It can be invoked with one of three subcommands:

control/controlclient/auto.go

@@ -7,6 +7,7 @@ import (
 	"context"
 	"errors"
 	"fmt"
+	"log"
 	"net/http"
 	"sync"
 	"sync/atomic"
@@ -423,6 +424,11 @@ type mapRoutineState struct {
 var _ NetmapDeltaUpdater = mapRoutineState{}
 
 func (mrs mapRoutineState) UpdateFullNetmap(nm *netmap.NetworkMap) {
+	for _, p := range nm.Peers {
+		if routes := p.PrimaryRoutes(); routes.Len() > 0 {
+			log.Printf("DEBUG: node %q routes %q", p.Name(), routes.AsSlice())
+		}
+	}
 	c := mrs.c
 
 	c.mu.Lock()

control/controlclient/direct.go

@@ -1643,6 +1643,56 @@ func (c *Direct) ReportHealthChange(w *health.Warnable, us *health.UnhealthyStat
 	res.Body.Close()
 }
 
+// SetDeviceAttrs does a synchronous call to the control plane to update
+// the node's attributes.
+//
+// See docs on [tailcfg.SetDeviceAttributesRequest] for background.
+func (c *Auto) SetDeviceAttrs(ctx context.Context, attrs tailcfg.AttrUpdate) error {
+	return c.direct.SetDeviceAttrs(ctx, attrs)
+}
+
+// SetDeviceAttrs does a synchronous call to the control plane to update
+// the node's attributes.
+//
+// See docs on [tailcfg.SetDeviceAttributesRequest] for background.
+func (c *Direct) SetDeviceAttrs(ctx context.Context, attrs tailcfg.AttrUpdate) error {
+	nc, err := c.getNoiseClient()
+	if err != nil {
+		return err
+	}
+	nodeKey, ok := c.GetPersist().PublicNodeKeyOK()
+	if !ok {
+		return errors.New("no node key")
+	}
+	if c.panicOnUse {
+		panic("tainted client")
+	}
+	req := &tailcfg.SetDeviceAttributesRequest{
+		NodeKey: nodeKey,
+		Version: tailcfg.CurrentCapabilityVersion,
+		Update:  attrs,
+	}
+
+	// TODO(bradfitz): unify the callers using doWithBody vs those using
+	// DoNoiseRequest. There seems to be a ~50/50 split and they're very close,
+	// but doWithBody sets the load balancing header and auto-JSON-encodes the
+	// body, but DoNoiseRequest is exported. Clean it up so they're consistent
+	// one way or another.
+
+	ctx, cancel := context.WithTimeout(ctx, 30*time.Second)
+	defer cancel()
+	res, err := nc.doWithBody(ctx, "PATCH", "/machine/set-device-attr", nodeKey, req)
+	if err != nil {
+		return err
+	}
+	defer res.Body.Close()
+	all, _ := io.ReadAll(res.Body)
+	if res.StatusCode != 200 {
+		return fmt.Errorf("HTTP error from control plane: %v: %s", res.Status, all)
+	}
+	return nil
+}
+
 func addLBHeader(req *http.Request, nodeKey key.NodePublic) {
 	if !nodeKey.IsZero() {
 		req.Header.Add(tailcfg.LBHeader, nodeKey.String())

control/controlclient/map.go

@@ -14,7 +14,6 @@ import (
 	"runtime"
 	"runtime/debug"
 	"slices"
-	"sort"
 	"strconv"
 	"sync"
 	"time"
@@ -31,6 +30,7 @@ import (
 	"tailscale.com/util/clientmetric"
 	"tailscale.com/util/mak"
 	"tailscale.com/util/set"
+	"tailscale.com/util/slicesx"
 	"tailscale.com/wgengine/filter"
 )
 
@@ -75,8 +75,7 @@ type mapSession struct {
 	lastPrintMap           time.Time
 	lastNode               tailcfg.NodeView
 	lastCapSet             set.Set[tailcfg.NodeCapability]
-	peers                  map[tailcfg.NodeID]*tailcfg.NodeView // pointer to view (oddly). same pointers as sortedPeers.
-	sortedPeers            []*tailcfg.NodeView                  // same pointers as peers, but sorted by Node.ID
+	peers                  map[tailcfg.NodeID]tailcfg.NodeView
 	lastDNSConfig          *tailcfg.DNSConfig
 	lastDERPMap            *tailcfg.DERPMap
 	lastUserProfile        map[tailcfg.UserID]tailcfg.UserProfile
@@ -366,16 +365,11 @@ var (
 	patchifiedPeerEqual = clientmetric.NewCounter("controlclient_patchified_peer_equal")
 )
 
-// updatePeersStateFromResponseres updates ms.peers and ms.sortedPeers from res. It takes ownership of res.
+// updatePeersStateFromResponseres updates ms.peers from resp.
+// It takes ownership of resp.
 func (ms *mapSession) updatePeersStateFromResponse(resp *tailcfg.MapResponse) (stats updateStats) {
-	defer func() {
-		if stats.removed > 0 || stats.added > 0 {
-			ms.rebuildSorted()
-		}
-	}()
-
 	if ms.peers == nil {
-		ms.peers = make(map[tailcfg.NodeID]*tailcfg.NodeView)
+		ms.peers = make(map[tailcfg.NodeID]tailcfg.NodeView)
 	}
 
 	if len(resp.Peers) > 0 {
@@ -384,12 +378,12 @@ func (ms *mapSession) updatePeersStateFromResponse(resp *tailcfg.MapResponse) (s
 		keep := make(map[tailcfg.NodeID]bool, len(resp.Peers))
 		for _, n := range resp.Peers {
 			keep[n.ID] = true
-			if vp, ok := ms.peers[n.ID]; ok {
+			lenBefore := len(ms.peers)
+			ms.peers[n.ID] = n.View()
+			if len(ms.peers) == lenBefore {
 				stats.changed++
-				*vp = n.View()
 			} else {
 				stats.added++
-				ms.peers[n.ID] = ptr.To(n.View())
 			}
 		}
 		for id := range ms.peers {
@@ -410,12 +404,12 @@ func (ms *mapSession) updatePeersStateFromResponse(resp *tailcfg.MapResponse) (s
 	}
 
 	for _, n := range resp.PeersChanged {
-		if vp, ok := ms.peers[n.ID]; ok {
+		lenBefore := len(ms.peers)
+		ms.peers[n.ID] = n.View()
+		if len(ms.peers) == lenBefore {
 			stats.changed++
-			*vp = n.View()
 		} else {
 			stats.added++
-			ms.peers[n.ID] = ptr.To(n.View())
 		}
 	}
 
@@ -427,7 +421,7 @@ func (ms *mapSession) updatePeersStateFromResponse(resp *tailcfg.MapResponse) (s
 			} else {
 				mut.LastSeen = nil
 			}
-			*vp = mut.View()
+			ms.peers[nodeID] = mut.View()
 			stats.changed++
 		}
 	}
@@ -436,7 +430,7 @@ func (ms *mapSession) updatePeersStateFromResponse(resp *tailcfg.MapResponse) (s
 		if vp, ok := ms.peers[nodeID]; ok {
 			mut := vp.AsStruct()
 			mut.Online = ptr.To(online)
-			*vp = mut.View()
+			ms.peers[nodeID] = mut.View()
 			stats.changed++
 		}
 	}
@@ -488,31 +482,12 @@ func (ms *mapSession) updatePeersStateFromResponse(resp *tailcfg.MapResponse) (s
 			mut.CapMap = v
 			patchCapMap.Add(1)
 		}
-		*vp = mut.View()
+		ms.peers[pc.NodeID] = mut.View()
 	}
 
 	return
 }
 
-// rebuildSorted rebuilds ms.sortedPeers from ms.peers. It should be called
-// after any additions or removals from peers.
-func (ms *mapSession) rebuildSorted() {
-	if ms.sortedPeers == nil {
-		ms.sortedPeers = make([]*tailcfg.NodeView, 0, len(ms.peers))
-	} else {
-		if len(ms.sortedPeers) > len(ms.peers) {
-			clear(ms.sortedPeers[len(ms.peers):])
-		}
-		ms.sortedPeers = ms.sortedPeers[:0]
-	}
-	for _, p := range ms.peers {
-		ms.sortedPeers = append(ms.sortedPeers, p)
-	}
-	sort.Slice(ms.sortedPeers, func(i, j int) bool {
-		return ms.sortedPeers[i].ID() < ms.sortedPeers[j].ID()
-	})
-}
-
 func (ms *mapSession) addUserProfile(nm *netmap.NetworkMap, userID tailcfg.UserID) {
 	if userID == 0 {
 		return
@@ -576,7 +551,7 @@ func (ms *mapSession) patchifyPeer(n *tailcfg.Node) (_ *tailcfg.PeerChange, ok b
 	if !ok {
 		return nil, false
 	}
-	return peerChangeDiff(*was, n)
+	return peerChangeDiff(was, n)
 }
 
 // peerChangeDiff returns the difference from 'was' to 'n', if possible.
@@ -688,21 +663,23 @@ func peerChangeDiff(was tailcfg.NodeView, n *tailcfg.Node) (_ *tailcfg.PeerChang
 			}
 		case "CapMap":
 			if len(n.CapMap) != was.CapMap().Len() {
+				// If they have different lengths, they're different.
 				if n.CapMap == nil {
 					pc().CapMap = make(tailcfg.NodeCapMap)
 				} else {
 					pc().CapMap = maps.Clone(n.CapMap)
 				}
-				break
-			}
-			was.CapMap().Range(func(k tailcfg.NodeCapability, v views.Slice[tailcfg.RawMessage]) bool {
-				nv, ok := n.CapMap[k]
-				if !ok || !views.SliceEqual(v, views.SliceOf(nv)) {
-					pc().CapMap = maps.Clone(n.CapMap)
-					return false
+			} else {
+				// If they have the same length, check that all their keys
+				// have the same values.
+				for k, v := range was.CapMap().All() {
+					nv, ok := n.CapMap[k]
+					if !ok || !views.SliceEqual(v, views.SliceOf(nv)) {
+						pc().CapMap = maps.Clone(n.CapMap)
+						break
+					}
 				}
-				return true
-			})
+			}
 		case "Tags":
 			if !views.SliceEqual(was.Tags(), views.SliceOf(n.Tags)) {
 				return nil, false
@@ -778,14 +755,19 @@ func peerChangeDiff(was tailcfg.NodeView, n *tailcfg.Node) (_ *tailcfg.PeerChang
 	return ret, true
 }
 
+func (ms *mapSession) sortedPeers() []tailcfg.NodeView {
+	ret := slicesx.MapValues(ms.peers)
+	slices.SortFunc(ret, func(a, b tailcfg.NodeView) int {
+		return cmp.Compare(a.ID(), b.ID())
+	})
+	return ret
+}
+
 // netmap returns a fully populated NetworkMap from the last state seen from
 // a call to updateStateFromResponse, filling in omitted
 // information from prior MapResponse values.
 func (ms *mapSession) netmap() *netmap.NetworkMap {
-	peerViews := make([]tailcfg.NodeView, len(ms.sortedPeers))
-	for i, vp := range ms.sortedPeers {
-		peerViews[i] = *vp
-	}
+	peerViews := ms.sortedPeers()
 
 	nm := &netmap.NetworkMap{
 		NodeKey:           ms.publicNodeKey,

control/controlclient/map_test.go

@@ -340,19 +340,18 @@ func TestUpdatePeersStateFromResponse(t *testing.T) {
 			}
 			ms := newTestMapSession(t, nil)
 			for _, n := range tt.prev {
-				mak.Set(&ms.peers, n.ID, ptr.To(n.View()))
+				mak.Set(&ms.peers, n.ID, n.View())
 			}
-			ms.rebuildSorted()
 
 			gotStats := ms.updatePeersStateFromResponse(tt.mapRes)
-
-			got := make([]*tailcfg.Node, len(ms.sortedPeers))
-			for i, vp := range ms.sortedPeers {
-				got[i] = vp.AsStruct()
-			}
 			if gotStats != tt.wantStats {
 				t.Errorf("got stats = %+v; want %+v", gotStats, tt.wantStats)
 			}
+
+			var got []*tailcfg.Node
+			for _, vp := range ms.sortedPeers() {
+				got = append(got, vp.AsStruct())
+			}
 			if !reflect.DeepEqual(got, tt.want) {
 				t.Errorf("wrong results\n got: %s\nwant: %s", formatNodes(got), formatNodes(tt.want))
 			}

control/controlclient/noise.go

@@ -11,6 +11,7 @@ import (
 	"errors"
 	"math"
 	"net/http"
+	"net/netip"
 	"net/url"
 	"sync"
 	"time"
@@ -111,24 +112,39 @@ type NoiseOpts struct {
 // netMon may be nil, if non-nil it's used to do faster interface lookups.
 // dialPlan may be nil
 func NewNoiseClient(opts NoiseOpts) (*NoiseClient, error) {
+	logf := opts.Logf
 	u, err := url.Parse(opts.ServerURL)
 	if err != nil {
 		return nil, err
 	}
+
+	if u.Scheme != "http" && u.Scheme != "https" {
+		return nil, errors.New("invalid ServerURL scheme, must be http or https")
+	}
+
 	var httpPort string
 	var httpsPort string
+	addr, _ := netip.ParseAddr(u.Hostname())
+	isPrivateHost := addr.IsPrivate() || addr.IsLoopback() || u.Hostname() == "localhost"
 	if port := u.Port(); port != "" {
-		// If there is an explicit port specified, trust the scheme and hope for the best
-		if u.Scheme == "http" {
+		// If there is an explicit port specified, entirely rely on the scheme,
+		// unless it's http with a private host in which case we never try using HTTPS.
+		if u.Scheme == "https" {
+			httpPort = ""
+			httpsPort = port
+		} else if u.Scheme == "http" {
 			httpPort = port
 			httpsPort = "443"
-			if u.Hostname() == "127.0.0.1" || u.Hostname() == "localhost" {
+			if isPrivateHost {
+				logf("setting empty HTTPS port with http scheme and private host %s", u.Hostname())
 				httpsPort = ""
 			}
-		} else {
-			httpPort = "80"
-			httpsPort = port
 		}
+	} else if u.Scheme == "http" && isPrivateHost {
+		// Whenever the scheme is http and the hostname is an IP address, do not set the HTTPS port,
+		// as there cannot be a TLS certificate issued for an IP, unless it's a public IP.
+		httpPort = "80"
+		httpsPort = ""
 	} else {
 		// Otherwise, use the standard ports
 		httpPort = "80"
@@ -380,17 +396,20 @@ func (nc *NoiseClient) dial(ctx context.Context) (*noiseconn.Conn, error) {
 // post does a POST to the control server at the given path, JSON-encoding body.
 // The provided nodeKey is an optional load balancing hint.
 func (nc *NoiseClient) post(ctx context.Context, path string, nodeKey key.NodePublic, body any) (*http.Response, error) {
+	return nc.doWithBody(ctx, "POST", path, nodeKey, body)
+}
+
+func (nc *NoiseClient) doWithBody(ctx context.Context, method, path string, nodeKey key.NodePublic, body any) (*http.Response, error) {
 	jbody, err := json.Marshal(body)
 	if err != nil {
 		return nil, err
 	}
-	req, err := http.NewRequestWithContext(ctx, "POST", "https://"+nc.host+path, bytes.NewReader(jbody))
+	req, err := http.NewRequestWithContext(ctx, method, "https://"+nc.host+path, bytes.NewReader(jbody))
 	if err != nil {
 		return nil, err
 	}
 	addLBHeader(req, nodeKey)
 	req.Header.Set("Content-Type", "application/json")
-
 	conn, err := nc.getConn(ctx)
 	if err != nil {
 		return nil, err

control/controlclient/noise_test.go

@@ -54,6 +54,123 @@ func TestNoiseClientHTTP2Upgrade_earlyPayload(t *testing.T) {
 	}.run(t)
 }
 
+func makeClientWithURL(t *testing.T, url string) *NoiseClient {
+	nc, err := NewNoiseClient(NoiseOpts{
+		Logf:      t.Logf,
+		ServerURL: url,
+	})
+	if err != nil {
+		t.Fatal(err)
+	}
+	return nc
+}
+
+func TestNoiseClientPortsAreSet(t *testing.T) {
+	tests := []struct {
+		name      string
+		url       string
+		wantHTTPS string
+		wantHTTP  string
+	}{
+		{
+			name:      "https-url",
+			url:       "https://example.com",
+			wantHTTPS: "443",
+			wantHTTP:  "80",
+		},
+		{
+			name:      "http-url",
+			url:       "http://example.com",
+			wantHTTPS: "443", // TODO(bradfitz): questionable; change?
+			wantHTTP:  "80",
+		},
+		{
+			name:      "https-url-custom-port",
+			url:       "https://example.com:123",
+			wantHTTPS: "123",
+			wantHTTP:  "",
+		},
+		{
+			name:      "http-url-custom-port",
+			url:       "http://example.com:123",
+			wantHTTPS: "443", // TODO(bradfitz): questionable; change?
+			wantHTTP:  "123",
+		},
+		{
+			name:      "http-loopback-no-port",
+			url:       "http://127.0.0.1",
+			wantHTTPS: "",
+			wantHTTP:  "80",
+		},
+		{
+			name:      "http-loopback-custom-port",
+			url:       "http://127.0.0.1:8080",
+			wantHTTPS: "",
+			wantHTTP:  "8080",
+		},
+		{
+			name:      "http-localhost-no-port",
+			url:       "http://localhost",
+			wantHTTPS: "",
+			wantHTTP:  "80",
+		},
+		{
+			name:      "http-localhost-custom-port",
+			url:       "http://localhost:8080",
+			wantHTTPS: "",
+			wantHTTP:  "8080",
+		},
+		{
+			name:      "http-private-ip-no-port",
+			url:       "http://192.168.2.3",
+			wantHTTPS: "",
+			wantHTTP:  "80",
+		},
+		{
+			name:      "http-private-ip-custom-port",
+			url:       "http://192.168.2.3:8080",
+			wantHTTPS: "",
+			wantHTTP:  "8080",
+		},
+		{
+			name:      "http-public-ip",
+			url:       "http://1.2.3.4",
+			wantHTTPS: "443", // TODO(bradfitz): questionable; change?
+			wantHTTP:  "80",
+		},
+		{
+			name:      "http-public-ip-custom-port",
+			url:       "http://1.2.3.4:8080",
+			wantHTTPS: "443", // TODO(bradfitz): questionable; change?
+			wantHTTP:  "8080",
+		},
+		{
+			name:      "https-public-ip",
+			url:       "https://1.2.3.4",
+			wantHTTPS: "443",
+			wantHTTP:  "80",
+		},
+		{
+			name:      "https-public-ip-custom-port",
+			url:       "https://1.2.3.4:8080",
+			wantHTTPS: "8080",
+			wantHTTP:  "",
+		},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			nc := makeClientWithURL(t, tt.url)
+			if nc.httpsPort != tt.wantHTTPS {
+				t.Errorf("nc.httpsPort = %q; want %q", nc.httpsPort, tt.wantHTTPS)
+			}
+			if nc.httpPort != tt.wantHTTP {
+				t.Errorf("nc.httpPort = %q; want %q", nc.httpPort, tt.wantHTTP)
+			}
+		})
+	}
+}
+
 func (tt noiseClientTest) run(t *testing.T) {
 	serverPrivate := key.NewMachine()
 	clientPrivate := key.NewMachine()
@@ -81,6 +198,7 @@ func (tt noiseClientTest) run(t *testing.T) {
 		ServerPubKey: serverPrivate.Public(),
 		ServerURL:    hs.URL,
 		Dialer:       dialer,
+		Logf:         t.Logf,
 	})
 	if err != nil {
 		t.Fatal(err)

docs/k8s/operator-architecture.md

@@ -0,0 +1,517 @@
+# Operator architecture diagrams
+
+The Tailscale [Kubernetes operator][kb-operator] has a collection of use-cases
+that can be mixed and matched as required. The following diagrams illustrate
+how the operator implements each use-case.
+
+In each diagram, the "tailscale" namespace is entirely managed by the operator
+once the operator itself has been deployed.
+
+Tailscale devices are highlighted as black nodes. The salient devices for each
+use-case are marked as "src" or "dst" to denote which node is a source or a
+destination in the context of ACL rules that will apply to network traffic.
+
+Note, in some cases, the config and the state Secret may be the same Kubernetes
+Secret.
+
+## API server proxy
+
+[Documentation][kb-operator-proxy]
+
+The operator runs the API server proxy in-process. If the proxy is running in
+"noauth" mode, it forwards HTTP requests unmodified. If the proxy is running in
+"auth" mode, it deletes any existing auth headers and adds
+[impersonation headers][k8s-impersonation] to the request before forwarding to
+the API server. A request with impersonation headers will look something like:
+
+```
+GET /api/v1/namespaces/default/pods HTTP/1.1
+Host: k8s-api.example.com
+Authorization: Bearer <operator-service-account-token>
+Impersonate-Group: tailnet-readers
+Accept: application/json
+```
+
+```mermaid
+%%{ init: { 'theme':'neutral' } }%%
+flowchart LR
+    classDef tsnode color:#fff,fill:#000;
+    classDef pod fill:#fff;
+
+    subgraph Key
+        ts[Tailscale device]:::tsnode
+        pod((Pod)):::pod
+        blank[" "]-->|WireGuard traffic| blank2[" "]
+        blank3[" "]-->|Other network traffic| blank4[" "]
+    end
+
+    subgraph k8s[Kubernetes cluster]
+        subgraph tailscale-ns[namespace=tailscale]
+            operator(("operator (dst)")):::tsnode
+        end
+
+        subgraph controlplane["Control plane"]
+            api[kube-apiserver]
+        end
+    end
+
+    client["client (src)"]:::tsnode --> operator
+    operator -->|"proxy (maybe with impersonation headers)"| api
+
+    linkStyle 0 stroke:red;
+    linkStyle 2 stroke:red;
+
+    linkStyle 1 stroke:blue;
+    linkStyle 3 stroke:blue;
+
+```
+
+## L3 ingress
+
+[Documentation][kb-operator-l3-ingress]
+
+The user deploys an app to the default namespace, and creates a normal Service
+that selects the app's Pods. Either add the annotation
+`tailscale.com/expose: "true"` or specify `.spec.type` as `Loadbalancer` and
+`.spec.loadBalancerClass` as `tailscale`. The operator will create an ingress
+proxy that allows devices anywhere on the tailnet to access the Service.
+
+The proxy Pod uses `iptables` or `nftables` rules to DNAT traffic bound for the
+proxy's tailnet IP to the Service's internal Cluster IP instead.
+
+```mermaid
+%%{ init: { 'theme':'neutral' } }%%
+flowchart TD
+    classDef tsnode color:#fff,fill:#000;
+    classDef pod fill:#fff;
+
+    subgraph Key
+        ts[Tailscale device]:::tsnode
+        pod((Pod)):::pod
+        blank[" "]-->|WireGuard traffic| blank2[" "]
+        blank3[" "]-->|Other network traffic| blank4[" "]
+    end
+
+    subgraph k8s[Kubernetes cluster]
+        subgraph tailscale-ns[namespace=tailscale]
+            operator((operator)):::tsnode
+            ingress-sts["StatefulSet"]
+            ingress(("ingress proxy (dst)")):::tsnode
+            config-secret["config Secret"]
+            state-secret["state Secret"]
+        end
+
+        subgraph defaultns[namespace=default]
+            svc[annotated Service]
+            svc --> pod1((pod1))
+            svc --> pod2((pod2))
+        end
+    end
+
+    client["client (src)"]:::tsnode --> ingress
+    ingress -->|forwards traffic| svc
+    operator -.->|creates| ingress-sts
+    ingress-sts -.->|manages| ingress
+    operator -.->|reads| svc
+    operator -.->|creates| config-secret
+    config-secret -.->|mounted| ingress
+    ingress -.->|stores state| state-secret
+
+    linkStyle 0 stroke:red;
+    linkStyle 4 stroke:red;
+
+    linkStyle 1 stroke:blue;
+    linkStyle 2 stroke:blue;
+    linkStyle 3 stroke:blue;
+    linkStyle 5 stroke:blue;
+
+```
+
+## L7 ingress
+
+[Documentation][kb-operator-l7-ingress]
+
+L7 ingress is relatively similar to L3 ingress. It is configured via an
+`Ingress` object instead of a `Service`, and uses `tailscale serve` to accept
+traffic instead of configuring `iptables` or `nftables` rules. Note that we use
+tailscaled's local API (`SetServeConfig`) to set serve config, not the
+`tailscale serve` command.
+
+```mermaid
+%%{ init: { 'theme':'neutral' } }%%
+flowchart TD
+    classDef tsnode color:#fff,fill:#000;
+    classDef pod fill:#fff;
+
+    subgraph Key
+        ts[Tailscale device]:::tsnode
+        pod((Pod)):::pod
+        blank[" "]-->|WireGuard traffic| blank2[" "]
+        blank3[" "]-->|Other network traffic| blank4[" "]
+    end
+
+    subgraph k8s[Kubernetes cluster]
+        subgraph tailscale-ns[namespace=tailscale]
+            operator((operator)):::tsnode
+            ingress-sts["StatefulSet"]
+            ingress-pod(("ingress proxy (dst)")):::tsnode
+            config-secret["config Secret"]
+            state-secret["state Secret"]
+        end
+
+        subgraph defaultns[namespace=default]
+            ingress[tailscale Ingress]
+            svc["Service"]
+            svc --> pod1((pod1))
+            svc --> pod2((pod2))
+        end
+    end
+
+    client["client (src)"]:::tsnode --> ingress-pod
+    ingress-pod -->|forwards /api prefix traffic| svc
+    operator -.->|creates| ingress-sts
+    ingress-sts -.->|manages| ingress-pod
+    operator -.->|reads| ingress
+    operator -.->|creates| config-secret
+    config-secret -.->|mounted| ingress-pod
+    ingress-pod -.->|stores state| state-secret
+    ingress -.->|/api prefix| svc
+
+    linkStyle 0 stroke:red;
+    linkStyle 4 stroke:red;
+
+    linkStyle 1 stroke:blue;
+    linkStyle 2 stroke:blue;
+    linkStyle 3 stroke:blue;
+    linkStyle 5 stroke:blue;
+
+```
+
+## L3 egress
+
+[Documentation][kb-operator-l3-egress]
+
+1. The user deploys a Service with `type: ExternalName` and an annotation 
+  `tailscale.com/tailnet-fqdn: db.tails-scales.ts.net`.
+1. The operator creates a proxy Pod managed by a single replica StatefulSet, and a headless Service pointing at the proxy Pod.
+1. The operator updates the `ExternalName` Service's `spec.externalName` field to point
+  at the headless Service it created in the previous step.
+
+(Optional) If the user also adds the `tailscale.com/proxy-group: egress-proxies`
+annotation to their `ExternalName` Service, the operator will skip creating a
+proxy Pod and instead point the headless Service at the existing ProxyGroup's
+pods. In this case, ports are also required in the `ExternalName` Service spec.
+See below for a more representative diagram.
+
+```mermaid
+%%{ init: { 'theme':'neutral' } }%%
+
+flowchart TD
+    classDef tsnode color:#fff,fill:#000;
+    classDef pod fill:#fff;
+
+    subgraph Key
+        ts[Tailscale device]:::tsnode
+        pod((Pod)):::pod
+        blank[" "]-->|WireGuard traffic| blank2[" "]
+        blank3[" "]-->|Other network traffic| blank4[" "]
+    end
+
+    subgraph k8s[Kubernetes cluster]
+        subgraph tailscale-ns[namespace=tailscale]
+            operator((operator)):::tsnode
+            egress(("egress proxy (src)")):::tsnode
+            egress-sts["StatefulSet"]
+            headless-svc[headless Service]
+            cfg-secret["config Secret"]
+            state-secret["state Secret"]
+        end
+
+        subgraph defaultns[namespace=default]
+            svc[ExternalName Service]
+            pod1((pod1)) --> svc
+            pod2((pod2)) --> svc
+        end
+    end
+
+    node["db.tails-scales.ts.net (dst)"]:::tsnode
+
+    svc -->|DNS points to| headless-svc
+    headless-svc -->|selects egress Pod| egress
+    egress -->|forwards traffic| node
+    operator -.->|creates| egress-sts
+    egress-sts -.->|manages| egress
+    operator -.->|creates| headless-svc
+    operator -.->|creates| cfg-secret
+    operator -.->|watches & updates| svc
+    cfg-secret -.->|mounted| egress
+    egress -.->|stores state| state-secret
+
+    linkStyle 0 stroke:red;
+    linkStyle 6 stroke:red;
+
+    linkStyle 1 stroke:blue;
+    linkStyle 2 stroke:blue;
+    linkStyle 3 stroke:blue;
+    linkStyle 4 stroke:blue;
+    linkStyle 5 stroke:blue;
+
+```
+
+## `ProxyGroup`
+
+[Documentation][kb-operator-l3-egress-proxygroup]
+
+The `ProxyGroup` custom resource manages a collection of proxy Pods that
+can be configured to egress traffic out of the cluster via ExternalName
+Services. A `ProxyGroup` is both a high availability (HA) version of L3
+egress, and a mechanism to serve multiple ExternalName Services on a single
+set of Tailscale devices (coalescing).
+
+In this diagram, the `ProxyGroup` is named `pg`. The Secrets associated with
+the `ProxyGroup` Pods are omitted for simplicity. They are similar to the L3
+egress case above, but there is a pair of config + state Secrets _per Pod_.
+
+Each ExternalName Service defines which ports should be mapped to their defined
+egress target. The operator maps from these ports to randomly chosen ephemeral
+ports via the ClusterIP Service and its EndpointSlice. The operator then
+generates the egress ConfigMap that tells the `ProxyGroup` Pods which incoming
+ports map to which egress targets.
+
+`ProxyGroups` currently only support egress.
+
+```mermaid
+%%{ init: { 'theme':'neutral' } }%%
+
+flowchart LR
+    classDef tsnode color:#fff,fill:#000;
+    classDef pod fill:#fff;
+
+    subgraph Key
+        ts[Tailscale device]:::tsnode
+        pod((Pod)):::pod
+        blank[" "]-->|WireGuard traffic| blank2[" "]
+        blank3[" "]-->|Other network traffic| blank4[" "]
+    end
+
+    subgraph k8s[Kubernetes cluster]
+        subgraph tailscale-ns[namespace=tailscale]
+            operator((operator)):::tsnode
+            pg-sts[StatefulSet]
+            pg-0(("pg-0 (src)")):::tsnode
+            pg-1(("pg-1 (src)")):::tsnode
+            db-cluster-ip[db ClusterIP Service]
+            api-cluster-ip[api ClusterIP Service]
+            egress-cm["egress ConfigMap"]
+        end
+
+        subgraph cluster-scope["Cluster scoped resources"]
+            pg["ProxyGroup 'pg'"]
+        end
+
+        subgraph defaultns[namespace=default]
+            db-svc[db ExternalName Service]
+            api-svc[api ExternalName Service]
+            pod1((pod1)) --> db-svc
+            pod2((pod2)) --> db-svc
+            pod1((pod1)) --> api-svc
+            pod2((pod2)) --> api-svc
+        end
+    end
+
+    db["db.tails-scales.ts.net (dst)"]:::tsnode
+    api["api.tails-scales.ts.net (dst)"]:::tsnode
+
+    db-svc -->|DNS points to| db-cluster-ip
+    api-svc -->|DNS points to| api-cluster-ip
+    db-cluster-ip -->|maps to ephemeral db ports| pg-0
+    db-cluster-ip -->|maps to ephemeral db ports| pg-1
+    api-cluster-ip -->|maps to ephemeral api ports| pg-0
+    api-cluster-ip -->|maps to ephemeral api ports| pg-1
+    pg-0 -->|forwards db port traffic| db
+    pg-0 -->|forwards api port traffic| api
+    pg-1 -->|forwards db port traffic| db
+    pg-1 -->|forwards api port traffic| api
+    operator -.->|creates & populates endpointslice| db-cluster-ip
+    operator -.->|creates & populates endpointslice| api-cluster-ip
+    operator -.->|stores port mapping| egress-cm
+    egress-cm -.->|mounted| pg-0
+    egress-cm -.->|mounted| pg-1
+    operator -.->|watches| pg
+    operator -.->|creates| pg-sts
+    pg-sts -.->|manages| pg-0
+    pg-sts -.->|manages| pg-1
+    operator -.->|watches| db-svc
+    operator -.->|watches| api-svc
+
+    linkStyle 0 stroke:red;
+    linkStyle 12 stroke:red;
+    linkStyle 13 stroke:red;
+    linkStyle 14 stroke:red;
+    linkStyle 15 stroke:red;
+
+    linkStyle 1 stroke:blue;
+    linkStyle 2 stroke:blue;
+    linkStyle 3 stroke:blue;
+    linkStyle 4 stroke:blue;
+    linkStyle 5 stroke:blue;
+    linkStyle 6 stroke:blue;
+    linkStyle 7 stroke:blue;
+    linkStyle 8 stroke:blue;
+    linkStyle 9 stroke:blue;
+    linkStyle 10 stroke:blue;
+    linkStyle 11 stroke:blue;
+
+```
+
+## Connector
+
+[Subnet router and exit node documentation][kb-operator-connector]
+
+[App connector documentation][kb-operator-app-connector]
+
+The Connector Custom Resource can deploy either a subnet router, an exit node,
+or an app connector. The following diagram shows all 3, but only one workflow
+can be configured per Connector resource.
+
+```mermaid
+%%{ init: { 'theme':'neutral' } }%%
+
+flowchart TD
+    classDef tsnode color:#fff,fill:#000;
+    classDef pod fill:#fff;
+    classDef hidden display:none;
+
+    subgraph Key
+        ts[Tailscale device]:::tsnode
+        pod((Pod)):::pod
+        blank[" "]-->|WireGuard traffic| blank2[" "]
+        blank3[" "]-->|Other network traffic| blank4[" "]
+    end
+
+    subgraph grouping[" "]
+        subgraph k8s[Kubernetes cluster]
+            subgraph tailscale-ns[namespace=tailscale]
+                operator((operator)):::tsnode
+                cn-sts[StatefulSet]
+                cn-pod(("tailscale (dst)")):::tsnode
+                cfg-secret["config Secret"]
+                state-secret["state Secret"]
+            end
+
+            subgraph cluster-scope["Cluster scoped resources"]
+                cn["Connector"]
+            end
+
+            subgraph defaultns["namespace=default"]
+                pod1
+            end
+        end
+
+        client["client (src)"]:::tsnode
+        Internet
+    end
+
+    client --> cn-pod
+    cn-pod -->|app connector or exit node routes| Internet
+    cn-pod -->|subnet route| pod1
+    operator -.->|watches| cn
+    operator -.->|creates| cn-sts
+    cn-sts -.->|manages| cn-pod
+    operator -.->|creates| cfg-secret
+    cfg-secret -.->|mounted| cn-pod
+    cn-pod -.->|stores state| state-secret
+
+    class grouping hidden
+
+    linkStyle 0 stroke:red;
+    linkStyle 2 stroke:red;
+
+    linkStyle 1 stroke:blue;
+    linkStyle 3 stroke:blue;
+    linkStyle 4 stroke:blue;
+
+```
+
+## Recorder nodes
+
+[Documentation][kb-operator-recorder]
+
+The `Recorder` custom resource makes it easier to deploy `tsrecorder` to a cluster.
+It currently only supports a single replica.
+
+```mermaid
+%%{ init: { 'theme':'neutral' } }%%
+
+flowchart TD
+    classDef tsnode color:#fff,fill:#000;
+    classDef pod fill:#fff;
+    classDef hidden display:none;
+
+    subgraph Key
+        ts[Tailscale device]:::tsnode
+        pod((Pod)):::pod
+        blank[" "]-->|WireGuard traffic| blank2[" "]
+        blank3[" "]-->|Other network traffic| blank4[" "]
+    end
+
+    subgraph grouping[" "]
+        subgraph k8s[Kubernetes cluster]
+            api["kube-apiserver"]
+
+            subgraph tailscale-ns[namespace=tailscale]
+                operator(("operator (dst)")):::tsnode
+                rec-sts[StatefulSet]
+                rec-0(("tsrecorder")):::tsnode
+                cfg-secret-0["config Secret"]
+                state-secret-0["state Secret"]
+            end
+
+            subgraph cluster-scope["Cluster scoped resources"]
+                rec["Recorder"]
+            end
+        end
+
+        client["client (src)"]:::tsnode
+        kubectl-exec["kubectl exec (src)"]:::tsnode
+        server["server (dst)"]:::tsnode
+        s3["S3-compatible storage"]
+    end
+
+    kubectl-exec -->|exec session| operator
+    operator -->|exec session recording| rec-0
+    operator -->|exec session| api
+    client -->|ssh session| server
+    server -->|ssh session recording| rec-0
+    rec-0 -->|session recordings| s3
+    operator -.->|watches| rec
+    operator -.->|creates| rec-sts
+    rec-sts -.->|manages| rec-0
+    operator -.->|creates| cfg-secret-0
+    cfg-secret-0 -.->|mounted| rec-0
+    rec-0 -.->|stores state| state-secret-0
+
+    class grouping hidden
+
+    linkStyle 0 stroke:red;
+    linkStyle 2 stroke:red;
+    linkStyle 3 stroke:red;
+    linkStyle 5 stroke:red;
+    linkStyle 6 stroke:red;
+
+    linkStyle 1 stroke:blue;
+    linkStyle 4 stroke:blue;
+    linkStyle 7 stroke:blue;
+
+```
+
+[kb-operator]: https://tailscale.com/kb/1236/kubernetes-operator
+[kb-operator-proxy]: https://tailscale.com/kb/1437/kubernetes-operator-api-server-proxy
+[kb-operator-l3-ingress]: https://tailscale.com/kb/1439/kubernetes-operator-cluster-ingress#exposing-a-cluster-workload-using-a-kubernetes-service
+[kb-operator-l7-ingress]: https://tailscale.com/kb/1439/kubernetes-operator-cluster-ingress#exposing-cluster-workloads-using-a-kubernetes-ingress
+[kb-operator-l3-egress]: https://tailscale.com/kb/1438/kubernetes-operator-cluster-egress
+[kb-operator-l3-egress-proxygroup]: https://tailscale.com/kb/1438/kubernetes-operator-cluster-egress#configure-an-egress-service-using-proxygroup
+[kb-operator-connector]: https://tailscale.com/kb/1441/kubernetes-operator-connector
+[kb-operator-app-connector]: https://tailscale.com/kb/1517/kubernetes-operator-app-connector
+[kb-operator-recorder]: https://tailscale.com/kb/1484/kubernetes-operator-deploying-tsrecorder
+[k8s-impersonation]: https://kubernetes.io/docs/reference/access-authn-authz/authentication/#user-impersonation

docs/windows/policy/en-US/tailscale.adml

@@ -31,7 +31,7 @@ See https://tailscale.com/kb/1315/mdm-keys#set-a-custom-control-server-url for m
             <string id="LogTarget_Help"><![CDATA[This policy can be used to require the use of a non-standard log server.
 Please note that using a non-standard log server will limit Tailscale Support's ability to diagnose problems.
 
-If you configure this policy, set it to the URL of your log server, beginning with https:// and ending with no trailing slash. If blank or "https://log.tailscale.io", the default log server will be used.
+If you configure this policy, set it to the URL of your log server, beginning with https:// and ending with no trailing slash. If blank or "https://log.tailscale.com", the default log server will be used.
 
 If you disable this policy, the Tailscale standard log server will be used by default, but a non-standard Tailscale log server can be configured using the TS_LOG_TARGET environment variable.]]></string>
             <string id="Tailnet">Specify which Tailnet should be used for Login</string>

envknob/logknob/logknob.go

@@ -11,7 +11,6 @@ import (
 	"tailscale.com/envknob"
 	"tailscale.com/tailcfg"
 	"tailscale.com/types/logger"
-	"tailscale.com/types/views"
 )
 
 // TODO(andrew-d): should we have a package-global registry of logknobs? It
@@ -59,7 +58,7 @@ func (lk *LogKnob) Set(v bool) {
 // about; we use this rather than a concrete type to avoid a circular
 // dependency.
 type NetMap interface {
-	SelfCapabilities() views.Slice[tailcfg.NodeCapability]
+	HasSelfCapability(tailcfg.NodeCapability) bool
 }
 
 // UpdateFromNetMap will enable logging if the SelfNode in the provided NetMap
@@ -68,8 +67,7 @@ func (lk *LogKnob) UpdateFromNetMap(nm NetMap) {
 	if lk.capName == "" {
 		return
 	}
-
-	lk.cap.Store(views.SliceContains(nm.SelfCapabilities(), lk.capName))
+	lk.cap.Store(nm.HasSelfCapability(lk.capName))
 }
 
 // Do will call log with the provided format and arguments if any of the

envknob/logknob/logknob_test.go

@@ -11,6 +11,7 @@ import (
 	"tailscale.com/envknob"
 	"tailscale.com/tailcfg"
 	"tailscale.com/types/netmap"
+	"tailscale.com/util/set"
 )
 
 var testKnob = NewLogKnob(
@@ -63,11 +64,7 @@ func TestLogKnob(t *testing.T) {
 		}
 
 		testKnob.UpdateFromNetMap(&netmap.NetworkMap{
-			SelfNode: (&tailcfg.Node{
-				Capabilities: []tailcfg.NodeCapability{
-					"https://tailscale.com/cap/testing",
-				},
-			}).View(),
+			AllCaps: set.Of(tailcfg.NodeCapability("https://tailscale.com/cap/testing")),
 		})
 		if !testKnob.shouldLog() {
 			t.Errorf("expected shouldLog()=true")

go.mod

@@ -34,7 +34,7 @@ require (
 	github.com/frankban/quicktest v1.14.6
 	github.com/fxamacker/cbor/v2 v2.6.0
 	github.com/gaissmai/bart v0.11.1
-	github.com/go-json-experiment/json v0.0.0-20231102232822-2e55bd4e08b0
+	github.com/go-json-experiment/json v0.0.0-20250103232110-6a9a0fde9288
 	github.com/go-logr/zapr v1.3.0
 	github.com/go-ole/go-ole v1.3.0
 	github.com/godbus/dbus/v5 v5.1.1-0.20230522191255-76236955d466
@@ -82,10 +82,10 @@ require (
 	github.com/tailscale/hujson v0.0.0-20221223112325-20486734a56a
 	github.com/tailscale/mkctr v0.0.0-20241111153353-1a38f6676f10
 	github.com/tailscale/netlink v1.1.1-0.20240822203006-4d49adab4de7
-	github.com/tailscale/peercred v0.0.0-20240214030740-b535050b2aa4
+	github.com/tailscale/peercred v0.0.0-20250107143737-35a0c7bd7edc
 	github.com/tailscale/web-client-prebuilt v0.0.0-20240226180453-5db17b287bf1
 	github.com/tailscale/wf v0.0.0-20240214030419-6fbb0a674ee6
-	github.com/tailscale/wireguard-go v0.0.0-20241113014420-4e883d38c8d3
+	github.com/tailscale/wireguard-go v0.0.0-20250107165329-0b8b35511f19
 	github.com/tailscale/xnet v0.0.0-20240729143630-8497ac4dab2e
 	github.com/tc-hib/winres v0.2.1
 	github.com/tcnksm/go-httpstat v0.2.0
@@ -95,13 +95,13 @@ require (
 	go.uber.org/zap v1.27.0
 	go4.org/mem v0.0.0-20220726221520-4f986261bf13
 	go4.org/netipx v0.0.0-20231129151722-fdeea329fbba
-	golang.org/x/crypto v0.30.0
-	golang.org/x/exp v0.0.0-20240119083558-1b970713d09a
+	golang.org/x/crypto v0.31.0
+	golang.org/x/exp v0.0.0-20240719175910-8a7402abbf56
 	golang.org/x/mod v0.19.0
-	golang.org/x/net v0.32.0
+	golang.org/x/net v0.33.0
 	golang.org/x/oauth2 v0.16.0
 	golang.org/x/sync v0.10.0
-	golang.org/x/sys v0.28.0
+	golang.org/x/sys v0.29.1-0.20250107080300-1c14dcadc3ab
 	golang.org/x/term v0.27.0
 	golang.org/x/time v0.5.0
 	golang.org/x/tools v0.23.0
@@ -135,7 +135,7 @@ require (
 	github.com/catenacyber/perfsprint v0.7.1 // indirect
 	github.com/ccojocar/zxcvbn-go v1.0.2 // indirect
 	github.com/ckaznocha/intrange v0.1.0 // indirect
-	github.com/cyphar/filepath-securejoin v0.2.4 // indirect
+	github.com/cyphar/filepath-securejoin v0.3.6 // indirect
 	github.com/dave/astrid v0.0.0-20170323122508-8c2895878b14 // indirect
 	github.com/dave/brenda v1.1.0 // indirect
 	github.com/docker/go-connections v0.5.0 // indirect
@@ -183,7 +183,7 @@ require (
 	github.com/Masterminds/semver v1.5.0 // indirect
 	github.com/Masterminds/semver/v3 v3.2.1 // indirect
 	github.com/Masterminds/sprig/v3 v3.2.3 // indirect
-	github.com/ProtonMail/go-crypto v1.0.0 // indirect
+	github.com/ProtonMail/go-crypto v1.1.3 // indirect
 	github.com/alexkohler/prealloc v1.0.0 // indirect
 	github.com/alingse/asasalint v0.0.11 // indirect
 	github.com/ashanbrown/forbidigo v1.6.0 // indirect
@@ -236,8 +236,8 @@ require (
 	github.com/fzipp/gocyclo v0.6.0 // indirect
 	github.com/go-critic/go-critic v0.11.2 // indirect
 	github.com/go-git/gcfg v1.5.1-0.20230307220236-3a3c6141e376 // indirect
-	github.com/go-git/go-billy/v5 v5.5.0 // indirect
-	github.com/go-git/go-git/v5 v5.11.0 // indirect
+	github.com/go-git/go-billy/v5 v5.6.1 // indirect
+	github.com/go-git/go-git/v5 v5.13.1 // indirect
 	github.com/go-logr/logr v1.4.2 // indirect
 	github.com/go-openapi/jsonpointer v0.20.2 // indirect
 	github.com/go-openapi/jsonreference v0.20.4 // indirect
@@ -343,13 +343,13 @@ require (
 	github.com/sashamelentyev/interfacebloat v1.1.0 // indirect
 	github.com/sashamelentyev/usestdlibvars v1.25.0 // indirect
 	github.com/securego/gosec/v2 v2.19.0 // indirect
-	github.com/sergi/go-diff v1.3.1 // indirect
+	github.com/sergi/go-diff v1.3.2-0.20230802210424-5b0b94c5c0d3 // indirect
 	github.com/shazow/go-diff v0.0.0-20160112020656-b6b7b6733b8c // indirect
 	github.com/shopspring/decimal v1.3.1 // indirect
 	github.com/sirupsen/logrus v1.9.3 // indirect
 	github.com/sivchari/containedctx v1.0.3 // indirect
 	github.com/sivchari/tenv v1.7.1 // indirect
-	github.com/skeema/knownhosts v1.2.1 // indirect
+	github.com/skeema/knownhosts v1.3.0 // indirect
 	github.com/sonatard/noctx v0.0.2 // indirect
 	github.com/sourcegraph/go-diff v0.7.0 // indirect
 	github.com/spf13/afero v1.11.0 // indirect
@@ -361,7 +361,7 @@ require (
 	github.com/ssgreg/nlreturn/v2 v2.2.1 // indirect
 	github.com/stbenjam/no-sprintf-host-port v0.1.1 // indirect
 	github.com/stretchr/objx v0.5.2 // indirect
-	github.com/stretchr/testify v1.9.0
+	github.com/stretchr/testify v1.10.0
 	github.com/subosito/gotenv v1.4.2 // indirect
 	github.com/t-yuki/gocover-cobertura v0.0.0-20180217150009-aaee18c8195c // indirect
 	github.com/tailscale/go-winio v0.0.0-20231025203758-c4f33415bf55

go.sum

@@ -83,8 +83,8 @@ github.com/Microsoft/go-winio v0.6.2 h1:F2VQgta7ecxGYO8k3ZZz3RS8fVIXVxONVUPlNERo
 github.com/Microsoft/go-winio v0.6.2/go.mod h1:yd8OoFMLzJbo9gZq8j5qaps8bJ9aShtEA8Ipt1oGCvU=
 github.com/OpenPeeDeeP/depguard/v2 v2.2.0 h1:vDfG60vDtIuf0MEOhmLlLLSzqaRM8EMcgJPdp74zmpA=
 github.com/OpenPeeDeeP/depguard/v2 v2.2.0/go.mod h1:CIzddKRvLBC4Au5aYP/i3nyaWQ+ClszLIuVocRiCYFQ=
-github.com/ProtonMail/go-crypto v1.0.0 h1:LRuvITjQWX+WIfr930YHG2HNfjR1uOfyf5vE0kC2U78=
-github.com/ProtonMail/go-crypto v1.0.0/go.mod h1:EjAoLdwvbIOoOQr3ihjnSoLZRtE8azugULFRteWMNc0=
+github.com/ProtonMail/go-crypto v1.1.3 h1:nRBOetoydLeUb4nHajyO2bKqMLfWQ/ZPwkXqXxPxCFk=
+github.com/ProtonMail/go-crypto v1.1.3/go.mod h1:rA3QumHc/FZ8pAHreoekgiAbzpNsfQAosU5td4SnOrE=
 github.com/ProtonMail/go-mime v0.0.0-20230322103455-7d82a3887f2f h1:tCbYj7/299ekTTXpdwKYF8eBlsYsDVoggDAuAjoK66k=
 github.com/ProtonMail/go-mime v0.0.0-20230322103455-7d82a3887f2f/go.mod h1:gcr0kNtGBqin9zDW9GOHcVntrwnjrK+qdJ06mWYBybw=
 github.com/ProtonMail/gopenpgp/v2 v2.7.1 h1:Awsg7MPc2gD3I7IFac2qE3Gdls0lZW8SzrFZ3k1oz0s=
@@ -200,7 +200,6 @@ github.com/butuzov/ireturn v0.3.0 h1:hTjMqWw3y5JC3kpnC5vXmFJAWI/m31jaCYQqzkS6PL0
 github.com/butuzov/ireturn v0.3.0/go.mod h1:A09nIiwiqzN/IoVo9ogpa0Hzi9fex1kd9PSD6edP5ZA=
 github.com/butuzov/mirror v1.1.0 h1:ZqX54gBVMXu78QLoiqdwpl2mgmoOJTk7s4p4o+0avZI=
 github.com/butuzov/mirror v1.1.0/go.mod h1:8Q0BdQU6rC6WILDiBM60DBfvV78OLJmMmixe7GF45AE=
-github.com/bwesterb/go-ristretto v1.2.3/go.mod h1:fUIoIZaG73pV5biE2Blr2xEzDoMj7NFEuV9ekS419A0=
 github.com/caarlos0/go-rpmutils v0.2.1-0.20211112020245-2cd62ff89b11 h1:IRrDwVlWQr6kS1U8/EtyA1+EHcc4yl8pndcqXWrEamg=
 github.com/caarlos0/go-rpmutils v0.2.1-0.20211112020245-2cd62ff89b11/go.mod h1:je2KZ+LxaCNvCoKg32jtOIULcFogJKcL1ZWUaIBjKj0=
 github.com/caarlos0/testfs v0.4.4 h1:3PHvzHi5Lt+g332CiShwS8ogTgS3HjrmzZxCm6JCDr8=
@@ -231,7 +230,6 @@ github.com/cilium/ebpf v0.15.0/go.mod h1:DHp1WyrLeiBh19Cf/tfiSMhqheEiK8fXFZ4No0P
 github.com/ckaznocha/intrange v0.1.0 h1:ZiGBhvrdsKpoEfzh9CjBfDSZof6QB0ORY5tXasUtiew=
 github.com/ckaznocha/intrange v0.1.0/go.mod h1:Vwa9Ekex2BrEQMg6zlrWwbs/FtYw7eS5838Q7UjK7TQ=
 github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw=
-github.com/cloudflare/circl v1.3.3/go.mod h1:5XYMA4rFBvNIrhs50XuiBJ15vF2pZn4nnUKZrLbUZFA=
 github.com/cloudflare/circl v1.3.7 h1:qlCDlTPz2n9fu58M0Nh1J/JzcFpfgkFHHX3O35r5vcU=
 github.com/cloudflare/circl v1.3.7/go.mod h1:sRTcRWXGLrKw6yIGJ+l7amYJFfAXbZG0kBSc8r4zxgA=
 github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc=
@@ -251,8 +249,8 @@ github.com/creack/pty v1.1.23 h1:4M6+isWdcStXEf15G/RbrMPOQj1dZ7HPZCGwE4kOeP0=
 github.com/creack/pty v1.1.23/go.mod h1:08sCNb52WyoAwi2QDyzUCTgcvVFhUzewun7wtTfvcwE=
 github.com/curioswitch/go-reassign v0.2.0 h1:G9UZyOcpk/d7Gd6mqYgd8XYWFMw/znxwGDUstnC9DIo=
 github.com/curioswitch/go-reassign v0.2.0/go.mod h1:x6OpXuWvgfQaMGks2BZybTngWjT84hqJfKoO8Tt/Roc=
-github.com/cyphar/filepath-securejoin v0.2.4 h1:Ugdm7cg7i6ZK6x3xDF1oEu1nfkyfH53EtKeQYTC3kyg=
-github.com/cyphar/filepath-securejoin v0.2.4/go.mod h1:aPGpWjXOXUn2NCNjFvBE6aRxGGx79pTxQpKOJNYHHl4=
+github.com/cyphar/filepath-securejoin v0.3.6 h1:4d9N5ykBnSp5Xn2JkhocYDkOpURL/18CYMpo6xB9uWM=
+github.com/cyphar/filepath-securejoin v0.3.6/go.mod h1:Sdj7gXlvMcPZsbhwhQ33GguGLDGQL7h7bg04C/+u9jI=
 github.com/daixiang0/gci v0.12.3 h1:yOZI7VAxAGPQmkb1eqt5g/11SUlwoat1fSblGLmdiQc=
 github.com/daixiang0/gci v0.12.3/go.mod h1:xtHP9N7AHdNvtRNfcx9gwTDfw7FRJx4bZUsiEfiNNAI=
 github.com/dave/astrid v0.0.0-20170323122508-8c2895878b14 h1:YI1gOOdmMk3xodBao7fehcvoZsEeOyy/cfhlpCSPgM4=
@@ -293,8 +291,8 @@ github.com/dsnet/try v0.0.3 h1:ptR59SsrcFUYbT/FhAbKTV6iLkeD6O18qfIWRml2fqI=
 github.com/dsnet/try v0.0.3/go.mod h1:WBM8tRpUmnXXhY1U6/S8dt6UWdHTQ7y8A5YSkRCkq40=
 github.com/elastic/crd-ref-docs v0.0.12 h1:F3seyncbzUz3rT3d+caeYWhumb5ojYQ6Bl0Z+zOp16M=
 github.com/elastic/crd-ref-docs v0.0.12/go.mod h1:X83mMBdJt05heJUYiS3T0yJ/JkCuliuhSUNav5Gjo/U=
-github.com/elazarl/goproxy v0.0.0-20230808193330-2592e75ae04a h1:mATvB/9r/3gvcejNsXKSkQ6lcIaNec2nyfOdlTBR2lU=
-github.com/elazarl/goproxy v0.0.0-20230808193330-2592e75ae04a/go.mod h1:Ro8st/ElPeALwNFlcTpWmkr6IoMFfkjXAvTHpevnDsM=
+github.com/elazarl/goproxy v1.2.3 h1:xwIyKHbaP5yfT6O9KIeYJR5549MXRQkoQMRXGztz8YQ=
+github.com/elazarl/goproxy v1.2.3/go.mod h1:YfEbZtqP4AetfO6d40vWchF3znWX7C7Vd6ZMfdL8z64=
 github.com/emicklei/go-restful/v3 v3.11.2 h1:1onLa9DcsMYO9P+CXaL0dStDqQ2EHHXLiz+BtnqkLAU=
 github.com/emicklei/go-restful/v3 v3.11.2/go.mod h1:6n3XBCmQQb25CM2LCACGz8ukIrRry+4bhvbpWn3mrbc=
 github.com/emirpasic/gods v1.18.1 h1:FXtiHYKDGKCW2KzwZKx0iC0PQmdlorYgdFG9jPXJ1Bc=
@@ -335,23 +333,23 @@ github.com/ghostiam/protogetter v0.3.5 h1:+f7UiF8XNd4w3a//4DnusQ2SZjPkUjxkMEfjbx
 github.com/ghostiam/protogetter v0.3.5/go.mod h1:7lpeDnEJ1ZjL/YtyoN99ljO4z0pd3H0d18/t2dPBxHw=
 github.com/github/fakeca v0.1.0 h1:Km/MVOFvclqxPM9dZBC4+QE564nU4gz4iZ0D9pMw28I=
 github.com/github/fakeca v0.1.0/go.mod h1:+bormgoGMMuamOscx7N91aOuUST7wdaJ2rNjeohylyo=
-github.com/gliderlabs/ssh v0.3.5 h1:OcaySEmAQJgyYcArR+gGGTHCyE7nvhEMTlYY+Dp8CpY=
-github.com/gliderlabs/ssh v0.3.5/go.mod h1:8XB4KraRrX39qHhT6yxPsHedjA08I/uBVwj4xC+/+z4=
+github.com/gliderlabs/ssh v0.3.8 h1:a4YXD1V7xMF9g5nTkdfnja3Sxy1PVDCj1Zg4Wb8vY6c=
+github.com/gliderlabs/ssh v0.3.8/go.mod h1:xYoytBv1sV0aL3CavoDuJIQNURXkkfPA/wxQ1pL1fAU=
 github.com/go-critic/go-critic v0.11.2 h1:81xH/2muBphEgPtcwH1p6QD+KzXl2tMSi3hXjBSxDnM=
 github.com/go-critic/go-critic v0.11.2/go.mod h1:OePaicfjsf+KPy33yq4gzv6CO7TEQ9Rom6ns1KsJnl8=
 github.com/go-git/gcfg v1.5.1-0.20230307220236-3a3c6141e376 h1:+zs/tPmkDkHx3U66DAb0lQFJrpS6731Oaa12ikc+DiI=
 github.com/go-git/gcfg v1.5.1-0.20230307220236-3a3c6141e376/go.mod h1:an3vInlBmSxCcxctByoQdvwPiA7DTK7jaaFDBTtu0ic=
-github.com/go-git/go-billy/v5 v5.5.0 h1:yEY4yhzCDuMGSv83oGxiBotRzhwhNr8VZyphhiu+mTU=
-github.com/go-git/go-billy/v5 v5.5.0/go.mod h1:hmexnoNsr2SJU1Ju67OaNz5ASJY3+sHgFRpCtpDCKow=
+github.com/go-git/go-billy/v5 v5.6.1 h1:u+dcrgaguSSkbjzHwelEjc0Yj300NUevrrPphk/SoRA=
+github.com/go-git/go-billy/v5 v5.6.1/go.mod h1:0AsLr1z2+Uksi4NlElmMblP5rPcDZNRCD8ujZCRR2BE=
 github.com/go-git/go-git-fixtures/v4 v4.3.2-0.20231010084843-55a94097c399 h1:eMje31YglSBqCdIqdhKBW8lokaMrL3uTkpGYlE2OOT4=
 github.com/go-git/go-git-fixtures/v4 v4.3.2-0.20231010084843-55a94097c399/go.mod h1:1OCfN199q1Jm3HZlxleg+Dw/mwps2Wbk9frAWm+4FII=
-github.com/go-git/go-git/v5 v5.11.0 h1:XIZc1p+8YzypNr34itUfSvYJcv+eYdTnTvOZ2vD3cA4=
-github.com/go-git/go-git/v5 v5.11.0/go.mod h1:6GFcX2P3NM7FPBfpePbpLd21XxsgdAt+lKqXmCUiUCY=
+github.com/go-git/go-git/v5 v5.13.1 h1:DAQ9APonnlvSWpvolXWIuV6Q6zXy2wHbN4cVlNR5Q+M=
+github.com/go-git/go-git/v5 v5.13.1/go.mod h1:qryJB4cSBoq3FRoBRf5A77joojuBcmPJ0qu3XXXVixc=
 github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU=
 github.com/go-gl/glfw/v3.3/glfw v0.0.0-20191125211704-12ad95a8df72/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8=
 github.com/go-gl/glfw/v3.3/glfw v0.0.0-20200222043503-6f7a984d4dc4/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8=
-github.com/go-json-experiment/json v0.0.0-20231102232822-2e55bd4e08b0 h1:ymLjT4f35nQbASLnvxEde4XOBL+Sn7rFuV+FOJqkljg=
-github.com/go-json-experiment/json v0.0.0-20231102232822-2e55bd4e08b0/go.mod h1:6daplAwHHGbUGib4990V3Il26O0OC4aRyvewaaAihaA=
+github.com/go-json-experiment/json v0.0.0-20250103232110-6a9a0fde9288 h1:KbX3Z3CgiYlbaavUq3Cj9/MjpO+88S7/AGXzynVDv84=
+github.com/go-json-experiment/json v0.0.0-20250103232110-6a9a0fde9288/go.mod h1:BWmvoE1Xia34f3l/ibJweyhrT+aROb/FQ6d+37F0e2s=
 github.com/go-kit/kit v0.8.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as=
 github.com/go-kit/kit v0.9.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as=
 github.com/go-kit/log v0.1.0/go.mod h1:zbhenjAZHb184qTLMA9ZjW7ThYL0H2mk7Q6pNt4vbaY=
@@ -745,8 +743,8 @@ github.com/onsi/ginkgo v1.16.5 h1:8xi0RTUf59SOSfEtZMvwTvXYMzG4gV23XVHOZiXNtnE=
 github.com/onsi/ginkgo v1.16.5/go.mod h1:+E8gABHa3K6zRBolWtd+ROzc/U5bkGt0FwiG042wbpU=
 github.com/onsi/ginkgo/v2 v2.17.1 h1:V++EzdbhI4ZV4ev0UTIj0PzhzOcReJFyJaLjtSF55M8=
 github.com/onsi/ginkgo/v2 v2.17.1/go.mod h1:llBI3WDLL9Z6taip6f33H76YcWtJv+7R3HigUjbIBOs=
-github.com/onsi/gomega v1.33.1 h1:dsYjIxxSR755MDmKVsaFQTE22ChNBcuuTWgkUDSubOk=
-github.com/onsi/gomega v1.33.1/go.mod h1:U4R44UsT+9eLIaYRB2a5qajjtQYn0hauxvRm16AVYg0=
+github.com/onsi/gomega v1.34.1 h1:EUMJIKUjM8sKjYbtxQI9A4z2o+rruxnzNvpknOXie6k=
+github.com/onsi/gomega v1.34.1/go.mod h1:kU1QgUvBDLXBJq618Xvm2LUX6rSAfRaFRTcdOeDLwwY=
 github.com/opencontainers/go-digest v1.0.0 h1:apOUWs51W5PlhuyGyz9FCeeBIOUDA/6nW8Oi/yOhh5U=
 github.com/opencontainers/go-digest v1.0.0/go.mod h1:0JzlMkj0TRzQZfJkVvzbP0HBR3IKzErnv2BNG4W4MAM=
 github.com/opencontainers/image-spec v1.1.0 h1:8SG7/vwALn54lVB/0yZ/MMwhFrPYtpEHQb2IpWsCzug=
@@ -846,8 +844,8 @@ github.com/sashamelentyev/usestdlibvars v1.25.0/go.mod h1:9nl0jgOfHKWNFS43Ojw0i7
 github.com/securego/gosec/v2 v2.19.0 h1:gl5xMkOI0/E6Hxx0XCY2XujA3V7SNSefA8sC+3f1gnk=
 github.com/securego/gosec/v2 v2.19.0/go.mod h1:hOkDcHz9J/XIgIlPDXalxjeVYsHxoWUc5zJSHxcB8YM=
 github.com/sergi/go-diff v1.0.0/go.mod h1:0CfEIISq7TuYL3j771MWULgwwjU+GofnZX9QAmXWZgo=
-github.com/sergi/go-diff v1.3.1 h1:xkr+Oxo4BOQKmkn/B9eMK0g5Kg/983T9DqqPHwYqD+8=
-github.com/sergi/go-diff v1.3.1/go.mod h1:aMJSSKb2lpPvRNec0+w3fl7LP9IOFzdc9Pa4NFbPK1I=
+github.com/sergi/go-diff v1.3.2-0.20230802210424-5b0b94c5c0d3 h1:n661drycOFuPLCN3Uc8sB6B/s6Z4t2xvBgU1htSHuq8=
+github.com/sergi/go-diff v1.3.2-0.20230802210424-5b0b94c5c0d3/go.mod h1:A0bzQcvG0E7Rwjx0REVgAGH58e96+X0MeOfepqsbeW4=
 github.com/shazow/go-diff v0.0.0-20160112020656-b6b7b6733b8c h1:W65qqJCIOVP4jpqPQ0YvHYKwcMEMVWIzWC5iNQQfBTU=
 github.com/shazow/go-diff v0.0.0-20160112020656-b6b7b6733b8c/go.mod h1:/PevMnwAxekIXwN8qQyfc5gl2NlkB3CQlkizAbOkeBs=
 github.com/shopspring/decimal v1.2.0/go.mod h1:DKyhrW/HYNuLGql+MJL6WCR6knT2jwCFRcu2hWCYk4o=
@@ -865,8 +863,8 @@ github.com/sivchari/containedctx v1.0.3 h1:x+etemjbsh2fB5ewm5FeLNi5bUjK0V8n0RB+W
 github.com/sivchari/containedctx v1.0.3/go.mod h1:c1RDvCbnJLtH4lLcYD/GqwiBSSf4F5Qk0xld2rBqzJ4=
 github.com/sivchari/tenv v1.7.1 h1:PSpuD4bu6fSmtWMxSGWcvqUUgIn7k3yOJhOIzVWn8Ak=
 github.com/sivchari/tenv v1.7.1/go.mod h1:64yStXKSOxDfX47NlhVwND4dHwfZDdbp2Lyl018Icvg=
-github.com/skeema/knownhosts v1.2.1 h1:SHWdIUa82uGZz+F+47k8SY4QhhI291cXCpopT1lK2AQ=
-github.com/skeema/knownhosts v1.2.1/go.mod h1:xYbVRSPxqBZFrdmDyMmsOs+uX1UZC3nTN3ThzgDxUwo=
+github.com/skeema/knownhosts v1.3.0 h1:AM+y0rI04VksttfwjkSTNQorvGqmwATnvnAHpSgc0LY=
+github.com/skeema/knownhosts v1.3.0/go.mod h1:sPINvnADmT/qYH1kfv+ePMmOBTH6Tbl7b5LvTDjFK7M=
 github.com/skip2/go-qrcode v0.0.0-20200617195104-da1b6568686e h1:MRM5ITcdelLK2j1vwZ3Je0FKVCfqOLp5zO6trqMLYs0=
 github.com/skip2/go-qrcode v0.0.0-20200617195104-da1b6568686e/go.mod h1:XV66xRDqSt+GTGFMVlhk3ULuV0y9ZmzeVGR4mloJI3M=
 github.com/smartystreets/assertions v1.13.1 h1:Ef7KhSmjZcK6AVf9YbJdvPYG9avaF0ZxudX+ThRdWfU=
@@ -909,8 +907,9 @@ github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/
 github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU=
 github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4=
 github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo=
-github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg=
 github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY=
+github.com/stretchr/testify v1.10.0 h1:Xv5erBjTwe/5IxqUQTdXv5kgmIvbHo3QQyRwhJsOfJA=
+github.com/stretchr/testify v1.10.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY=
 github.com/studio-b12/gowebdav v0.9.0 h1:1j1sc9gQnNxbXXM4M/CebPOX4aXYtr7MojAVcN4dHjU=
 github.com/studio-b12/gowebdav v0.9.0/go.mod h1:bHA7t77X/QFExdeAnDzK6vKM34kEZAcE1OX4MfiwjkE=
 github.com/subosito/gotenv v1.4.2 h1:X1TuBLAMDFbaTAChgCBLu3DU3UPyELpnF2jjJ2cz/S8=
@@ -935,14 +934,14 @@ github.com/tailscale/mkctr v0.0.0-20241111153353-1a38f6676f10 h1:ZB47BgnHcEHQJOD
 github.com/tailscale/mkctr v0.0.0-20241111153353-1a38f6676f10/go.mod h1:iDx/0Rr9VV/KanSUDpJ6I/ROf0sQ7OqljXc/esl0UIA=
 github.com/tailscale/netlink v1.1.1-0.20240822203006-4d49adab4de7 h1:uFsXVBE9Qr4ZoF094vE6iYTLDl0qCiKzYXlL6UeWObU=
 github.com/tailscale/netlink v1.1.1-0.20240822203006-4d49adab4de7/go.mod h1:NzVQi3Mleb+qzq8VmcWpSkcSYxXIg0DkI6XDzpVkhJ0=
-github.com/tailscale/peercred v0.0.0-20240214030740-b535050b2aa4 h1:Gz0rz40FvFVLTBk/K8UNAenb36EbDSnh+q7Z9ldcC8w=
-github.com/tailscale/peercred v0.0.0-20240214030740-b535050b2aa4/go.mod h1:phI29ccmHQBc+wvroosENp1IF9195449VDnFDhJ4rJU=
+github.com/tailscale/peercred v0.0.0-20250107143737-35a0c7bd7edc h1:24heQPtnFR+yfntqhI3oAu9i27nEojcQ4NuBQOo5ZFA=
+github.com/tailscale/peercred v0.0.0-20250107143737-35a0c7bd7edc/go.mod h1:f93CXfllFsO9ZQVq+Zocb1Gp4G5Fz0b0rXHLOzt/Djc=
 github.com/tailscale/web-client-prebuilt v0.0.0-20240226180453-5db17b287bf1 h1:tdUdyPqJ0C97SJfjB9tW6EylTtreyee9C44de+UBG0g=
 github.com/tailscale/web-client-prebuilt v0.0.0-20240226180453-5db17b287bf1/go.mod h1:agQPE6y6ldqCOui2gkIh7ZMztTkIQKH049tv8siLuNQ=
 github.com/tailscale/wf v0.0.0-20240214030419-6fbb0a674ee6 h1:l10Gi6w9jxvinoiq15g8OToDdASBni4CyJOdHY1Hr8M=
 github.com/tailscale/wf v0.0.0-20240214030419-6fbb0a674ee6/go.mod h1:ZXRML051h7o4OcI0d3AaILDIad/Xw0IkXaHM17dic1Y=
-github.com/tailscale/wireguard-go v0.0.0-20241113014420-4e883d38c8d3 h1:dmoPb3dG27tZgMtrvqfD/LW4w7gA6BSWl8prCPNmkCQ=
-github.com/tailscale/wireguard-go v0.0.0-20241113014420-4e883d38c8d3/go.mod h1:BOm5fXUBFM+m9woLNBoxI9TaBXXhGNP50LX/TGIvGb4=
+github.com/tailscale/wireguard-go v0.0.0-20250107165329-0b8b35511f19 h1:BcEJP2ewTIK2ZCsqgl6YGpuO6+oKqqag5HHb7ehljKw=
+github.com/tailscale/wireguard-go v0.0.0-20250107165329-0b8b35511f19/go.mod h1:BOm5fXUBFM+m9woLNBoxI9TaBXXhGNP50LX/TGIvGb4=
 github.com/tailscale/xnet v0.0.0-20240729143630-8497ac4dab2e h1:zOGKqN5D5hHhiYUp091JqK7DPCqSARyUfduhGUY8Bek=
 github.com/tailscale/xnet v0.0.0-20240729143630-8497ac4dab2e/go.mod h1:orPd6JZXXRyuDusYilywte7k094d7dycXXU5YnWsrwg=
 github.com/tc-hib/winres v0.2.1 h1:YDE0FiP0VmtRaDn7+aaChp1KiF4owBiJa5l964l5ujA=
@@ -1060,10 +1059,8 @@ golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5y
 golang.org/x/crypto v0.0.0-20220622213112-05595931fe9d/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4=
 golang.org/x/crypto v0.1.0/go.mod h1:RecgLatLF4+eUMCP1PoPZQb+cVrJcOPbHkTkbkB9sbw=
 golang.org/x/crypto v0.3.0/go.mod h1:hebNnKkNXi2UzZN1eVRvBB7co0a+JxK6XbPiWVs/3J4=
-golang.org/x/crypto v0.3.1-0.20221117191849-2c476679df9a/go.mod h1:hebNnKkNXi2UzZN1eVRvBB7co0a+JxK6XbPiWVs/3J4=
-golang.org/x/crypto v0.7.0/go.mod h1:pYwdfH91IfpZVANVyUOhSIPZaFoJGxTFbZhFTx+dXZU=
-golang.org/x/crypto v0.30.0 h1:RwoQn3GkWiMkzlX562cLB7OxWvjH1L8xutO2WoJcRoY=
-golang.org/x/crypto v0.30.0/go.mod h1:kDsLvtWBEx7MV9tJOj9bnXsPbxwJQ6csT/x4KIN4Ssk=
+golang.org/x/crypto v0.31.0 h1:ihbySMvVjLAeSH1IbfcRTkD/iNscyz8rGzjF/E5hV6U=
+golang.org/x/crypto v0.31.0/go.mod h1:kDsLvtWBEx7MV9tJOj9bnXsPbxwJQ6csT/x4KIN4Ssk=
 golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
 golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
 golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8=
@@ -1074,8 +1071,8 @@ golang.org/x/exp v0.0.0-20191227195350-da58074b4299/go.mod h1:2RIsYlXP63K8oxa1u0
 golang.org/x/exp v0.0.0-20200119233911-0405dc783f0a/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4=
 golang.org/x/exp v0.0.0-20200207192155-f17229e696bd/go.mod h1:J/WKrq2StrnmMY6+EHIKF9dgMWnmCNThgcyBT1FY9mM=
 golang.org/x/exp v0.0.0-20200224162631-6cc2880d07d6/go.mod h1:3jZMyOhIsHpP37uCMkUooju7aAi5cS1Q23tOzKc+0MU=
-golang.org/x/exp v0.0.0-20240119083558-1b970713d09a h1:Q8/wZp0KX97QFTc2ywcOE0YRjZPVIx+MXInMzdvQqcA=
-golang.org/x/exp v0.0.0-20240119083558-1b970713d09a/go.mod h1:idGWGoKP1toJGkd5/ig9ZLuPcZBC3ewk7SzmH0uou08=
+golang.org/x/exp v0.0.0-20240719175910-8a7402abbf56 h1:2dVuKD2vS7b0QIHQbpyTISPd0LeHDbnYEryqj5Q1ug8=
+golang.org/x/exp v0.0.0-20240719175910-8a7402abbf56/go.mod h1:M4RDyNAINzryxdtnbRXRL/OHtkFuWGRjvuhBJpk2IlY=
 golang.org/x/exp/typeparams v0.0.0-20220428152302-39d4317da171/go.mod h1:AbB0pIl9nAr9wVwH+Z2ZpaocVmF5I4GyWCDIsVjR0bk=
 golang.org/x/exp/typeparams v0.0.0-20230203172020-98cc5a0785f9/go.mod h1:AbB0pIl9nAr9wVwH+Z2ZpaocVmF5I4GyWCDIsVjR0bk=
 golang.org/x/exp/typeparams v0.0.0-20240314144324-c7f7c6466f7f h1:phY1HzDcf18Aq9A8KkmRtY9WvOFIxN8wgfvy6Zm1DV8=
@@ -1152,9 +1149,8 @@ golang.org/x/net v0.1.0/go.mod h1:Cx3nUiGt4eDBEyega/BKRp+/AlGL8hYe7U9odMt2Cco=
 golang.org/x/net v0.2.0/go.mod h1:KqCZLdyyvdV855qA2rE3GC2aiw5xGR5TEjj8smXukLY=
 golang.org/x/net v0.5.0/go.mod h1:DivGGAXEgPSlEBzxGzZI+ZLohi+xUj054jfeKui00ws=
 golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs=
-golang.org/x/net v0.8.0/go.mod h1:QVkue5JL9kW//ek3r6jTKnTFis1tRmNAW2P1shuFdJc=
-golang.org/x/net v0.32.0 h1:ZqPmj8Kzc+Y6e0+skZsuACbx+wzMgo5MQsJh9Qd6aYI=
-golang.org/x/net v0.32.0/go.mod h1:CwU0IoeOlnQQWJ6ioyFrfRuomB8GKF6KbYXZVyeXNfs=
+golang.org/x/net v0.33.0 h1:74SYHlV8BIgHIFC/LrYkOGIwL19eTYXQ5wc6TBuO36I=
+golang.org/x/net v0.33.0/go.mod h1:HXLR5J+9DxmrqMwG9qjGCxZ+zKXxBru04zlTvWlWuN4=
 golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
 golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
 golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
@@ -1234,20 +1230,18 @@ golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBc
 golang.org/x/sys v0.0.0-20220817070843-5a390386f1f2/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.2.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
-golang.org/x/sys v0.3.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.4.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.4.1-0.20230131160137-e7d7f63158de/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
-golang.org/x/sys v0.28.0 h1:Fksou7UEQUWlKvIdsqzJmUmCX3cZuD2+P3XyyzwMhlA=
-golang.org/x/sys v0.28.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
+golang.org/x/sys v0.29.1-0.20250107080300-1c14dcadc3ab h1:BMkEEWYOjkvOX7+YKOGbp6jCyQ5pR2j0Ah47p1Vdsx4=
+golang.org/x/sys v0.29.1-0.20250107080300-1c14dcadc3ab/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
 golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
 golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
 golang.org/x/term v0.1.0/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
 golang.org/x/term v0.2.0/go.mod h1:TVmDHMZPmdnySmBfhjOoOdhjzdE1h4u1VwSiw2l1Nuc=
 golang.org/x/term v0.4.0/go.mod h1:9P2UbLfCdcvo3p/nzKvsmas4TnlujnuoV9hGgYzW1lQ=
 golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k=
-golang.org/x/term v0.6.0/go.mod h1:m6U89DPEgQRMq3DNkDClhWw02AUbt2daBVO4cn4Hv9U=
 golang.org/x/term v0.27.0 h1:WP60Sv1nlK1T6SupCHbXzSaN0b9wUmsPoRS9b61A23Q=
 golang.org/x/term v0.27.0/go.mod h1:iMsnZpn0cago0GOrHO2+Y7u7JPn5AylBrcoWkElMTSM=
 golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
@@ -1261,7 +1255,6 @@ golang.org/x/text v0.3.8/go.mod h1:E6s5w1FMmriuDzIBO73fBruAKo1PCIq6d2Q6DHfQ8WQ=
 golang.org/x/text v0.4.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8=
 golang.org/x/text v0.6.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8=
 golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8=
-golang.org/x/text v0.8.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8=
 golang.org/x/text v0.21.0 h1:zyQAAkrwaneQ066sspRyJaG9VNi/YJ1NfzcGB3hZ/qo=
 golang.org/x/text v0.21.0/go.mod h1:4IBbMaMmOPCJ8SecivzSH54+73PCFmPWxNTLm+vZkEQ=
 golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=

go.toolchain.rev

@@ -1 +1 @@
-96578f73d04e1a231fa2a495ad3fa97747785bc6
+161c3b79ed91039e65eb148f2547dea6b91e2247

hostinfo/hostinfo.go

@@ -233,7 +233,6 @@ func desktop() (ret opt.Bool) {
 	seenDesktop := false
 	for lr := range lineiter.File("/proc/net/unix") {
 		line, _ := lr.Value()
-		seenDesktop = seenDesktop || mem.Contains(mem.B(line), mem.S(" @/tmp/dbus-"))
 		seenDesktop = seenDesktop || mem.Contains(mem.B(line), mem.S(".X11-unix"))
 		seenDesktop = seenDesktop || mem.Contains(mem.B(line), mem.S("/wayland-1"))
 	}

ipn/backend.go

@@ -58,23 +58,29 @@ type EngineStatus struct {
 // to subscribe to.
 type NotifyWatchOpt uint64
 
+// NotifyWatchOpt values.
+//
+// These aren't declared using Go's iota because they're not purely internal to
+// the process and iota should not be used for values that are serialized to
+// disk or network. In this case, these values come over the network via the
+// LocalAPI, a mostly stable API.
 const (
 	// NotifyWatchEngineUpdates, if set, causes Engine updates to be sent to the
 	// client either regularly or when they change, without having to ask for
 	// each one via Engine.RequestStatus.
-	NotifyWatchEngineUpdates NotifyWatchOpt = 1 << iota
+	NotifyWatchEngineUpdates NotifyWatchOpt = 1 << 0
 
-	NotifyInitialState  // if set, the first Notify message (sent immediately) will contain the current State + BrowseToURL + SessionID
-	NotifyInitialPrefs  // if set, the first Notify message (sent immediately) will contain the current Prefs
-	NotifyInitialNetMap // if set, the first Notify message (sent immediately) will contain the current NetMap
+	NotifyInitialState  NotifyWatchOpt = 1 << 1 // if set, the first Notify message (sent immediately) will contain the current State + BrowseToURL + SessionID
+	NotifyInitialPrefs  NotifyWatchOpt = 1 << 2 // if set, the first Notify message (sent immediately) will contain the current Prefs
+	NotifyInitialNetMap NotifyWatchOpt = 1 << 3 // if set, the first Notify message (sent immediately) will contain the current NetMap
 
-	NotifyNoPrivateKeys        // if set, private keys that would normally be sent in updates are zeroed out
-	NotifyInitialDriveShares   // if set, the first Notify message (sent immediately) will contain the current Taildrive Shares
-	NotifyInitialOutgoingFiles // if set, the first Notify message (sent immediately) will contain the current Taildrop OutgoingFiles
+	NotifyNoPrivateKeys        NotifyWatchOpt = 1 << 4 // if set, private keys that would normally be sent in updates are zeroed out
+	NotifyInitialDriveShares   NotifyWatchOpt = 1 << 5 // if set, the first Notify message (sent immediately) will contain the current Taildrive Shares
+	NotifyInitialOutgoingFiles NotifyWatchOpt = 1 << 6 // if set, the first Notify message (sent immediately) will contain the current Taildrop OutgoingFiles
 
-	NotifyInitialHealthState // if set, the first Notify message (sent immediately) will contain the current health.State of the client
+	NotifyInitialHealthState NotifyWatchOpt = 1 << 7 // if set, the first Notify message (sent immediately) will contain the current health.State of the client
 
-	NotifyRateLimit // if set, rate limit spammy netmap updates to every few seconds
+	NotifyRateLimit NotifyWatchOpt = 1 << 8 // if set, rate limit spammy netmap updates to every few seconds
 )
 
 // Notify is a communication from a backend (e.g. tailscaled) to a frontend
@@ -147,7 +153,7 @@ type Notify struct {
 	// any changes to the user in the UI.
 	Health *health.State `json:",omitempty"`
 
-	// type is mirrored in xcode/Shared/IPN.swift
+	// type is mirrored in xcode/IPN/Core/LocalAPI/Model/LocalAPIModel.swift
 }
 
 func (n Notify) String() string {

ipn/ipnlocal/local.go

@@ -11,6 +11,7 @@ import (
 	"context"
 	"crypto/sha256"
 	"encoding/base64"
+	"encoding/hex"
 	"encoding/json"
 	"errors"
 	"fmt"
@@ -38,7 +39,6 @@ import (
 
 	"go4.org/mem"
 	"go4.org/netipx"
-	xmaps "golang.org/x/exp/maps"
 	"golang.org/x/net/dns/dnsmessage"
 	"gvisor.dev/gvisor/pkg/tcpip"
 	"tailscale.com/appc"
@@ -95,8 +95,10 @@ import (
 	"tailscale.com/types/preftype"
 	"tailscale.com/types/ptr"
 	"tailscale.com/types/views"
+	"tailscale.com/util/clientmetric"
 	"tailscale.com/util/deephash"
 	"tailscale.com/util/dnsname"
+	"tailscale.com/util/goroutines"
 	"tailscale.com/util/httpm"
 	"tailscale.com/util/mak"
 	"tailscale.com/util/multierr"
@@ -104,6 +106,7 @@ import (
 	"tailscale.com/util/osuser"
 	"tailscale.com/util/rands"
 	"tailscale.com/util/set"
+	"tailscale.com/util/slicesx"
 	"tailscale.com/util/syspolicy"
 	"tailscale.com/util/syspolicy/rsop"
 	"tailscale.com/util/systemd"
@@ -163,7 +166,7 @@ type watchSession struct {
 	ch        chan *ipn.Notify
 	owner     ipnauth.Actor // or nil
 	sessionID string
-	cancel    func() // call to signal that the session must be terminated
+	cancel    context.CancelFunc // to shut down the session
 }
 
 // LocalBackend is the glue between the major pieces of the Tailscale
@@ -178,7 +181,7 @@ type watchSession struct {
 // state machine generates events back out to zero or more components.
 type LocalBackend struct {
 	// Elements that are thread-safe or constant after construction.
-	ctx                      context.Context    // canceled by Close
+	ctx                      context.Context    // canceled by [LocalBackend.Shutdown]
 	ctxCancel                context.CancelFunc // cancels ctx
 	logf                     logger.Logf        // general logging
 	keyLogf                  logger.Logf        // for printing list of peers on change
@@ -231,6 +234,10 @@ type LocalBackend struct {
 	shouldInterceptTCPPortAtomic syncs.AtomicValue[func(uint16) bool]
 	numClientStatusCalls         atomic.Uint32
 
+	// goTracker accounts for all goroutines started by LocalBacked, primarily
+	// for testing and graceful shutdown purposes.
+	goTracker goroutines.Tracker
+
 	// The mutex protects the following elements.
 	mu             sync.Mutex
 	conf           *conffile.Config // latest parsed config, or nil if not in declarative mode
@@ -362,7 +369,7 @@ type LocalBackend struct {
 	allowedSuggestedExitNodes   set.Set[tailcfg.StableNodeID]
 
 	// refreshAutoExitNode indicates if the exit node should be recomputed when the next netcheck report is available.
-	refreshAutoExitNode bool
+	refreshAutoExitNode bool // guarded by mu
 
 	// captiveCtx and captiveCancel are used to control captive portal
 	// detection. They are protected by 'mu' and can be changed during the
@@ -866,7 +873,7 @@ func (b *LocalBackend) linkChange(delta *netmon.ChangeDelta) {
 			// TODO(raggi,tailscale/corp#22574): authReconfig should be refactored such that we can call the
 			// necessary operations here and avoid the need for asynchronous behavior that is racy and hard
 			// to test here, and do less extra work in these conditions.
-			go b.authReconfig()
+			b.goTracker.Go(b.authReconfig)
 		}
 	}
 
@@ -879,7 +886,7 @@ func (b *LocalBackend) linkChange(delta *netmon.ChangeDelta) {
 		want := b.netMap.GetAddresses().Len()
 		if len(b.peerAPIListeners) < want {
 			b.logf("linkChange: peerAPIListeners too low; trying again")
-			go b.initPeerAPIListener()
+			b.goTracker.Go(b.initPeerAPIListener)
 		}
 	}
 }
@@ -1004,6 +1011,33 @@ func (b *LocalBackend) Shutdown() {
 	b.ctxCancel()
 	b.e.Close()
 	<-b.e.Done()
+	b.awaitNoGoroutinesInTest()
+}
+
+func (b *LocalBackend) awaitNoGoroutinesInTest() {
+	if !testenv.InTest() {
+		return
+	}
+	ctx, cancel := context.WithTimeout(context.Background(), 8*time.Second)
+	defer cancel()
+
+	ch := make(chan bool, 1)
+	defer b.goTracker.AddDoneCallback(func() { ch <- true })()
+
+	for {
+		n := b.goTracker.RunningGoroutines()
+		if n == 0 {
+			return
+		}
+		select {
+		case <-ctx.Done():
+			// TODO(bradfitz): pass down some TB-like failer interface from
+			// tests, without depending on testing from here?
+			// But this is fine in tests too:
+			panic(fmt.Sprintf("timeout waiting for %d goroutines to stop", n))
+		case <-ch:
+		}
+	}
 }
 
 func stripKeysFromPrefs(p ipn.PrefsView) ipn.PrefsView {
@@ -1126,11 +1160,10 @@ func (b *LocalBackend) UpdateStatus(sb *ipnstate.StatusBuilder) {
 					ss.Capabilities = make([]tailcfg.NodeCapability, 1, cm.Len()+1)
 					ss.Capabilities[0] = "HTTPS://TAILSCALE.COM/s/DEPRECATED-NODE-CAPS#see-https://github.com/tailscale/tailscale/issues/11508"
 					ss.CapMap = make(tailcfg.NodeCapMap, sn.CapMap().Len())
-					cm.Range(func(k tailcfg.NodeCapability, v views.Slice[tailcfg.RawMessage]) bool {
+					for k, v := range cm.All() {
 						ss.CapMap[k] = v.AsSlice()
 						ss.Capabilities = append(ss.Capabilities, k)
-						return true
-					})
+					}
 					slices.Sort(ss.Capabilities[1:])
 				}
 			}
@@ -1192,10 +1225,9 @@ func (b *LocalBackend) populatePeerStatusLocked(sb *ipnstate.StatusBuilder) {
 		}
 		if cm := p.CapMap(); cm.Len() > 0 {
 			ps.CapMap = make(tailcfg.NodeCapMap, cm.Len())
-			cm.Range(func(k tailcfg.NodeCapability, v views.Slice[tailcfg.RawMessage]) bool {
+			for k, v := range cm.All() {
 				ps.CapMap[k] = v.AsSlice()
-				return true
-			})
+			}
 		}
 		peerStatusFromNode(ps, p)
 
@@ -1782,8 +1814,9 @@ func (b *LocalBackend) UpdateNetmapDelta(muts []netmap.NodeMutation) (handled bo
 			b.send(*notify)
 		}
 	}()
-	unlock := b.lockAndGetUnlock()
-	defer unlock()
+	b.mu.Lock()
+	defer b.mu.Unlock()
+
 	if !b.updateNetmapDeltaLocked(muts) {
 		return false
 	}
@@ -1791,14 +1824,8 @@ func (b *LocalBackend) UpdateNetmapDelta(muts []netmap.NodeMutation) (handled bo
 	if b.netMap != nil && mutationsAreWorthyOfTellingIPNBus(muts) {
 		nm := ptr.To(*b.netMap) // shallow clone
 		nm.Peers = make([]tailcfg.NodeView, 0, len(b.peers))
-		shouldAutoExitNode := shouldAutoExitNode()
 		for _, p := range b.peers {
 			nm.Peers = append(nm.Peers, p)
-			// If the auto exit node currently set goes offline, find another auto exit node.
-			if shouldAutoExitNode && b.pm.prefs.ExitNodeID() == p.StableID() && p.Online() != nil && !*p.Online() {
-				b.setAutoExitNodeIDLockedOnEntry(unlock)
-				return false
-			}
 		}
 		slices.SortFunc(nm.Peers, func(a, b tailcfg.NodeView) int {
 			return cmp.Compare(a.ID(), b.ID())
@@ -1829,6 +1856,20 @@ func mutationsAreWorthyOfTellingIPNBus(muts []netmap.NodeMutation) bool {
 	return false
 }
 
+// pickNewAutoExitNode picks a new automatic exit node if needed.
+func (b *LocalBackend) pickNewAutoExitNode() {
+	unlock := b.lockAndGetUnlock()
+	defer unlock()
+
+	newPrefs := b.setAutoExitNodeIDLockedOnEntry(unlock)
+	if !newPrefs.Valid() {
+		// Unchanged.
+		return
+	}
+
+	b.send(ipn.Notify{Prefs: &newPrefs})
+}
+
 func (b *LocalBackend) updateNetmapDeltaLocked(muts []netmap.NodeMutation) (handled bool) {
 	if b.netMap == nil || len(b.peers) == 0 {
 		return false
@@ -1851,6 +1892,12 @@ func (b *LocalBackend) updateNetmapDeltaLocked(muts []netmap.NodeMutation) (hand
 			mak.Set(&mutableNodes, nv.ID(), n)
 		}
 		m.Apply(n)
+
+		// If our exit node went offline, we need to schedule picking
+		// a new one.
+		if mo, ok := m.(netmap.NodeMutationOnline); ok && !mo.Online && n.StableID == b.pm.prefs.ExitNodeID() && shouldAutoExitNode() {
+			b.goTracker.Go(b.pickNewAutoExitNode)
+		}
 	}
 	for nid, n := range mutableNodes {
 		b.peers[nid] = n.View()
@@ -2022,7 +2069,7 @@ func (b *LocalBackend) DisablePortMapperForTest() {
 func (b *LocalBackend) PeersForTest() []tailcfg.NodeView {
 	b.mu.Lock()
 	defer b.mu.Unlock()
-	ret := xmaps.Values(b.peers)
+	ret := slicesx.MapValues(b.peers)
 	slices.SortFunc(ret, func(a, b tailcfg.NodeView) int {
 		return cmp.Compare(a.ID(), b.ID())
 	})
@@ -2154,7 +2201,7 @@ func (b *LocalBackend) Start(opts ipn.Options) error {
 
 	if b.portpoll != nil {
 		b.portpollOnce.Do(func() {
-			go b.readPoller()
+			b.goTracker.Go(b.readPoller)
 		})
 	}
 
@@ -2368,7 +2415,7 @@ func (b *LocalBackend) updateFilterLocked(netMap *netmap.NetworkMap, prefs ipn.P
 	b.e.SetJailedFilter(filter.NewShieldsUpFilter(localNets, logNets, oldJailedFilter, b.logf))
 
 	if b.sshServer != nil {
-		go b.sshServer.OnPolicyChange()
+		b.goTracker.Go(b.sshServer.OnPolicyChange)
 	}
 }
 
@@ -2817,6 +2864,9 @@ func (b *LocalBackend) WatchNotificationsAs(ctx context.Context, actor ipnauth.A
 	mak.Set(&b.notifyWatchers, sessionID, session)
 	b.mu.Unlock()
 
+	metricCurrentWatchIPNBus.Add(1)
+	defer metricCurrentWatchIPNBus.Add(-1)
+
 	defer func() {
 		b.mu.Lock()
 		delete(b.notifyWatchers, sessionID)
@@ -2845,7 +2895,7 @@ func (b *LocalBackend) WatchNotificationsAs(ctx context.Context, actor ipnauth.A
 	// request every 2 seconds.
 	// TODO(bradfitz): plumb this further and only send a Notify on change.
 	if mask&ipn.NotifyWatchEngineUpdates != 0 {
-		go b.pollRequestEngineStatus(ctx)
+		b.goTracker.Go(func() { b.pollRequestEngineStatus(ctx) })
 	}
 
 	// TODO(marwan-at-work): streaming background logs?
@@ -3852,7 +3902,7 @@ func (b *LocalBackend) editPrefsLockedOnEntry(mp *ipn.MaskedPrefs, unlock unlock
 	if mp.EggSet {
 		mp.EggSet = false
 		b.egg = true
-		go b.doSetHostinfoFilterServices()
+		b.goTracker.Go(b.doSetHostinfoFilterServices)
 	}
 	p0 := b.pm.CurrentPrefs()
 	p1 := b.pm.CurrentPrefs().AsStruct()
@@ -3945,7 +3995,7 @@ func (b *LocalBackend) setPrefsLockedOnEntry(newp *ipn.Prefs, unlock unlockOnce)
 
 	if oldp.ShouldSSHBeRunning() && !newp.ShouldSSHBeRunning() {
 		if b.sshServer != nil {
-			go b.sshServer.Shutdown()
+			b.goTracker.Go(b.sshServer.Shutdown)
 			b.sshServer = nil
 		}
 	}
@@ -4287,8 +4337,14 @@ func (b *LocalBackend) authReconfig() {
 	dcfg := dnsConfigForNetmap(nm, b.peers, prefs, b.keyExpired, b.logf, version.OS())
 	// If the current node is an app connector, ensure the app connector machine is started
 	b.reconfigAppConnectorLocked(nm, prefs)
+	closing := b.shutdownCalled
 	b.mu.Unlock()
 
+	if closing {
+		b.logf("[v1] authReconfig: skipping because in shutdown")
+		return
+	}
+
 	if blocked {
 		b.logf("[v1] authReconfig: blocked, skipping.")
 		return
@@ -4753,7 +4809,7 @@ func (b *LocalBackend) initPeerAPIListener() {
 		b.peerAPIListeners = append(b.peerAPIListeners, pln)
 	}
 
-	go b.doSetHostinfoFilterServices()
+	b.goTracker.Go(b.doSetHostinfoFilterServices)
 }
 
 // magicDNSRootDomains returns the subset of nm.DNS.Domains that are the search domains for MagicDNS.
@@ -4966,13 +5022,7 @@ func (b *LocalBackend) applyPrefsToHostinfoLocked(hi *tailcfg.Hostinfo, prefs ip
 	}
 	hi.SSH_HostKeys = sshHostKeys
 
-	services := vipServicesFromPrefs(prefs)
-	if len(services) > 0 {
-		buf, _ := json.Marshal(services)
-		hi.ServicesHash = fmt.Sprintf("%02x", sha256.Sum256(buf))
-	} else {
-		hi.ServicesHash = ""
-	}
+	hi.ServicesHash = b.vipServiceHashLocked(prefs)
 
 	// The Hostinfo.WantIngress field tells control whether this node wants to
 	// be wired up for ingress connections. If harmless if it's accidentally
@@ -5022,7 +5072,7 @@ func (b *LocalBackend) enterStateLockedOnEntry(newState ipn.State, unlock unlock
 		// can be shut down if we transition away from Running.
 		if b.captiveCancel == nil {
 			b.captiveCtx, b.captiveCancel = context.WithCancel(b.ctx)
-			go b.checkCaptivePortalLoop(b.captiveCtx)
+			b.goTracker.Go(func() { b.checkCaptivePortalLoop(b.captiveCtx) })
 		}
 	} else if oldState == ipn.Running {
 		// Transitioning away from running.
@@ -5274,7 +5324,7 @@ func (b *LocalBackend) requestEngineStatusAndWait() {
 	b.statusLock.Lock()
 	defer b.statusLock.Unlock()
 
-	go b.e.RequestStatus()
+	b.goTracker.Go(b.e.RequestStatus)
 	b.logf("requestEngineStatusAndWait: waiting...")
 	b.statusChanged.Wait() // temporarily releases lock while waiting
 	b.logf("requestEngineStatusAndWait: got status update.")
@@ -5385,7 +5435,7 @@ func (b *LocalBackend) setWebClientAtomicBoolLocked(nm *netmap.NetworkMap) {
 	shouldRun := !nm.HasCap(tailcfg.NodeAttrDisableWebClient)
 	wasRunning := b.webClientAtomicBool.Swap(shouldRun)
 	if wasRunning && !shouldRun {
-		go b.webClientShutdown() // stop web client
+		b.goTracker.Go(b.webClientShutdown) // stop web client
 	}
 }
 
@@ -5506,29 +5556,34 @@ func (b *LocalBackend) setNetInfo(ni *tailcfg.NetInfo) {
 	}
 }
 
-func (b *LocalBackend) setAutoExitNodeIDLockedOnEntry(unlock unlockOnce) {
+func (b *LocalBackend) setAutoExitNodeIDLockedOnEntry(unlock unlockOnce) (newPrefs ipn.PrefsView) {
+	var zero ipn.PrefsView
 	defer unlock()
 
 	prefs := b.pm.CurrentPrefs()
 	if !prefs.Valid() {
 		b.logf("[unexpected]: received tailnet exit node ID pref change callback but current prefs are nil")
-		return
+		return zero
 	}
 	prefsClone := prefs.AsStruct()
 	newSuggestion, err := b.suggestExitNodeLocked(nil)
 	if err != nil {
 		b.logf("setAutoExitNodeID: %v", err)
-		return
+		return zero
+	}
+	if prefsClone.ExitNodeID == newSuggestion.ID {
+		return zero
 	}
 	prefsClone.ExitNodeID = newSuggestion.ID
-	_, err = b.editPrefsLockedOnEntry(&ipn.MaskedPrefs{
+	newPrefs, err = b.editPrefsLockedOnEntry(&ipn.MaskedPrefs{
 		Prefs:         *prefsClone,
 		ExitNodeIDSet: true,
 	}, unlock)
 	if err != nil {
 		b.logf("setAutoExitNodeID: failed to apply exit node ID preference: %v", err)
-		return
+		return zero
 	}
+	return newPrefs
 }
 
 // setNetMapLocked updates the LocalBackend state to reflect the newly
@@ -5884,12 +5939,11 @@ func (b *LocalBackend) setTCPPortsInterceptedFromNetmapAndPrefsLocked(prefs ipn.
 	b.reloadServeConfigLocked(prefs)
 	if b.serveConfig.Valid() {
 		servePorts := make([]uint16, 0, 3)
-		b.serveConfig.RangeOverTCPs(func(port uint16, _ ipn.TCPPortHandlerView) bool {
+		for port := range b.serveConfig.TCPs() {
 			if port > 0 {
 				servePorts = append(servePorts, uint16(port))
 			}
-			return true
-		})
+		}
 		handlePorts = append(handlePorts, servePorts...)
 
 		b.setServeProxyHandlersLocked()
@@ -5903,7 +5957,7 @@ func (b *LocalBackend) setTCPPortsInterceptedFromNetmapAndPrefsLocked(prefs ipn.
 	if wire := b.wantIngressLocked(); b.hostinfo != nil && b.hostinfo.WireIngress != wire {
 		b.logf("Hostinfo.WireIngress changed to %v", wire)
 		b.hostinfo.WireIngress = wire
-		go b.doSetHostinfoFilterServices()
+		b.goTracker.Go(b.doSetHostinfoFilterServices)
 	}
 
 	b.setTCPPortsIntercepted(handlePorts)
@@ -5917,16 +5971,16 @@ func (b *LocalBackend) setServeProxyHandlersLocked() {
 		return
 	}
 	var backends map[string]bool
-	b.serveConfig.RangeOverWebs(func(_ ipn.HostPort, conf ipn.WebServerConfigView) (cont bool) {
-		conf.Handlers().Range(func(_ string, h ipn.HTTPHandlerView) (cont bool) {
+	for _, conf := range b.serveConfig.Webs() {
+		for _, h := range conf.Handlers().All() {
 			backend := h.Proxy()
 			if backend == "" {
 				// Only create proxy handlers for servers with a proxy backend.
-				return true
+				continue
 			}
 			mak.Set(&backends, backend, true)
 			if _, ok := b.serveProxyHandlers.Load(backend); ok {
-				return true
+				continue
 			}
 
 			b.logf("serve: creating a new proxy handler for %s", backend)
@@ -5935,13 +5989,11 @@ func (b *LocalBackend) setServeProxyHandlersLocked() {
 				// The backend endpoint (h.Proxy) should have been validated by expandProxyTarget
 				// in the CLI, so just log the error here.
 				b.logf("[unexpected] could not create proxy for %v: %s", backend, err)
-				return true
+				continue
 			}
 			b.serveProxyHandlers.Store(backend, p)
-			return true
-		})
-		return true
-	})
+		}
+	}
 
 	// Clean up handlers for proxy backends that are no longer present
 	// in configuration.
@@ -6408,6 +6460,20 @@ func (b *LocalBackend) SetExpirySooner(ctx context.Context, expiry time.Time) er
 	return cc.SetExpirySooner(ctx, expiry)
 }
 
+// SetDeviceAttrs does a synchronous call to the control plane to update
+// the node's attributes.
+//
+// See docs on [tailcfg.SetDeviceAttributesRequest] for background.
+func (b *LocalBackend) SetDeviceAttrs(ctx context.Context, attrs tailcfg.AttrUpdate) error {
+	b.mu.Lock()
+	cc := b.ccAuto
+	b.mu.Unlock()
+	if cc == nil {
+		return errors.New("not running")
+	}
+	return cc.SetDeviceAttrs(ctx, attrs)
+}
+
 // exitNodeCanProxyDNS reports the DoH base URL ("http://foo/dns-query") without query parameters
 // to exitNodeID's DoH service, if available.
 //
@@ -7361,9 +7427,9 @@ func suggestExitNode(report *netcheck.Report, netMap *netmap.NetworkMap, prevSug
 	// First, try to select an exit node that has the closest DERP home, based on lastReport's DERP latency.
 	// If there are no latency values, it returns an arbitrary region
 	if len(candidatesByRegion) > 0 {
-		minRegion := minLatencyDERPRegion(xmaps.Keys(candidatesByRegion), report)
+		minRegion := minLatencyDERPRegion(slicesx.MapKeys(candidatesByRegion), report)
 		if minRegion == 0 {
-			minRegion = selectRegion(views.SliceOf(xmaps.Keys(candidatesByRegion)))
+			minRegion = selectRegion(views.SliceOf(slicesx.MapKeys(candidatesByRegion)))
 		}
 		regionCandidates, ok := candidatesByRegion[minRegion]
 		if !ok {
@@ -7592,28 +7658,38 @@ func maybeUsernameOf(actor ipnauth.Actor) string {
 func (b *LocalBackend) VIPServices() []*tailcfg.VIPService {
 	b.mu.Lock()
 	defer b.mu.Unlock()
-	return vipServicesFromPrefs(b.pm.CurrentPrefs())
+	return b.vipServicesFromPrefsLocked(b.pm.CurrentPrefs())
 }
 
-func vipServicesFromPrefs(prefs ipn.PrefsView) []*tailcfg.VIPService {
+func (b *LocalBackend) vipServiceHashLocked(prefs ipn.PrefsView) string {
+	services := b.vipServicesFromPrefsLocked(prefs)
+	if len(services) == 0 {
+		return ""
+	}
+	buf, err := json.Marshal(services)
+	if err != nil {
+		b.logf("vipServiceHashLocked: %v", err)
+		return ""
+	}
+	hash := sha256.Sum256(buf)
+	return hex.EncodeToString(hash[:])
+}
+
+func (b *LocalBackend) vipServicesFromPrefsLocked(prefs ipn.PrefsView) []*tailcfg.VIPService {
 	// keyed by service name
 	var services map[string]*tailcfg.VIPService
-
-	// TODO(naman): this envknob will be replaced with service-specific port
-	// information once we start storing that.
-	var allPortsServices []string
-	if env := envknob.String("TS_DEBUG_ALLPORTS_SERVICES"); env != "" {
-		allPortsServices = strings.Split(env, ",")
+	if !b.serveConfig.Valid() {
+		return nil
 	}
 
-	for _, s := range allPortsServices {
-		mak.Set(&services, s, &tailcfg.VIPService{
-			Name:  s,
-			Ports: []tailcfg.ProtoPortRange{{Ports: tailcfg.PortRangeAny}},
+	for svc, config := range b.serveConfig.Services().All() {
+		mak.Set(&services, svc, &tailcfg.VIPService{
+			Name:  svc,
+			Ports: config.ServicePortRange(),
 		})
 	}
 
-	for _, s := range prefs.AdvertiseServices().AsSlice() {
+	for _, s := range prefs.AdvertiseServices().All() {
 		if services == nil || services[s] == nil {
 			mak.Set(&services, s, &tailcfg.VIPService{
 				Name: s,
@@ -7622,5 +7698,9 @@ func vipServicesFromPrefs(prefs ipn.PrefsView) []*tailcfg.VIPService {
 		services[s].Active = true
 	}
 
-	return slices.Collect(maps.Values(services))
+	return slicesx.MapValues(services)
 }
+
+var (
+	metricCurrentWatchIPNBus = clientmetric.NewGauge("localbackend_current_watch_ipn_bus")
+)

ipn/ipnlocal/local_test.go

@@ -30,7 +30,6 @@ import (
 	"tailscale.com/control/controlclient"
 	"tailscale.com/drive"
 	"tailscale.com/drive/driveimpl"
-	"tailscale.com/envknob"
 	"tailscale.com/health"
 	"tailscale.com/hostinfo"
 	"tailscale.com/ipn"
@@ -1867,16 +1866,16 @@ func TestUpdateNetmapDeltaAutoExitNode(t *testing.T) {
 		PreferredDERP: 2,
 	}
 	tests := []struct {
-		name                      string
-		lastSuggestedExitNode     tailcfg.StableNodeID
-		netmap                    *netmap.NetworkMap
-		muts                      []*tailcfg.PeerChange
-		exitNodeIDWant            tailcfg.StableNodeID
-		updateNetmapDeltaResponse bool
-		report                    *netcheck.Report
+		name                  string
+		lastSuggestedExitNode tailcfg.StableNodeID
+		netmap                *netmap.NetworkMap
+		muts                  []*tailcfg.PeerChange
+		exitNodeIDWant        tailcfg.StableNodeID
+		report                *netcheck.Report
 	}{
 		{
-			name:                  "selected auto exit node goes offline",
+			// selected auto exit node goes offline
+			name:                  "exit-node-goes-offline",
 			lastSuggestedExitNode: peer1.StableID(),
 			netmap: &netmap.NetworkMap{
 				Peers: []tailcfg.NodeView{
@@ -1895,12 +1894,12 @@ func TestUpdateNetmapDeltaAutoExitNode(t *testing.T) {
 					Online: ptr.To(true),
 				},
 			},
-			exitNodeIDWant:            peer2.StableID(),
-			updateNetmapDeltaResponse: false,
-			report:                    report,
+			exitNodeIDWant: peer2.StableID(),
+			report:         report,
 		},
 		{
-			name:                  "other exit node goes offline doesn't change selected auto exit node that's still online",
+			// other exit node goes offline doesn't change selected auto exit node that's still online
+			name:                  "other-node-goes-offline",
 			lastSuggestedExitNode: peer2.StableID(),
 			netmap: &netmap.NetworkMap{
 				Peers: []tailcfg.NodeView{
@@ -1919,9 +1918,8 @@ func TestUpdateNetmapDeltaAutoExitNode(t *testing.T) {
 					Online: ptr.To(true),
 				},
 			},
-			exitNodeIDWant:            peer2.StableID(),
-			updateNetmapDeltaResponse: true,
-			report:                    report,
+			exitNodeIDWant: peer2.StableID(),
+			report:         report,
 		},
 	}
 
@@ -1939,6 +1937,20 @@ func TestUpdateNetmapDeltaAutoExitNode(t *testing.T) {
 			b.lastSuggestedExitNode = tt.lastSuggestedExitNode
 			b.sys.MagicSock.Get().SetLastNetcheckReportForTest(b.ctx, tt.report)
 			b.SetPrefsForTest(b.pm.CurrentPrefs().AsStruct())
+
+			allDone := make(chan bool, 1)
+			defer b.goTracker.AddDoneCallback(func() {
+				b.mu.Lock()
+				defer b.mu.Unlock()
+				if b.goTracker.RunningGoroutines() > 0 {
+					return
+				}
+				select {
+				case allDone <- true:
+				default:
+				}
+			})()
+
 			someTime := time.Unix(123, 0)
 			muts, ok := netmap.MutationsFromMapResponse(&tailcfg.MapResponse{
 				PeersChangedPatch: tt.muts,
@@ -1946,16 +1958,34 @@ func TestUpdateNetmapDeltaAutoExitNode(t *testing.T) {
 			if !ok {
 				t.Fatal("netmap.MutationsFromMapResponse failed")
 			}
+
 			if b.pm.prefs.ExitNodeID() != tt.lastSuggestedExitNode {
 				t.Fatalf("did not set exit node ID to last suggested exit node despite auto policy")
 			}
 
+			was := b.goTracker.StartedGoroutines()
 			got := b.UpdateNetmapDelta(muts)
-			if got != tt.updateNetmapDeltaResponse {
-				t.Fatalf("got %v expected %v from UpdateNetmapDelta", got, tt.updateNetmapDeltaResponse)
+			if !got {
+				t.Error("got false from UpdateNetmapDelta")
 			}
-			if b.pm.prefs.ExitNodeID() != tt.exitNodeIDWant {
-				t.Fatalf("did not get expected exit node id after UpdateNetmapDelta")
+			startedGoroutine := b.goTracker.StartedGoroutines() != was
+
+			wantChange := tt.exitNodeIDWant != tt.lastSuggestedExitNode
+			if startedGoroutine != wantChange {
+				t.Errorf("got startedGoroutine %v, want %v", startedGoroutine, wantChange)
+			}
+			if startedGoroutine {
+				select {
+				case <-time.After(5 * time.Second):
+					t.Fatal("timed out waiting for goroutine to finish")
+				case <-allDone:
+				}
+			}
+			b.mu.Lock()
+			gotExitNode := b.pm.prefs.ExitNodeID()
+			b.mu.Unlock()
+			if gotExitNode != tt.exitNodeIDWant {
+				t.Fatalf("exit node ID after UpdateNetmapDelta = %v; want %v", gotExitNode, tt.exitNodeIDWant)
 			}
 		})
 	}
@@ -4478,15 +4508,15 @@ func TestConfigFileReload(t *testing.T) {
 
 func TestGetVIPServices(t *testing.T) {
 	tests := []struct {
-		name       string
-		advertised []string
-		mapped     []string
-		want       []*tailcfg.VIPService
+		name        string
+		advertised  []string
+		serveConfig *ipn.ServeConfig
+		want        []*tailcfg.VIPService
 	}{
 		{
 			"advertised-only",
 			[]string{"svc:abc", "svc:def"},
-			[]string{},
+			&ipn.ServeConfig{},
 			[]*tailcfg.VIPService{
 				{
 					Name:   "svc:abc",
@@ -4499,9 +4529,13 @@ func TestGetVIPServices(t *testing.T) {
 			},
 		},
 		{
-			"mapped-only",
+			"served-only",
 			[]string{},
-			[]string{"svc:abc"},
+			&ipn.ServeConfig{
+				Services: map[string]*ipn.ServiceConfig{
+					"svc:abc": {Tun: true},
+				},
+			},
 			[]*tailcfg.VIPService{
 				{
 					Name:  "svc:abc",
@@ -4510,9 +4544,13 @@ func TestGetVIPServices(t *testing.T) {
 			},
 		},
 		{
-			"mapped-and-advertised",
-			[]string{"svc:abc"},
+			"served-and-advertised",
 			[]string{"svc:abc"},
+			&ipn.ServeConfig{
+				Services: map[string]*ipn.ServiceConfig{
+					"svc:abc": {Tun: true},
+				},
+			},
 			[]*tailcfg.VIPService{
 				{
 					Name:   "svc:abc",
@@ -4522,9 +4560,13 @@ func TestGetVIPServices(t *testing.T) {
 			},
 		},
 		{
-			"mapped-and-advertised-separately",
+			"served-and-advertised-different-service",
 			[]string{"svc:def"},
-			[]string{"svc:abc"},
+			&ipn.ServeConfig{
+				Services: map[string]*ipn.ServiceConfig{
+					"svc:abc": {Tun: true},
+				},
+			},
 			[]*tailcfg.VIPService{
 				{
 					Name:  "svc:abc",
@@ -4536,14 +4578,78 @@ func TestGetVIPServices(t *testing.T) {
 				},
 			},
 		},
+		{
+			"served-with-port-ranges-one-range-single",
+			[]string{},
+			&ipn.ServeConfig{
+				Services: map[string]*ipn.ServiceConfig{
+					"svc:abc": {TCP: map[uint16]*ipn.TCPPortHandler{
+						80: {HTTPS: true},
+					}},
+				},
+			},
+			[]*tailcfg.VIPService{
+				{
+					Name:  "svc:abc",
+					Ports: []tailcfg.ProtoPortRange{{Proto: 6, Ports: tailcfg.PortRange{First: 80, Last: 80}}},
+				},
+			},
+		},
+		{
+			"served-with-port-ranges-one-range-multiple",
+			[]string{},
+			&ipn.ServeConfig{
+				Services: map[string]*ipn.ServiceConfig{
+					"svc:abc": {TCP: map[uint16]*ipn.TCPPortHandler{
+						80: {HTTPS: true},
+						81: {HTTPS: true},
+						82: {HTTPS: true},
+					}},
+				},
+			},
+			[]*tailcfg.VIPService{
+				{
+					Name:  "svc:abc",
+					Ports: []tailcfg.ProtoPortRange{{Proto: 6, Ports: tailcfg.PortRange{First: 80, Last: 82}}},
+				},
+			},
+		},
+		{
+			"served-with-port-ranges-multiple-ranges",
+			[]string{},
+			&ipn.ServeConfig{
+				Services: map[string]*ipn.ServiceConfig{
+					"svc:abc": {TCP: map[uint16]*ipn.TCPPortHandler{
+						80:   {HTTPS: true},
+						81:   {HTTPS: true},
+						82:   {HTTPS: true},
+						1212: {HTTPS: true},
+						1213: {HTTPS: true},
+						1214: {HTTPS: true},
+					}},
+				},
+			},
+			[]*tailcfg.VIPService{
+				{
+					Name: "svc:abc",
+					Ports: []tailcfg.ProtoPortRange{
+						{Proto: 6, Ports: tailcfg.PortRange{First: 80, Last: 82}},
+						{Proto: 6, Ports: tailcfg.PortRange{First: 1212, Last: 1214}},
+					},
+				},
+			},
+		},
 	}
 	for _, tt := range tests {
 		t.Run(tt.name, func(t *testing.T) {
-			envknob.Setenv("TS_DEBUG_ALLPORTS_SERVICES", strings.Join(tt.mapped, ","))
+			lb := newLocalBackendWithTestControl(t, false, func(tb testing.TB, opts controlclient.Options) controlclient.Client {
+				return newClient(tb, opts)
+			})
+			lb.serveConfig = tt.serveConfig.View()
 			prefs := &ipn.Prefs{
 				AdvertiseServices: tt.advertised,
 			}
-			got := vipServicesFromPrefs(prefs.View())
+			got := lb.vipServicesFromPrefsLocked(prefs.View())
 			slices.SortFunc(got, func(a, b *tailcfg.VIPService) int {
 				return strings.Compare(a.Name, b.Name)
 			})

ipn/ipnlocal/serve.go

@@ -326,7 +326,7 @@ func (b *LocalBackend) setServeConfigLocked(config *ipn.ServeConfig, etag string
 		if b.serveConfig.Valid() {
 			has = b.serveConfig.Foreground().Contains
 		}
-		prevConfig.Foreground().Range(func(k string, v ipn.ServeConfigView) (cont bool) {
+		for k := range prevConfig.Foreground().All() {
 			if !has(k) {
 				for _, sess := range b.notifyWatchers {
 					if sess.sessionID == k {
@@ -334,8 +334,7 @@ func (b *LocalBackend) setServeConfigLocked(config *ipn.ServeConfig, etag string
 					}
 				}
 			}
-			return true
-		})
+		}
 	}
 
 	return nil

ipn/ipnserver/proxyconnect.go

@@ -14,7 +14,7 @@ import (
 )
 
 // handleProxyConnectConn handles a CONNECT request to
-// log.tailscale.io (or whatever the configured log server is). This
+// log.tailscale.com (or whatever the configured log server is). This
 // is intended for use by the Windows GUI client to log via when an
 // exit node is in use, so the logs don't go out via the exit node and
 // instead go directly, like tailscaled's. The dialer tried to do that

ipn/localapi/localapi.go

@@ -83,6 +83,7 @@ var handler = map[string]localAPIHandler{
 
 	// The other /localapi/v0/NAME handlers are exact matches and contain only NAME
 	// without a trailing slash:
+	"alpha-set-device-attrs":      (*Handler).serveSetDeviceAttrs, // see tailscale/corp#24690
 	"bugreport":                   (*Handler).serveBugReport,
 	"check-ip-forwarding":         (*Handler).serveCheckIPForwarding,
 	"check-prefs":                 (*Handler).serveCheckPrefs,
@@ -446,6 +447,33 @@ func (h *Handler) serveWhoIs(w http.ResponseWriter, r *http.Request) {
 	h.serveWhoIsWithBackend(w, r, h.b)
 }
 
+// serveSetDeviceAttrs is (as of 2024-12-30) an experimental LocalAPI handler to
+// set device attributes via the control plane.
+//
+// See tailscale/corp#24690.
+func (h *Handler) serveSetDeviceAttrs(w http.ResponseWriter, r *http.Request) {
+	ctx := r.Context()
+	if !h.PermitWrite {
+		http.Error(w, "set-device-attrs access denied", http.StatusForbidden)
+		return
+	}
+	if r.Method != "PATCH" {
+		http.Error(w, "only PATCH allowed", http.StatusMethodNotAllowed)
+		return
+	}
+	var req map[string]any
+	if err := json.NewDecoder(r.Body).Decode(&req); err != nil {
+		http.Error(w, err.Error(), http.StatusBadRequest)
+		return
+	}
+	if err := h.b.SetDeviceAttrs(ctx, req); err != nil {
+		http.Error(w, err.Error(), http.StatusInternalServerError)
+		return
+	}
+	w.Header().Set("Content-Type", "application/json")
+	io.WriteString(w, "{}\n")
+}
+
 // localBackendWhoIsMethods is the subset of ipn.LocalBackend as needed
 // by the localapi WhoIs method.
 type localBackendWhoIsMethods interface {

ipn/serve.go

@@ -6,6 +6,7 @@ package ipn
 import (
 	"errors"
 	"fmt"
+	"iter"
 	"net"
 	"net/netip"
 	"net/url"
@@ -15,7 +16,9 @@ import (
 
 	"tailscale.com/ipn/ipnstate"
 	"tailscale.com/tailcfg"
+	"tailscale.com/types/ipproto"
 	"tailscale.com/util/mak"
+	"tailscale.com/util/set"
 )
 
 // ServeConfigKey returns a StateKey that stores the
@@ -564,58 +567,53 @@ func ExpandProxyTargetValue(target string, supportedSchemes []string, defaultSch
 	return u.String(), nil
 }
 
-// RangeOverTCPs ranges over both background and foreground TCPs.
-// If the returned bool from the given f is false, then this function stops
-// iterating immediately and does not check other foreground configs.
-func (v ServeConfigView) RangeOverTCPs(f func(port uint16, _ TCPPortHandlerView) bool) {
-	parentCont := true
-	v.TCP().Range(func(k uint16, v TCPPortHandlerView) (cont bool) {
-		parentCont = f(k, v)
-		return parentCont
-	})
-	v.Foreground().Range(func(k string, v ServeConfigView) (cont bool) {
-		if !parentCont {
-			return false
+// TCPs returns an iterator over both background and foreground TCP
+// listeners.
+//
+// The key is the port number.
+func (v ServeConfigView) TCPs() iter.Seq2[uint16, TCPPortHandlerView] {
+	return func(yield func(uint16, TCPPortHandlerView) bool) {
+		for k, v := range v.TCP().All() {
+			if !yield(k, v) {
+				return
+			}
 		}
-		v.TCP().Range(func(k uint16, v TCPPortHandlerView) (cont bool) {
-			parentCont = f(k, v)
-			return parentCont
-		})
-		return parentCont
-	})
+		for _, conf := range v.Foreground().All() {
+			for k, v := range conf.TCP().All() {
+				if !yield(k, v) {
+					return
+				}
+			}
+		}
+	}
 }
 
-// RangeOverWebs ranges over both background and foreground Webs.
-// If the returned bool from the given f is false, then this function stops
-// iterating immediately and does not check other foreground configs.
-func (v ServeConfigView) RangeOverWebs(f func(_ HostPort, conf WebServerConfigView) bool) {
-	parentCont := true
-	v.Web().Range(func(k HostPort, v WebServerConfigView) (cont bool) {
-		parentCont = f(k, v)
-		return parentCont
-	})
-	v.Foreground().Range(func(k string, v ServeConfigView) (cont bool) {
-		if !parentCont {
-			return false
+// Webs returns an iterator over both background and foreground Web configurations.
+func (v ServeConfigView) Webs() iter.Seq2[HostPort, WebServerConfigView] {
+	return func(yield func(HostPort, WebServerConfigView) bool) {
+		for k, v := range v.Web().All() {
+			if !yield(k, v) {
+				return
+			}
 		}
-		v.Web().Range(func(k HostPort, v WebServerConfigView) (cont bool) {
-			parentCont = f(k, v)
-			return parentCont
-		})
-		return parentCont
-	})
+		for _, conf := range v.Foreground().All() {
+			for k, v := range conf.Web().All() {
+				if !yield(k, v) {
+					return
+				}
+			}
+		}
+	}
 }
 
 // FindTCP returns the first TCP that matches with the given port. It
 // prefers a foreground match first followed by a background search if none
 // existed.
 func (v ServeConfigView) FindTCP(port uint16) (res TCPPortHandlerView, ok bool) {
-	v.Foreground().Range(func(_ string, v ServeConfigView) (cont bool) {
-		res, ok = v.TCP().GetOk(port)
-		return !ok
-	})
-	if ok {
-		return res, ok
+	for _, conf := range v.Foreground().All() {
+		if res, ok := conf.TCP().GetOk(port); ok {
+			return res, ok
+		}
 	}
 	return v.TCP().GetOk(port)
 }
@@ -624,12 +622,10 @@ func (v ServeConfigView) FindTCP(port uint16) (res TCPPortHandlerView, ok bool)
 // prefers a foreground match first followed by a background search if none
 // existed.
 func (v ServeConfigView) FindWeb(hp HostPort) (res WebServerConfigView, ok bool) {
-	v.Foreground().Range(func(_ string, v ServeConfigView) (cont bool) {
-		res, ok = v.Web().GetOk(hp)
-		return !ok
-	})
-	if ok {
-		return res, ok
+	for _, conf := range v.Foreground().All() {
+		if res, ok := conf.Web().GetOk(hp); ok {
+			return res, ok
+		}
 	}
 	return v.Web().GetOk(hp)
 }
@@ -637,14 +633,15 @@ func (v ServeConfigView) FindWeb(hp HostPort) (res WebServerConfigView, ok bool)
 // HasAllowFunnel returns whether this config has at least one AllowFunnel
 // set in the background or foreground configs.
 func (v ServeConfigView) HasAllowFunnel() bool {
-	return v.AllowFunnel().Len() > 0 || func() bool {
-		var exists bool
-		v.Foreground().Range(func(k string, v ServeConfigView) (cont bool) {
-			exists = v.AllowFunnel().Len() > 0
-			return !exists
-		})
-		return exists
-	}()
+	if v.AllowFunnel().Len() > 0 {
+		return true
+	}
+	for _, conf := range v.Foreground().All() {
+		if conf.AllowFunnel().Len() > 0 {
+			return true
+		}
+	}
+	return false
 }
 
 // FindFunnel reports whether target exists in either the background AllowFunnel
@@ -653,12 +650,48 @@ func (v ServeConfigView) HasFunnelForTarget(target HostPort) bool {
 	if v.AllowFunnel().Get(target) {
 		return true
 	}
-	var exists bool
-	v.Foreground().Range(func(_ string, v ServeConfigView) (cont bool) {
-		if exists = v.AllowFunnel().Get(target); exists {
-			return false
+	for _, conf := range v.Foreground().All() {
+		if conf.AllowFunnel().Get(target) {
+			return true
 		}
-		return true
-	})
-	return exists
+	}
+	return false
+}
+
+// ServicePortRange returns the list of tailcfg.ProtoPortRange that represents
+// the proto/ports pairs that are being served by the service.
+//
+// Right now Tun mode is the only thing supports UDP, otherwise serve only supports TCP.
+func (v ServiceConfigView) ServicePortRange() []tailcfg.ProtoPortRange {
+	if v.Tun() {
+		// If the service is in Tun mode, means service accept TCP/UDP on all ports.
+		return []tailcfg.ProtoPortRange{{Ports: tailcfg.PortRangeAny}}
+	}
+	tcp := int(ipproto.TCP)
+
+	// Deduplicate the ports.
+	servePorts := make(set.Set[uint16])
+	for port := range v.TCP().All() {
+		if port > 0 {
+			servePorts.Add(uint16(port))
+		}
+	}
+	dedupedServePorts := servePorts.Slice()
+	slices.Sort(dedupedServePorts)
+
+	var ranges []tailcfg.ProtoPortRange
+	for _, p := range dedupedServePorts {
+		if n := len(ranges); n > 0 && p == ranges[n-1].Ports.Last+1 {
+			ranges[n-1].Ports.Last = p
+			continue
+		}
+		ranges = append(ranges, tailcfg.ProtoPortRange{
+			Proto: tcp,
+			Ports: tailcfg.PortRange{
+				First: p,
+				Last:  p,
+			},
+		})
+	}
+	return ranges
 }

k8s-operator/api.md

@@ -313,6 +313,37 @@ _Appears in:_
 
 
 
+#### LabelValue
+
+_Underlying type:_ _string_
+
+
+
+_Validation:_
+- MaxLength: 63
+- Pattern: `^(([a-zA-Z0-9][-._a-zA-Z0-9]*)?[a-zA-Z0-9])?$`
+- Type: string
+
+_Appears in:_
+- [Labels](#labels)
+
+
+
+#### Labels
+
+_Underlying type:_ _[map[string]LabelValue](#map[string]labelvalue)_
+
+
+
+
+
+_Appears in:_
+- [Pod](#pod)
+- [ServiceMonitor](#servicemonitor)
+- [StatefulSet](#statefulset)
+
+
+
 #### Metrics
 
 
@@ -407,7 +438,7 @@ _Appears in:_
 
 | Field | Description | Default | Validation |
 | --- | --- | --- | --- |
-| `labels` _object (keys:string, values:string)_ | Labels that will be added to the proxy Pod.<br />Any labels specified here will be merged with the default labels<br />applied to the Pod by the Tailscale Kubernetes operator.<br />Label keys and values must be valid Kubernetes label keys and values.<br />https://kubernetes.io/docs/concepts/overview/working-with-objects/labels/#syntax-and-character-set |  |  |
+| `labels` _[Labels](#labels)_ | Labels that will be added to the proxy Pod.<br />Any labels specified here will be merged with the default labels<br />applied to the Pod by the Tailscale Kubernetes operator.<br />Label keys and values must be valid Kubernetes label keys and values.<br />https://kubernetes.io/docs/concepts/overview/working-with-objects/labels/#syntax-and-character-set |  |  |
 | `annotations` _object (keys:string, values:string)_ | Annotations that will be added to the proxy Pod.<br />Any annotations specified here will be merged with the default<br />annotations applied to the Pod by the Tailscale Kubernetes operator.<br />Annotations must be valid Kubernetes annotations.<br />https://kubernetes.io/docs/concepts/overview/working-with-objects/annotations/#syntax-and-character-set |  |  |
 | `affinity` _[Affinity](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.3/#affinity-v1-core)_ | Proxy Pod's affinity rules.<br />By default, the Tailscale Kubernetes operator does not apply any affinity rules.<br />https://kubernetes.io/docs/reference/kubernetes-api/workload-resources/pod-v1/#affinity |  |  |
 | `tailscaleContainer` _[Container](#container)_ | Configuration for the proxy container running tailscale. |  |  |
@@ -508,7 +539,16 @@ _Appears in:_
 
 
 
+ProxyGroup defines a set of Tailscale devices that will act as proxies.
+Currently only egress ProxyGroups are supported.
 
+Use the tailscale.com/proxy-group annotation on a Service to specify that
+the egress proxy should be implemented by a ProxyGroup instead of a single
+dedicated proxy. In addition to running a highly available set of proxies,
+ProxyGroup also allows for serving many annotated Services from a single
+set of proxies to minimise resource consumption.
+
+More info: https://tailscale.com/kb/1438/kubernetes-operator-cluster-egress
 
 
 
@@ -559,9 +599,9 @@ _Appears in:_
 
 | Field | Description | Default | Validation |
 | --- | --- | --- | --- |
-| `type` _[ProxyGroupType](#proxygrouptype)_ | Type of the ProxyGroup proxies. Currently the only supported type is egress. |  | Enum: [egress] <br />Type: string <br /> |
+| `type` _[ProxyGroupType](#proxygrouptype)_ | Type of the ProxyGroup proxies. Supported types are egress and ingress.<br />Type is immutable once a ProxyGroup is created. |  | Enum: [egress ingress] <br />Type: string <br /> |
 | `tags` _[Tags](#tags)_ | Tags that the Tailscale devices will be tagged with. Defaults to [tag:k8s].<br />If you specify custom tags here, make sure you also make the operator<br />an owner of these tags.<br />See  https://tailscale.com/kb/1236/kubernetes-operator/#setting-up-the-kubernetes-operator.<br />Tags cannot be changed once a ProxyGroup device has been created.<br />Tag values must be in form ^tag:[a-zA-Z][a-zA-Z0-9-]*$. |  | Pattern: `^tag:[a-zA-Z][a-zA-Z0-9-]*$` <br />Type: string <br /> |
-| `replicas` _integer_ | Replicas specifies how many replicas to create the StatefulSet with.<br />Defaults to 2. |  |  |
+| `replicas` _integer_ | Replicas specifies how many replicas to create the StatefulSet with.<br />Defaults to 2. |  | Minimum: 0 <br /> |
 | `hostnamePrefix` _[HostnamePrefix](#hostnameprefix)_ | HostnamePrefix is the hostname prefix to use for tailnet devices created<br />by the ProxyGroup. Each device will have the integer number from its<br />StatefulSet pod appended to this prefix to form the full hostname.<br />HostnamePrefix can contain lower case letters, numbers and dashes, it<br />must not start with a dash and must be between 1 and 62 characters long. |  | Pattern: `^[a-z0-9][a-z0-9-]{0,61}$` <br />Type: string <br /> |
 | `proxyClass` _string_ | ProxyClass is the name of the ProxyClass custom resource that contains<br />configuration options that should be applied to the resources created<br />for this ProxyGroup. If unset, and there is no default ProxyClass<br />configured, the operator will create resources with the default<br />configuration. |  |  |
 
@@ -590,7 +630,7 @@ _Underlying type:_ _string_
 
 
 _Validation:_
-- Enum: [egress]
+- Enum: [egress ingress]
 - Type: string
 
 _Appears in:_
@@ -602,7 +642,11 @@ _Appears in:_
 
 
 
+Recorder defines a tsrecorder device for recording SSH sessions. By default,
+it will store recordings in a local ephemeral volume. If you want to persist
+recordings, you can configure an S3-compatible API for storage.
 
+More info: https://tailscale.com/kb/1484/kubernetes-operator-deploying-tsrecorder
 
 
 
@@ -851,6 +895,7 @@ _Appears in:_
 | Field | Description | Default | Validation |
 | --- | --- | --- | --- |
 | `enable` _boolean_ | If Enable is set to true, a Prometheus ServiceMonitor will be created. Enable can only be set to true if metrics are enabled. |  |  |
+| `labels` _[Labels](#labels)_ | Labels to add to the ServiceMonitor.<br />Labels must be valid Kubernetes labels.<br />https://kubernetes.io/docs/concepts/overview/working-with-objects/labels/#syntax-and-character-set |  |  |
 
 
 #### StatefulSet
@@ -866,7 +911,7 @@ _Appears in:_
 
 | Field | Description | Default | Validation |
 | --- | --- | --- | --- |
-| `labels` _object (keys:string, values:string)_ | Labels that will be added to the StatefulSet created for the proxy.<br />Any labels specified here will be merged with the default labels<br />applied to the StatefulSet by the Tailscale Kubernetes operator as<br />well as any other labels that might have been applied by other<br />actors.<br />Label keys and values must be valid Kubernetes label keys and values.<br />https://kubernetes.io/docs/concepts/overview/working-with-objects/labels/#syntax-and-character-set |  |  |
+| `labels` _[Labels](#labels)_ | Labels that will be added to the StatefulSet created for the proxy.<br />Any labels specified here will be merged with the default labels<br />applied to the StatefulSet by the Tailscale Kubernetes operator as<br />well as any other labels that might have been applied by other<br />actors.<br />Label keys and values must be valid Kubernetes label keys and values.<br />https://kubernetes.io/docs/concepts/overview/working-with-objects/labels/#syntax-and-character-set |  |  |
 | `annotations` _object (keys:string, values:string)_ | Annotations that will be added to the StatefulSet created for the proxy.<br />Any Annotations specified here will be merged with the default annotations<br />applied to the StatefulSet by the Tailscale Kubernetes operator as<br />well as any other annotations that might have been applied by other<br />actors.<br />Annotations must be valid Kubernetes annotations.<br />https://kubernetes.io/docs/concepts/overview/working-with-objects/annotations/#syntax-and-character-set |  |  |
 | `pod` _[Pod](#pod)_ | Configuration for the proxy Pod. |  |  |
 

k8s-operator/apis/v1alpha1/types_proxyclass.go

@@ -87,7 +87,7 @@ type StatefulSet struct {
 	// Label keys and values must be valid Kubernetes label keys and values.
 	// https://kubernetes.io/docs/concepts/overview/working-with-objects/labels/#syntax-and-character-set
 	// +optional
-	Labels map[string]string `json:"labels,omitempty"`
+	Labels Labels `json:"labels,omitempty"`
 	// Annotations that will be added to the StatefulSet created for the proxy.
 	// Any Annotations specified here will be merged with the default annotations
 	// applied to the StatefulSet by the Tailscale Kubernetes operator as
@@ -109,7 +109,7 @@ type Pod struct {
 	// Label keys and values must be valid Kubernetes label keys and values.
 	// https://kubernetes.io/docs/concepts/overview/working-with-objects/labels/#syntax-and-character-set
 	// +optional
-	Labels map[string]string `json:"labels,omitempty"`
+	Labels Labels `json:"labels,omitempty"`
 	// Annotations that will be added to the proxy Pod.
 	// Any annotations specified here will be merged with the default
 	// annotations applied to the Pod by the Tailscale Kubernetes operator.
@@ -188,8 +188,34 @@ type Metrics struct {
 type ServiceMonitor struct {
 	// If Enable is set to true, a Prometheus ServiceMonitor will be created. Enable can only be set to true if metrics are enabled.
 	Enable bool `json:"enable"`
+	// Labels to add to the ServiceMonitor.
+	// Labels must be valid Kubernetes labels.
+	// https://kubernetes.io/docs/concepts/overview/working-with-objects/labels/#syntax-and-character-set
+	// +optional
+	Labels Labels `json:"labels"`
 }
 
+type Labels map[string]LabelValue
+
+func (l Labels) Parse() map[string]string {
+	if l == nil {
+		return nil
+	}
+	m := make(map[string]string, len(l))
+	for k, v := range l {
+		m[k] = string(v)
+	}
+	return m
+}
+
+// We do not validate the values of the label keys here - it is done by the ProxyClass
+// reconciler because the validation rules are too complex for a CRD validation markers regex.
+
+// +kubebuilder:validation:Type=string
+// +kubebuilder:validation:Pattern=`^(([a-zA-Z0-9][-._a-zA-Z0-9]*)?[a-zA-Z0-9])?$`
+// +kubebuilder:validation:MaxLength=63
+type LabelValue string
+
 type Container struct {
 	// List of environment variables to set in the container.
 	// https://kubernetes.io/docs/reference/kubernetes-api/workload-resources/pod-v1/#environment-variables

k8s-operator/apis/v1alpha1/types_proxygroup.go

@@ -13,7 +13,18 @@ import (
 // +kubebuilder:subresource:status
 // +kubebuilder:resource:scope=Cluster,shortName=pg
 // +kubebuilder:printcolumn:name="Status",type="string",JSONPath=`.status.conditions[?(@.type == "ProxyGroupReady")].reason`,description="Status of the deployed ProxyGroup resources."
+// +kubebuilder:printcolumn:name="Type",type="string",JSONPath=`.spec.type`,description="ProxyGroup type."
 
+// ProxyGroup defines a set of Tailscale devices that will act as proxies.
+// Currently only egress ProxyGroups are supported.
+//
+// Use the tailscale.com/proxy-group annotation on a Service to specify that
+// the egress proxy should be implemented by a ProxyGroup instead of a single
+// dedicated proxy. In addition to running a highly available set of proxies,
+// ProxyGroup also allows for serving many annotated Services from a single
+// set of proxies to minimise resource consumption.
+//
+// More info: https://tailscale.com/kb/1438/kubernetes-operator-cluster-egress
 type ProxyGroup struct {
 	metav1.TypeMeta   `json:",inline"`
 	metav1.ObjectMeta `json:"metadata,omitempty"`
@@ -37,7 +48,9 @@ type ProxyGroupList struct {
 }
 
 type ProxyGroupSpec struct {
-	// Type of the ProxyGroup proxies. Currently the only supported type is egress.
+	// Type of the ProxyGroup proxies. Supported types are egress and ingress.
+	// Type is immutable once a ProxyGroup is created.
+	// +kubebuilder:validation:XValidation:rule="self == oldSelf",message="ProxyGroup type is immutable"
 	Type ProxyGroupType `json:"type"`
 
 	// Tags that the Tailscale devices will be tagged with. Defaults to [tag:k8s].
@@ -52,6 +65,7 @@ type ProxyGroupSpec struct {
 	// Replicas specifies how many replicas to create the StatefulSet with.
 	// Defaults to 2.
 	// +optional
+	// +kubebuilder:validation:Minimum=0
 	Replicas *int32 `json:"replicas,omitempty"`
 
 	// HostnamePrefix is the hostname prefix to use for tailnet devices created
@@ -99,11 +113,12 @@ type TailnetDevice struct {
 }
 
 // +kubebuilder:validation:Type=string
-// +kubebuilder:validation:Enum=egress
+// +kubebuilder:validation:Enum=egress;ingress
 type ProxyGroupType string
 
 const (
-	ProxyGroupTypeEgress ProxyGroupType = "egress"
+	ProxyGroupTypeEgress  ProxyGroupType = "egress"
+	ProxyGroupTypeIngress ProxyGroupType = "ingress"
 )
 
 // +kubebuilder:validation:Type=string

k8s-operator/apis/v1alpha1/types_recorder.go

@@ -16,6 +16,11 @@ import (
 // +kubebuilder:printcolumn:name="Status",type="string",JSONPath=`.status.conditions[?(@.type == "RecorderReady")].reason`,description="Status of the deployed Recorder resources."
 // +kubebuilder:printcolumn:name="URL",type="string",JSONPath=`.status.devices[?(@.url != "")].url`,description="URL on which the UI is exposed if enabled."
 
+// Recorder defines a tsrecorder device for recording SSH sessions. By default,
+// it will store recordings in a local ephemeral volume. If you want to persist
+// recordings, you can configure an S3-compatible API for storage.
+//
+// More info: https://tailscale.com/kb/1484/kubernetes-operator-deploying-tsrecorder
 type Recorder struct {
 	metav1.TypeMeta   `json:",inline"`
 	metav1.ObjectMeta `json:"metadata,omitempty"`

k8s-operator/apis/v1alpha1/zz_generated.deepcopy.go

@@ -316,13 +316,34 @@ func (in *Env) DeepCopy() *Env {
 	return out
 }
 
+// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
+func (in Labels) DeepCopyInto(out *Labels) {
+	{
+		in := &in
+		*out = make(Labels, len(*in))
+		for key, val := range *in {
+			(*out)[key] = val
+		}
+	}
+}
+
+// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new Labels.
+func (in Labels) DeepCopy() Labels {
+	if in == nil {
+		return nil
+	}
+	out := new(Labels)
+	in.DeepCopyInto(out)
+	return *out
+}
+
 // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
 func (in *Metrics) DeepCopyInto(out *Metrics) {
 	*out = *in
 	if in.ServiceMonitor != nil {
 		in, out := &in.ServiceMonitor, &out.ServiceMonitor
 		*out = new(ServiceMonitor)
-		**out = **in
+		(*in).DeepCopyInto(*out)
 	}
 }
 
@@ -391,7 +412,7 @@ func (in *Pod) DeepCopyInto(out *Pod) {
 	*out = *in
 	if in.Labels != nil {
 		in, out := &in.Labels, &out.Labels
-		*out = make(map[string]string, len(*in))
+		*out = make(Labels, len(*in))
 		for key, val := range *in {
 			(*out)[key] = val
 		}
@@ -999,6 +1020,13 @@ func (in *S3Secret) DeepCopy() *S3Secret {
 // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
 func (in *ServiceMonitor) DeepCopyInto(out *ServiceMonitor) {
 	*out = *in
+	if in.Labels != nil {
+		in, out := &in.Labels, &out.Labels
+		*out = make(Labels, len(*in))
+		for key, val := range *in {
+			(*out)[key] = val
+		}
+	}
 }
 
 // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ServiceMonitor.
@@ -1016,7 +1044,7 @@ func (in *StatefulSet) DeepCopyInto(out *StatefulSet) {
 	*out = *in
 	if in.Labels != nil {
 		in, out := &in.Labels, &out.Labels
-		*out = make(map[string]string, len(*in))
+		*out = make(Labels, len(*in))
 		for key, val := range *in {
 			(*out)[key] = val
 		}

logpolicy/logpolicy.go

@@ -889,7 +889,7 @@ func (opts TransportOptions) New() http.RoundTripper {
 
 	host := cmp.Or(opts.Host, logtail.DefaultHost)
 	tr.TLSClientConfig = tlsdial.Config(host, opts.Health, tr.TLSClientConfig)
-	// Force TLS 1.3 since we know log.tailscale.io supports it.
+	// Force TLS 1.3 since we know log.tailscale.com supports it.
 	tr.TLSClientConfig.MinVersion = tls.VersionTLS13
 
 	return tr

logpolicy/logpolicy_test.go

@@ -7,6 +7,8 @@ import (
 	"os"
 	"reflect"
 	"testing"
+
+	"tailscale.com/logtail"
 )
 
 func TestLogHost(t *testing.T) {
@@ -20,7 +22,7 @@ func TestLogHost(t *testing.T) {
 		env  string
 		want string
 	}{
-		{"", "log.tailscale.io"},
+		{"", logtail.DefaultHost},
 		{"http://foo.com", "foo.com"},
 		{"https://foo.com", "foo.com"},
 		{"https://foo.com/", "foo.com"},

logtail/api.md

@@ -6,14 +6,14 @@ retrieving, and processing log entries.
 # Overview
 
 HTTP requests are received at the service **base URL**
-[https://log.tailscale.io](https://log.tailscale.io), and return JSON-encoded
+[https://log.tailscale.com](https://log.tailscale.com), and return JSON-encoded
 responses using standard HTTP response codes.
 
 Authorization for the configuration and retrieval APIs is done with a secret
 API key passed as the HTTP basic auth username. Secret keys are generated via
 the web UI at base URL. An example of using basic auth with curl:
 
-    curl -u <log_api_key>: https://log.tailscale.io/collections
+    curl -u <log_api_key>: https://log.tailscale.com/collections
 
 In the future, an HTTP header will allow using MessagePack instead of JSON.
 

logtail/example/logadopt/logadopt.go

@@ -25,7 +25,7 @@ func main() {
 	}
 	log.SetFlags(0)
 
-	req, err := http.NewRequest("POST", "https://log.tailscale.io/instances", strings.NewReader(url.Values{
+	req, err := http.NewRequest("POST", "https://log.tailscale.com/instances", strings.NewReader(url.Values{
 		"collection": []string{*collection},
 		"instances":  []string{*publicID},
 		"adopt":      []string{"true"},

logtail/example/logreprocess/demo.sh

@@ -13,7 +13,7 @@
 #
 # Then generate a LOGTAIL_API_KEY and two test collections by visiting:
 #
-#        https://log.tailscale.io
+#        https://log.tailscale.com
 #
 # Then set the three variables below.
 trap 'rv=$?; [ "$rv" = 0 ] || echo "-- exiting with code $rv"; exit $rv' EXIT

logtail/example/logreprocess/logreprocess.go

@@ -37,7 +37,7 @@ func main() {
 		}()
 	}
 
-	req, err := http.NewRequest("GET", "https://log.tailscale.io/c/"+*collection+"?stream=true", nil)
+	req, err := http.NewRequest("GET", "https://log.tailscale.com/c/"+*collection+"?stream=true", nil)
 	if err != nil {
 		log.Fatal(err)
 	}

logtail/logtail.go

@@ -1,7 +1,7 @@
 // Copyright (c) Tailscale Inc & AUTHORS
 // SPDX-License-Identifier: BSD-3-Clause
 
-// Package logtail sends logs to log.tailscale.io.
+// Package logtail sends logs to log.tailscale.com.
 package logtail
 
 import (
@@ -55,7 +55,7 @@ const bufferSize = 4 << 10
 
 // DefaultHost is the default host name to upload logs to when
 // Config.BaseURL isn't provided.
-const DefaultHost = "log.tailscale.io"
+const DefaultHost = "log.tailscale.com"
 
 const defaultFlushDelay = 2 * time.Second
 
@@ -69,7 +69,7 @@ type Config struct {
 	Collection     string          // collection name, a domain name
 	PrivateID      logid.PrivateID // private ID for the primary log stream
 	CopyPrivateID  logid.PrivateID // private ID for a log stream that is a superset of this log stream
-	BaseURL        string          // if empty defaults to "https://log.tailscale.io"
+	BaseURL        string          // if empty defaults to "https://log.tailscale.com"
 	HTTPC          *http.Client    // if empty defaults to http.DefaultClient
 	SkipClientTime bool            // if true, client_time is not written to logs
 	LowMemory      bool            // if true, logtail minimizes memory use
@@ -507,7 +507,7 @@ func (l *Logger) upload(ctx context.Context, body []byte, origlen int) (retryAft
 	}
 	if runtime.GOOS == "js" {
 		// We once advertised we'd accept optional client certs (for internal use)
-		// on log.tailscale.io but then Tailscale SSH js/wasm clients prompted
+		// on log.tailscale.com but then Tailscale SSH js/wasm clients prompted
 		// users (on some browsers?) to pick a client cert. We'll fix the server's
 		// TLS ServerHello, but we can also fix it client side for good measure.
 		//

metrics/metrics.go

@@ -11,6 +11,9 @@ import (
 	"io"
 	"slices"
 	"strings"
+	"sync"
+
+	"tailscale.com/syncs"
 )
 
 // Set is a string-to-Var map variable that satisfies the expvar.Var
@@ -37,6 +40,8 @@ type Set struct {
 type LabelMap struct {
 	Label string
 	expvar.Map
+	// shardedIntMu orders the initialization of new shardedint keys
+	shardedIntMu sync.Mutex
 }
 
 // SetInt64 sets the *Int value stored under the given map key.
@@ -44,6 +49,19 @@ func (m *LabelMap) SetInt64(key string, v int64) {
 	m.Get(key).Set(v)
 }
 
+// Add adds delta to the any int-like value stored under the given map key.
+func (m *LabelMap) Add(key string, delta int64) {
+	type intAdder interface {
+		Add(delta int64)
+	}
+	o := m.Map.Get(key)
+	if o == nil {
+		m.Map.Add(key, delta)
+		return
+	}
+	o.(intAdder).Add(delta)
+}
+
 // Get returns a direct pointer to the expvar.Int for key, creating it
 // if necessary.
 func (m *LabelMap) Get(key string) *expvar.Int {
@@ -51,6 +69,23 @@ func (m *LabelMap) Get(key string) *expvar.Int {
 	return m.Map.Get(key).(*expvar.Int)
 }
 
+// GetShardedInt returns a direct pointer to the syncs.ShardedInt for key,
+// creating it if necessary.
+func (m *LabelMap) GetShardedInt(key string) *syncs.ShardedInt {
+	i := m.Map.Get(key)
+	if i == nil {
+		m.shardedIntMu.Lock()
+		defer m.shardedIntMu.Unlock()
+		i = m.Map.Get(key)
+		if i != nil {
+			return i.(*syncs.ShardedInt)
+		}
+		i = syncs.NewShardedInt()
+		m.Set(key, i)
+	}
+	return i.(*syncs.ShardedInt)
+}
+
 // GetIncrFunc returns a function that increments the expvar.Int named by key.
 //
 // Most callers should not need this; it exists to satisfy an

metrics/metrics_test.go

@@ -21,6 +21,15 @@ func TestLabelMap(t *testing.T) {
 	if g, w := m.Get("bar").Value(), int64(2); g != w {
 		t.Errorf("bar = %v; want %v", g, w)
 	}
+	m.GetShardedInt("sharded").Add(5)
+	if g, w := m.GetShardedInt("sharded").Value(), int64(5); g != w {
+		t.Errorf("sharded = %v; want %v", g, w)
+	}
+	m.Add("sharded", 1)
+	if g, w := m.GetShardedInt("sharded").Value(), int64(6); g != w {
+		t.Errorf("sharded = %v; want %v", g, w)
+	}
+	m.Add("neverbefore", 1)
 }
 
 func TestCurrentFileDescriptors(t *testing.T) {

net/dns/manager.go

@@ -19,7 +19,6 @@ import (
 	"sync/atomic"
 	"time"
 
-	xmaps "golang.org/x/exp/maps"
 	"tailscale.com/control/controlknobs"
 	"tailscale.com/health"
 	"tailscale.com/net/dns/resolver"
@@ -31,6 +30,7 @@ import (
 	"tailscale.com/types/logger"
 	"tailscale.com/util/clientmetric"
 	"tailscale.com/util/dnsname"
+	"tailscale.com/util/slicesx"
 )
 
 var (
@@ -204,7 +204,7 @@ func compileHostEntries(cfg Config) (hosts []*HostEntry) {
 	if len(hostsMap) == 0 {
 		return nil
 	}
-	hosts = xmaps.Values(hostsMap)
+	hosts = slicesx.MapValues(hostsMap)
 	slices.SortFunc(hosts, func(a, b *HostEntry) int {
 		if len(a.Hosts) == 0 && len(b.Hosts) == 0 {
 			return 0

net/dns/resolver/forwarder.go

@@ -6,9 +6,7 @@ package resolver
 import (
 	"bytes"
 	"context"
-	"crypto/sha256"
 	"crypto/tls"
-	"encoding/base64"
 	"encoding/binary"
 	"errors"
 	"fmt"
@@ -23,6 +21,7 @@ import (
 	"sync/atomic"
 	"time"
 
+	"golang.org/x/net/dns/dnsmessage"
 	dns "golang.org/x/net/dns/dnsmessage"
 	"tailscale.com/control/controlknobs"
 	"tailscale.com/envknob"
@@ -938,6 +937,11 @@ func (f *forwarder) forwardWithDestChan(ctx context.Context, query packet, respo
 
 	if len(resolvers) == 0 {
 		resolvers = f.resolvers(domain)
+		names := make([]string, len(resolvers))
+		for _, r := range resolvers {
+			names = append(names, r.name.Addr)
+		}
+		f.logf("DEBUG: resolvers for %q: %q", domain, names)
 		if len(resolvers) == 0 {
 			metricDNSFwdErrorNoUpstream.Add(1)
 			f.health.SetUnhealthy(dnsForwarderFailing, health.Args{health.ArgDNSServers: ""})
@@ -974,9 +978,9 @@ func (f *forwarder) forwardWithDestChan(ctx context.Context, query packet, respo
 	defer fq.closeOnCtxDone.Close()
 
 	if verboseDNSForward() {
-		domainSha256 := sha256.Sum256([]byte(domain))
-		domainSig := base64.RawStdEncoding.EncodeToString(domainSha256[:3])
-		f.logf("request(%d, %v, %d, %s) %d...", fq.txid, typ, len(domain), domainSig, len(fq.packet))
+		//domainSha256 := sha256.Sum256([]byte(domain))
+		//domainSig := base64.RawStdEncoding.EncodeToString(domainSha256[:3])
+		f.logf("request(%d, %v, %q, %s) %d...", fq.txid, typ, len(domain), domain, len(fq.packet))
 	}
 
 	resc := make(chan []byte, 1) // it's fine buffered or not
@@ -1018,8 +1022,29 @@ func (f *forwarder) forwardWithDestChan(ctx context.Context, query packet, respo
 				metricDNSFwdErrorContext.Add(1)
 				return fmt.Errorf("waiting to send response: %w", ctx.Err())
 			case responseChan <- packet{v, query.family, query.addr}:
-				if verboseDNSForward() {
-					f.logf("response(%d, %v, %d) = %d, nil", fq.txid, typ, len(domain), len(v))
+				answers, err := func() ([]string, error) {
+					p := new(dnsmessage.Parser)
+					_, err := p.Start(v)
+					if err != nil {
+						return nil, err
+					}
+					if err := p.SkipAllQuestions(); err != nil {
+						return nil, err
+					}
+					ans, err := p.AllAnswers()
+					if err != nil {
+						return nil, err
+					}
+					var res []string
+					for _, a := range ans {
+						res = append(res, a.Body.GoString())
+					}
+					return res, nil
+				}()
+				if err != nil {
+					f.logf("failed to parse DNS response: %v", err)
+				} else {
+					f.logf("response(%d, %v, %q) = %q, nil", fq.txid, typ, domain, answers)
 				}
 				metricDNSFwdSuccess.Add(1)
 				f.health.SetHealthy(dnsForwarderFailing)

net/dns/resolver/tsdns.go

@@ -1297,6 +1297,7 @@ func (r *Resolver) respond(query []byte) ([]byte, error) {
 	if rcode == dns.RCodeRefused {
 		return nil, errNotOurName // sentinel error return value: it requests forwarding
 	}
+	r.logf("DEBUG: resolveLocal(%q) ip: %q code: %q", name, ip, rcode)
 
 	resp := parser.response()
 	resp.Header.RCode = rcode

net/netcheck/netcheck.go

@@ -23,7 +23,6 @@ import (
 	"syscall"
 	"time"
 
-	"github.com/tcnksm/go-httpstat"
 	"tailscale.com/derp/derphttp"
 	"tailscale.com/envknob"
 	"tailscale.com/net/captivedetection"
@@ -1110,10 +1109,11 @@ func (c *Client) runHTTPOnlyChecks(ctx context.Context, last *Report, rs *report
 	return nil
 }
 
+// measureHTTPSLatency measures HTTP request latency to the DERP region, but
+// only returns success if an HTTPS request to the region succeeds.
 func (c *Client) measureHTTPSLatency(ctx context.Context, reg *tailcfg.DERPRegion) (time.Duration, netip.Addr, error) {
 	metricHTTPSend.Add(1)
-	var result httpstat.Result
-	ctx, cancel := context.WithTimeout(httpstat.WithHTTPStat(ctx, &result), httpsProbeTimeout)
+	ctx, cancel := context.WithTimeout(ctx, httpsProbeTimeout)
 	defer cancel()
 
 	var ip netip.Addr
@@ -1121,6 +1121,8 @@ func (c *Client) measureHTTPSLatency(ctx context.Context, reg *tailcfg.DERPRegio
 	dc := derphttp.NewNetcheckClient(c.logf, c.NetMon)
 	defer dc.Close()
 
+	// DialRegionTLS may dial multiple times if a node is not available, as such
+	// it does not have stable timing to measure.
 	tlsConn, tcpConn, node, err := dc.DialRegionTLS(ctx, reg)
 	if err != nil {
 		return 0, ip, err
@@ -1138,6 +1140,8 @@ func (c *Client) measureHTTPSLatency(ctx context.Context, reg *tailcfg.DERPRegio
 	connc := make(chan *tls.Conn, 1)
 	connc <- tlsConn
 
+	// make an HTTP request to measure, as this enables us to account for MITM
+	// overhead in e.g. corp environments that have HTTP MITM in front of DERP.
 	tr := &http.Transport{
 		DialContext: func(ctx context.Context, network, addr string) (net.Conn, error) {
 			return nil, errors.New("unexpected DialContext dial")
@@ -1153,12 +1157,17 @@ func (c *Client) measureHTTPSLatency(ctx context.Context, reg *tailcfg.DERPRegio
 	}
 	hc := &http.Client{Transport: tr}
 
+	// This is the request that will be measured, the request and response
+	// should be small enough to fit into a single packet each way unless the
+	// connection has already become unstable.
 	req, err := http.NewRequestWithContext(ctx, "GET", "https://"+node.HostName+"/derp/latency-check", nil)
 	if err != nil {
 		return 0, ip, err
 	}
 
+	startTime := c.timeNow()
 	resp, err := hc.Do(req)
+	reqDur := c.timeNow().Sub(startTime)
 	if err != nil {
 		return 0, ip, err
 	}
@@ -1175,11 +1184,12 @@ func (c *Client) measureHTTPSLatency(ctx context.Context, reg *tailcfg.DERPRegio
 	if err != nil {
 		return 0, ip, err
 	}
-	result.End(c.timeNow())
 
-	// TODO: decide best timing heuristic here.
-	// Maybe the server should return the tcpinfo_rtt?
-	return result.ServerProcessing, ip, nil
+	// return the connection duration, not the request duration, as this is the
+	// best approximation of the RTT latency to the node. Note that the
+	// connection setup performs happy-eyeballs and TLS so there are additional
+	// overheads.
+	return reqDur, ip, nil
 }
 
 func (c *Client) measureAllICMPLatency(ctx context.Context, rs *reportState, need []*tailcfg.DERPRegion) error {

net/netmon/netmon_darwin.go

@@ -56,18 +56,7 @@ func (m *darwinRouteMon) Receive() (message, error) {
 		if err != nil {
 			return nil, err
 		}
-		msgs, err := func() (msgs []route.Message, err error) {
-			defer func() {
-				// TODO(raggi,#14201): remove once we've got a fix from
-				// golang/go#70528.
-				msg := recover()
-				if msg != nil {
-					msgs = nil
-					err = fmt.Errorf("panic in route.ParseRIB: %s", msg)
-				}
-			}()
-			return route.ParseRIB(route.RIBTypeRoute, m.buf[:n])
-		}()
+		msgs, err := route.ParseRIB(route.RIBTypeRoute, m.buf[:n])
 		if err != nil {
 			if debugRouteMessages {
 				m.logf("read %d bytes (% 02x), failed to parse RIB: %v", n, m.buf[:n], err)

net/tlsdial/tlsdial.go

@@ -89,8 +89,8 @@ func Config(host string, ht *health.Tracker, base *tls.Config) *tls.Config {
 	// (with the baked-in fallback root) in the VerifyConnection hook.
 	conf.InsecureSkipVerify = true
 	conf.VerifyConnection = func(cs tls.ConnectionState) (retErr error) {
-		if host == "log.tailscale.io" && hostinfo.IsNATLabGuestVM() {
-			// Allow log.tailscale.io TLS MITM for integration tests when
+		if host == "log.tailscale.com" && hostinfo.IsNATLabGuestVM() {
+			// Allow log.tailscale.com TLS MITM for integration tests when
 			// the client's running within a NATLab VM.
 			return nil
 		}

net/tshttpproxy/tshttpproxy_synology.go

@@ -47,7 +47,7 @@ func synologyProxyFromConfigCached(req *http.Request) (*url.URL, error) {
 	var err error
 	modtime := mtime(synologyProxyConfigPath)
 
-	if modtime != cache.updated {
+	if !modtime.Equal(cache.updated) {
 		cache.httpProxy, cache.httpsProxy, err = synologyProxiesFromConfig()
 		cache.updated = modtime
 	}

net/tshttpproxy/tshttpproxy_synology_test.go

@@ -41,7 +41,7 @@ func TestSynologyProxyFromConfigCached(t *testing.T) {
 			t.Fatalf("got %s, %v; want nil, nil", val, err)
 		}
 
-		if got, want := cache.updated, time.Unix(0, 0); got != want {
+		if got, want := cache.updated.UTC(), time.Unix(0, 0).UTC(); !got.Equal(want) {
 			t.Fatalf("got %s, want %s", got, want)
 		}
 		if cache.httpProxy != nil {

prober/derp.go

@@ -8,24 +8,34 @@ import (
 	"cmp"
 	"context"
 	crand "crypto/rand"
+	"encoding/binary"
 	"encoding/json"
 	"errors"
 	"expvar"
 	"fmt"
+	"io"
 	"log"
+	"maps"
 	"net"
 	"net/http"
+	"net/netip"
+	"slices"
 	"strconv"
 	"strings"
 	"sync"
 	"time"
 
 	"github.com/prometheus/client_golang/prometheus"
+	wgconn "github.com/tailscale/wireguard-go/conn"
+	"github.com/tailscale/wireguard-go/device"
+	"github.com/tailscale/wireguard-go/tun"
+	"go4.org/netipx"
 	"tailscale.com/client/tailscale"
 	"tailscale.com/derp"
 	"tailscale.com/derp/derphttp"
 	"tailscale.com/net/netmon"
 	"tailscale.com/net/stun"
+	"tailscale.com/net/tstun"
 	"tailscale.com/syncs"
 	"tailscale.com/tailcfg"
 	"tailscale.com/types/key"
@@ -42,8 +52,13 @@ type derpProber struct {
 	tlsInterval  time.Duration
 
 	// Optional bandwidth probing.
-	bwInterval  time.Duration
-	bwProbeSize int64
+	bwInterval      time.Duration
+	bwProbeSize     int64
+	bwTUNIPv4Prefix *netip.Prefix // or nil to not use TUN
+
+	// Optional queuing delay probing.
+	qdPacketsPerSecond int // in packets per second
+	qdPacketTimeout    time.Duration
 
 	// Optionally restrict probes to a single regionCode.
 	regionCode string
@@ -56,6 +71,7 @@ type derpProber struct {
 	udpProbeFn  func(string, int) ProbeClass
 	meshProbeFn func(string, string) ProbeClass
 	bwProbeFn   func(string, string, int64) ProbeClass
+	qdProbeFn   func(string, string, int, time.Duration) ProbeClass
 
 	sync.Mutex
 	lastDERPMap   *tailcfg.DERPMap
@@ -68,11 +84,30 @@ type DERPOpt func(*derpProber)
 
 // WithBandwidthProbing enables bandwidth probing. When enabled, a payload of
 // `size` bytes will be regularly transferred through each DERP server, and each
-// pair of DERP servers in every region.
-func WithBandwidthProbing(interval time.Duration, size int64) DERPOpt {
+// pair of DERP servers in every region. If tunAddress is specified, probes will
+// use a TCP connection over a TUN device at this address in order to exercise
+// TCP-in-TCP in similar fashion to TCP over Tailscale via DERP.
+func WithBandwidthProbing(interval time.Duration, size int64, tunAddress string) DERPOpt {
 	return func(d *derpProber) {
 		d.bwInterval = interval
 		d.bwProbeSize = size
+		if tunAddress != "" {
+			prefix, err := netip.ParsePrefix(fmt.Sprintf("%s/30", tunAddress))
+			if err != nil {
+				log.Fatalf("failed to parse IP prefix from bw-tun-ipv4-addr: %v", err)
+			}
+			d.bwTUNIPv4Prefix = &prefix
+		}
+	}
+}
+
+// WithQueuingDelayProbing enables/disables queuing delay probing. qdSendRate
+// is the number of packets sent per second. qdTimeout is the amount of time
+// after which a sent packet is considered to have timed out.
+func WithQueuingDelayProbing(qdPacketsPerSecond int, qdPacketTimeout time.Duration) DERPOpt {
+	return func(d *derpProber) {
+		d.qdPacketsPerSecond = qdPacketsPerSecond
+		d.qdPacketTimeout = qdPacketTimeout
 	}
 }
 
@@ -130,6 +165,7 @@ func DERP(p *Prober, derpMapURL string, opts ...DERPOpt) (*derpProber, error) {
 	d.udpProbeFn = d.ProbeUDP
 	d.meshProbeFn = d.probeMesh
 	d.bwProbeFn = d.probeBandwidth
+	d.qdProbeFn = d.probeQueuingDelay
 	return d, nil
 }
 
@@ -196,14 +232,27 @@ func (d *derpProber) probeMapFn(ctx context.Context) error {
 					}
 				}
 
-				if d.bwInterval > 0 && d.bwProbeSize > 0 {
+				if d.bwInterval != 0 && d.bwProbeSize > 0 {
 					n := fmt.Sprintf("derp/%s/%s/%s/bw", region.RegionCode, server.Name, to.Name)
 					wantProbes[n] = true
 					if d.probes[n] == nil {
-						log.Printf("adding DERP bandwidth probe for %s->%s (%s) %v bytes every %v", server.Name, to.Name, region.RegionName, d.bwProbeSize, d.bwInterval)
+						tunString := ""
+						if d.bwTUNIPv4Prefix != nil {
+							tunString = " (TUN)"
+						}
+						log.Printf("adding%s DERP bandwidth probe for %s->%s (%s) %v bytes every %v", tunString, server.Name, to.Name, region.RegionName, d.bwProbeSize, d.bwInterval)
 						d.probes[n] = d.p.Run(n, d.bwInterval, labels, d.bwProbeFn(server.Name, to.Name, d.bwProbeSize))
 					}
 				}
+
+				if d.qdPacketsPerSecond > 0 {
+					n := fmt.Sprintf("derp/%s/%s/%s/qd", region.RegionCode, server.Name, to.Name)
+					wantProbes[n] = true
+					if d.probes[n] == nil {
+						log.Printf("adding DERP queuing delay probe for %s->%s (%s)", server.Name, to.Name, region.RegionName)
+						d.probes[n] = d.p.Run(n, -10*time.Second, labels, d.qdProbeFn(server.Name, to.Name, d.qdPacketsPerSecond, d.qdPacketTimeout))
+					}
+				}
 			}
 		}
 	}
@@ -219,7 +268,7 @@ func (d *derpProber) probeMapFn(ctx context.Context) error {
 	return nil
 }
 
-// probeMesh returs a probe class that sends a test packet through a pair of DERP
+// probeMesh returns a probe class that sends a test packet through a pair of DERP
 // servers (or just one server, if 'from' and 'to' are the same). 'from' and 'to'
 // are expected to be names (DERPNode.Name) of two DERP servers in the same region.
 func (d *derpProber) probeMesh(from, to string) ProbeClass {
@@ -242,7 +291,7 @@ func (d *derpProber) probeMesh(from, to string) ProbeClass {
 	}
 }
 
-// probeBandwidth returs a probe class that sends a payload of a given size
+// probeBandwidth returns a probe class that sends a payload of a given size
 // through a pair of DERP servers (or just one server, if 'from' and 'to' are
 // the same). 'from' and 'to' are expected to be names (DERPNode.Name) of two
 // DERP servers in the same region.
@@ -251,26 +300,217 @@ func (d *derpProber) probeBandwidth(from, to string, size int64) ProbeClass {
 	if from == to {
 		derpPath = "single"
 	}
-	var transferTime expvar.Float
+	var transferTimeSeconds expvar.Float
 	return ProbeClass{
 		Probe: func(ctx context.Context) error {
 			fromN, toN, err := d.getNodePair(from, to)
 			if err != nil {
 				return err
 			}
-			return derpProbeBandwidth(ctx, d.lastDERPMap, fromN, toN, size, &transferTime)
+			return derpProbeBandwidth(ctx, d.lastDERPMap, fromN, toN, size, &transferTimeSeconds, d.bwTUNIPv4Prefix)
+		},
+		Class: "derp_bw",
+		Labels: Labels{
+			"derp_path":  derpPath,
+			"tcp_in_tcp": strconv.FormatBool(d.bwTUNIPv4Prefix != nil),
 		},
-		Class:  "derp_bw",
-		Labels: Labels{"derp_path": derpPath},
 		Metrics: func(l prometheus.Labels) []prometheus.Metric {
 			return []prometheus.Metric{
 				prometheus.MustNewConstMetric(prometheus.NewDesc("derp_bw_probe_size_bytes", "Payload size of the bandwidth prober", nil, l), prometheus.GaugeValue, float64(size)),
-				prometheus.MustNewConstMetric(prometheus.NewDesc("derp_bw_transfer_time_seconds_total", "Time it took to transfer data", nil, l), prometheus.CounterValue, transferTime.Value()),
+				prometheus.MustNewConstMetric(prometheus.NewDesc("derp_bw_transfer_time_seconds_total", "Time it took to transfer data", nil, l), prometheus.CounterValue, transferTimeSeconds.Value()),
+				prometheus.MustNewConstMetric(prometheus.NewDesc("derp_bw_bytes_total", "Amount of data transferred", nil, l), prometheus.CounterValue, float64(size)),
 			}
 		},
 	}
 }
 
+// probeQueuingDelay returns a probe class that continuously sends packets
+// through a pair of DERP servers (or just one server, if 'from' and 'to' are
+// the same) at a rate of `packetsPerSecond` packets per second in order to
+// measure queuing delays. Packets arriving after `packetTimeout` don't contribute
+// to the queuing delay measurement and are recorded as dropped. 'from' and 'to' are
+// expected to be names (DERPNode.Name) of two DERP servers in the same region,
+// and may refer to the same server.
+func (d *derpProber) probeQueuingDelay(from, to string, packetsPerSecond int, packetTimeout time.Duration) ProbeClass {
+	derpPath := "mesh"
+	if from == to {
+		derpPath = "single"
+	}
+	var packetsDropped expvar.Float
+	qdh := newHistogram([]float64{.005, .01, .025, .05, .1, .25, .5, 1})
+	return ProbeClass{
+		Probe: func(ctx context.Context) error {
+			fromN, toN, err := d.getNodePair(from, to)
+			if err != nil {
+				return err
+			}
+			return derpProbeQueuingDelay(ctx, d.lastDERPMap, fromN, toN, packetsPerSecond, packetTimeout, &packetsDropped, qdh)
+		},
+		Class:  "derp_qd",
+		Labels: Labels{"derp_path": derpPath},
+		Metrics: func(l prometheus.Labels) []prometheus.Metric {
+			qdh.mx.Lock()
+			result := []prometheus.Metric{
+				prometheus.MustNewConstMetric(prometheus.NewDesc("derp_qd_probe_dropped_packets", "Total packets dropped", nil, l), prometheus.CounterValue, float64(packetsDropped.Value())),
+				prometheus.MustNewConstHistogram(prometheus.NewDesc("derp_qd_probe_delays_seconds", "Distribution of queuing delays", nil, l), qdh.count, qdh.sum, maps.Clone(qdh.bucketedCounts)),
+			}
+			qdh.mx.Unlock()
+			return result
+		},
+	}
+}
+
+// derpProbeQueuingDelay continuously sends data between two local DERP clients
+// connected to two DERP servers in order to measure queuing delays. From and to
+// can be the same server.
+func derpProbeQueuingDelay(ctx context.Context, dm *tailcfg.DERPMap, from, to *tailcfg.DERPNode, packetsPerSecond int, packetTimeout time.Duration, packetsDropped *expvar.Float, qdh *histogram) (err error) {
+	// This probe uses clients with isProber=false to avoid spamming the derper
+	// logs with every packet sent by the queuing delay probe.
+	fromc, err := newConn(ctx, dm, from, false)
+	if err != nil {
+		return err
+	}
+	defer fromc.Close()
+	toc, err := newConn(ctx, dm, to, false)
+	if err != nil {
+		return err
+	}
+	defer toc.Close()
+
+	// Wait a bit for from's node to hear about to existing on the
+	// other node in the region, in the case where the two nodes
+	// are different.
+	if from.Name != to.Name {
+		time.Sleep(100 * time.Millisecond) // pretty arbitrary
+	}
+
+	if err := runDerpProbeQueuingDelayContinously(ctx, from, to, fromc, toc, packetsPerSecond, packetTimeout, packetsDropped, qdh); err != nil {
+		// Record pubkeys on failed probes to aid investigation.
+		return fmt.Errorf("%s -> %s: %w",
+			fromc.SelfPublicKey().ShortString(),
+			toc.SelfPublicKey().ShortString(), err)
+	}
+	return nil
+}
+
+func runDerpProbeQueuingDelayContinously(ctx context.Context, from, to *tailcfg.DERPNode, fromc, toc *derphttp.Client, packetsPerSecond int, packetTimeout time.Duration, packetsDropped *expvar.Float, qdh *histogram) error {
+	// Make sure all goroutines have finished.
+	var wg sync.WaitGroup
+	defer wg.Wait()
+
+	// Close the clients to make sure goroutines that are reading/writing from them terminate.
+	defer fromc.Close()
+	defer toc.Close()
+
+	type txRecord struct {
+		at  time.Time
+		seq uint64
+	}
+	// txRecords is sized to hold enough transmission records to keep timings
+	// for packets up to their timeout. As records age out of the front of this
+	// list, if the associated packet arrives, we won't have a txRecord for it
+	// and will consider it to have timed out.
+	txRecords := make([]txRecord, 0, packetsPerSecond*int(packetTimeout.Seconds()))
+	var txRecordsMu sync.Mutex
+
+	// Send the packets.
+	sendErrC := make(chan error, 1)
+	// TODO: construct a disco CallMeMaybe in the same fashion as magicsock, e.g. magic bytes, src pub, seal payload.
+	// DERP server handling of disco may vary from non-disco, and we may want to measure queue delay of both.
+	pkt := make([]byte, 260) // the same size as a CallMeMaybe packet observed on a Tailscale client.
+	crand.Read(pkt)
+
+	wg.Add(1)
+	go func() {
+		defer wg.Done()
+		t := time.NewTicker(time.Second / time.Duration(packetsPerSecond))
+		defer t.Stop()
+
+		seq := uint64(0)
+		for {
+			select {
+			case <-ctx.Done():
+				return
+			case <-t.C:
+				txRecordsMu.Lock()
+				if len(txRecords) == cap(txRecords) {
+					txRecords = slices.Delete(txRecords, 0, 1)
+					packetsDropped.Add(1)
+				}
+				txRecords = append(txRecords, txRecord{time.Now(), seq})
+				txRecordsMu.Unlock()
+				binary.BigEndian.PutUint64(pkt, seq)
+				seq++
+				if err := fromc.Send(toc.SelfPublicKey(), pkt); err != nil {
+					sendErrC <- fmt.Errorf("sending packet %w", err)
+					return
+				}
+			}
+		}
+	}()
+
+	// Receive the packets.
+	recvFinishedC := make(chan error, 1)
+	wg.Add(1)
+	go func() {
+		defer wg.Done()
+		defer close(recvFinishedC) // to break out of 'select' below.
+		for {
+			m, err := toc.Recv()
+			if err != nil {
+				recvFinishedC <- err
+				return
+			}
+			switch v := m.(type) {
+			case derp.ReceivedPacket:
+				now := time.Now()
+				if v.Source != fromc.SelfPublicKey() {
+					recvFinishedC <- fmt.Errorf("got data packet from unexpected source, %v", v.Source)
+					return
+				}
+				seq := binary.BigEndian.Uint64(v.Data)
+				txRecordsMu.Lock()
+			findTxRecord:
+				for i, record := range txRecords {
+					switch {
+					case record.seq == seq:
+						rtt := now.Sub(record.at)
+						qdh.add(rtt.Seconds())
+						txRecords = slices.Delete(txRecords, i, i+1)
+						break findTxRecord
+					case record.seq > seq:
+						// No sent time found, probably a late arrival already
+						// recorded as drop by sender when deleted.
+						break findTxRecord
+					case record.seq < seq:
+						continue
+					}
+				}
+				txRecordsMu.Unlock()
+
+			case derp.KeepAliveMessage:
+				// Silently ignore.
+
+			default:
+				log.Printf("%v: ignoring Recv frame type %T", to.Name, v)
+				// Loop.
+			}
+		}
+	}()
+
+	select {
+	case <-ctx.Done():
+		return fmt.Errorf("timeout: %w", ctx.Err())
+	case err := <-sendErrC:
+		return fmt.Errorf("error sending via %q: %w", from.Name, err)
+	case err := <-recvFinishedC:
+		if err != nil {
+			return fmt.Errorf("error receiving from %q: %w", to.Name, err)
+		}
+	}
+	return nil
+}
+
 // getNodePair returns DERPNode objects for two DERP servers based on their
 // short names.
 func (d *derpProber) getNodePair(n1, n2 string) (ret1, ret2 *tailcfg.DERPNode, _ error) {
@@ -412,8 +652,10 @@ func derpProbeUDP(ctx context.Context, ipStr string, port int) error {
 }
 
 // derpProbeBandwidth sends a payload of a given size between two local
-// DERP clients connected to two DERP servers.
-func derpProbeBandwidth(ctx context.Context, dm *tailcfg.DERPMap, from, to *tailcfg.DERPNode, size int64, transferTime *expvar.Float) (err error) {
+// DERP clients connected to two DERP servers.If tunIPv4Address is specified,
+// probes will use a TCP connection over a TUN device at this address in order
+// to exercise TCP-in-TCP in similar fashion to TCP over Tailscale via DERP.
+func derpProbeBandwidth(ctx context.Context, dm *tailcfg.DERPMap, from, to *tailcfg.DERPNode, size int64, transferTimeSeconds *expvar.Float, tunIPv4Prefix *netip.Prefix) (err error) {
 	// This probe uses clients with isProber=false to avoid spamming the derper logs with every packet
 	// sent by the bandwidth probe.
 	fromc, err := newConn(ctx, dm, from, false)
@@ -434,10 +676,13 @@ func derpProbeBandwidth(ctx context.Context, dm *tailcfg.DERPMap, from, to *tail
 		time.Sleep(100 * time.Millisecond) // pretty arbitrary
 	}
 
-	start := time.Now()
-	defer func() { transferTime.Add(time.Since(start).Seconds()) }()
+	if tunIPv4Prefix != nil {
+		err = derpProbeBandwidthTUN(ctx, transferTimeSeconds, from, to, fromc, toc, size, tunIPv4Prefix)
+	} else {
+		err = derpProbeBandwidthDirect(ctx, transferTimeSeconds, from, to, fromc, toc, size)
+	}
 
-	if err := runDerpProbeNodePair(ctx, from, to, fromc, toc, size); err != nil {
+	if err != nil {
 		// Record pubkeys on failed probes to aid investigation.
 		return fmt.Errorf("%s -> %s: %w",
 			fromc.SelfPublicKey().ShortString(),
@@ -544,6 +789,8 @@ func runDerpProbeNodePair(ctx context.Context, from, to *tailcfg.DERPNode, fromc
 					recvc <- fmt.Errorf("got data packet %d from unexpected source, %v", idx, v.Source)
 					return
 				}
+				// This assumes that the packets are received reliably and in order.
+				// The DERP protocol does not guarantee this, but this probe assumes it.
 				if got, want := v.Data, pkts[idx]; !bytes.Equal(got, want) {
 					recvc <- fmt.Errorf("unexpected data packet %d (out of %d)", idx, len(pkts))
 					return
@@ -577,6 +824,272 @@ func runDerpProbeNodePair(ctx context.Context, from, to *tailcfg.DERPNode, fromc
 	return nil
 }
 
+// derpProbeBandwidthDirect takes two DERP clients (fromc and toc) connected to two
+// DERP servers (from and to) and sends a test payload of a given size from one
+// to another using runDerpProbeNodePair. The time taken to finish the transfer is
+// recorded in `transferTimeSeconds`.
+func derpProbeBandwidthDirect(ctx context.Context, transferTimeSeconds *expvar.Float, from, to *tailcfg.DERPNode, fromc, toc *derphttp.Client, size int64) error {
+	start := time.Now()
+	defer func() { transferTimeSeconds.Add(time.Since(start).Seconds()) }()
+
+	return runDerpProbeNodePair(ctx, from, to, fromc, toc, size)
+}
+
+// derpProbeBandwidthTUNMu ensures that TUN bandwidth probes don't run concurrently.
+// This is necessary to avoid conflicts trying to create the TUN device, and
+// it also has the nice benefit of preventing concurrent bandwidth probes from
+// influencing each other's results.
+//
+// This guards derpProbeBandwidthTUN.
+var derpProbeBandwidthTUNMu sync.Mutex
+
+// derpProbeBandwidthTUN takes two DERP clients (fromc and toc) connected to two
+// DERP servers (from and to) and sends a test payload of a given size from one
+// to another over a TUN device at an address at the start of the usable host IP
+// range that the given tunAddress lives in. The time taken to finish the transfer
+// is recorded in `transferTimeSeconds`.
+func derpProbeBandwidthTUN(ctx context.Context, transferTimeSeconds *expvar.Float, from, to *tailcfg.DERPNode, fromc, toc *derphttp.Client, size int64, prefix *netip.Prefix) error {
+	// Make sure all goroutines have finished.
+	var wg sync.WaitGroup
+	defer wg.Wait()
+
+	// Close the clients to make sure goroutines that are reading/writing from them terminate.
+	defer fromc.Close()
+	defer toc.Close()
+
+	ipRange := netipx.RangeOfPrefix(*prefix)
+	// Start of the usable host IP range from the address we have been passed in.
+	ifAddr := ipRange.From().Next()
+	// Destination address to dial. This is the next address in the range from
+	// our ifAddr to ensure that the underlying networking stack is actually being
+	// utilized instead of being optimized away and treated as a loopback. Packets
+	// sent to this address will be routed over the TUN.
+	destinationAddr := ifAddr.Next()
+
+	derpProbeBandwidthTUNMu.Lock()
+	defer derpProbeBandwidthTUNMu.Unlock()
+
+	// Temporarily set up a TUN device with which to simulate a real client TCP connection
+	// tunneling over DERP. Use `tstun.DefaultTUNMTU()` (e.g., 1280) as our MTU as this is
+	// the minimum safe MTU used by Tailscale.
+	dev, err := tun.CreateTUN(tunName, int(tstun.DefaultTUNMTU()))
+	if err != nil {
+		return fmt.Errorf("failed to create TUN device: %w", err)
+	}
+	defer func() {
+		if err := dev.Close(); err != nil {
+			log.Printf("failed to close TUN device: %s", err)
+		}
+	}()
+	mtu, err := dev.MTU()
+	if err != nil {
+		return fmt.Errorf("failed to get TUN MTU: %w", err)
+	}
+
+	name, err := dev.Name()
+	if err != nil {
+		return fmt.Errorf("failed to get device name: %w", err)
+	}
+
+	// Perform platform specific configuration of the TUN device.
+	err = configureTUN(*prefix, name)
+	if err != nil {
+		return fmt.Errorf("failed to configure tun: %w", err)
+	}
+
+	// Depending on platform, we need some space for headers at the front
+	// of TUN I/O op buffers. The below constant is more than enough space
+	// for any platform that this might run on.
+	tunStartOffset := device.MessageTransportHeaderSize
+
+	// This goroutine reads packets from the TUN device and evaluates if they
+	// are IPv4 packets destined for loopback via DERP. If so, it performs L3 NAT
+	// (swap src/dst) and writes them towards DERP in order to loopback via the
+	// `toc` DERP client. It only reports errors to `tunReadErrC`.
+	wg.Add(1)
+	tunReadErrC := make(chan error, 1)
+	go func() {
+		defer wg.Done()
+
+		numBufs := wgconn.IdealBatchSize
+		bufs := make([][]byte, 0, numBufs)
+		sizes := make([]int, numBufs)
+		for range numBufs {
+			bufs = append(bufs, make([]byte, mtu+tunStartOffset))
+		}
+
+		destinationAddrBytes := destinationAddr.AsSlice()
+		scratch := make([]byte, 4)
+		for {
+			n, err := dev.Read(bufs, sizes, tunStartOffset)
+			if err != nil {
+				tunReadErrC <- err
+				return
+			}
+
+			for i := range n {
+				pkt := bufs[i][tunStartOffset : sizes[i]+tunStartOffset]
+				// Skip everything except valid IPv4 packets
+				if len(pkt) < 20 {
+					// Doesn't even have a full IPv4 header
+					continue
+				}
+				if pkt[0]>>4 != 4 {
+					// Not IPv4
+					continue
+				}
+
+				if !bytes.Equal(pkt[16:20], destinationAddrBytes) {
+					// Unexpected dst address
+					continue
+				}
+
+				copy(scratch, pkt[12:16])
+				copy(pkt[12:16], pkt[16:20])
+				copy(pkt[16:20], scratch)
+
+				if err := fromc.Send(toc.SelfPublicKey(), pkt); err != nil {
+					tunReadErrC <- err
+					return
+				}
+			}
+		}
+	}()
+
+	// This goroutine reads packets from the `toc` DERP client and writes them towards the TUN.
+	// It only reports errors to `recvErrC` channel.
+	wg.Add(1)
+	recvErrC := make(chan error, 1)
+	go func() {
+		defer wg.Done()
+
+		buf := make([]byte, mtu+tunStartOffset)
+		bufs := make([][]byte, 1)
+
+		for {
+			m, err := toc.Recv()
+			if err != nil {
+				recvErrC <- fmt.Errorf("failed to receive: %w", err)
+				return
+			}
+			switch v := m.(type) {
+			case derp.ReceivedPacket:
+				if v.Source != fromc.SelfPublicKey() {
+					recvErrC <- fmt.Errorf("got data packet from unexpected source, %v", v.Source)
+					return
+				}
+				pkt := v.Data
+				copy(buf[tunStartOffset:], pkt)
+				bufs[0] = buf[:len(pkt)+tunStartOffset]
+				if _, err := dev.Write(bufs, tunStartOffset); err != nil {
+					recvErrC <- fmt.Errorf("failed to write to TUN device: %w", err)
+					return
+				}
+			case derp.KeepAliveMessage:
+				// Silently ignore.
+			default:
+				log.Printf("%v: ignoring Recv frame type %T", to.Name, v)
+				// Loop.
+			}
+		}
+	}()
+
+	// Start a listener to receive the data
+	l, err := net.Listen("tcp", net.JoinHostPort(ifAddr.String(), "0"))
+	if err != nil {
+		return fmt.Errorf("failed to listen: %s", err)
+	}
+	defer l.Close()
+
+	// 128KB by default
+	const writeChunkSize = 128 << 10
+
+	randData := make([]byte, writeChunkSize)
+	_, err = crand.Read(randData)
+	if err != nil {
+		return fmt.Errorf("failed to initialize random data: %w", err)
+	}
+
+	// Dial ourselves
+	_, port, err := net.SplitHostPort(l.Addr().String())
+	if err != nil {
+		return fmt.Errorf("failed to split address %q: %w", l.Addr().String(), err)
+	}
+
+	connAddr := net.JoinHostPort(destinationAddr.String(), port)
+	conn, err := net.Dial("tcp", connAddr)
+	if err != nil {
+		return fmt.Errorf("failed to dial address %q: %w", connAddr, err)
+	}
+	defer conn.Close()
+
+	// Timing only includes the actual sending and receiving of data.
+	start := time.Now()
+
+	// This goroutine reads data from the TCP stream being looped back via DERP.
+	// It reports to `readFinishedC` when `size` bytes have been read, or if an
+	// error occurs.
+	wg.Add(1)
+	readFinishedC := make(chan error, 1)
+	go func() {
+		defer wg.Done()
+
+		readConn, err := l.Accept()
+		if err != nil {
+			readFinishedC <- err
+		}
+		defer readConn.Close()
+		deadline, ok := ctx.Deadline()
+		if ok {
+			// Don't try reading past our context's deadline.
+			if err := readConn.SetReadDeadline(deadline); err != nil {
+				readFinishedC <- fmt.Errorf("unable to set read deadline: %w", err)
+			}
+		}
+		_, err = io.CopyN(io.Discard, readConn, size)
+		// Measure transfer time irrespective of whether it succeeded or failed.
+		transferTimeSeconds.Add(time.Since(start).Seconds())
+		readFinishedC <- err
+	}()
+
+	// This goroutine sends data to the TCP stream being looped back via DERP.
+	// It only reports errors to `sendErrC`.
+	wg.Add(1)
+	sendErrC := make(chan error, 1)
+	go func() {
+		defer wg.Done()
+
+		for wrote := 0; wrote < int(size); wrote += len(randData) {
+			b := randData
+			if wrote+len(randData) > int(size) {
+				// This is the last chunk and we don't need the whole thing
+				b = b[0 : int(size)-wrote]
+			}
+			if _, err := conn.Write(b); err != nil {
+				sendErrC <- fmt.Errorf("failed to write to conn: %w", err)
+				return
+			}
+		}
+	}()
+
+	select {
+	case <-ctx.Done():
+		return fmt.Errorf("timeout: %w", ctx.Err())
+	case err := <-tunReadErrC:
+		return fmt.Errorf("error reading from TUN via %q: %w", from.Name, err)
+	case err := <-sendErrC:
+		return fmt.Errorf("error sending via %q: %w", from.Name, err)
+	case err := <-recvErrC:
+		return fmt.Errorf("error receiving from %q: %w", to.Name, err)
+	case err := <-readFinishedC:
+		if err != nil {
+			return fmt.Errorf("error reading from %q to TUN: %w", to.Name, err)
+		}
+	}
+
+	return nil
+}
+
 func newConn(ctx context.Context, dm *tailcfg.DERPMap, n *tailcfg.DERPNode, isProber bool) (*derphttp.Client, error) {
 	// To avoid spamming the log with regular connection messages.
 	l := logger.Filtered(log.Printf, func(s string) bool {

prober/histogram.go

@@ -0,0 +1,49 @@
+// Copyright (c) Tailscale Inc & AUTHORS
+// SPDX-License-Identifier: BSD-3-Clause
+
+package prober
+
+import (
+	"slices"
+	"sync"
+)
+
+// histogram serves as an adapter to the Prometheus histogram datatype.
+// The prober framework passes labels at custom metric collection time that
+// it expects to be coupled with the returned metrics. See ProbeClass.Metrics
+// and its call sites. Native prometheus histograms cannot be collected while
+// injecting more labels. Instead we use this type and pass observations +
+// collection labels to prometheus.MustNewConstHistogram() at prometheus
+// metric collection time.
+type histogram struct {
+	count          uint64
+	sum            float64
+	buckets        []float64
+	bucketedCounts map[float64]uint64
+	mx             sync.Mutex
+}
+
+// newHistogram constructs a histogram that buckets data based on the given
+// slice of upper bounds.
+func newHistogram(buckets []float64) *histogram {
+	slices.Sort(buckets)
+	return &histogram{
+		buckets:        buckets,
+		bucketedCounts: make(map[float64]uint64, len(buckets)),
+	}
+}
+
+func (h *histogram) add(v float64) {
+	h.mx.Lock()
+	defer h.mx.Unlock()
+
+	h.count++
+	h.sum += v
+
+	for _, b := range h.buckets {
+		if v > b {
+			continue
+		}
+		h.bucketedCounts[b] += 1
+	}
+}

prober/histogram_test.go

@@ -0,0 +1,29 @@
+// Copyright (c) Tailscale Inc & AUTHORS
+// SPDX-License-Identifier: BSD-3-Clause
+
+package prober
+
+import (
+	"testing"
+
+	"github.com/google/go-cmp/cmp"
+)
+
+func TestHistogram(t *testing.T) {
+	h := newHistogram([]float64{1, 2})
+	h.add(0.5)
+	h.add(1)
+	h.add(1.5)
+	h.add(2)
+	h.add(2.5)
+
+	if diff := cmp.Diff(h.count, uint64(5)); diff != "" {
+		t.Errorf("wrong count; (-got+want):%v", diff)
+	}
+	if diff := cmp.Diff(h.sum, 7.5); diff != "" {
+		t.Errorf("wrong sum; (-got+want):%v", diff)
+	}
+	if diff := cmp.Diff(h.bucketedCounts, map[float64]uint64{1: 2, 2: 4}); diff != "" {
+		t.Errorf("wrong bucketedCounts; (-got+want):%v", diff)
+	}
+}

prober/prober.go

@@ -94,6 +94,9 @@ func newForTest(now func() time.Time, newTicker func(time.Duration) ticker) *Pro
 
 // Run executes probe class function every interval, and exports probe results under probeName.
 //
+// If interval is negative, the probe will run continuously. If it encounters a failure while
+// running continuously, it will pause for -1*interval and then retry.
+//
 // Registering a probe under an already-registered name panics.
 func (p *Prober) Run(name string, interval time.Duration, labels Labels, pc ProbeClass) *Probe {
 	p.mu.Lock()
@@ -256,6 +259,11 @@ type Probe struct {
 	latencyHist *ring.Ring
 }
 
+// IsContinuous indicates that this is a continuous probe.
+func (p *Probe) IsContinuous() bool {
+	return p.interval < 0
+}
+
 // Close shuts down the Probe and unregisters it from its Prober.
 // It is safe to Run a new probe of the same name after Close returns.
 func (p *Probe) Close() error {
@@ -288,6 +296,22 @@ func (p *Probe) loop() {
 		return
 	}
 
+	if p.IsContinuous() {
+		// Probe function is going to run continuously.
+		for {
+			p.run()
+			// Wait and then retry if probe fails. We use the inverse of the
+			// configured negative interval as our sleep period.
+			// TODO(percy):implement exponential backoff, possibly using logtail/backoff.
+			select {
+			case <-time.After(-1 * p.interval):
+				p.run()
+			case <-p.ctx.Done():
+				return
+			}
+		}
+	}
+
 	p.tick = p.prober.newTicker(p.interval)
 	defer p.tick.Stop()
 	for {
@@ -323,9 +347,13 @@ func (p *Probe) run() (pi ProbeInfo, err error) {
 			p.recordEnd(err)
 		}
 	}()
-	timeout := time.Duration(float64(p.interval) * 0.8)
-	ctx, cancel := context.WithTimeout(p.ctx, timeout)
-	defer cancel()
+	ctx := p.ctx
+	if !p.IsContinuous() {
+		timeout := time.Duration(float64(p.interval) * 0.8)
+		var cancel func()
+		ctx, cancel = context.WithTimeout(ctx, timeout)
+		defer cancel()
+	}
 
 	err = p.probeClass.Probe(ctx)
 	p.recordEnd(err)
@@ -365,6 +393,16 @@ func (p *Probe) recordEnd(err error) {
 	p.successHist = p.successHist.Next()
 }
 
+// ProbeStatus indicates the status of a probe.
+type ProbeStatus string
+
+const (
+	ProbeStatusUnknown   = "unknown"
+	ProbeStatusRunning   = "running"
+	ProbeStatusFailed    = "failed"
+	ProbeStatusSucceeded = "succeeded"
+)
+
 // ProbeInfo is a snapshot of the configuration and state of a Probe.
 type ProbeInfo struct {
 	Name            string
@@ -374,7 +412,7 @@ type ProbeInfo struct {
 	Start           time.Time
 	End             time.Time
 	Latency         time.Duration
-	Result          bool
+	Status          ProbeStatus
 	Error           string
 	RecentResults   []bool
 	RecentLatencies []time.Duration
@@ -402,6 +440,10 @@ func (pb ProbeInfo) RecentMedianLatency() time.Duration {
 	return pb.RecentLatencies[len(pb.RecentLatencies)/2]
 }
 
+func (pb ProbeInfo) Continuous() bool {
+	return pb.Interval < 0
+}
+
 // ProbeInfo returns the state of all probes.
 func (p *Prober) ProbeInfo() map[string]ProbeInfo {
 	out := map[string]ProbeInfo{}
@@ -429,9 +471,14 @@ func (probe *Probe) probeInfoLocked() ProbeInfo {
 		Labels:   probe.metricLabels,
 		Start:    probe.start,
 		End:      probe.end,
-		Result:   probe.succeeded,
 	}
-	if probe.lastErr != nil {
+	inf.Status = ProbeStatusUnknown
+	if probe.end.Before(probe.start) {
+		inf.Status = ProbeStatusRunning
+	} else if probe.succeeded {
+		inf.Status = ProbeStatusSucceeded
+	} else if probe.lastErr != nil {
+		inf.Status = ProbeStatusFailed
 		inf.Error = probe.lastErr.Error()
 	}
 	if probe.latency > 0 {
@@ -467,7 +514,7 @@ func (p *Prober) RunHandler(w http.ResponseWriter, r *http.Request) error {
 	p.mu.Lock()
 	probe, ok := p.probes[name]
 	p.mu.Unlock()
-	if !ok {
+	if !ok || probe.IsContinuous() {
 		return tsweb.Error(http.StatusNotFound, fmt.Sprintf("unknown probe %q", name), nil)
 	}
 
@@ -531,7 +578,8 @@ func (p *Probe) Collect(ch chan<- prometheus.Metric) {
 	if !p.start.IsZero() {
 		ch <- prometheus.MustNewConstMetric(p.mStartTime, prometheus.GaugeValue, float64(p.start.Unix()))
 	}
-	if p.end.IsZero() {
+	// For periodic probes that haven't ended, don't collect probe metrics yet.
+	if p.end.IsZero() && !p.IsContinuous() {
 		return
 	}
 	ch <- prometheus.MustNewConstMetric(p.mEndTime, prometheus.GaugeValue, float64(p.end.Unix()))

prober/prober_test.go

@@ -316,7 +316,7 @@ func TestProberProbeInfo(t *testing.T) {
 			Interval:        probeInterval,
 			Labels:          map[string]string{"class": "", "name": "probe1"},
 			Latency:         500 * time.Millisecond,
-			Result:          true,
+			Status:          ProbeStatusSucceeded,
 			RecentResults:   []bool{true},
 			RecentLatencies: []time.Duration{500 * time.Millisecond},
 		},
@@ -324,6 +324,7 @@ func TestProberProbeInfo(t *testing.T) {
 			Name:            "probe2",
 			Interval:        probeInterval,
 			Labels:          map[string]string{"class": "", "name": "probe2"},
+			Status:          ProbeStatusFailed,
 			Error:           "error2",
 			RecentResults:   []bool{false},
 			RecentLatencies: nil, // no latency for failed probes
@@ -349,7 +350,7 @@ func TestProbeInfoRecent(t *testing.T) {
 	}{
 		{
 			name:                    "no_runs",
-			wantProbeInfo:           ProbeInfo{},
+			wantProbeInfo:           ProbeInfo{Status: ProbeStatusUnknown},
 			wantRecentSuccessRatio:  0,
 			wantRecentMedianLatency: 0,
 		},
@@ -358,7 +359,7 @@ func TestProbeInfoRecent(t *testing.T) {
 			results: []probeResult{{latency: 100 * time.Millisecond, err: nil}},
 			wantProbeInfo: ProbeInfo{
 				Latency:         100 * time.Millisecond,
-				Result:          true,
+				Status:          ProbeStatusSucceeded,
 				RecentResults:   []bool{true},
 				RecentLatencies: []time.Duration{100 * time.Millisecond},
 			},
@@ -369,7 +370,7 @@ func TestProbeInfoRecent(t *testing.T) {
 			name:    "single_failure",
 			results: []probeResult{{latency: 100 * time.Millisecond, err: errors.New("error123")}},
 			wantProbeInfo: ProbeInfo{
-				Result:          false,
+				Status:          ProbeStatusFailed,
 				RecentResults:   []bool{false},
 				RecentLatencies: nil,
 				Error:           "error123",
@@ -390,7 +391,7 @@ func TestProbeInfoRecent(t *testing.T) {
 				{latency: 80 * time.Millisecond, err: nil},
 			},
 			wantProbeInfo: ProbeInfo{
-				Result:        true,
+				Status:        ProbeStatusSucceeded,
 				Latency:       80 * time.Millisecond,
 				RecentResults: []bool{false, true, true, false, true, true, false, true},
 				RecentLatencies: []time.Duration{
@@ -420,7 +421,7 @@ func TestProbeInfoRecent(t *testing.T) {
 				{latency: 110 * time.Millisecond, err: nil},
 			},
 			wantProbeInfo: ProbeInfo{
-				Result:        true,
+				Status:        ProbeStatusSucceeded,
 				Latency:       110 * time.Millisecond,
 				RecentResults: []bool{true, true, true, true, true, true, true, true, true, true},
 				RecentLatencies: []time.Duration{
@@ -483,7 +484,7 @@ func TestProberRunHandler(t *testing.T) {
 				ProbeInfo: ProbeInfo{
 					Name:          "success",
 					Interval:      probeInterval,
-					Result:        true,
+					Status:        ProbeStatusSucceeded,
 					RecentResults: []bool{true, true},
 				},
 				PreviousSuccessRatio: 1,
@@ -498,7 +499,7 @@ func TestProberRunHandler(t *testing.T) {
 				ProbeInfo: ProbeInfo{
 					Name:          "failure",
 					Interval:      probeInterval,
-					Result:        false,
+					Status:        ProbeStatusFailed,
 					Error:         "error123",
 					RecentResults: []bool{false, false},
 				},

prober/status.go

@@ -62,8 +62,9 @@ func (p *Prober) StatusHandler(opts ...statusHandlerOpt) tsweb.ReturnHandlerFunc
 	return func(w http.ResponseWriter, r *http.Request) error {
 		type probeStatus struct {
 			ProbeInfo
-			TimeSinceLast time.Duration
-			Links         map[string]template.URL
+			TimeSinceLastStart time.Duration
+			TimeSinceLastEnd   time.Duration
+			Links              map[string]template.URL
 		}
 		vars := struct {
 			Title           string
@@ -81,12 +82,15 @@ func (p *Prober) StatusHandler(opts ...statusHandlerOpt) tsweb.ReturnHandlerFunc
 
 		for name, info := range p.ProbeInfo() {
 			vars.TotalProbes++
-			if !info.Result {
+			if info.Error != "" {
 				vars.UnhealthyProbes++
 			}
 			s := probeStatus{ProbeInfo: info}
+			if !info.Start.IsZero() {
+				s.TimeSinceLastStart = time.Since(info.Start).Truncate(time.Second)
+			}
 			if !info.End.IsZero() {
-				s.TimeSinceLast = time.Since(info.End).Truncate(time.Second)
+				s.TimeSinceLastEnd = time.Since(info.End).Truncate(time.Second)
 			}
 			for textTpl, urlTpl := range params.probeLinks {
 				text, err := renderTemplate(textTpl, info)

prober/status.html

@@ -73,8 +73,9 @@
             <th>Name</th>
             <th>Probe Class & Labels</th>
             <th>Interval</th>
-            <th>Last Attempt</th>
-            <th>Success</th>
+            <th>Last Finished</th>
+            <th>Last Started</th>
+            <th>Status</th>
             <th>Latency</th>
             <th>Last Error</th>
         </tr></thead>
@@ -85,9 +86,11 @@
                 {{$name}}
                 {{range $text, $url := $probeInfo.Links}}
                 <br/>
-                <button onclick="location.href='{{$url}}';" type="button">
-                    {{$text}}
-                </button>
+                {{if not $probeInfo.Continuous}}
+                    <button onclick="location.href='{{$url}}';" type="button">
+                        {{$text}}
+                    </button>
+                {{end}}
                 {{end}}
             </td>
             <td>{{$probeInfo.Class}}<br/>
@@ -97,28 +100,48 @@
                 {{end}}
                 </div>
             </td>
-            <td>{{$probeInfo.Interval}}</td>
-            <td data-sort="{{$probeInfo.TimeSinceLast.Milliseconds}}">
-                {{if $probeInfo.TimeSinceLast}}
-                    {{$probeInfo.TimeSinceLast.String}} ago<br/>
+            <td>
+                {{if $probeInfo.Continuous}}
+                    Continuous
+                {{else}}
+                    {{$probeInfo.Interval}}
+                {{end}}
+            </td>
+            <td data-sort="{{$probeInfo.TimeSinceLastEnd.Milliseconds}}">
+                {{if $probeInfo.TimeSinceLastEnd}}
+                    {{$probeInfo.TimeSinceLastEnd.String}} ago<br/>
                     <span class="small">{{$probeInfo.End.Format "2006-01-02T15:04:05Z07:00"}}</span>
                 {{else}}
                     Never
                 {{end}}
             </td>
-            <td>
-                {{if $probeInfo.Result}}
-                    {{$probeInfo.Result}}
+            <td data-sort="{{$probeInfo.TimeSinceLastStart.Milliseconds}}">
+                {{if $probeInfo.TimeSinceLastStart}}
+                    {{$probeInfo.TimeSinceLastStart.String}} ago<br/>
+                    <span class="small">{{$probeInfo.Start.Format "2006-01-02T15:04:05Z07:00"}}</span>
                 {{else}}
-                    <span class="error">{{$probeInfo.Result}}</span>
+                    Never
+                {{end}}
+            </td>
+            <td>
+                {{if $probeInfo.Error}}
+                    <span class="error">{{$probeInfo.Status}}</span>
+                {{else}}
+                    {{$probeInfo.Status}}
                 {{end}}<br/>
-                <div class="small">Recent: {{$probeInfo.RecentResults}}</div>
-                <div class="small">Mean: {{$probeInfo.RecentSuccessRatio}}</div>
+                {{if not $probeInfo.Continuous}}
+                    <div class="small">Recent: {{$probeInfo.RecentResults}}</div>
+                    <div class="small">Mean: {{$probeInfo.RecentSuccessRatio}}</div>
+                {{end}}
             </td>
             <td data-sort="{{$probeInfo.Latency.Milliseconds}}">
-                {{$probeInfo.Latency.String}}
-                <div class="small">Recent: {{$probeInfo.RecentLatencies}}</div>
-                <div class="small">Median: {{$probeInfo.RecentMedianLatency}}</div>
+                {{if $probeInfo.Continuous}}
+                    n/a
+                {{else}}
+                    {{$probeInfo.Latency.String}}
+                    <div class="small">Recent: {{$probeInfo.RecentLatencies}}</div>
+                    <div class="small">Median: {{$probeInfo.RecentMedianLatency}}</div>
+                {{end}}
             </td>
             <td class="small">{{$probeInfo.Error}}</td>
         </tr>

prober/tun_darwin.go

@@ -0,0 +1,35 @@
+// Copyright (c) Tailscale Inc & AUTHORS
+// SPDX-License-Identifier: BSD-3-Clause
+
+//go:build darwin
+
+package prober
+
+import (
+	"fmt"
+	"net/netip"
+	"os/exec"
+
+	"go4.org/netipx"
+)
+
+const tunName = "utun"
+
+func configureTUN(addr netip.Prefix, tunname string) error {
+	cmd := exec.Command("ifconfig", tunname, "inet", addr.String(), addr.Addr().String())
+	res, err := cmd.CombinedOutput()
+	if err != nil {
+		return fmt.Errorf("failed to add address: %w (%s)", err, string(res))
+	}
+
+	net := netipx.PrefixIPNet(addr)
+	nip := net.IP.Mask(net.Mask)
+	nstr := fmt.Sprintf("%v/%d", nip, addr.Bits())
+	cmd = exec.Command("route", "-q", "-n", "add", "-inet", nstr, "-iface", addr.Addr().String())
+	res, err = cmd.CombinedOutput()
+	if err != nil {
+		return fmt.Errorf("failed to add route: %w (%s)", err, string(res))
+	}
+
+	return nil
+}

prober/tun_default.go

@@ -0,0 +1,18 @@
+// Copyright (c) Tailscale Inc & AUTHORS
+// SPDX-License-Identifier: BSD-3-Clause
+
+//go:build !linux && !darwin
+
+package prober
+
+import (
+	"fmt"
+	"net/netip"
+	"runtime"
+)
+
+const tunName = "unused"
+
+func configureTUN(addr netip.Prefix, tunname string) error {
+	return fmt.Errorf("not implemented on " + runtime.GOOS)
+}

prober/tun_linux.go

@@ -0,0 +1,36 @@
+// Copyright (c) Tailscale Inc & AUTHORS
+// SPDX-License-Identifier: BSD-3-Clause
+
+//go:build linux
+
+package prober
+
+import (
+	"fmt"
+	"net/netip"
+
+	"github.com/tailscale/netlink"
+	"go4.org/netipx"
+)
+
+const tunName = "derpprobe"
+
+func configureTUN(addr netip.Prefix, tunname string) error {
+	link, err := netlink.LinkByName(tunname)
+	if err != nil {
+		return fmt.Errorf("failed to look up link %q: %w", tunname, err)
+	}
+
+	// We need to bring the TUN device up before assigning an address. This
+	// allows the OS to automatically create a route for it. Otherwise, we'd
+	// have to manually create the route.
+	if err := netlink.LinkSetUp(link); err != nil {
+		return fmt.Errorf("failed to bring tun %q up: %w", tunname, err)
+	}
+
+	if err := netlink.AddrReplace(link, &netlink.Addr{IPNet: netipx.PrefixIPNet(addr)}); err != nil {
+		return fmt.Errorf("failed to add address: %w", err)
+	}
+
+	return nil
+}

scripts/installer.sh

@@ -68,6 +68,14 @@ main() {
 				if [ -z "${VERSION_ID:-}" ]; then
 					# rolling release. If you haven't kept current, that's on you.
 					APT_KEY_TYPE="keyring"
+				# Parrot Security is a special case that uses ID=debian
+				elif [ "$NAME" = "Parrot Security" ]; then
+					# All versions new enough to have this behaviour prefer keyring
+					# and their VERSION_ID is not consistent with Debian.
+					APT_KEY_TYPE="keyring"
+					# They don't specify the Debian version they're based off in os-release
+					# but Parrot 6 is based on Debian 12 Bookworm.
+					VERSION=bookworm
 				elif [ "$VERSION_ID" -lt 11 ]; then
 					APT_KEY_TYPE="legacy"
 				else
@@ -154,7 +162,7 @@ main() {
 					APT_KEY_TYPE="keyring"
 				fi
 				;;
-			Deepin)  # https://github.com/tailscale/tailscale/issues/7862
+			Deepin|deepin)  # https://github.com/tailscale/tailscale/issues/7862
 				OS="debian"
 				PACKAGETYPE="apt"
 				if [ "$VERSION_ID" -lt 20 ]; then
@@ -165,6 +173,19 @@ main() {
 					VERSION="bullseye"
 				fi
 				;;
+			pika)
+				PACKAGETYPE="apt"
+				# All versions of PikaOS are new enough to prefer keyring
+				APT_KEY_TYPE="keyring"
+				# Older versions of PikaOS are based on Ubuntu rather than Debian
+				if [ "$VERSION_ID" -lt 4 ]; then
+					OS="ubuntu"
+					VERSION="$UBUNTU_CODENAME"
+				else
+					OS="debian"
+					VERSION="$DEBIAN_CODENAME"
+				fi
+				;;
 			centos)
 				OS="$ID"
 				VERSION="$VERSION_ID"
@@ -224,7 +245,7 @@ main() {
 				VERSION="leap/15.4"
 				PACKAGETYPE="zypper"
 				;;
-			arch|archarm|endeavouros|blendos|garuda|archcraft)
+			arch|archarm|endeavouros|blendos|garuda|archcraft|cachyos)
 				OS="arch"
 				VERSION="" # rolling release
 				PACKAGETYPE="pacman"

ssh/tailssh/incubator.go

@@ -1014,10 +1014,10 @@ func (ss *sshSession) startWithStdPipes() (err error) {
 
 func envForUser(u *userMeta) []string {
 	return []string{
-		fmt.Sprintf("SHELL=" + u.LoginShell()),
-		fmt.Sprintf("USER=" + u.Username),
-		fmt.Sprintf("HOME=" + u.HomeDir),
-		fmt.Sprintf("PATH=" + defaultPathForUser(&u.User)),
+		fmt.Sprintf("SHELL=%s", u.LoginShell()),
+		fmt.Sprintf("USER=%s", u.Username),
+		fmt.Sprintf("HOME=%s", u.HomeDir),
+		fmt.Sprintf("PATH=%s", defaultPathForUser(&u.User)),
 	}
 }